1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/FunctionLoweringInfo.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineConstantPool.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/RuntimeLibcallUtil.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetFrameLowering.h"
46#include "llvm/CodeGen/TargetLowering.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGen/ValueTypes.h"
50#include "llvm/CodeGenTypes/MachineValueType.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfoMetadata.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
61#include "llvm/Support/Casting.h"
62#include "llvm/Support/CodeGen.h"
63#include "llvm/Support/Compiler.h"
64#include "llvm/Support/Debug.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/KnownFPClass.h"
68#include "llvm/Support/MathExtras.h"
69#include "llvm/Support/raw_ostream.h"
70#include "llvm/Target/TargetMachine.h"
71#include "llvm/Target/TargetOptions.h"
72#include "llvm/TargetParser/Triple.h"
73#include "llvm/Transforms/Utils/SizeOpts.h"
74#include <algorithm>
75#include <cassert>
76#include <cstdint>
77#include <cstdlib>
78#include <limits>
79#include <optional>
80#include <string>
81#include <utility>
82#include <vector>
83
84using namespace llvm;
85using namespace llvm::SDPatternMatch;
86
87/// makeVTList - Return an instance of the SDVTList struct initialized with the
88/// specified members.
89static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
90 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
91 return Res;
92}
93
94// Default null implementations of the callbacks.
95void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
96void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
97void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
98
99void SelectionDAG::DAGNodeDeletedListener::anchor() {}
100void SelectionDAG::DAGNodeInsertedListener::anchor() {}
101
102#define DEBUG_TYPE "selectiondag"
103
104static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
105 cl::Hidden, cl::init(Val: true),
106 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
107
108static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
109 cl::desc("Number limit for gluing ld/st of memcpy."),
110 cl::Hidden, cl::init(Val: 0));
111
112static cl::opt<unsigned>
113 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
114 cl::desc("DAG combiner limit number of steps when searching DAG "
115 "for predecessor nodes"));
116
117static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
118 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
119}
120
121unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
122
123//===----------------------------------------------------------------------===//
124// ConstantFPSDNode Class
125//===----------------------------------------------------------------------===//
126
127/// isExactlyValue - We don't rely on operator== working on double values, as
128/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
129/// As such, this method can be used to do an exact bit-for-bit comparison of
130/// two floating point values.
131bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
132 return getValueAPF().bitwiseIsEqual(RHS: V);
133}
134
135bool ConstantFPSDNode::isValueValidForType(EVT VT,
136 const APFloat& Val) {
137 assert(VT.isFloatingPoint() && "Can only convert between FP types");
138
139 // convert modifies in place, so make a copy.
140 APFloat Val2 = APFloat(Val);
141 bool losesInfo;
142 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
143 losesInfo: &losesInfo);
144 return !losesInfo;
145}
146
147//===----------------------------------------------------------------------===//
148// ISD Namespace
149//===----------------------------------------------------------------------===//
150
151bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
152 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
153 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
154 unsigned EltSize =
155 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
156 SplatVal = OptAPInt->trunc(width: EltSize);
157 return true;
158 }
159 }
160
161 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
162 if (!BV)
163 return false;
164
165 APInt SplatUndef;
166 unsigned SplatBitSize;
167 bool HasUndefs;
168 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
169 // Endianness does not matter here. We are checking for a splat given the
170 // element size of the vector, and if we find such a splat for little endian
171 // layout, then that should be valid also for big endian (as the full vector
172 // size is known to be a multiple of the element size).
173 const bool IsBigEndian = false;
174 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
175 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
176 EltSize == SplatBitSize;
177}
178
179// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
180// specializations of the more general isConstantSplatVector()?
181
182bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
183 // Look through a bit convert.
184 while (N->getOpcode() == ISD::BITCAST)
185 N = N->getOperand(Num: 0).getNode();
186
187 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
188 APInt SplatVal;
189 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
190 }
191
192 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
193
194 unsigned i = 0, e = N->getNumOperands();
195
196 // Skip over all of the undef values.
197 while (i != e && N->getOperand(Num: i).isUndef())
198 ++i;
199
200 // Do not accept an all-undef vector.
201 if (i == e) return false;
202
203 // Do not accept build_vectors that aren't all constants or which have non-~0
204 // elements. We have to be a bit careful here, as the type of the constant
205 // may not be the same as the type of the vector elements due to type
206 // legalization (the elements are promoted to a legal type for the target and
207 // a vector of a type may be legal when the base element type is not).
208 // We only want to check enough bits to cover the vector elements, because
209 // we care if the resultant vector is all ones, not whether the individual
210 // constants are.
211 SDValue NotZero = N->getOperand(Num: i);
212 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
213 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
214 if (OptAPInt->countr_one() < EltSize)
215 return false;
216 } else
217 return false;
218
219 // Okay, we have at least one ~0 value, check to see if the rest match or are
220 // undefs. Even with the above element type twiddling, this should be OK, as
221 // the same type legalization should have applied to all the elements.
222 for (++i; i != e; ++i)
223 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
224 return false;
225 return true;
226}
227
228bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
229 // Look through a bit convert.
230 while (N->getOpcode() == ISD::BITCAST)
231 N = N->getOperand(Num: 0).getNode();
232
233 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
234 APInt SplatVal;
235 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
236 }
237
238 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
239
240 bool IsAllUndef = true;
241 for (const SDValue &Op : N->op_values()) {
242 if (Op.isUndef())
243 continue;
244 IsAllUndef = false;
245 // Do not accept build_vectors that aren't all constants or which have non-0
246 // elements. We have to be a bit careful here, as the type of the constant
247 // may not be the same as the type of the vector elements due to type
248 // legalization (the elements are promoted to a legal type for the target
249 // and a vector of a type may be legal when the base element type is not).
250 // We only want to check enough bits to cover the vector elements, because
251 // we care if the resultant vector is all zeros, not whether the individual
252 // constants are.
253 if (auto OptAPInt = Op->bitcastToAPInt()) {
254 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
255 if (OptAPInt->countr_zero() < EltSize)
256 return false;
257 } else
258 return false;
259 }
260
261 // Do not accept an all-undef vector.
262 if (IsAllUndef)
263 return false;
264 return true;
265}
266
267bool ISD::isBuildVectorAllOnes(const SDNode *N) {
268 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
269}
270
271bool ISD::isBuildVectorAllZeros(const SDNode *N) {
272 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
273}
274
275bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
276 if (N->getOpcode() != ISD::BUILD_VECTOR)
277 return false;
278
279 for (const SDValue &Op : N->op_values()) {
280 if (Op.isUndef())
281 continue;
282 if (!isa<ConstantSDNode>(Val: Op))
283 return false;
284 }
285 return true;
286}
287
288bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
289 if (N->getOpcode() != ISD::BUILD_VECTOR)
290 return false;
291
292 for (const SDValue &Op : N->op_values()) {
293 if (Op.isUndef())
294 continue;
295 if (!isa<ConstantFPSDNode>(Val: Op))
296 return false;
297 }
298 return true;
299}
300
301bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
302 bool Signed) {
303 assert(N->getValueType(0).isVector() && "Expected a vector!");
304
305 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
306 if (EltSize <= NewEltSize)
307 return false;
308
309 if (N->getOpcode() == ISD::ZERO_EXTEND) {
310 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
311 NewEltSize) &&
312 !Signed;
313 }
314 if (N->getOpcode() == ISD::SIGN_EXTEND) {
315 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
316 NewEltSize) &&
317 Signed;
318 }
319 if (N->getOpcode() != ISD::BUILD_VECTOR)
320 return false;
321
322 for (const SDValue &Op : N->op_values()) {
323 if (Op.isUndef())
324 continue;
325 if (!isa<ConstantSDNode>(Val: Op))
326 return false;
327
328 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
329 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
330 return false;
331 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
332 return false;
333 }
334
335 return true;
336}
337
338bool ISD::allOperandsUndef(const SDNode *N) {
339 // Return false if the node has no operands.
340 // This is "logically inconsistent" with the definition of "all" but
341 // is probably the desired behavior.
342 if (N->getNumOperands() == 0)
343 return false;
344 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
345}
346
347bool ISD::isFreezeUndef(const SDNode *N) {
348 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
349}
350
351template <typename ConstNodeType>
352bool ISD::matchUnaryPredicateImpl(SDValue Op,
353 std::function<bool(ConstNodeType *)> Match,
354 bool AllowUndefs, bool AllowTruncation) {
355 // FIXME: Add support for scalar UNDEF cases?
356 if (auto *C = dyn_cast<ConstNodeType>(Op))
357 return Match(C);
358
359 // FIXME: Add support for vector UNDEF cases?
360 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
361 ISD::SPLAT_VECTOR != Op.getOpcode())
362 return false;
363
364 EVT SVT = Op.getValueType().getScalarType();
365 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
366 if (AllowUndefs && Op.getOperand(i).isUndef()) {
367 if (!Match(nullptr))
368 return false;
369 continue;
370 }
371
372 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
373 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
374 !Match(Cst))
375 return false;
376 }
377 return true;
378}
379// Build used template types.
380template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
381 SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);
382template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
383 SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);
384
385bool ISD::matchBinaryPredicate(
386 SDValue LHS, SDValue RHS,
387 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
388 bool AllowUndefs, bool AllowTypeMismatch) {
389 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
390 return false;
391
392 // TODO: Add support for scalar UNDEF cases?
393 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
394 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
395 return Match(LHSCst, RHSCst);
396
397 // TODO: Add support for vector UNDEF cases?
398 if (LHS.getOpcode() != RHS.getOpcode() ||
399 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
400 LHS.getOpcode() != ISD::SPLAT_VECTOR))
401 return false;
402
403 EVT SVT = LHS.getValueType().getScalarType();
404 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
405 SDValue LHSOp = LHS.getOperand(i);
406 SDValue RHSOp = RHS.getOperand(i);
407 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
408 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
409 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
410 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
411 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
412 return false;
413 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
414 LHSOp.getValueType() != RHSOp.getValueType()))
415 return false;
416 if (!Match(LHSCst, RHSCst))
417 return false;
418 }
419 return true;
420}
421
422ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
423 switch (MinMaxOpc) {
424 default:
425 llvm_unreachable("unrecognized opcode");
426 case ISD::UMIN:
427 return ISD::UMAX;
428 case ISD::UMAX:
429 return ISD::UMIN;
430 case ISD::SMIN:
431 return ISD::SMAX;
432 case ISD::SMAX:
433 return ISD::SMIN;
434 }
435}
436
437ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
438 switch (MinMaxOpc) {
439 default:
440 llvm_unreachable("unrecognized min/max opcode");
441 case ISD::SMIN:
442 return ISD::UMIN;
443 case ISD::SMAX:
444 return ISD::UMAX;
445 case ISD::UMIN:
446 return ISD::SMIN;
447 case ISD::UMAX:
448 return ISD::SMAX;
449 }
450}
451
452ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
453 switch (VecReduceOpcode) {
454 default:
455 llvm_unreachable("Expected VECREDUCE opcode");
456 case ISD::VECREDUCE_FADD:
457 case ISD::VECREDUCE_SEQ_FADD:
458 case ISD::VP_REDUCE_FADD:
459 case ISD::VP_REDUCE_SEQ_FADD:
460 return ISD::FADD;
461 case ISD::VECREDUCE_FMUL:
462 case ISD::VECREDUCE_SEQ_FMUL:
463 case ISD::VP_REDUCE_FMUL:
464 case ISD::VP_REDUCE_SEQ_FMUL:
465 return ISD::FMUL;
466 case ISD::VECREDUCE_ADD:
467 case ISD::VP_REDUCE_ADD:
468 return ISD::ADD;
469 case ISD::VECREDUCE_MUL:
470 case ISD::VP_REDUCE_MUL:
471 return ISD::MUL;
472 case ISD::VECREDUCE_AND:
473 case ISD::VP_REDUCE_AND:
474 return ISD::AND;
475 case ISD::VECREDUCE_OR:
476 case ISD::VP_REDUCE_OR:
477 return ISD::OR;
478 case ISD::VECREDUCE_XOR:
479 case ISD::VP_REDUCE_XOR:
480 return ISD::XOR;
481 case ISD::VECREDUCE_SMAX:
482 case ISD::VP_REDUCE_SMAX:
483 return ISD::SMAX;
484 case ISD::VECREDUCE_SMIN:
485 case ISD::VP_REDUCE_SMIN:
486 return ISD::SMIN;
487 case ISD::VECREDUCE_UMAX:
488 case ISD::VP_REDUCE_UMAX:
489 return ISD::UMAX;
490 case ISD::VECREDUCE_UMIN:
491 case ISD::VP_REDUCE_UMIN:
492 return ISD::UMIN;
493 case ISD::VECREDUCE_FMAX:
494 case ISD::VP_REDUCE_FMAX:
495 return ISD::FMAXNUM;
496 case ISD::VECREDUCE_FMIN:
497 case ISD::VP_REDUCE_FMIN:
498 return ISD::FMINNUM;
499 case ISD::VECREDUCE_FMAXIMUM:
500 case ISD::VP_REDUCE_FMAXIMUM:
501 return ISD::FMAXIMUM;
502 case ISD::VECREDUCE_FMINIMUM:
503 case ISD::VP_REDUCE_FMINIMUM:
504 return ISD::FMINIMUM;
505 }
506}
507
508bool ISD::isVPOpcode(unsigned Opcode) {
509 switch (Opcode) {
510 default:
511 return false;
512#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
513 case ISD::VPSD: \
514 return true;
515#include "llvm/IR/VPIntrinsics.def"
516 }
517}
518
519bool ISD::isVPBinaryOp(unsigned Opcode) {
520 switch (Opcode) {
521 default:
522 break;
523#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
524#define VP_PROPERTY_BINARYOP return true;
525#define END_REGISTER_VP_SDNODE(VPSD) break;
526#include "llvm/IR/VPIntrinsics.def"
527 }
528 return false;
529}
530
531bool ISD::isVPReduction(unsigned Opcode) {
532 switch (Opcode) {
533 default:
534 return false;
535 case ISD::VP_REDUCE_ADD:
536 case ISD::VP_REDUCE_MUL:
537 case ISD::VP_REDUCE_AND:
538 case ISD::VP_REDUCE_OR:
539 case ISD::VP_REDUCE_XOR:
540 case ISD::VP_REDUCE_SMAX:
541 case ISD::VP_REDUCE_SMIN:
542 case ISD::VP_REDUCE_UMAX:
543 case ISD::VP_REDUCE_UMIN:
544 case ISD::VP_REDUCE_FMAX:
545 case ISD::VP_REDUCE_FMIN:
546 case ISD::VP_REDUCE_FMAXIMUM:
547 case ISD::VP_REDUCE_FMINIMUM:
548 case ISD::VP_REDUCE_FADD:
549 case ISD::VP_REDUCE_FMUL:
550 case ISD::VP_REDUCE_SEQ_FADD:
551 case ISD::VP_REDUCE_SEQ_FMUL:
552 return true;
553 }
554}
555
556/// The operand position of the vector mask.
557std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
558 switch (Opcode) {
559 default:
560 return std::nullopt;
561#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
562 case ISD::VPSD: \
563 return MASKPOS;
564#include "llvm/IR/VPIntrinsics.def"
565 }
566}
567
568/// The operand position of the explicit vector length parameter.
569std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
570 switch (Opcode) {
571 default:
572 return std::nullopt;
573#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
574 case ISD::VPSD: \
575 return EVLPOS;
576#include "llvm/IR/VPIntrinsics.def"
577 }
578}
579
580std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
581 bool hasFPExcept) {
582 // FIXME: Return strict opcodes in case of fp exceptions.
583 switch (VPOpcode) {
584 default:
585 return std::nullopt;
586#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
587#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
588#define END_REGISTER_VP_SDNODE(VPOPC) break;
589#include "llvm/IR/VPIntrinsics.def"
590 }
591 return std::nullopt;
592}
593
594std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
595 switch (Opcode) {
596 default:
597 return std::nullopt;
598#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
599#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
600#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
601#include "llvm/IR/VPIntrinsics.def"
602 }
603}
604
605ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
606 switch (ExtType) {
607 case ISD::EXTLOAD:
608 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
609 case ISD::SEXTLOAD:
610 return ISD::SIGN_EXTEND;
611 case ISD::ZEXTLOAD:
612 return ISD::ZERO_EXTEND;
613 default:
614 break;
615 }
616
617 llvm_unreachable("Invalid LoadExtType");
618}
619
620ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
621 // To perform this operation, we just need to swap the L and G bits of the
622 // operation.
623 unsigned OldL = (Operation >> 2) & 1;
624 unsigned OldG = (Operation >> 1) & 1;
625 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
626 (OldL << 1) | // New G bit
627 (OldG << 2)); // New L bit.
628}
629
630static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
631 unsigned Operation = Op;
632 if (isIntegerLike)
633 Operation ^= 7; // Flip L, G, E bits, but not U.
634 else
635 Operation ^= 15; // Flip all of the condition bits.
636
637 if (Operation > ISD::SETTRUE2)
638 Operation &= ~8; // Don't let N and U bits get set.
639
640 return ISD::CondCode(Operation);
641}
642
643ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
644 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
645}
646
647ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
648 bool isIntegerLike) {
649 return getSetCCInverseImpl(Op, isIntegerLike);
650}
651
652/// For an integer comparison, return 1 if the comparison is a signed operation
653/// and 2 if the result is an unsigned comparison. Return zero if the operation
654/// does not depend on the sign of the input (setne and seteq).
655static int isSignedOp(ISD::CondCode Opcode) {
656 switch (Opcode) {
657 default: llvm_unreachable("Illegal integer setcc operation!");
658 case ISD::SETEQ:
659 case ISD::SETNE: return 0;
660 case ISD::SETLT:
661 case ISD::SETLE:
662 case ISD::SETGT:
663 case ISD::SETGE: return 1;
664 case ISD::SETULT:
665 case ISD::SETULE:
666 case ISD::SETUGT:
667 case ISD::SETUGE: return 2;
668 }
669}
670
671ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
672 EVT Type) {
673 bool IsInteger = Type.isInteger();
674 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
675 // Cannot fold a signed integer setcc with an unsigned integer setcc.
676 return ISD::SETCC_INVALID;
677
678 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
679
680 // If the N and U bits get set, then the resultant comparison DOES suddenly
681 // care about orderedness, and it is true when ordered.
682 if (Op > ISD::SETTRUE2)
683 Op &= ~16; // Clear the U bit if the N bit is set.
684
685 // Canonicalize illegal integer setcc's.
686 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
687 Op = ISD::SETNE;
688
689 return ISD::CondCode(Op);
690}
691
692ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
693 EVT Type) {
694 bool IsInteger = Type.isInteger();
695 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
696 // Cannot fold a signed setcc with an unsigned setcc.
697 return ISD::SETCC_INVALID;
698
699 // Combine all of the condition bits.
700 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
701
702 // Canonicalize illegal integer setcc's.
703 if (IsInteger) {
704 switch (Result) {
705 default: break;
706 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
707 case ISD::SETOEQ: // SETEQ & SETU[LG]E
708 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
709 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
710 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
711 }
712 }
713
714 return Result;
715}
716
717//===----------------------------------------------------------------------===//
718// SDNode Profile Support
719//===----------------------------------------------------------------------===//
720
721/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
722static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
723 ID.AddInteger(I: OpC);
724}
725
726/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
727/// solely with their pointer.
728static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
729 ID.AddPointer(Ptr: VTList.VTs);
730}
731
732/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
733static void AddNodeIDOperands(FoldingSetNodeID &ID,
734 ArrayRef<SDValue> Ops) {
735 for (const auto &Op : Ops) {
736 ID.AddPointer(Ptr: Op.getNode());
737 ID.AddInteger(I: Op.getResNo());
738 }
739}
740
741/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
742static void AddNodeIDOperands(FoldingSetNodeID &ID,
743 ArrayRef<SDUse> Ops) {
744 for (const auto &Op : Ops) {
745 ID.AddPointer(Ptr: Op.getNode());
746 ID.AddInteger(I: Op.getResNo());
747 }
748}
749
750static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
751 SDVTList VTList, ArrayRef<SDValue> OpList) {
752 AddNodeIDOpcode(ID, OpC);
753 AddNodeIDValueTypes(ID, VTList);
754 AddNodeIDOperands(ID, Ops: OpList);
755}
756
757/// If this is an SDNode with special info, add this info to the NodeID data.
758static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
759 switch (N->getOpcode()) {
760 case ISD::TargetExternalSymbol:
761 case ISD::ExternalSymbol:
762 case ISD::MCSymbol:
763 llvm_unreachable("Should only be used on nodes with operands");
764 default: break; // Normal nodes don't need extra info.
765 case ISD::TargetConstant:
766 case ISD::Constant: {
767 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
768 ID.AddPointer(Ptr: C->getConstantIntValue());
769 ID.AddBoolean(B: C->isOpaque());
770 break;
771 }
772 case ISD::TargetConstantFP:
773 case ISD::ConstantFP:
774 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
775 break;
776 case ISD::TargetGlobalAddress:
777 case ISD::GlobalAddress:
778 case ISD::TargetGlobalTLSAddress:
779 case ISD::GlobalTLSAddress: {
780 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
781 ID.AddPointer(Ptr: GA->getGlobal());
782 ID.AddInteger(I: GA->getOffset());
783 ID.AddInteger(I: GA->getTargetFlags());
784 break;
785 }
786 case ISD::BasicBlock:
787 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
788 break;
789 case ISD::Register:
790 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
791 break;
792 case ISD::RegisterMask:
793 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
794 break;
795 case ISD::SRCVALUE:
796 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
797 break;
798 case ISD::FrameIndex:
799 case ISD::TargetFrameIndex:
800 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
801 break;
802 case ISD::PSEUDO_PROBE:
803 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
804 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
805 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
806 break;
807 case ISD::JumpTable:
808 case ISD::TargetJumpTable:
809 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
810 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
811 break;
812 case ISD::ConstantPool:
813 case ISD::TargetConstantPool: {
814 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
815 ID.AddInteger(I: CP->getAlign().value());
816 ID.AddInteger(I: CP->getOffset());
817 if (CP->isMachineConstantPoolEntry())
818 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
819 else
820 ID.AddPointer(Ptr: CP->getConstVal());
821 ID.AddInteger(I: CP->getTargetFlags());
822 break;
823 }
824 case ISD::TargetIndex: {
825 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
826 ID.AddInteger(I: TI->getIndex());
827 ID.AddInteger(I: TI->getOffset());
828 ID.AddInteger(I: TI->getTargetFlags());
829 break;
830 }
831 case ISD::LOAD: {
832 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
833 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
834 ID.AddInteger(I: LD->getRawSubclassData());
835 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
836 ID.AddInteger(I: LD->getMemOperand()->getFlags());
837 break;
838 }
839 case ISD::STORE: {
840 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
841 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
842 ID.AddInteger(I: ST->getRawSubclassData());
843 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
844 ID.AddInteger(I: ST->getMemOperand()->getFlags());
845 break;
846 }
847 case ISD::VP_LOAD: {
848 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
849 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
850 ID.AddInteger(I: ELD->getRawSubclassData());
851 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
852 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
853 break;
854 }
855 case ISD::VP_LOAD_FF: {
856 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
857 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
858 ID.AddInteger(I: LD->getRawSubclassData());
859 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
860 ID.AddInteger(I: LD->getMemOperand()->getFlags());
861 break;
862 }
863 case ISD::VP_STORE: {
864 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
865 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
866 ID.AddInteger(I: EST->getRawSubclassData());
867 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
868 ID.AddInteger(I: EST->getMemOperand()->getFlags());
869 break;
870 }
871 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
872 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
873 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
874 ID.AddInteger(I: SLD->getRawSubclassData());
875 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
876 break;
877 }
878 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
879 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
880 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
881 ID.AddInteger(I: SST->getRawSubclassData());
882 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
883 break;
884 }
885 case ISD::VP_GATHER: {
886 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
887 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
888 ID.AddInteger(I: EG->getRawSubclassData());
889 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
890 ID.AddInteger(I: EG->getMemOperand()->getFlags());
891 break;
892 }
893 case ISD::VP_SCATTER: {
894 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
895 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
896 ID.AddInteger(I: ES->getRawSubclassData());
897 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
898 ID.AddInteger(I: ES->getMemOperand()->getFlags());
899 break;
900 }
901 case ISD::MLOAD: {
902 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
903 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
904 ID.AddInteger(I: MLD->getRawSubclassData());
905 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
906 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
907 break;
908 }
909 case ISD::MSTORE: {
910 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
911 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
912 ID.AddInteger(I: MST->getRawSubclassData());
913 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
914 ID.AddInteger(I: MST->getMemOperand()->getFlags());
915 break;
916 }
917 case ISD::MGATHER: {
918 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
919 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
920 ID.AddInteger(I: MG->getRawSubclassData());
921 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
922 ID.AddInteger(I: MG->getMemOperand()->getFlags());
923 break;
924 }
925 case ISD::MSCATTER: {
926 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
927 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
928 ID.AddInteger(I: MS->getRawSubclassData());
929 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
930 ID.AddInteger(I: MS->getMemOperand()->getFlags());
931 break;
932 }
933 case ISD::ATOMIC_CMP_SWAP:
934 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
935 case ISD::ATOMIC_SWAP:
936 case ISD::ATOMIC_LOAD_ADD:
937 case ISD::ATOMIC_LOAD_SUB:
938 case ISD::ATOMIC_LOAD_AND:
939 case ISD::ATOMIC_LOAD_CLR:
940 case ISD::ATOMIC_LOAD_OR:
941 case ISD::ATOMIC_LOAD_XOR:
942 case ISD::ATOMIC_LOAD_NAND:
943 case ISD::ATOMIC_LOAD_MIN:
944 case ISD::ATOMIC_LOAD_MAX:
945 case ISD::ATOMIC_LOAD_UMIN:
946 case ISD::ATOMIC_LOAD_UMAX:
947 case ISD::ATOMIC_LOAD:
948 case ISD::ATOMIC_STORE: {
949 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
950 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
951 ID.AddInteger(I: AT->getRawSubclassData());
952 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
953 ID.AddInteger(I: AT->getMemOperand()->getFlags());
954 break;
955 }
956 case ISD::VECTOR_SHUFFLE: {
957 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
958 for (int M : Mask)
959 ID.AddInteger(I: M);
960 break;
961 }
962 case ISD::ADDRSPACECAST: {
963 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
964 ID.AddInteger(I: ASC->getSrcAddressSpace());
965 ID.AddInteger(I: ASC->getDestAddressSpace());
966 break;
967 }
968 case ISD::TargetBlockAddress:
969 case ISD::BlockAddress: {
970 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
971 ID.AddPointer(Ptr: BA->getBlockAddress());
972 ID.AddInteger(I: BA->getOffset());
973 ID.AddInteger(I: BA->getTargetFlags());
974 break;
975 }
976 case ISD::AssertAlign:
977 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
978 break;
979 case ISD::PREFETCH:
980 case ISD::INTRINSIC_VOID:
981 case ISD::INTRINSIC_W_CHAIN:
982 // Handled by MemIntrinsicSDNode check after the switch.
983 break;
984 case ISD::MDNODE_SDNODE:
985 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
986 break;
987 } // end switch (N->getOpcode())
988
989 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
990 // to check.
991 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
992 ID.AddInteger(I: MN->getRawSubclassData());
993 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
994 for (const MachineMemOperand *MMO : MN->memoperands()) {
995 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
996 ID.AddInteger(I: MMO->getFlags());
997 }
998 }
999}
1000
1001/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
1002/// data.
1003static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1004 AddNodeIDOpcode(ID, OpC: N->getOpcode());
1005 // Add the return value info.
1006 AddNodeIDValueTypes(ID, VTList: N->getVTList());
1007 // Add the operand info.
1008 AddNodeIDOperands(ID, Ops: N->ops());
1009
1010 // Handle SDNode leafs with special info.
1011 AddNodeIDCustom(ID, N);
1012}
1013
1014//===----------------------------------------------------------------------===//
1015// SelectionDAG Class
1016//===----------------------------------------------------------------------===//
1017
1018/// doNotCSE - Return true if CSE should not be performed for this node.
1019static bool doNotCSE(SDNode *N) {
1020 if (N->getValueType(ResNo: 0) == MVT::Glue)
1021 return true; // Never CSE anything that produces a glue result.
1022
1023 switch (N->getOpcode()) {
1024 default: break;
1025 case ISD::HANDLENODE:
1026 case ISD::EH_LABEL:
1027 return true; // Never CSE these nodes.
1028 }
1029
1030 // Check that remaining values produced are not flags.
1031 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1032 if (N->getValueType(ResNo: i) == MVT::Glue)
1033 return true; // Never CSE anything that produces a glue result.
1034
1035 return false;
1036}
1037
1038/// Construct a DemandedElts mask which demands all elements of \p V.
1039/// If \p V is not a fixed-length vector, then this will return a single bit.
1040static APInt getDemandAllEltsMask(SDValue V) {
1041 EVT VT = V.getValueType();
1042 // Since the number of lanes in a scalable vector is unknown at compile time,
1043 // we track one bit which is implicitly broadcast to all lanes. This means
1044 // that all lanes in a scalable vector are considered demanded.
1045 return VT.isFixedLengthVector() ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1046 : APInt(1, 1);
1047}
1048
1049/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1050/// SelectionDAG.
1051void SelectionDAG::RemoveDeadNodes() {
1052 // Create a dummy node (which is not added to allnodes), that adds a reference
1053 // to the root node, preventing it from being deleted.
1054 HandleSDNode Dummy(getRoot());
1055
1056 SmallVector<SDNode*, 128> DeadNodes;
1057
1058 // Add all obviously-dead nodes to the DeadNodes worklist.
1059 for (SDNode &Node : allnodes())
1060 if (Node.use_empty())
1061 DeadNodes.push_back(Elt: &Node);
1062
1063 RemoveDeadNodes(DeadNodes);
1064
1065 // If the root changed (e.g. it was a dead load, update the root).
1066 setRoot(Dummy.getValue());
1067}
1068
1069/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1070/// given list, and any nodes that become unreachable as a result.
1071void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1072
1073 // Process the worklist, deleting the nodes and adding their uses to the
1074 // worklist.
1075 while (!DeadNodes.empty()) {
1076 SDNode *N = DeadNodes.pop_back_val();
1077 // Skip to next node if we've already managed to delete the node. This could
1078 // happen if replacing a node causes a node previously added to the node to
1079 // be deleted.
1080 if (N->getOpcode() == ISD::DELETED_NODE)
1081 continue;
1082
1083 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1084 DUL->NodeDeleted(N, nullptr);
1085
1086 // Take the node out of the appropriate CSE map.
1087 RemoveNodeFromCSEMaps(N);
1088
1089 // Next, brutally remove the operand list. This is safe to do, as there are
1090 // no cycles in the graph.
1091 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1092 SDUse &Use = *I++;
1093 SDNode *Operand = Use.getNode();
1094 Use.set(SDValue());
1095
1096 // Now that we removed this operand, see if there are no uses of it left.
1097 if (Operand->use_empty())
1098 DeadNodes.push_back(Elt: Operand);
1099 }
1100
1101 DeallocateNode(N);
1102 }
1103}
1104
1105void SelectionDAG::RemoveDeadNode(SDNode *N){
1106 SmallVector<SDNode*, 16> DeadNodes(1, N);
1107
1108 // Create a dummy node that adds a reference to the root node, preventing
1109 // it from being deleted. (This matters if the root is an operand of the
1110 // dead node.)
1111 HandleSDNode Dummy(getRoot());
1112
1113 RemoveDeadNodes(DeadNodes);
1114}
1115
1116void SelectionDAG::DeleteNode(SDNode *N) {
1117 // First take this out of the appropriate CSE map.
1118 RemoveNodeFromCSEMaps(N);
1119
1120 // Finally, remove uses due to operands of this node, remove from the
1121 // AllNodes list, and delete the node.
1122 DeleteNodeNotInCSEMaps(N);
1123}
1124
1125void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1126 assert(N->getIterator() != AllNodes.begin() &&
1127 "Cannot delete the entry node!");
1128 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1129
1130 // Drop all of the operands and decrement used node's use counts.
1131 N->DropOperands();
1132
1133 DeallocateNode(N);
1134}
1135
1136void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1137 assert(!(V->isVariadic() && isParameter));
1138 if (isParameter)
1139 ByvalParmDbgValues.push_back(Elt: V);
1140 else
1141 DbgValues.push_back(Elt: V);
1142 for (const SDNode *Node : V->getSDNodes())
1143 if (Node)
1144 DbgValMap[Node].push_back(Elt: V);
1145}
1146
1147void SDDbgInfo::erase(const SDNode *Node) {
1148 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1149 if (I == DbgValMap.end())
1150 return;
1151 for (auto &Val: I->second)
1152 Val->setIsInvalidated();
1153 DbgValMap.erase(I);
1154}
1155
1156void SelectionDAG::DeallocateNode(SDNode *N) {
1157 // If we have operands, deallocate them.
1158 removeOperands(Node: N);
1159
1160 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1161
1162 // Set the opcode to DELETED_NODE to help catch bugs when node
1163 // memory is reallocated.
1164 // FIXME: There are places in SDag that have grown a dependency on the opcode
1165 // value in the released node.
1166 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1167 N->NodeType = ISD::DELETED_NODE;
1168
1169 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1170 // them and forget about that node.
1171 DbgInfo->erase(Node: N);
1172
1173 // Invalidate extra info.
1174 SDEI.erase(Val: N);
1175}
1176
1177#ifndef NDEBUG
1178/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1179void SelectionDAG::verifyNode(SDNode *N) const {
1180 switch (N->getOpcode()) {
1181 default:
1182 if (N->isTargetOpcode())
1183 getSelectionDAGInfo().verifyTargetNode(*this, N);
1184 break;
1185 case ISD::BUILD_PAIR: {
1186 EVT VT = N->getValueType(0);
1187 assert(N->getNumValues() == 1 && "Too many results!");
1188 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1189 "Wrong return type!");
1190 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1191 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1192 "Mismatched operand types!");
1193 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1194 "Wrong operand type!");
1195 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1196 "Wrong return type size");
1197 break;
1198 }
1199 case ISD::BUILD_VECTOR: {
1200 assert(N->getNumValues() == 1 && "Too many results!");
1201 assert(N->getValueType(0).isVector() && "Wrong return type!");
1202 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1203 "Wrong number of operands!");
1204 EVT EltVT = N->getValueType(0).getVectorElementType();
1205 for (const SDUse &Op : N->ops()) {
1206 assert((Op.getValueType() == EltVT ||
1207 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1208 EltVT.bitsLE(Op.getValueType()))) &&
1209 "Wrong operand type!");
1210 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1211 "Operands must all have the same type");
1212 }
1213 break;
1214 }
1215 case ISD::SADDO:
1216 case ISD::UADDO:
1217 case ISD::SSUBO:
1218 case ISD::USUBO:
1219 assert(N->getNumValues() == 2 && "Wrong number of results!");
1220 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1221 "Invalid add/sub overflow op!");
1222 assert(N->getVTList().VTs[0].isInteger() &&
1223 N->getVTList().VTs[1].isInteger() &&
1224 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1225 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1226 "Binary operator types must match!");
1227 break;
1228 }
1229}
1230#endif // NDEBUG
1231
1232/// Insert a newly allocated node into the DAG.
1233///
1234/// Handles insertion into the all nodes list and CSE map, as well as
1235/// verification and other common operations when a new node is allocated.
1236void SelectionDAG::InsertNode(SDNode *N) {
1237 AllNodes.push_back(val: N);
1238#ifndef NDEBUG
1239 N->PersistentId = NextPersistentId++;
1240 verifyNode(N);
1241#endif
1242 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1243 DUL->NodeInserted(N);
1244}
1245
1246/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1247/// correspond to it. This is useful when we're about to delete or repurpose
1248/// the node. We don't want future request for structurally identical nodes
1249/// to return N anymore.
1250bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1251 bool Erased = false;
1252 switch (N->getOpcode()) {
1253 case ISD::HANDLENODE: return false; // noop.
1254 case ISD::CONDCODE:
1255 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1256 "Cond code doesn't exist!");
1257 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1258 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1259 break;
1260 case ISD::ExternalSymbol:
1261 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1262 break;
1263 case ISD::TargetExternalSymbol: {
1264 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1265 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1266 ESN->getSymbol(), ESN->getTargetFlags()));
1267 break;
1268 }
1269 case ISD::MCSymbol: {
1270 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1271 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1272 break;
1273 }
1274 case ISD::VALUETYPE: {
1275 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1276 if (VT.isExtended()) {
1277 Erased = ExtendedValueTypeNodes.erase(x: VT);
1278 } else {
1279 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1280 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1281 }
1282 break;
1283 }
1284 default:
1285 // Remove it from the CSE Map.
1286 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1287 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1288 Erased = CSEMap.RemoveNode(N);
1289 break;
1290 }
1291#ifndef NDEBUG
1292 // Verify that the node was actually in one of the CSE maps, unless it has a
1293 // glue result (which cannot be CSE'd) or is one of the special cases that are
1294 // not subject to CSE.
1295 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1296 !N->isMachineOpcode() && !doNotCSE(N)) {
1297 N->dump(this);
1298 dbgs() << "\n";
1299 llvm_unreachable("Node is not in map!");
1300 }
1301#endif
1302 return Erased;
1303}
1304
1305/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1306/// maps and modified in place. Add it back to the CSE maps, unless an identical
1307/// node already exists, in which case transfer all its users to the existing
1308/// node. This transfer can potentially trigger recursive merging.
1309void
1310SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1311 // For node types that aren't CSE'd, just act as if no identical node
1312 // already exists.
1313 if (!doNotCSE(N)) {
1314 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1315 if (Existing != N) {
1316 // If there was already an existing matching node, use ReplaceAllUsesWith
1317 // to replace the dead one with the existing one. This can cause
1318 // recursive merging of other unrelated nodes down the line.
1319 Existing->intersectFlagsWith(Flags: N->getFlags());
1320 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing))
1321 MemNode->refineRanges(NewMMOs: cast<MemSDNode>(Val: N)->memoperands());
1322 ReplaceAllUsesWith(From: N, To: Existing);
1323
1324 // N is now dead. Inform the listeners and delete it.
1325 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1326 DUL->NodeDeleted(N, Existing);
1327 DeleteNodeNotInCSEMaps(N);
1328 return;
1329 }
1330 }
1331
1332 // If the node doesn't already exist, we updated it. Inform listeners.
1333 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1334 DUL->NodeUpdated(N);
1335}
1336
1337/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1338/// were replaced with those specified. If this node is never memoized,
1339/// return null, otherwise return a pointer to the slot it would take. If a
1340/// node already exists with these operands, the slot will be non-null.
1341SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1342 void *&InsertPos) {
1343 if (doNotCSE(N))
1344 return nullptr;
1345
1346 SDValue Ops[] = { Op };
1347 FoldingSetNodeID ID;
1348 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1349 AddNodeIDCustom(ID, N);
1350 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1351 if (Node)
1352 Node->intersectFlagsWith(Flags: N->getFlags());
1353 return Node;
1354}
1355
1356/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1357/// were replaced with those specified. If this node is never memoized,
1358/// return null, otherwise return a pointer to the slot it would take. If a
1359/// node already exists with these operands, the slot will be non-null.
1360SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1361 SDValue Op1, SDValue Op2,
1362 void *&InsertPos) {
1363 if (doNotCSE(N))
1364 return nullptr;
1365
1366 SDValue Ops[] = { Op1, Op2 };
1367 FoldingSetNodeID ID;
1368 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1369 AddNodeIDCustom(ID, N);
1370 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1371 if (Node)
1372 Node->intersectFlagsWith(Flags: N->getFlags());
1373 return Node;
1374}
1375
1376/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1377/// were replaced with those specified. If this node is never memoized,
1378/// return null, otherwise return a pointer to the slot it would take. If a
1379/// node already exists with these operands, the slot will be non-null.
1380SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1381 void *&InsertPos) {
1382 if (doNotCSE(N))
1383 return nullptr;
1384
1385 FoldingSetNodeID ID;
1386 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1387 AddNodeIDCustom(ID, N);
1388 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1389 if (Node)
1390 Node->intersectFlagsWith(Flags: N->getFlags());
1391 return Node;
1392}
1393
1394Align SelectionDAG::getEVTAlign(EVT VT) const {
1395 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1396 : VT.getTypeForEVT(Context&: *getContext());
1397
1398 return getDataLayout().getABITypeAlign(Ty);
1399}
1400
1401// EntryNode could meaningfully have debug info if we can find it...
1402SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1403 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1404 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1405 Root(getEntryNode()) {
1406 InsertNode(N: &EntryNode);
1407 DbgInfo = new SDDbgInfo();
1408}
1409
1410void SelectionDAG::init(MachineFunction &NewMF,
1411 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1412 const TargetLibraryInfo *LibraryInfo,
1413 const LibcallLoweringInfo *LibcallsInfo,
1414 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1415 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
1416 FunctionVarLocs const *VarLocs) {
1417 MF = &NewMF;
1418 SDAGISelPass = PassPtr;
1419 ORE = &NewORE;
1420 TLI = getSubtarget().getTargetLowering();
1421 TSI = getSubtarget().getSelectionDAGInfo();
1422 LibInfo = LibraryInfo;
1423 Libcalls = LibcallsInfo;
1424 Context = &MF->getFunction().getContext();
1425 UA = NewUA;
1426 PSI = PSIin;
1427 BFI = BFIin;
1428 MMI = &MMIin;
1429 FnVarLocs = VarLocs;
1430}
1431
1432SelectionDAG::~SelectionDAG() {
1433 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1434 allnodes_clear();
1435 OperandRecycler.clear(OperandAllocator);
1436 delete DbgInfo;
1437}
1438
1439bool SelectionDAG::shouldOptForSize() const {
1440 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1441}
1442
1443void SelectionDAG::allnodes_clear() {
1444 assert(&*AllNodes.begin() == &EntryNode);
1445 AllNodes.remove(IT: AllNodes.begin());
1446 while (!AllNodes.empty())
1447 DeallocateNode(N: &AllNodes.front());
1448#ifndef NDEBUG
1449 NextPersistentId = 0;
1450#endif
1451}
1452
1453SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1454 void *&InsertPos) {
1455 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1456 if (N) {
1457 switch (N->getOpcode()) {
1458 default: break;
1459 case ISD::Constant:
1460 case ISD::ConstantFP:
1461 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1462 "debug location. Use another overload.");
1463 }
1464 }
1465 return N;
1466}
1467
1468SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1469 const SDLoc &DL, void *&InsertPos) {
1470 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1471 if (N) {
1472 switch (N->getOpcode()) {
1473 case ISD::Constant:
1474 case ISD::ConstantFP:
1475 // Erase debug location from the node if the node is used at several
1476 // different places. Do not propagate one location to all uses as it
1477 // will cause a worse single stepping debugging experience.
1478 if (N->getDebugLoc() != DL.getDebugLoc())
1479 N->setDebugLoc(DebugLoc());
1480 break;
1481 default:
1482 // When the node's point of use is located earlier in the instruction
1483 // sequence than its prior point of use, update its debug info to the
1484 // earlier location.
1485 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1486 N->setDebugLoc(DL.getDebugLoc());
1487 break;
1488 }
1489 }
1490 return N;
1491}
1492
1493void SelectionDAG::clear() {
1494 allnodes_clear();
1495 OperandRecycler.clear(OperandAllocator);
1496 OperandAllocator.Reset();
1497 CSEMap.clear();
1498
1499 ExtendedValueTypeNodes.clear();
1500 ExternalSymbols.clear();
1501 TargetExternalSymbols.clear();
1502 MCSymbols.clear();
1503 SDEI.clear();
1504 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1505 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1506
1507 EntryNode.UseList = nullptr;
1508 InsertNode(N: &EntryNode);
1509 Root = getEntryNode();
1510 DbgInfo->clear();
1511}
1512
1513SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1514 return VT.bitsGT(VT: Op.getValueType())
1515 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1516 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1517 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1518}
1519
1520std::pair<SDValue, SDValue>
1521SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1522 const SDLoc &DL, EVT VT) {
1523 assert(!VT.bitsEq(Op.getValueType()) &&
1524 "Strict no-op FP extend/round not allowed.");
1525 SDValue Res =
1526 VT.bitsGT(VT: Op.getValueType())
1527 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1528 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1529 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1530
1531 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1532}
1533
1534SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1535 return VT.bitsGT(VT: Op.getValueType()) ?
1536 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1537 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1538}
1539
1540SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1541 return VT.bitsGT(VT: Op.getValueType()) ?
1542 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1543 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1544}
1545
1546SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1547 return VT.bitsGT(VT: Op.getValueType()) ?
1548 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1549 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1550}
1551
1552SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1553 EVT VT) {
1554 assert(!VT.isVector());
1555 auto Type = Op.getValueType();
1556 SDValue DestOp;
1557 if (Type == VT)
1558 return Op;
1559 auto Size = Op.getValueSizeInBits();
1560 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1561 if (DestOp.getValueType() == VT)
1562 return DestOp;
1563
1564 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1565}
1566
1567SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1568 EVT VT) {
1569 assert(!VT.isVector());
1570 auto Type = Op.getValueType();
1571 SDValue DestOp;
1572 if (Type == VT)
1573 return Op;
1574 auto Size = Op.getValueSizeInBits();
1575 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1576 if (DestOp.getValueType() == VT)
1577 return DestOp;
1578
1579 return getSExtOrTrunc(Op: DestOp, DL, VT);
1580}
1581
1582SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1583 EVT VT) {
1584 assert(!VT.isVector());
1585 auto Type = Op.getValueType();
1586 SDValue DestOp;
1587 if (Type == VT)
1588 return Op;
1589 auto Size = Op.getValueSizeInBits();
1590 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1591 if (DestOp.getValueType() == VT)
1592 return DestOp;
1593
1594 return getZExtOrTrunc(Op: DestOp, DL, VT);
1595}
1596
1597SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1598 EVT OpVT) {
1599 if (VT.bitsLE(VT: Op.getValueType()))
1600 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1601
1602 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1603 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1604}
1605
1606SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1607 EVT OpVT = Op.getValueType();
1608 assert(VT.isInteger() && OpVT.isInteger() &&
1609 "Cannot getZeroExtendInReg FP types");
1610 assert(VT.isVector() == OpVT.isVector() &&
1611 "getZeroExtendInReg type should be vector iff the operand "
1612 "type is vector!");
1613 assert((!VT.isVector() ||
1614 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1615 "Vector element counts must match in getZeroExtendInReg");
1616 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1617 if (OpVT == VT)
1618 return Op;
1619 // TODO: Use computeKnownBits instead of AssertZext.
1620 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1621 ->getVT()
1622 .getScalarType()
1623 .bitsLE(VT: VT.getScalarType()))
1624 return Op;
1625 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1626 loBitsSet: VT.getScalarSizeInBits());
1627 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1628}
1629
1630SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,
1631 SDValue EVL, const SDLoc &DL,
1632 EVT VT) {
1633 EVT OpVT = Op.getValueType();
1634 assert(VT.isInteger() && OpVT.isInteger() &&
1635 "Cannot getVPZeroExtendInReg FP types");
1636 assert(VT.isVector() && OpVT.isVector() &&
1637 "getVPZeroExtendInReg type and operand type should be vector!");
1638 assert(VT.getVectorElementCount() == OpVT.getVectorElementCount() &&
1639 "Vector element counts must match in getZeroExtendInReg");
1640 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1641 if (OpVT == VT)
1642 return Op;
1643 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1644 loBitsSet: VT.getScalarSizeInBits());
1645 return getNode(Opcode: ISD::VP_AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT), N3: Mask,
1646 N4: EVL);
1647}
1648
1649SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1650 // Only unsigned pointer semantics are supported right now. In the future this
1651 // might delegate to TLI to check pointer signedness.
1652 return getZExtOrTrunc(Op, DL, VT);
1653}
1654
1655SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1656 // Only unsigned pointer semantics are supported right now. In the future this
1657 // might delegate to TLI to check pointer signedness.
1658 return getZeroExtendInReg(Op, DL, VT);
1659}
1660
1661SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1662 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1663}
1664
1665/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1666SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1667 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1668}
1669
1670SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1671 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1672 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1673}
1674
1675SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1676 SDValue Mask, SDValue EVL, EVT VT) {
1677 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1678 return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1679}
1680
1681SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1682 SDValue Mask, SDValue EVL) {
1683 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1684}
1685
1686SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1687 SDValue Mask, SDValue EVL) {
1688 if (VT.bitsGT(VT: Op.getValueType()))
1689 return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1690 if (VT.bitsLT(VT: Op.getValueType()))
1691 return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1692 return Op;
1693}
1694
1695SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1696 EVT OpVT) {
1697 if (!V)
1698 return getConstant(Val: 0, DL, VT);
1699
1700 switch (TLI->getBooleanContents(Type: OpVT)) {
1701 case TargetLowering::ZeroOrOneBooleanContent:
1702 case TargetLowering::UndefinedBooleanContent:
1703 return getConstant(Val: 1, DL, VT);
1704 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1705 return getAllOnesConstant(DL, VT);
1706 }
1707 llvm_unreachable("Unexpected boolean content enum!");
1708}
1709
1710SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1711 bool isT, bool isO) {
1712 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1713 DL, VT, isTarget: isT, isOpaque: isO);
1714}
1715
1716SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1717 bool isT, bool isO) {
1718 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1719}
1720
1721SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1722 EVT VT, bool isT, bool isO) {
1723 assert(VT.isInteger() && "Cannot create FP integer constant!");
1724
1725 EVT EltVT = VT.getScalarType();
1726 const ConstantInt *Elt = &Val;
1727
1728 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1729 // to-be-splatted scalar ConstantInt.
1730 if (isa<VectorType>(Val: Elt->getType()))
1731 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1732
1733 // In some cases the vector type is legal but the element type is illegal and
1734 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1735 // inserted value (the type does not need to match the vector element type).
1736 // Any extra bits introduced will be truncated away.
1737 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1738 TargetLowering::TypePromoteInteger) {
1739 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1740 APInt NewVal;
1741 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1742 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1743 else
1744 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1745 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1746 }
1747 // In other cases the element type is illegal and needs to be expanded, for
1748 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1749 // the value into n parts and use a vector type with n-times the elements.
1750 // Then bitcast to the type requested.
1751 // Legalizing constants too early makes the DAGCombiner's job harder so we
1752 // only legalize if the DAG tells us we must produce legal types.
1753 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1754 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1755 TargetLowering::TypeExpandInteger) {
1756 const APInt &NewVal = Elt->getValue();
1757 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1758 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1759
1760 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1761 if (VT.isScalableVector() ||
1762 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1763 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1764 "Can only handle an even split!");
1765 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1766
1767 SmallVector<SDValue, 2> ScalarParts;
1768 for (unsigned i = 0; i != Parts; ++i)
1769 ScalarParts.push_back(Elt: getConstant(
1770 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1771 VT: ViaEltVT, isT, isO));
1772
1773 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1774 }
1775
1776 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1777 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1778
1779 // Check the temporary vector is the correct size. If this fails then
1780 // getTypeToTransformTo() probably returned a type whose size (in bits)
1781 // isn't a power-of-2 factor of the requested type size.
1782 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1783
1784 SmallVector<SDValue, 2> EltParts;
1785 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1786 EltParts.push_back(Elt: getConstant(
1787 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1788 VT: ViaEltVT, isT, isO));
1789
1790 // EltParts is currently in little endian order. If we actually want
1791 // big-endian order then reverse it now.
1792 if (getDataLayout().isBigEndian())
1793 std::reverse(first: EltParts.begin(), last: EltParts.end());
1794
1795 // The elements must be reversed when the element order is different
1796 // to the endianness of the elements (because the BITCAST is itself a
1797 // vector shuffle in this situation). However, we do not need any code to
1798 // perform this reversal because getConstant() is producing a vector
1799 // splat.
1800 // This situation occurs in MIPS MSA.
1801
1802 SmallVector<SDValue, 8> Ops;
1803 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1804 llvm::append_range(C&: Ops, R&: EltParts);
1805
1806 SDValue V =
1807 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1808 return V;
1809 }
1810
1811 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1812 "APInt size does not match type size!");
1813 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1814 SDVTList VTs = getVTList(VT: EltVT);
1815 FoldingSetNodeID ID;
1816 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1817 ID.AddPointer(Ptr: Elt);
1818 ID.AddBoolean(B: isO);
1819 void *IP = nullptr;
1820 SDNode *N = nullptr;
1821 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1822 if (!VT.isVector())
1823 return SDValue(N, 0);
1824
1825 if (!N) {
1826 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1827 CSEMap.InsertNode(N, InsertPos: IP);
1828 InsertNode(N);
1829 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1830 }
1831
1832 SDValue Result(N, 0);
1833 if (VT.isVector())
1834 Result = getSplat(VT, DL, Op: Result);
1835 return Result;
1836}
1837
1838SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1839 bool isT, bool isO) {
1840 unsigned Size = VT.getScalarSizeInBits();
1841 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1842}
1843
1844SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1845 bool IsOpaque) {
1846 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1847 isT: IsTarget, isO: IsOpaque);
1848}
1849
1850SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1851 bool isTarget) {
1852 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1853}
1854
1855SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1856 const SDLoc &DL) {
1857 assert(VT.isInteger() && "Shift amount is not an integer type!");
1858 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1859 return getConstant(Val, DL, VT: ShiftVT);
1860}
1861
1862SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1863 const SDLoc &DL) {
1864 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1865 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1866}
1867
1868SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1869 bool isTarget) {
1870 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1871}
1872
1873SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1874 bool isTarget) {
1875 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1876}
1877
1878SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1879 EVT VT, bool isTarget) {
1880 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1881
1882 EVT EltVT = VT.getScalarType();
1883 const ConstantFP *Elt = &V;
1884
1885 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1886 // the to-be-splatted scalar ConstantFP.
1887 if (isa<VectorType>(Val: Elt->getType()))
1888 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1889
1890 // Do the map lookup using the actual bit pattern for the floating point
1891 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1892 // we don't have issues with SNANs.
1893 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1894 SDVTList VTs = getVTList(VT: EltVT);
1895 FoldingSetNodeID ID;
1896 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1897 ID.AddPointer(Ptr: Elt);
1898 void *IP = nullptr;
1899 SDNode *N = nullptr;
1900 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1901 if (!VT.isVector())
1902 return SDValue(N, 0);
1903
1904 if (!N) {
1905 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1906 CSEMap.InsertNode(N, InsertPos: IP);
1907 InsertNode(N);
1908 }
1909
1910 SDValue Result(N, 0);
1911 if (VT.isVector())
1912 Result = getSplat(VT, DL, Op: Result);
1913 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1914 return Result;
1915}
1916
1917SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1918 bool isTarget) {
1919 EVT EltVT = VT.getScalarType();
1920 if (EltVT == MVT::f32)
1921 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1922 if (EltVT == MVT::f64)
1923 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1924 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1925 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1926 bool Ignored;
1927 APFloat APF = APFloat(Val);
1928 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1929 losesInfo: &Ignored);
1930 return getConstantFP(V: APF, DL, VT, isTarget);
1931 }
1932 llvm_unreachable("Unsupported type in getConstantFP");
1933}
1934
1935SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1936 EVT VT, int64_t Offset, bool isTargetGA,
1937 unsigned TargetFlags) {
1938 assert((TargetFlags == 0 || isTargetGA) &&
1939 "Cannot set target flags on target-independent globals");
1940
1941 // Truncate (with sign-extension) the offset value to the pointer size.
1942 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1943 if (BitWidth < 64)
1944 Offset = SignExtend64(X: Offset, B: BitWidth);
1945
1946 unsigned Opc;
1947 if (GV->isThreadLocal())
1948 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1949 else
1950 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1951
1952 SDVTList VTs = getVTList(VT);
1953 FoldingSetNodeID ID;
1954 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1955 ID.AddPointer(Ptr: GV);
1956 ID.AddInteger(I: Offset);
1957 ID.AddInteger(I: TargetFlags);
1958 void *IP = nullptr;
1959 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1960 return SDValue(E, 0);
1961
1962 auto *N = newSDNode<GlobalAddressSDNode>(
1963 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1964 CSEMap.InsertNode(N, InsertPos: IP);
1965 InsertNode(N);
1966 return SDValue(N, 0);
1967}
1968
1969SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1970 SDVTList VTs = getVTList(VT: MVT::Untyped);
1971 FoldingSetNodeID ID;
1972 AddNodeIDNode(ID, OpC: ISD::DEACTIVATION_SYMBOL, VTList: VTs, OpList: {});
1973 ID.AddPointer(Ptr: GV);
1974 void *IP = nullptr;
1975 if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
1976 return SDValue(E, 0);
1977
1978 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1979 CSEMap.InsertNode(N, InsertPos: IP);
1980 InsertNode(N);
1981 return SDValue(N, 0);
1982}
1983
1984SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1985 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1986 SDVTList VTs = getVTList(VT);
1987 FoldingSetNodeID ID;
1988 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1989 ID.AddInteger(I: FI);
1990 void *IP = nullptr;
1991 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1992 return SDValue(E, 0);
1993
1994 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1995 CSEMap.InsertNode(N, InsertPos: IP);
1996 InsertNode(N);
1997 return SDValue(N, 0);
1998}
1999
2000SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
2001 unsigned TargetFlags) {
2002 assert((TargetFlags == 0 || isTarget) &&
2003 "Cannot set target flags on target-independent jump tables");
2004 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
2005 SDVTList VTs = getVTList(VT);
2006 FoldingSetNodeID ID;
2007 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2008 ID.AddInteger(I: JTI);
2009 ID.AddInteger(I: TargetFlags);
2010 void *IP = nullptr;
2011 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2012 return SDValue(E, 0);
2013
2014 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
2015 CSEMap.InsertNode(N, InsertPos: IP);
2016 InsertNode(N);
2017 return SDValue(N, 0);
2018}
2019
2020SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
2021 const SDLoc &DL) {
2022 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
2023 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
2024 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
2025}
2026
2027SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2028 MaybeAlign Alignment, int Offset,
2029 bool isTarget, unsigned TargetFlags) {
2030 assert((TargetFlags == 0 || isTarget) &&
2031 "Cannot set target flags on target-independent globals");
2032 if (!Alignment)
2033 Alignment = shouldOptForSize()
2034 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2035 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2036 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2037 SDVTList VTs = getVTList(VT);
2038 FoldingSetNodeID ID;
2039 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2040 ID.AddInteger(I: Alignment->value());
2041 ID.AddInteger(I: Offset);
2042 ID.AddPointer(Ptr: C);
2043 ID.AddInteger(I: TargetFlags);
2044 void *IP = nullptr;
2045 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2046 return SDValue(E, 0);
2047
2048 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2049 Args&: TargetFlags);
2050 CSEMap.InsertNode(N, InsertPos: IP);
2051 InsertNode(N);
2052 SDValue V = SDValue(N, 0);
2053 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2054 return V;
2055}
2056
2057SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2058 MaybeAlign Alignment, int Offset,
2059 bool isTarget, unsigned TargetFlags) {
2060 assert((TargetFlags == 0 || isTarget) &&
2061 "Cannot set target flags on target-independent globals");
2062 if (!Alignment)
2063 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2064 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2065 SDVTList VTs = getVTList(VT);
2066 FoldingSetNodeID ID;
2067 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2068 ID.AddInteger(I: Alignment->value());
2069 ID.AddInteger(I: Offset);
2070 C->addSelectionDAGCSEId(ID);
2071 ID.AddInteger(I: TargetFlags);
2072 void *IP = nullptr;
2073 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2074 return SDValue(E, 0);
2075
2076 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2077 Args&: TargetFlags);
2078 CSEMap.InsertNode(N, InsertPos: IP);
2079 InsertNode(N);
2080 return SDValue(N, 0);
2081}
2082
2083SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2084 FoldingSetNodeID ID;
2085 AddNodeIDNode(ID, OpC: ISD::BasicBlock, VTList: getVTList(VT: MVT::Other), OpList: {});
2086 ID.AddPointer(Ptr: MBB);
2087 void *IP = nullptr;
2088 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2089 return SDValue(E, 0);
2090
2091 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2092 CSEMap.InsertNode(N, InsertPos: IP);
2093 InsertNode(N);
2094 return SDValue(N, 0);
2095}
2096
2097SDValue SelectionDAG::getValueType(EVT VT) {
2098 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2099 ValueTypeNodes.size())
2100 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2101
2102 SDNode *&N = VT.isExtended() ?
2103 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2104
2105 if (N) return SDValue(N, 0);
2106 N = newSDNode<VTSDNode>(Args&: VT);
2107 InsertNode(N);
2108 return SDValue(N, 0);
2109}
2110
2111SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2112 SDNode *&N = ExternalSymbols[Sym];
2113 if (N) return SDValue(N, 0);
2114 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2115 InsertNode(N);
2116 return SDValue(N, 0);
2117}
2118
2119SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2120 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2121 return getExternalSymbol(Sym: SymName.data(), VT);
2122}
2123
2124SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2125 SDNode *&N = MCSymbols[Sym];
2126 if (N)
2127 return SDValue(N, 0);
2128 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2129 InsertNode(N);
2130 return SDValue(N, 0);
2131}
2132
2133SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2134 unsigned TargetFlags) {
2135 SDNode *&N =
2136 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2137 if (N) return SDValue(N, 0);
2138 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2139 InsertNode(N);
2140 return SDValue(N, 0);
2141}
2142
2143SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2144 EVT VT, unsigned TargetFlags) {
2145 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2146 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2147}
2148
2149SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2150 if ((unsigned)Cond >= CondCodeNodes.size())
2151 CondCodeNodes.resize(new_size: Cond+1);
2152
2153 if (!CondCodeNodes[Cond]) {
2154 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2155 CondCodeNodes[Cond] = N;
2156 InsertNode(N);
2157 }
2158
2159 return SDValue(CondCodeNodes[Cond], 0);
2160}
2161
2162SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2163 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2164 "APInt size does not match type size!");
2165
2166 if (MulImm == 0)
2167 return getConstant(Val: 0, DL, VT);
2168
2169 const MachineFunction &MF = getMachineFunction();
2170 const Function &F = MF.getFunction();
2171 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2172 if (const APInt *C = CR.getSingleElement())
2173 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2174
2175 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2176}
2177
2178/// \returns a value of type \p VT that represents the runtime value of \p
2179/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2180/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2181/// or TypeSize.
2182template <typename Ty>
2183static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2184 EVT VT, Ty Quantity) {
2185 if (Quantity.isScalable())
2186 return DAG.getVScale(
2187 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2188
2189 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2190}
2191
2192SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2193 ElementCount EC) {
2194 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2195}
2196
2197SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2198 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2199}
2200
2201SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2202 ElementCount EC) {
2203 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2204 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2205 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2206 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2207}
2208
2209SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2210 APInt One(ResVT.getScalarSizeInBits(), 1);
2211 return getStepVector(DL, ResVT, StepVal: One);
2212}
2213
2214SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2215 const APInt &StepVal) {
2216 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2217 if (ResVT.isScalableVector())
2218 return getNode(
2219 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2220 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2221
2222 SmallVector<SDValue, 16> OpsStepConstants;
2223 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2224 OpsStepConstants.push_back(
2225 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2226 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2227}
2228
2229/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2230/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2231static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2232 std::swap(a&: N1, b&: N2);
2233 ShuffleVectorSDNode::commuteMask(Mask: M);
2234}
2235
2236SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2237 SDValue N2, ArrayRef<int> Mask) {
2238 assert(VT.getVectorNumElements() == Mask.size() &&
2239 "Must have the same number of vector elements as mask elements!");
2240 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2241 "Invalid VECTOR_SHUFFLE");
2242
2243 // Canonicalize shuffle undef, undef -> undef
2244 if (N1.isUndef() && N2.isUndef())
2245 return getUNDEF(VT);
2246
2247 // Validate that all indices in Mask are within the range of the elements
2248 // input to the shuffle.
2249 int NElts = Mask.size();
2250 assert(llvm::all_of(Mask,
2251 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2252 "Index out of range");
2253
2254 // Copy the mask so we can do any needed cleanup.
2255 SmallVector<int, 8> MaskVec(Mask);
2256
2257 // Canonicalize shuffle v, v -> v, undef
2258 if (N1 == N2) {
2259 N2 = getUNDEF(VT);
2260 for (int i = 0; i != NElts; ++i)
2261 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2262 }
2263
2264 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2265 if (N1.isUndef())
2266 commuteShuffle(N1, N2, M: MaskVec);
2267
2268 if (TLI->hasVectorBlend()) {
2269 // If shuffling a splat, try to blend the splat instead. We do this here so
2270 // that even when this arises during lowering we don't have to re-handle it.
2271 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2272 BitVector UndefElements;
2273 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2274 if (!Splat)
2275 return;
2276
2277 for (int i = 0; i < NElts; ++i) {
2278 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2279 continue;
2280
2281 // If this input comes from undef, mark it as such.
2282 if (UndefElements[MaskVec[i] - Offset]) {
2283 MaskVec[i] = -1;
2284 continue;
2285 }
2286
2287 // If we can blend a non-undef lane, use that instead.
2288 if (!UndefElements[i])
2289 MaskVec[i] = i + Offset;
2290 }
2291 };
2292 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2293 BlendSplat(N1BV, 0);
2294 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2295 BlendSplat(N2BV, NElts);
2296 }
2297
2298 // Canonicalize all index into lhs, -> shuffle lhs, undef
2299 // Canonicalize all index into rhs, -> shuffle rhs, undef
2300 bool AllLHS = true, AllRHS = true;
2301 bool N2Undef = N2.isUndef();
2302 for (int i = 0; i != NElts; ++i) {
2303 if (MaskVec[i] >= NElts) {
2304 if (N2Undef)
2305 MaskVec[i] = -1;
2306 else
2307 AllLHS = false;
2308 } else if (MaskVec[i] >= 0) {
2309 AllRHS = false;
2310 }
2311 }
2312 if (AllLHS && AllRHS)
2313 return getUNDEF(VT);
2314 if (AllLHS && !N2Undef)
2315 N2 = getUNDEF(VT);
2316 if (AllRHS) {
2317 N1 = getUNDEF(VT);
2318 commuteShuffle(N1, N2, M: MaskVec);
2319 }
2320 // Reset our undef status after accounting for the mask.
2321 N2Undef = N2.isUndef();
2322 // Re-check whether both sides ended up undef.
2323 if (N1.isUndef() && N2Undef)
2324 return getUNDEF(VT);
2325
2326 // If Identity shuffle return that node.
2327 bool Identity = true, AllSame = true;
2328 for (int i = 0; i != NElts; ++i) {
2329 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2330 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2331 }
2332 if (Identity && NElts)
2333 return N1;
2334
2335 // Shuffling a constant splat doesn't change the result.
2336 if (N2Undef) {
2337 SDValue V = N1;
2338
2339 // Look through any bitcasts. We check that these don't change the number
2340 // (and size) of elements and just changes their types.
2341 while (V.getOpcode() == ISD::BITCAST)
2342 V = V->getOperand(Num: 0);
2343
2344 // A splat should always show up as a build vector node.
2345 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2346 BitVector UndefElements;
2347 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2348 // If this is a splat of an undef, shuffling it is also undef.
2349 if (Splat && Splat.isUndef())
2350 return getUNDEF(VT);
2351
2352 bool SameNumElts =
2353 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2354
2355 // We only have a splat which can skip shuffles if there is a splatted
2356 // value and no undef lanes rearranged by the shuffle.
2357 if (Splat && UndefElements.none()) {
2358 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2359 // number of elements match or the value splatted is a zero constant.
2360 if (SameNumElts || isNullConstant(V: Splat))
2361 return N1;
2362 }
2363
2364 // If the shuffle itself creates a splat, build the vector directly.
2365 if (AllSame && SameNumElts) {
2366 EVT BuildVT = BV->getValueType(ResNo: 0);
2367 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2368 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2369
2370 // We may have jumped through bitcasts, so the type of the
2371 // BUILD_VECTOR may not match the type of the shuffle.
2372 if (BuildVT != VT)
2373 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2374 return NewBV;
2375 }
2376 }
2377 }
2378
2379 SDVTList VTs = getVTList(VT);
2380 FoldingSetNodeID ID;
2381 SDValue Ops[2] = { N1, N2 };
2382 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: VTs, OpList: Ops);
2383 for (int i = 0; i != NElts; ++i)
2384 ID.AddInteger(I: MaskVec[i]);
2385
2386 void* IP = nullptr;
2387 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2388 return SDValue(E, 0);
2389
2390 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2391 // SDNode doesn't have access to it. This memory will be "leaked" when
2392 // the node is deallocated, but recovered when the NodeAllocator is released.
2393 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2394 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2395
2396 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2397 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2398 createOperands(Node: N, Vals: Ops);
2399
2400 CSEMap.InsertNode(N, InsertPos: IP);
2401 InsertNode(N);
2402 SDValue V = SDValue(N, 0);
2403 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2404 return V;
2405}
2406
2407SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2408 EVT VT = SV.getValueType(ResNo: 0);
2409 SmallVector<int, 8> MaskVec(SV.getMask());
2410 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2411
2412 SDValue Op0 = SV.getOperand(Num: 0);
2413 SDValue Op1 = SV.getOperand(Num: 1);
2414 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2415}
2416
2417SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2418 SDVTList VTs = getVTList(VT);
2419 FoldingSetNodeID ID;
2420 AddNodeIDNode(ID, OpC: ISD::Register, VTList: VTs, OpList: {});
2421 ID.AddInteger(I: Reg.id());
2422 void *IP = nullptr;
2423 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2424 return SDValue(E, 0);
2425
2426 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2427 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2428 CSEMap.InsertNode(N, InsertPos: IP);
2429 InsertNode(N);
2430 return SDValue(N, 0);
2431}
2432
2433SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2434 FoldingSetNodeID ID;
2435 AddNodeIDNode(ID, OpC: ISD::RegisterMask, VTList: getVTList(VT: MVT::Untyped), OpList: {});
2436 ID.AddPointer(Ptr: RegMask);
2437 void *IP = nullptr;
2438 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2439 return SDValue(E, 0);
2440
2441 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2442 CSEMap.InsertNode(N, InsertPos: IP);
2443 InsertNode(N);
2444 return SDValue(N, 0);
2445}
2446
2447SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2448 MCSymbol *Label) {
2449 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2450}
2451
2452SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2453 SDValue Root, MCSymbol *Label) {
2454 FoldingSetNodeID ID;
2455 SDValue Ops[] = { Root };
2456 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT: MVT::Other), OpList: Ops);
2457 ID.AddPointer(Ptr: Label);
2458 void *IP = nullptr;
2459 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2460 return SDValue(E, 0);
2461
2462 auto *N =
2463 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2464 createOperands(Node: N, Vals: Ops);
2465
2466 CSEMap.InsertNode(N, InsertPos: IP);
2467 InsertNode(N);
2468 return SDValue(N, 0);
2469}
2470
2471SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2472 int64_t Offset, bool isTarget,
2473 unsigned TargetFlags) {
2474 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2475 SDVTList VTs = getVTList(VT);
2476
2477 FoldingSetNodeID ID;
2478 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2479 ID.AddPointer(Ptr: BA);
2480 ID.AddInteger(I: Offset);
2481 ID.AddInteger(I: TargetFlags);
2482 void *IP = nullptr;
2483 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2484 return SDValue(E, 0);
2485
2486 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2487 CSEMap.InsertNode(N, InsertPos: IP);
2488 InsertNode(N);
2489 return SDValue(N, 0);
2490}
2491
2492SDValue SelectionDAG::getSrcValue(const Value *V) {
2493 FoldingSetNodeID ID;
2494 AddNodeIDNode(ID, OpC: ISD::SRCVALUE, VTList: getVTList(VT: MVT::Other), OpList: {});
2495 ID.AddPointer(Ptr: V);
2496
2497 void *IP = nullptr;
2498 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2499 return SDValue(E, 0);
2500
2501 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2502 CSEMap.InsertNode(N, InsertPos: IP);
2503 InsertNode(N);
2504 return SDValue(N, 0);
2505}
2506
2507SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2508 FoldingSetNodeID ID;
2509 AddNodeIDNode(ID, OpC: ISD::MDNODE_SDNODE, VTList: getVTList(VT: MVT::Other), OpList: {});
2510 ID.AddPointer(Ptr: MD);
2511
2512 void *IP = nullptr;
2513 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2514 return SDValue(E, 0);
2515
2516 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2517 CSEMap.InsertNode(N, InsertPos: IP);
2518 InsertNode(N);
2519 return SDValue(N, 0);
2520}
2521
2522SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2523 if (VT == V.getValueType())
2524 return V;
2525
2526 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2527}
2528
2529SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2530 unsigned SrcAS, unsigned DestAS) {
2531 SDVTList VTs = getVTList(VT);
2532 SDValue Ops[] = {Ptr};
2533 FoldingSetNodeID ID;
2534 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: VTs, OpList: Ops);
2535 ID.AddInteger(I: SrcAS);
2536 ID.AddInteger(I: DestAS);
2537
2538 void *IP = nullptr;
2539 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2540 return SDValue(E, 0);
2541
2542 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2543 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2544 createOperands(Node: N, Vals: Ops);
2545
2546 CSEMap.InsertNode(N, InsertPos: IP);
2547 InsertNode(N);
2548 return SDValue(N, 0);
2549}
2550
2551SDValue SelectionDAG::getFreeze(SDValue V) {
2552 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2553}
2554
2555SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2556 bool PoisonOnly) {
2557 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, PoisonOnly))
2558 return V;
2559 return getFreeze(V);
2560}
2561
2562/// getShiftAmountOperand - Return the specified value casted to
2563/// the target's desired shift amount type.
2564SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2565 EVT OpTy = Op.getValueType();
2566 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2567 if (OpTy == ShTy || OpTy.isVector()) return Op;
2568
2569 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2570}
2571
2572SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2573 SDLoc dl(Node);
2574 const TargetLowering &TLI = getTargetLoweringInfo();
2575 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2576 EVT VT = Node->getValueType(ResNo: 0);
2577 SDValue Tmp1 = Node->getOperand(Num: 0);
2578 SDValue Tmp2 = Node->getOperand(Num: 1);
2579 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2580
2581 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2582 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2583 SDValue VAList = VAListLoad;
2584
2585 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2586 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2587 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2588
2589 VAList = getNode(
2590 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2591 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2592 }
2593
2594 // Increment the pointer, VAList, to the next vaarg
2595 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2596 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2597 Ty: VT.getTypeForEVT(Context&: *getContext())),
2598 DL: dl, VT: VAList.getValueType()));
2599 // Store the incremented VAList to the legalized pointer
2600 Tmp1 =
2601 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2602 // Load the actual argument out of the pointer VAList
2603 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2604}
2605
2606SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2607 SDLoc dl(Node);
2608 const TargetLowering &TLI = getTargetLoweringInfo();
2609 // This defaults to loading a pointer from the input and storing it to the
2610 // output, returning the chain.
2611 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2612 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2613 SDValue Tmp1 =
2614 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2615 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2616 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2617 PtrInfo: MachinePointerInfo(VD));
2618}
2619
2620Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2621 const DataLayout &DL = getDataLayout();
2622 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2623 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2624
2625 if (TLI->isTypeLegal(VT) || !VT.isVector())
2626 return RedAlign;
2627
2628 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2629 const Align StackAlign = TFI->getStackAlign();
2630
2631 // See if we can choose a smaller ABI alignment in cases where it's an
2632 // illegal vector type that will get broken down.
2633 if (RedAlign > StackAlign) {
2634 EVT IntermediateVT;
2635 MVT RegisterVT;
2636 unsigned NumIntermediates;
2637 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2638 NumIntermediates, RegisterVT);
2639 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2640 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2641 if (RedAlign2 < RedAlign)
2642 RedAlign = RedAlign2;
2643
2644 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2645 // If the stack is not realignable, the alignment should be limited to the
2646 // StackAlignment
2647 RedAlign = std::min(a: RedAlign, b: StackAlign);
2648 }
2649
2650 return RedAlign;
2651}
2652
2653SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2654 MachineFrameInfo &MFI = MF->getFrameInfo();
2655 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2656 int StackID = 0;
2657 if (Bytes.isScalable())
2658 StackID = TFI->getStackIDForScalableVectors();
2659 // The stack id gives an indication of whether the object is scalable or
2660 // not, so it's safe to pass in the minimum size here.
2661 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2662 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2663 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2664}
2665
2666SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2667 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2668 Align StackAlign =
2669 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2670 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2671}
2672
2673SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2674 TypeSize VT1Size = VT1.getStoreSize();
2675 TypeSize VT2Size = VT2.getStoreSize();
2676 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2677 "Don't know how to choose the maximum size when creating a stack "
2678 "temporary");
2679 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2680 ? VT1Size
2681 : VT2Size;
2682
2683 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2684 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2685 const DataLayout &DL = getDataLayout();
2686 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2687 return CreateStackTemporary(Bytes, Alignment: Align);
2688}
2689
2690SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2691 ISD::CondCode Cond, const SDLoc &dl,
2692 SDNodeFlags Flags) {
2693 EVT OpVT = N1.getValueType();
2694
2695 auto GetUndefBooleanConstant = [&]() {
2696 if (VT.getScalarType() == MVT::i1 ||
2697 TLI->getBooleanContents(Type: OpVT) ==
2698 TargetLowering::UndefinedBooleanContent)
2699 return getUNDEF(VT);
2700 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2701 // so we cannot use getUNDEF(). Return zero instead.
2702 return getConstant(Val: 0, DL: dl, VT);
2703 };
2704
2705 // These setcc operations always fold.
2706 switch (Cond) {
2707 default: break;
2708 case ISD::SETFALSE:
2709 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2710 case ISD::SETTRUE:
2711 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2712
2713 case ISD::SETOEQ:
2714 case ISD::SETOGT:
2715 case ISD::SETOGE:
2716 case ISD::SETOLT:
2717 case ISD::SETOLE:
2718 case ISD::SETONE:
2719 case ISD::SETO:
2720 case ISD::SETUO:
2721 case ISD::SETUEQ:
2722 case ISD::SETUNE:
2723 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2724 break;
2725 }
2726
2727 if (OpVT.isInteger()) {
2728 // For EQ and NE, we can always pick a value for the undef to make the
2729 // predicate pass or fail, so we can return undef.
2730 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2731 // icmp eq/ne X, undef -> undef.
2732 if ((N1.isUndef() || N2.isUndef()) &&
2733 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2734 return GetUndefBooleanConstant();
2735
2736 // If both operands are undef, we can return undef for int comparison.
2737 // icmp undef, undef -> undef.
2738 if (N1.isUndef() && N2.isUndef())
2739 return GetUndefBooleanConstant();
2740
2741 // icmp X, X -> true/false
2742 // icmp X, undef -> true/false because undef could be X.
2743 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2744 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2745 }
2746
2747 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2748 const APInt &C2 = N2C->getAPIntValue();
2749 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2750 const APInt &C1 = N1C->getAPIntValue();
2751
2752 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2753 DL: dl, VT, OpVT);
2754 }
2755 }
2756
2757 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2758 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2759
2760 if (N1CFP && N2CFP) {
2761 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2762 switch (Cond) {
2763 default: break;
2764 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2765 return GetUndefBooleanConstant();
2766 [[fallthrough]];
2767 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2768 OpVT);
2769 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2770 return GetUndefBooleanConstant();
2771 [[fallthrough]];
2772 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2773 R==APFloat::cmpLessThan, DL: dl, VT,
2774 OpVT);
2775 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2776 return GetUndefBooleanConstant();
2777 [[fallthrough]];
2778 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2779 OpVT);
2780 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2781 return GetUndefBooleanConstant();
2782 [[fallthrough]];
2783 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2784 VT, OpVT);
2785 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2786 return GetUndefBooleanConstant();
2787 [[fallthrough]];
2788 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2789 R==APFloat::cmpEqual, DL: dl, VT,
2790 OpVT);
2791 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2792 return GetUndefBooleanConstant();
2793 [[fallthrough]];
2794 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2795 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2796 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2797 OpVT);
2798 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2799 OpVT);
2800 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2801 R==APFloat::cmpEqual, DL: dl, VT,
2802 OpVT);
2803 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2804 OpVT);
2805 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2806 R==APFloat::cmpLessThan, DL: dl, VT,
2807 OpVT);
2808 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2809 R==APFloat::cmpUnordered, DL: dl, VT,
2810 OpVT);
2811 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2812 VT, OpVT);
2813 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2814 OpVT);
2815 }
2816 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2817 // Ensure that the constant occurs on the RHS.
2818 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2819 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2820 return SDValue();
2821 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chian=*/Chain: {},
2822 /*IsSignaling=*/false, Flags);
2823 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2824 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2825 // If an operand is known to be a nan (or undef that could be a nan), we can
2826 // fold it.
2827 // Choosing NaN for the undef will always make unordered comparison succeed
2828 // and ordered comparison fails.
2829 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2830 switch (ISD::getUnorderedFlavor(Cond)) {
2831 default:
2832 llvm_unreachable("Unknown flavor!");
2833 case 0: // Known false.
2834 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2835 case 1: // Known true.
2836 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2837 case 2: // Undefined.
2838 return GetUndefBooleanConstant();
2839 }
2840 }
2841
2842 // Could not fold it.
2843 return SDValue();
2844}
2845
2846/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2847/// use this predicate to simplify operations downstream.
2848bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2849 unsigned BitWidth = Op.getScalarValueSizeInBits();
2850 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2851}
2852
2853// TODO: Should have argument to specify if sign bit of nan is ignorable.
2854bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2855 if (Depth >= MaxRecursionDepth)
2856 return false; // Limit search depth.
2857
2858 unsigned Opc = Op.getOpcode();
2859 switch (Opc) {
2860 case ISD::FABS:
2861 return true;
2862 case ISD::AssertNoFPClass: {
2863 FPClassTest NoFPClass =
2864 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2865
2866 const FPClassTest TestMask = fcNan | fcNegative;
2867 return (NoFPClass & TestMask) == TestMask;
2868 }
2869 case ISD::ARITH_FENCE:
2870 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2871 case ISD::FEXP:
2872 case ISD::FEXP2:
2873 case ISD::FEXP10:
2874 return Op->getFlags().hasNoNaNs();
2875 case ISD::FMINNUM:
2876 case ISD::FMINNUM_IEEE:
2877 case ISD::FMINIMUM:
2878 case ISD::FMINIMUMNUM:
2879 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2880 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2881 case ISD::FMAXNUM:
2882 case ISD::FMAXNUM_IEEE:
2883 case ISD::FMAXIMUM:
2884 case ISD::FMAXIMUMNUM:
2885 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2886 // is sufficient.
2887 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2888 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2889 default:
2890 return false;
2891 }
2892
2893 llvm_unreachable("covered opcode switch");
2894}
2895
2896/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2897/// this predicate to simplify operations downstream. Mask is known to be zero
2898/// for bits that V cannot have.
2899bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2900 unsigned Depth) const {
2901 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2902}
2903
2904/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2905/// DemandedElts. We use this predicate to simplify operations downstream.
2906/// Mask is known to be zero for bits that V cannot have.
2907bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2908 const APInt &DemandedElts,
2909 unsigned Depth) const {
2910 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2911}
2912
2913/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2914/// DemandedElts. We use this predicate to simplify operations downstream.
2915bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2916 unsigned Depth /* = 0 */) const {
2917 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2918}
2919
2920/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2921bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2922 unsigned Depth) const {
2923 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2924}
2925
2926APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2927 const APInt &DemandedElts,
2928 unsigned Depth) const {
2929 EVT VT = Op.getValueType();
2930 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2931
2932 unsigned NumElts = VT.getVectorNumElements();
2933 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2934
2935 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2936 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2937 if (!DemandedElts[EltIdx])
2938 continue; // Don't query elements that are not demanded.
2939 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2940 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2941 KnownZeroElements.setBit(EltIdx);
2942 }
2943 return KnownZeroElements;
2944}
2945
2946/// isSplatValue - Return true if the vector V has the same value
2947/// across all DemandedElts. For scalable vectors, we don't know the
2948/// number of lanes at compile time. Instead, we use a 1 bit APInt
2949/// to represent a conservative value for all lanes; that is, that
2950/// one bit value is implicitly splatted across all lanes.
2951bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2952 APInt &UndefElts, unsigned Depth) const {
2953 unsigned Opcode = V.getOpcode();
2954 EVT VT = V.getValueType();
2955 assert(VT.isVector() && "Vector type expected");
2956 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2957 "scalable demanded bits are ignored");
2958
2959 if (!DemandedElts)
2960 return false; // No demanded elts, better to assume we don't know anything.
2961
2962 if (Depth >= MaxRecursionDepth)
2963 return false; // Limit search depth.
2964
2965 // Deal with some common cases here that work for both fixed and scalable
2966 // vector types.
2967 switch (Opcode) {
2968 case ISD::SPLAT_VECTOR:
2969 UndefElts = V.getOperand(i: 0).isUndef()
2970 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2971 : APInt(DemandedElts.getBitWidth(), 0);
2972 return true;
2973 case ISD::ADD:
2974 case ISD::SUB:
2975 case ISD::AND:
2976 case ISD::XOR:
2977 case ISD::OR: {
2978 APInt UndefLHS, UndefRHS;
2979 SDValue LHS = V.getOperand(i: 0);
2980 SDValue RHS = V.getOperand(i: 1);
2981 // Only recognize splats with the same demanded undef elements for both
2982 // operands, otherwise we might fail to handle binop-specific undef
2983 // handling.
2984 // e.g. (and undef, 0) -> 0 etc.
2985 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2986 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2987 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2988 UndefElts = UndefLHS | UndefRHS;
2989 return true;
2990 }
2991 return false;
2992 }
2993 case ISD::ABS:
2994 case ISD::TRUNCATE:
2995 case ISD::SIGN_EXTEND:
2996 case ISD::ZERO_EXTEND:
2997 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
2998 default:
2999 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3000 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3001 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3002 Depth);
3003 break;
3004 }
3005
3006 // We don't support other cases than those above for scalable vectors at
3007 // the moment.
3008 if (VT.isScalableVector())
3009 return false;
3010
3011 unsigned NumElts = VT.getVectorNumElements();
3012 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3013 UndefElts = APInt::getZero(numBits: NumElts);
3014
3015 switch (Opcode) {
3016 case ISD::BUILD_VECTOR: {
3017 SDValue Scl;
3018 for (unsigned i = 0; i != NumElts; ++i) {
3019 SDValue Op = V.getOperand(i);
3020 if (Op.isUndef()) {
3021 UndefElts.setBit(i);
3022 continue;
3023 }
3024 if (!DemandedElts[i])
3025 continue;
3026 if (Scl && Scl != Op)
3027 return false;
3028 Scl = Op;
3029 }
3030 return true;
3031 }
3032 case ISD::VECTOR_SHUFFLE: {
3033 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3034 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3035 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3036 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3037 for (int i = 0; i != (int)NumElts; ++i) {
3038 int M = Mask[i];
3039 if (M < 0) {
3040 UndefElts.setBit(i);
3041 continue;
3042 }
3043 if (!DemandedElts[i])
3044 continue;
3045 if (M < (int)NumElts)
3046 DemandedLHS.setBit(M);
3047 else
3048 DemandedRHS.setBit(M - NumElts);
3049 }
3050
3051 // If we aren't demanding either op, assume there's no splat.
3052 // If we are demanding both ops, assume there's no splat.
3053 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3054 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3055 return false;
3056
3057 // See if the demanded elts of the source op is a splat or we only demand
3058 // one element, which should always be a splat.
3059 // TODO: Handle source ops splats with undefs.
3060 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3061 APInt SrcUndefs;
3062 return (SrcElts.popcount() == 1) ||
3063 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3064 (SrcElts & SrcUndefs).isZero());
3065 };
3066 if (!DemandedLHS.isZero())
3067 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3068 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3069 }
3070 case ISD::EXTRACT_SUBVECTOR: {
3071 // Offset the demanded elts by the subvector index.
3072 SDValue Src = V.getOperand(i: 0);
3073 // We don't support scalable vectors at the moment.
3074 if (Src.getValueType().isScalableVector())
3075 return false;
3076 uint64_t Idx = V.getConstantOperandVal(i: 1);
3077 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3078 APInt UndefSrcElts;
3079 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3080 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3081 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3082 return true;
3083 }
3084 break;
3085 }
3086 case ISD::ANY_EXTEND_VECTOR_INREG:
3087 case ISD::SIGN_EXTEND_VECTOR_INREG:
3088 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3089 // Widen the demanded elts by the src element count.
3090 SDValue Src = V.getOperand(i: 0);
3091 // We don't support scalable vectors at the moment.
3092 if (Src.getValueType().isScalableVector())
3093 return false;
3094 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3095 APInt UndefSrcElts;
3096 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3097 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3098 UndefElts = UndefSrcElts.trunc(width: NumElts);
3099 return true;
3100 }
3101 break;
3102 }
3103 case ISD::BITCAST: {
3104 SDValue Src = V.getOperand(i: 0);
3105 EVT SrcVT = Src.getValueType();
3106 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3107 unsigned BitWidth = VT.getScalarSizeInBits();
3108
3109 // Ignore bitcasts from unsupported types.
3110 // TODO: Add fp support?
3111 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3112 break;
3113
3114 // Bitcast 'small element' vector to 'large element' vector.
3115 if ((BitWidth % SrcBitWidth) == 0) {
3116 // See if each sub element is a splat.
3117 unsigned Scale = BitWidth / SrcBitWidth;
3118 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3119 APInt ScaledDemandedElts =
3120 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3121 for (unsigned I = 0; I != Scale; ++I) {
3122 APInt SubUndefElts;
3123 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3124 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3125 SubDemandedElts &= ScaledDemandedElts;
3126 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3127 return false;
3128 // TODO: Add support for merging sub undef elements.
3129 if (!SubUndefElts.isZero())
3130 return false;
3131 }
3132 return true;
3133 }
3134 break;
3135 }
3136 }
3137
3138 return false;
3139}
3140
3141/// Helper wrapper to main isSplatValue function.
3142bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3143 EVT VT = V.getValueType();
3144 assert(VT.isVector() && "Vector type expected");
3145
3146 APInt UndefElts;
3147 // Since the number of lanes in a scalable vector is unknown at compile time,
3148 // we track one bit which is implicitly broadcast to all lanes. This means
3149 // that all lanes in a scalable vector are considered demanded.
3150 APInt DemandedElts
3151 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3152 return isSplatValue(V, DemandedElts, UndefElts) &&
3153 (AllowUndefs || !UndefElts);
3154}
3155
3156SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3157 V = peekThroughExtractSubvectors(V);
3158
3159 EVT VT = V.getValueType();
3160 unsigned Opcode = V.getOpcode();
3161 switch (Opcode) {
3162 default: {
3163 APInt UndefElts;
3164 // Since the number of lanes in a scalable vector is unknown at compile time,
3165 // we track one bit which is implicitly broadcast to all lanes. This means
3166 // that all lanes in a scalable vector are considered demanded.
3167 APInt DemandedElts
3168 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3169
3170 if (isSplatValue(V, DemandedElts, UndefElts)) {
3171 if (VT.isScalableVector()) {
3172 // DemandedElts and UndefElts are ignored for scalable vectors, since
3173 // the only supported cases are SPLAT_VECTOR nodes.
3174 SplatIdx = 0;
3175 } else {
3176 // Handle case where all demanded elements are UNDEF.
3177 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3178 SplatIdx = 0;
3179 return getUNDEF(VT);
3180 }
3181 SplatIdx = (UndefElts & DemandedElts).countr_one();
3182 }
3183 return V;
3184 }
3185 break;
3186 }
3187 case ISD::SPLAT_VECTOR:
3188 SplatIdx = 0;
3189 return V;
3190 case ISD::VECTOR_SHUFFLE: {
3191 assert(!VT.isScalableVector());
3192 // Check if this is a shuffle node doing a splat.
3193 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3194 // getTargetVShiftNode currently struggles without the splat source.
3195 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3196 if (!SVN->isSplat())
3197 break;
3198 int Idx = SVN->getSplatIndex();
3199 int NumElts = V.getValueType().getVectorNumElements();
3200 SplatIdx = Idx % NumElts;
3201 return V.getOperand(i: Idx / NumElts);
3202 }
3203 }
3204
3205 return SDValue();
3206}
3207
3208SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3209 int SplatIdx;
3210 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3211 EVT SVT = SrcVector.getValueType().getScalarType();
3212 EVT LegalSVT = SVT;
3213 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3214 if (!SVT.isInteger())
3215 return SDValue();
3216 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3217 if (LegalSVT.bitsLT(VT: SVT))
3218 return SDValue();
3219 }
3220 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3221 }
3222 return SDValue();
3223}
3224
3225std::optional<ConstantRange>
3226SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3227 unsigned Depth) const {
3228 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3229 V.getOpcode() == ISD::SRA) &&
3230 "Unknown shift node");
3231 // Shifting more than the bitwidth is not valid.
3232 unsigned BitWidth = V.getScalarValueSizeInBits();
3233
3234 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3235 const APInt &ShAmt = Cst->getAPIntValue();
3236 if (ShAmt.uge(RHS: BitWidth))
3237 return std::nullopt;
3238 return ConstantRange(ShAmt);
3239 }
3240
3241 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3242 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3243 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3244 if (!DemandedElts[i])
3245 continue;
3246 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3247 if (!SA) {
3248 MinAmt = MaxAmt = nullptr;
3249 break;
3250 }
3251 const APInt &ShAmt = SA->getAPIntValue();
3252 if (ShAmt.uge(RHS: BitWidth))
3253 return std::nullopt;
3254 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3255 MinAmt = &ShAmt;
3256 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3257 MaxAmt = &ShAmt;
3258 }
3259 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3260 "Failed to find matching min/max shift amounts");
3261 if (MinAmt && MaxAmt)
3262 return ConstantRange(*MinAmt, *MaxAmt + 1);
3263 }
3264
3265 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3266 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3267 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3268 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3269 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3270
3271 return std::nullopt;
3272}
3273
3274std::optional<unsigned>
3275SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3276 unsigned Depth) const {
3277 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3278 V.getOpcode() == ISD::SRA) &&
3279 "Unknown shift node");
3280 if (std::optional<ConstantRange> AmtRange =
3281 getValidShiftAmountRange(V, DemandedElts, Depth))
3282 if (const APInt *ShAmt = AmtRange->getSingleElement())
3283 return ShAmt->getZExtValue();
3284 return std::nullopt;
3285}
3286
3287std::optional<unsigned>
3288SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3289 APInt DemandedElts = getDemandAllEltsMask(V);
3290 return getValidShiftAmount(V, DemandedElts, Depth);
3291}
3292
3293std::optional<unsigned>
3294SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3295 unsigned Depth) const {
3296 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3297 V.getOpcode() == ISD::SRA) &&
3298 "Unknown shift node");
3299 if (std::optional<ConstantRange> AmtRange =
3300 getValidShiftAmountRange(V, DemandedElts, Depth))
3301 return AmtRange->getUnsignedMin().getZExtValue();
3302 return std::nullopt;
3303}
3304
3305std::optional<unsigned>
3306SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3307 APInt DemandedElts = getDemandAllEltsMask(V);
3308 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3309}
3310
3311std::optional<unsigned>
3312SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3313 unsigned Depth) const {
3314 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3315 V.getOpcode() == ISD::SRA) &&
3316 "Unknown shift node");
3317 if (std::optional<ConstantRange> AmtRange =
3318 getValidShiftAmountRange(V, DemandedElts, Depth))
3319 return AmtRange->getUnsignedMax().getZExtValue();
3320 return std::nullopt;
3321}
3322
3323std::optional<unsigned>
3324SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3325 APInt DemandedElts = getDemandAllEltsMask(V);
3326 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3327}
3328
3329/// Determine which bits of Op are known to be either zero or one and return
3330/// them in Known. For vectors, the known bits are those that are shared by
3331/// every vector element.
3332KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3333 APInt DemandedElts = getDemandAllEltsMask(V: Op);
3334 return computeKnownBits(Op, DemandedElts, Depth);
3335}
3336
3337/// Determine which bits of Op are known to be either zero or one and return
3338/// them in Known. The DemandedElts argument allows us to only collect the known
3339/// bits that are shared by the requested vector elements.
3340KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3341 unsigned Depth) const {
3342 unsigned BitWidth = Op.getScalarValueSizeInBits();
3343
3344 KnownBits Known(BitWidth); // Don't know anything.
3345
3346 if (auto OptAPInt = Op->bitcastToAPInt()) {
3347 // We know all of the bits for a constant!
3348 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3349 }
3350
3351 if (Depth >= MaxRecursionDepth)
3352 return Known; // Limit search depth.
3353
3354 KnownBits Known2;
3355 unsigned NumElts = DemandedElts.getBitWidth();
3356 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3357 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3358 assert((!Op.getValueType().isFixedLengthVector() ||
3359 NumElts == Op.getValueType().getVectorNumElements()) &&
3360 "Unexpected vector size");
3361
3362 if (!DemandedElts)
3363 return Known; // No demanded elts, better to assume we don't know anything.
3364
3365 unsigned Opcode = Op.getOpcode();
3366 switch (Opcode) {
3367 case ISD::MERGE_VALUES:
3368 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3369 Depth: Depth + 1);
3370 case ISD::SPLAT_VECTOR: {
3371 SDValue SrcOp = Op.getOperand(i: 0);
3372 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3373 "Expected SPLAT_VECTOR implicit truncation");
3374 // Implicitly truncate the bits to match the official semantics of
3375 // SPLAT_VECTOR.
3376 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3377 break;
3378 }
3379 case ISD::SPLAT_VECTOR_PARTS: {
3380 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3381 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3382 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3383 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3384 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3385 }
3386 break;
3387 }
3388 case ISD::STEP_VECTOR: {
3389 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3390
3391 if (Step.isPowerOf2())
3392 Known.Zero.setLowBits(Step.logBase2());
3393
3394 const Function &F = getMachineFunction().getFunction();
3395
3396 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3397 break;
3398 const APInt MinNumElts =
3399 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3400
3401 bool Overflow;
3402 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3403 .getUnsignedMax()
3404 .umul_ov(RHS: MinNumElts, Overflow);
3405 if (Overflow)
3406 break;
3407
3408 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3409 if (Overflow)
3410 break;
3411
3412 Known.Zero.setHighBits(MaxValue.countl_zero());
3413 break;
3414 }
3415 case ISD::BUILD_VECTOR:
3416 assert(!Op.getValueType().isScalableVector());
3417 // Collect the known bits that are shared by every demanded vector element.
3418 Known.setAllConflict();
3419 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3420 if (!DemandedElts[i])
3421 continue;
3422
3423 SDValue SrcOp = Op.getOperand(i);
3424 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3425
3426 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3427 if (SrcOp.getValueSizeInBits() != BitWidth) {
3428 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3429 "Expected BUILD_VECTOR implicit truncation");
3430 Known2 = Known2.trunc(BitWidth);
3431 }
3432
3433 // Known bits are the values that are shared by every demanded element.
3434 Known = Known.intersectWith(RHS: Known2);
3435
3436 // If we don't know any bits, early out.
3437 if (Known.isUnknown())
3438 break;
3439 }
3440 break;
3441 case ISD::VECTOR_COMPRESS: {
3442 SDValue Vec = Op.getOperand(i: 0);
3443 SDValue PassThru = Op.getOperand(i: 2);
3444 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3445 // If we don't know any bits, early out.
3446 if (Known.isUnknown())
3447 break;
3448 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3449 Known = Known.intersectWith(RHS: Known2);
3450 break;
3451 }
3452 case ISD::VECTOR_SHUFFLE: {
3453 assert(!Op.getValueType().isScalableVector());
3454 // Collect the known bits that are shared by every vector element referenced
3455 // by the shuffle.
3456 APInt DemandedLHS, DemandedRHS;
3457 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3458 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3459 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3460 DemandedLHS, DemandedRHS))
3461 break;
3462
3463 // Known bits are the values that are shared by every demanded element.
3464 Known.setAllConflict();
3465 if (!!DemandedLHS) {
3466 SDValue LHS = Op.getOperand(i: 0);
3467 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3468 Known = Known.intersectWith(RHS: Known2);
3469 }
3470 // If we don't know any bits, early out.
3471 if (Known.isUnknown())
3472 break;
3473 if (!!DemandedRHS) {
3474 SDValue RHS = Op.getOperand(i: 1);
3475 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3476 Known = Known.intersectWith(RHS: Known2);
3477 }
3478 break;
3479 }
3480 case ISD::VSCALE: {
3481 const Function &F = getMachineFunction().getFunction();
3482 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3483 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3484 break;
3485 }
3486 case ISD::CONCAT_VECTORS: {
3487 if (Op.getValueType().isScalableVector())
3488 break;
3489 // Split DemandedElts and test each of the demanded subvectors.
3490 Known.setAllConflict();
3491 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3492 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3493 unsigned NumSubVectors = Op.getNumOperands();
3494 for (unsigned i = 0; i != NumSubVectors; ++i) {
3495 APInt DemandedSub =
3496 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3497 if (!!DemandedSub) {
3498 SDValue Sub = Op.getOperand(i);
3499 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3500 Known = Known.intersectWith(RHS: Known2);
3501 }
3502 // If we don't know any bits, early out.
3503 if (Known.isUnknown())
3504 break;
3505 }
3506 break;
3507 }
3508 case ISD::INSERT_SUBVECTOR: {
3509 if (Op.getValueType().isScalableVector())
3510 break;
3511 // Demand any elements from the subvector and the remainder from the src its
3512 // inserted into.
3513 SDValue Src = Op.getOperand(i: 0);
3514 SDValue Sub = Op.getOperand(i: 1);
3515 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3516 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3517 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3518 APInt DemandedSrcElts = DemandedElts;
3519 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3520
3521 Known.setAllConflict();
3522 if (!!DemandedSubElts) {
3523 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3524 if (Known.isUnknown())
3525 break; // early-out.
3526 }
3527 if (!!DemandedSrcElts) {
3528 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3529 Known = Known.intersectWith(RHS: Known2);
3530 }
3531 break;
3532 }
3533 case ISD::EXTRACT_SUBVECTOR: {
3534 // Offset the demanded elts by the subvector index.
3535 SDValue Src = Op.getOperand(i: 0);
3536
3537 APInt DemandedSrcElts;
3538 if (Src.getValueType().isScalableVector())
3539 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3540 else {
3541 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3542 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3543 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3544 }
3545 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3546 break;
3547 }
3548 case ISD::SCALAR_TO_VECTOR: {
3549 if (Op.getValueType().isScalableVector())
3550 break;
3551 // We know about scalar_to_vector as much as we know about it source,
3552 // which becomes the first element of otherwise unknown vector.
3553 if (DemandedElts != 1)
3554 break;
3555
3556 SDValue N0 = Op.getOperand(i: 0);
3557 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3558 if (N0.getValueSizeInBits() != BitWidth)
3559 Known = Known.trunc(BitWidth);
3560
3561 break;
3562 }
3563 case ISD::BITCAST: {
3564 if (Op.getValueType().isScalableVector())
3565 break;
3566
3567 SDValue N0 = Op.getOperand(i: 0);
3568 EVT SubVT = N0.getValueType();
3569 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3570
3571 // Ignore bitcasts from unsupported types.
3572 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3573 break;
3574
3575 // Fast handling of 'identity' bitcasts.
3576 if (BitWidth == SubBitWidth) {
3577 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3578 break;
3579 }
3580
3581 bool IsLE = getDataLayout().isLittleEndian();
3582
3583 // Bitcast 'small element' vector to 'large element' scalar/vector.
3584 if ((BitWidth % SubBitWidth) == 0) {
3585 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3586
3587 // Collect known bits for the (larger) output by collecting the known
3588 // bits from each set of sub elements and shift these into place.
3589 // We need to separately call computeKnownBits for each set of
3590 // sub elements as the knownbits for each is likely to be different.
3591 unsigned SubScale = BitWidth / SubBitWidth;
3592 APInt SubDemandedElts(NumElts * SubScale, 0);
3593 for (unsigned i = 0; i != NumElts; ++i)
3594 if (DemandedElts[i])
3595 SubDemandedElts.setBit(i * SubScale);
3596
3597 for (unsigned i = 0; i != SubScale; ++i) {
3598 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3599 Depth: Depth + 1);
3600 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3601 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3602 }
3603 }
3604
3605 // Bitcast 'large element' scalar/vector to 'small element' vector.
3606 if ((SubBitWidth % BitWidth) == 0) {
3607 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3608
3609 // Collect known bits for the (smaller) output by collecting the known
3610 // bits from the overlapping larger input elements and extracting the
3611 // sub sections we actually care about.
3612 unsigned SubScale = SubBitWidth / BitWidth;
3613 APInt SubDemandedElts =
3614 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3615 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3616
3617 Known.setAllConflict();
3618 for (unsigned i = 0; i != NumElts; ++i)
3619 if (DemandedElts[i]) {
3620 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3621 unsigned Offset = (Shifts % SubScale) * BitWidth;
3622 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3623 // If we don't know any bits, early out.
3624 if (Known.isUnknown())
3625 break;
3626 }
3627 }
3628 break;
3629 }
3630 case ISD::AND:
3631 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3632 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3633
3634 Known &= Known2;
3635 break;
3636 case ISD::OR:
3637 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3638 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3639
3640 Known |= Known2;
3641 break;
3642 case ISD::XOR:
3643 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3644 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3645
3646 Known ^= Known2;
3647 break;
3648 case ISD::MUL: {
3649 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3650 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3651 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3652 // TODO: SelfMultiply can be poison, but not undef.
3653 if (SelfMultiply)
3654 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3655 Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
3656 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3657
3658 // If the multiplication is known not to overflow, the product of a number
3659 // with itself is non-negative. Only do this if we didn't already computed
3660 // the opposite value for the sign bit.
3661 if (Op->getFlags().hasNoSignedWrap() &&
3662 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3663 !Known.isNegative())
3664 Known.makeNonNegative();
3665 break;
3666 }
3667 case ISD::MULHU: {
3668 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3669 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3670 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3671 break;
3672 }
3673 case ISD::MULHS: {
3674 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3675 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3676 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3677 break;
3678 }
3679 case ISD::ABDU: {
3680 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3681 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3682 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3683 break;
3684 }
3685 case ISD::ABDS: {
3686 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3687 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3688 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3689 unsigned SignBits1 =
3690 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3691 if (SignBits1 == 1)
3692 break;
3693 unsigned SignBits0 =
3694 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3695 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3696 break;
3697 }
3698 case ISD::UMUL_LOHI: {
3699 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3700 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3701 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3702 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3703 if (Op.getResNo() == 0)
3704 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3705 else
3706 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3707 break;
3708 }
3709 case ISD::SMUL_LOHI: {
3710 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3711 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3712 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3713 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3714 if (Op.getResNo() == 0)
3715 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3716 else
3717 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3718 break;
3719 }
3720 case ISD::AVGFLOORU: {
3721 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3722 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3723 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3724 break;
3725 }
3726 case ISD::AVGCEILU: {
3727 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3728 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3729 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3730 break;
3731 }
3732 case ISD::AVGFLOORS: {
3733 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3734 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3735 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3736 break;
3737 }
3738 case ISD::AVGCEILS: {
3739 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3740 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3741 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3742 break;
3743 }
3744 case ISD::SELECT:
3745 case ISD::VSELECT:
3746 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3747 // If we don't know any bits, early out.
3748 if (Known.isUnknown())
3749 break;
3750 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3751
3752 // Only known if known in both the LHS and RHS.
3753 Known = Known.intersectWith(RHS: Known2);
3754 break;
3755 case ISD::SELECT_CC:
3756 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3757 // If we don't know any bits, early out.
3758 if (Known.isUnknown())
3759 break;
3760 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3761
3762 // Only known if known in both the LHS and RHS.
3763 Known = Known.intersectWith(RHS: Known2);
3764 break;
3765 case ISD::SMULO:
3766 case ISD::UMULO:
3767 if (Op.getResNo() != 1)
3768 break;
3769 // The boolean result conforms to getBooleanContents.
3770 // If we know the result of a setcc has the top bits zero, use this info.
3771 // We know that we have an integer-based boolean since these operations
3772 // are only available for integer.
3773 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3774 TargetLowering::ZeroOrOneBooleanContent &&
3775 BitWidth > 1)
3776 Known.Zero.setBitsFrom(1);
3777 break;
3778 case ISD::SETCC:
3779 case ISD::SETCCCARRY:
3780 case ISD::STRICT_FSETCC:
3781 case ISD::STRICT_FSETCCS: {
3782 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3783 // If we know the result of a setcc has the top bits zero, use this info.
3784 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3785 TargetLowering::ZeroOrOneBooleanContent &&
3786 BitWidth > 1)
3787 Known.Zero.setBitsFrom(1);
3788 break;
3789 }
3790 case ISD::SHL: {
3791 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3792 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3793
3794 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3795 bool NSW = Op->getFlags().hasNoSignedWrap();
3796
3797 bool ShAmtNonZero = Known2.isNonZero();
3798
3799 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3800
3801 // Minimum shift low bits are known zero.
3802 if (std::optional<unsigned> ShMinAmt =
3803 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3804 Known.Zero.setLowBits(*ShMinAmt);
3805 break;
3806 }
3807 case ISD::SRL:
3808 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3809 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3810 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3811 Exact: Op->getFlags().hasExact());
3812
3813 // Minimum shift high bits are known zero.
3814 if (std::optional<unsigned> ShMinAmt =
3815 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3816 Known.Zero.setHighBits(*ShMinAmt);
3817 break;
3818 case ISD::SRA:
3819 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3820 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3821 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3822 Exact: Op->getFlags().hasExact());
3823 break;
3824 case ISD::ROTL:
3825 case ISD::ROTR:
3826 if (ConstantSDNode *C =
3827 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3828 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3829
3830 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3831
3832 // Canonicalize to ROTR.
3833 if (Opcode == ISD::ROTL && Amt != 0)
3834 Amt = BitWidth - Amt;
3835
3836 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3837 Known.One = Known.One.rotr(rotateAmt: Amt);
3838 }
3839 break;
3840 case ISD::FSHL:
3841 case ISD::FSHR:
3842 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3843 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3844
3845 // For fshl, 0-shift returns the 1st arg.
3846 // For fshr, 0-shift returns the 2nd arg.
3847 if (Amt == 0) {
3848 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3849 DemandedElts, Depth: Depth + 1);
3850 break;
3851 }
3852
3853 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3854 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3855 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3856 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3857 if (Opcode == ISD::FSHL) {
3858 Known <<= Amt;
3859 Known2 >>= BitWidth - Amt;
3860 } else {
3861 Known <<= BitWidth - Amt;
3862 Known2 >>= Amt;
3863 }
3864 Known = Known.unionWith(RHS: Known2);
3865 }
3866 break;
3867 case ISD::SHL_PARTS:
3868 case ISD::SRA_PARTS:
3869 case ISD::SRL_PARTS: {
3870 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3871
3872 // Collect lo/hi source values and concatenate.
3873 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3874 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3875 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3876 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3877 Known = Known2.concat(Lo: Known);
3878
3879 // Collect shift amount.
3880 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3881
3882 if (Opcode == ISD::SHL_PARTS)
3883 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3884 else if (Opcode == ISD::SRA_PARTS)
3885 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3886 else // if (Opcode == ISD::SRL_PARTS)
3887 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3888
3889 // TODO: Minimum shift low/high bits are known zero.
3890
3891 if (Op.getResNo() == 0)
3892 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3893 else
3894 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3895 break;
3896 }
3897 case ISD::SIGN_EXTEND_INREG: {
3898 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3899 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3900 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3901 break;
3902 }
3903 case ISD::CTTZ:
3904 case ISD::CTTZ_ZERO_UNDEF: {
3905 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3906 // If we have a known 1, its position is our upper bound.
3907 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3908 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3909 Known.Zero.setBitsFrom(LowBits);
3910 break;
3911 }
3912 case ISD::CTLZ:
3913 case ISD::CTLZ_ZERO_UNDEF: {
3914 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3915 // If we have a known 1, its position is our upper bound.
3916 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3917 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3918 Known.Zero.setBitsFrom(LowBits);
3919 break;
3920 }
3921 case ISD::CTLS: {
3922 unsigned MinRedundantSignBits =
3923 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3924 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3925 APInt(BitWidth, BitWidth));
3926 Known = Range.toKnownBits();
3927 break;
3928 }
3929 case ISD::CTPOP: {
3930 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3931 // If we know some of the bits are zero, they can't be one.
3932 unsigned PossibleOnes = Known2.countMaxPopulation();
3933 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3934 break;
3935 }
3936 case ISD::PARITY: {
3937 // Parity returns 0 everywhere but the LSB.
3938 Known.Zero.setBitsFrom(1);
3939 break;
3940 }
3941 case ISD::CLMUL: {
3942 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3943 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3944 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3945 break;
3946 }
3947 case ISD::MGATHER:
3948 case ISD::MLOAD: {
3949 ISD::LoadExtType ETy =
3950 (Opcode == ISD::MGATHER)
3951 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3952 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3953 if (ETy == ISD::ZEXTLOAD) {
3954 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3955 KnownBits Known0(MemVT.getScalarSizeInBits());
3956 return Known0.zext(BitWidth);
3957 }
3958 break;
3959 }
3960 case ISD::LOAD: {
3961 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3962 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3963 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3964 // Determine any common known bits from the loaded constant pool value.
3965 Type *CstTy = Cst->getType();
3966 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3967 !Op.getValueType().isScalableVector()) {
3968 // If its a vector splat, then we can (quickly) reuse the scalar path.
3969 // NOTE: We assume all elements match and none are UNDEF.
3970 if (CstTy->isVectorTy()) {
3971 if (const Constant *Splat = Cst->getSplatValue()) {
3972 Cst = Splat;
3973 CstTy = Cst->getType();
3974 }
3975 }
3976 // TODO - do we need to handle different bitwidths?
3977 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3978 // Iterate across all vector elements finding common known bits.
3979 Known.setAllConflict();
3980 for (unsigned i = 0; i != NumElts; ++i) {
3981 if (!DemandedElts[i])
3982 continue;
3983 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
3984 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
3985 const APInt &Value = CInt->getValue();
3986 Known.One &= Value;
3987 Known.Zero &= ~Value;
3988 continue;
3989 }
3990 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
3991 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3992 Known.One &= Value;
3993 Known.Zero &= ~Value;
3994 continue;
3995 }
3996 }
3997 Known.One.clearAllBits();
3998 Known.Zero.clearAllBits();
3999 break;
4000 }
4001 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4002 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4003 Known = KnownBits::makeConstant(C: CInt->getValue());
4004 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4005 Known =
4006 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4007 }
4008 }
4009 }
4010 } else if (Op.getResNo() == 0) {
4011 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4012 KnownBits KnownScalarMemory(ScalarMemorySize);
4013 if (const MDNode *MD = LD->getRanges())
4014 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4015
4016 // Extend the Known bits from memory to the size of the scalar result.
4017 if (ISD::isZEXTLoad(N: Op.getNode()))
4018 Known = KnownScalarMemory.zext(BitWidth);
4019 else if (ISD::isSEXTLoad(N: Op.getNode()))
4020 Known = KnownScalarMemory.sext(BitWidth);
4021 else if (ISD::isEXTLoad(N: Op.getNode()))
4022 Known = KnownScalarMemory.anyext(BitWidth);
4023 else
4024 Known = KnownScalarMemory;
4025 assert(Known.getBitWidth() == BitWidth);
4026 return Known;
4027 }
4028 break;
4029 }
4030 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4031 if (Op.getValueType().isScalableVector())
4032 break;
4033 EVT InVT = Op.getOperand(i: 0).getValueType();
4034 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4035 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4036 Known = Known.zext(BitWidth);
4037 break;
4038 }
4039 case ISD::ZERO_EXTEND: {
4040 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4041 Known = Known.zext(BitWidth);
4042 break;
4043 }
4044 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4045 if (Op.getValueType().isScalableVector())
4046 break;
4047 EVT InVT = Op.getOperand(i: 0).getValueType();
4048 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4049 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4050 // If the sign bit is known to be zero or one, then sext will extend
4051 // it to the top bits, else it will just zext.
4052 Known = Known.sext(BitWidth);
4053 break;
4054 }
4055 case ISD::SIGN_EXTEND: {
4056 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4057 // If the sign bit is known to be zero or one, then sext will extend
4058 // it to the top bits, else it will just zext.
4059 Known = Known.sext(BitWidth);
4060 break;
4061 }
4062 case ISD::ANY_EXTEND_VECTOR_INREG: {
4063 if (Op.getValueType().isScalableVector())
4064 break;
4065 EVT InVT = Op.getOperand(i: 0).getValueType();
4066 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4067 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4068 Known = Known.anyext(BitWidth);
4069 break;
4070 }
4071 case ISD::ANY_EXTEND: {
4072 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4073 Known = Known.anyext(BitWidth);
4074 break;
4075 }
4076 case ISD::TRUNCATE: {
4077 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4078 Known = Known.trunc(BitWidth);
4079 break;
4080 }
4081 case ISD::TRUNCATE_SSAT_S: {
4082 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4083 Known = Known.truncSSat(BitWidth);
4084 break;
4085 }
4086 case ISD::TRUNCATE_SSAT_U: {
4087 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4088 Known = Known.truncSSatU(BitWidth);
4089 break;
4090 }
4091 case ISD::TRUNCATE_USAT_U: {
4092 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4093 Known = Known.truncUSat(BitWidth);
4094 break;
4095 }
4096 case ISD::AssertZext: {
4097 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4098 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4099 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4100 Known.Zero |= (~InMask);
4101 Known.One &= (~Known.Zero);
4102 break;
4103 }
4104 case ISD::AssertAlign: {
4105 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4106 assert(LogOfAlign != 0);
4107
4108 // TODO: Should use maximum with source
4109 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4110 // well as clearing one bits.
4111 Known.Zero.setLowBits(LogOfAlign);
4112 Known.One.clearLowBits(loBits: LogOfAlign);
4113 break;
4114 }
4115 case ISD::AssertNoFPClass: {
4116 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4117
4118 FPClassTest NoFPClass =
4119 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4120 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4121 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4122 // Cannot be negative.
4123 Known.makeNonNegative();
4124 }
4125
4126 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4127 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4128 // Cannot be positive.
4129 Known.makeNegative();
4130 }
4131
4132 break;
4133 }
4134 case ISD::FABS:
4135 // fabs clears the sign bit
4136 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4137 Known.makeNonNegative();
4138 break;
4139 case ISD::FGETSIGN:
4140 // All bits are zero except the low bit.
4141 Known.Zero.setBitsFrom(1);
4142 break;
4143 case ISD::ADD: {
4144 SDNodeFlags Flags = Op.getNode()->getFlags();
4145 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4146 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4147 bool SelfAdd = Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
4148 isGuaranteedNotToBeUndefOrPoison(
4149 Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
4150 Known = KnownBits::add(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4151 NUW: Flags.hasNoUnsignedWrap(), SelfAdd);
4152 break;
4153 }
4154 case ISD::SUB: {
4155 SDNodeFlags Flags = Op.getNode()->getFlags();
4156 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4157 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4158 Known = KnownBits::sub(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4159 NUW: Flags.hasNoUnsignedWrap());
4160 break;
4161 }
4162 case ISD::USUBO:
4163 case ISD::SSUBO:
4164 case ISD::USUBO_CARRY:
4165 case ISD::SSUBO_CARRY:
4166 if (Op.getResNo() == 1) {
4167 // If we know the result of a setcc has the top bits zero, use this info.
4168 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4169 TargetLowering::ZeroOrOneBooleanContent &&
4170 BitWidth > 1)
4171 Known.Zero.setBitsFrom(1);
4172 break;
4173 }
4174 [[fallthrough]];
4175 case ISD::SUBC: {
4176 assert(Op.getResNo() == 0 &&
4177 "We only compute knownbits for the difference here.");
4178
4179 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4180 KnownBits Borrow(1);
4181 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4182 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4183 // Borrow has bit width 1
4184 Borrow = Borrow.trunc(BitWidth: 1);
4185 } else {
4186 Borrow.setAllZero();
4187 }
4188
4189 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4190 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4191 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4192 break;
4193 }
4194 case ISD::UADDO:
4195 case ISD::SADDO:
4196 case ISD::UADDO_CARRY:
4197 case ISD::SADDO_CARRY:
4198 if (Op.getResNo() == 1) {
4199 // If we know the result of a setcc has the top bits zero, use this info.
4200 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4201 TargetLowering::ZeroOrOneBooleanContent &&
4202 BitWidth > 1)
4203 Known.Zero.setBitsFrom(1);
4204 break;
4205 }
4206 [[fallthrough]];
4207 case ISD::ADDC:
4208 case ISD::ADDE: {
4209 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4210
4211 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4212 KnownBits Carry(1);
4213 if (Opcode == ISD::ADDE)
4214 // Can't track carry from glue, set carry to unknown.
4215 Carry.resetAll();
4216 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4217 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4218 // Carry has bit width 1
4219 Carry = Carry.trunc(BitWidth: 1);
4220 } else {
4221 Carry.setAllZero();
4222 }
4223
4224 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4225 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4226 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4227 break;
4228 }
4229 case ISD::UDIV: {
4230 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4231 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4232 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4233 break;
4234 }
4235 case ISD::SDIV: {
4236 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4237 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4238 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4239 break;
4240 }
4241 case ISD::SREM: {
4242 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4243 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4244 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4245 break;
4246 }
4247 case ISD::UREM: {
4248 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4249 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4250 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4251 break;
4252 }
4253 case ISD::EXTRACT_ELEMENT: {
4254 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4255 const unsigned Index = Op.getConstantOperandVal(i: 1);
4256 const unsigned EltBitWidth = Op.getValueSizeInBits();
4257
4258 // Remove low part of known bits mask
4259 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4260 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4261
4262 // Remove high part of known bit mask
4263 Known = Known.trunc(BitWidth: EltBitWidth);
4264 break;
4265 }
4266 case ISD::EXTRACT_VECTOR_ELT: {
4267 SDValue InVec = Op.getOperand(i: 0);
4268 SDValue EltNo = Op.getOperand(i: 1);
4269 EVT VecVT = InVec.getValueType();
4270 // computeKnownBits not yet implemented for scalable vectors.
4271 if (VecVT.isScalableVector())
4272 break;
4273 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4274 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4275
4276 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4277 // anything about the extended bits.
4278 if (BitWidth > EltBitWidth)
4279 Known = Known.trunc(BitWidth: EltBitWidth);
4280
4281 // If we know the element index, just demand that vector element, else for
4282 // an unknown element index, ignore DemandedElts and demand them all.
4283 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4284 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4285 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4286 DemandedSrcElts =
4287 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4288
4289 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4290 if (BitWidth > EltBitWidth)
4291 Known = Known.anyext(BitWidth);
4292 break;
4293 }
4294 case ISD::INSERT_VECTOR_ELT: {
4295 if (Op.getValueType().isScalableVector())
4296 break;
4297
4298 // If we know the element index, split the demand between the
4299 // source vector and the inserted element, otherwise assume we need
4300 // the original demanded vector elements and the value.
4301 SDValue InVec = Op.getOperand(i: 0);
4302 SDValue InVal = Op.getOperand(i: 1);
4303 SDValue EltNo = Op.getOperand(i: 2);
4304 bool DemandedVal = true;
4305 APInt DemandedVecElts = DemandedElts;
4306 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4307 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4308 unsigned EltIdx = CEltNo->getZExtValue();
4309 DemandedVal = !!DemandedElts[EltIdx];
4310 DemandedVecElts.clearBit(BitPosition: EltIdx);
4311 }
4312 Known.setAllConflict();
4313 if (DemandedVal) {
4314 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4315 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4316 }
4317 if (!!DemandedVecElts) {
4318 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4319 Known = Known.intersectWith(RHS: Known2);
4320 }
4321 break;
4322 }
4323 case ISD::BITREVERSE: {
4324 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4325 Known = Known2.reverseBits();
4326 break;
4327 }
4328 case ISD::BSWAP: {
4329 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4330 Known = Known2.byteSwap();
4331 break;
4332 }
4333 case ISD::ABS: {
4334 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4335 Known = Known2.abs();
4336 Known.Zero.setHighBits(
4337 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4338 break;
4339 }
4340 case ISD::USUBSAT: {
4341 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4342 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4343 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4344 break;
4345 }
4346 case ISD::UMIN: {
4347 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4348 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4349 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4350 break;
4351 }
4352 case ISD::UMAX: {
4353 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4354 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4355 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4356 break;
4357 }
4358 case ISD::SMIN:
4359 case ISD::SMAX: {
4360 // If we have a clamp pattern, we know that the number of sign bits will be
4361 // the minimum of the clamp min/max range.
4362 bool IsMax = (Opcode == ISD::SMAX);
4363 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4364 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4365 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4366 CstHigh =
4367 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4368 if (CstLow && CstHigh) {
4369 if (!IsMax)
4370 std::swap(a&: CstLow, b&: CstHigh);
4371
4372 const APInt &ValueLow = CstLow->getAPIntValue();
4373 const APInt &ValueHigh = CstHigh->getAPIntValue();
4374 if (ValueLow.sle(RHS: ValueHigh)) {
4375 unsigned LowSignBits = ValueLow.getNumSignBits();
4376 unsigned HighSignBits = ValueHigh.getNumSignBits();
4377 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4378 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4379 Known.One.setHighBits(MinSignBits);
4380 break;
4381 }
4382 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4383 Known.Zero.setHighBits(MinSignBits);
4384 break;
4385 }
4386 }
4387 }
4388
4389 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4390 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4391 if (IsMax)
4392 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4393 else
4394 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4395
4396 // For SMAX, if CstLow is non-negative we know the result will be
4397 // non-negative and thus all sign bits are 0.
4398 // TODO: There's an equivalent of this for smin with negative constant for
4399 // known ones.
4400 if (IsMax && CstLow) {
4401 const APInt &ValueLow = CstLow->getAPIntValue();
4402 if (ValueLow.isNonNegative()) {
4403 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4404 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4405 }
4406 }
4407
4408 break;
4409 }
4410 case ISD::UINT_TO_FP: {
4411 Known.makeNonNegative();
4412 break;
4413 }
4414 case ISD::SINT_TO_FP: {
4415 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4416 if (Known2.isNonNegative())
4417 Known.makeNonNegative();
4418 else if (Known2.isNegative())
4419 Known.makeNegative();
4420 break;
4421 }
4422 case ISD::FP_TO_UINT_SAT: {
4423 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4424 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4425 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4426 break;
4427 }
4428 case ISD::ATOMIC_LOAD: {
4429 // If we are looking at the loaded value.
4430 if (Op.getResNo() == 0) {
4431 auto *AT = cast<AtomicSDNode>(Val&: Op);
4432 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4433 KnownBits KnownScalarMemory(ScalarMemorySize);
4434 if (const MDNode *MD = AT->getRanges())
4435 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4436
4437 switch (AT->getExtensionType()) {
4438 case ISD::ZEXTLOAD:
4439 Known = KnownScalarMemory.zext(BitWidth);
4440 break;
4441 case ISD::SEXTLOAD:
4442 Known = KnownScalarMemory.sext(BitWidth);
4443 break;
4444 case ISD::EXTLOAD:
4445 switch (TLI->getExtendForAtomicOps()) {
4446 case ISD::ZERO_EXTEND:
4447 Known = KnownScalarMemory.zext(BitWidth);
4448 break;
4449 case ISD::SIGN_EXTEND:
4450 Known = KnownScalarMemory.sext(BitWidth);
4451 break;
4452 default:
4453 Known = KnownScalarMemory.anyext(BitWidth);
4454 break;
4455 }
4456 break;
4457 case ISD::NON_EXTLOAD:
4458 Known = KnownScalarMemory;
4459 break;
4460 }
4461 assert(Known.getBitWidth() == BitWidth);
4462 }
4463 break;
4464 }
4465 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4466 if (Op.getResNo() == 1) {
4467 // The boolean result conforms to getBooleanContents.
4468 // If we know the result of a setcc has the top bits zero, use this info.
4469 // We know that we have an integer-based boolean since these operations
4470 // are only available for integer.
4471 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4472 TargetLowering::ZeroOrOneBooleanContent &&
4473 BitWidth > 1)
4474 Known.Zero.setBitsFrom(1);
4475 break;
4476 }
4477 [[fallthrough]];
4478 case ISD::ATOMIC_CMP_SWAP:
4479 case ISD::ATOMIC_SWAP:
4480 case ISD::ATOMIC_LOAD_ADD:
4481 case ISD::ATOMIC_LOAD_SUB:
4482 case ISD::ATOMIC_LOAD_AND:
4483 case ISD::ATOMIC_LOAD_CLR:
4484 case ISD::ATOMIC_LOAD_OR:
4485 case ISD::ATOMIC_LOAD_XOR:
4486 case ISD::ATOMIC_LOAD_NAND:
4487 case ISD::ATOMIC_LOAD_MIN:
4488 case ISD::ATOMIC_LOAD_MAX:
4489 case ISD::ATOMIC_LOAD_UMIN:
4490 case ISD::ATOMIC_LOAD_UMAX: {
4491 // If we are looking at the loaded value.
4492 if (Op.getResNo() == 0) {
4493 auto *AT = cast<AtomicSDNode>(Val&: Op);
4494 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4495
4496 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4497 Known.Zero.setBitsFrom(MemBits);
4498 }
4499 break;
4500 }
4501 case ISD::FrameIndex:
4502 case ISD::TargetFrameIndex:
4503 TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4504 Known, MF: getMachineFunction());
4505 break;
4506
4507 default:
4508 if (Opcode < ISD::BUILTIN_OP_END)
4509 break;
4510 [[fallthrough]];
4511 case ISD::INTRINSIC_WO_CHAIN:
4512 case ISD::INTRINSIC_W_CHAIN:
4513 case ISD::INTRINSIC_VOID:
4514 // TODO: Probably okay to remove after audit; here to reduce change size
4515 // in initial enablement patch for scalable vectors
4516 if (Op.getValueType().isScalableVector())
4517 break;
4518
4519 // Allow the target to implement this method for its nodes.
4520 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4521 break;
4522 }
4523
4524 return Known;
4525}
4526
4527/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4528static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4529 switch (OR) {
4530 case ConstantRange::OverflowResult::MayOverflow:
4531 return SelectionDAG::OFK_Sometime;
4532 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4533 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4534 return SelectionDAG::OFK_Always;
4535 case ConstantRange::OverflowResult::NeverOverflows:
4536 return SelectionDAG::OFK_Never;
4537 }
4538 llvm_unreachable("Unknown OverflowResult");
4539}
4540
4541SelectionDAG::OverflowKind
4542SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4543 // X + 0 never overflow
4544 if (isNullConstant(V: N1))
4545 return OFK_Never;
4546
4547 // If both operands each have at least two sign bits, the addition
4548 // cannot overflow.
4549 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4550 return OFK_Never;
4551
4552 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4553 return OFK_Sometime;
4554}
4555
4556SelectionDAG::OverflowKind
4557SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4558 // X + 0 never overflow
4559 if (isNullConstant(V: N1))
4560 return OFK_Never;
4561
4562 // mulhi + 1 never overflow
4563 KnownBits N1Known = computeKnownBits(Op: N1);
4564 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4565 N1Known.getMaxValue().ult(RHS: 2))
4566 return OFK_Never;
4567
4568 KnownBits N0Known = computeKnownBits(Op: N0);
4569 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4570 N0Known.getMaxValue().ult(RHS: 2))
4571 return OFK_Never;
4572
4573 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4574 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4575 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4576 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4577}
4578
4579SelectionDAG::OverflowKind
4580SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4581 // X - 0 never overflow
4582 if (isNullConstant(V: N1))
4583 return OFK_Never;
4584
4585 // If both operands each have at least two sign bits, the subtraction
4586 // cannot overflow.
4587 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4588 return OFK_Never;
4589
4590 KnownBits N0Known = computeKnownBits(Op: N0);
4591 KnownBits N1Known = computeKnownBits(Op: N1);
4592 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4593 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4594 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4595}
4596
4597SelectionDAG::OverflowKind
4598SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4599 // X - 0 never overflow
4600 if (isNullConstant(V: N1))
4601 return OFK_Never;
4602
4603 ConstantRange N0Range =
4604 computeConstantRangeIncludingKnownBits(Op: N0, /*ForSigned=*/false);
4605 ConstantRange N1Range =
4606 computeConstantRangeIncludingKnownBits(Op: N1, /*ForSigned=*/false);
4607 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4608}
4609
4610SelectionDAG::OverflowKind
4611SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4612 // X * 0 and X * 1 never overflow.
4613 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4614 return OFK_Never;
4615
4616 ConstantRange N0Range = computeConstantRangeIncludingKnownBits(Op: N0, ForSigned: false);
4617 ConstantRange N1Range = computeConstantRangeIncludingKnownBits(Op: N1, ForSigned: false);
4618 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4619}
4620
4621SelectionDAG::OverflowKind
4622SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4623 // X * 0 and X * 1 never overflow.
4624 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4625 return OFK_Never;
4626
4627 // Get the size of the result.
4628 unsigned BitWidth = N0.getScalarValueSizeInBits();
4629
4630 // Sum of the sign bits.
4631 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4632
4633 // If we have enough sign bits, then there's no overflow.
4634 if (SignBits > BitWidth + 1)
4635 return OFK_Never;
4636
4637 if (SignBits == BitWidth + 1) {
4638 // The overflow occurs when the true multiplication of the
4639 // the operands is the minimum negative number.
4640 KnownBits N0Known = computeKnownBits(Op: N0);
4641 KnownBits N1Known = computeKnownBits(Op: N1);
4642 // If one of the operands is non-negative, then there's no
4643 // overflow.
4644 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4645 return OFK_Never;
4646 }
4647
4648 return OFK_Sometime;
4649}
4650
4651ConstantRange SelectionDAG::computeConstantRange(SDValue Op, bool ForSigned,
4652 unsigned Depth) const {
4653 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4654 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4655}
4656
4657ConstantRange SelectionDAG::computeConstantRange(SDValue Op,
4658 const APInt &DemandedElts,
4659 bool ForSigned,
4660 unsigned Depth) const {
4661 EVT VT = Op.getValueType();
4662 unsigned BitWidth = VT.getScalarSizeInBits();
4663
4664 if (Depth >= MaxRecursionDepth)
4665 return ConstantRange::getFull(BitWidth);
4666
4667 if (ConstantSDNode *C = isConstOrConstSplat(N: Op, DemandedElts))
4668 return ConstantRange(C->getAPIntValue());
4669
4670 unsigned Opcode = Op.getOpcode();
4671 switch (Opcode) {
4672 case ISD::VSCALE: {
4673 const Function &F = getMachineFunction().getFunction();
4674 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
4675 return getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier);
4676 }
4677 default:
4678 break;
4679 }
4680
4681 return ConstantRange::getFull(BitWidth);
4682}
4683
4684ConstantRange
4685SelectionDAG::computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned,
4686 unsigned Depth) const {
4687 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4688 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4689 Depth);
4690}
4691
4692ConstantRange SelectionDAG::computeConstantRangeIncludingKnownBits(
4693 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4694 unsigned Depth) const {
4695 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4696 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, IsSigned: ForSigned);
4697 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4698 ConstantRange::PreferredRangeType RangeType =
4699 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4700 return CR1.intersectWith(CR: CR2, Type: RangeType);
4701}
4702
4703bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4704 unsigned Depth) const {
4705 APInt DemandedElts = getDemandAllEltsMask(V: Val);
4706 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4707}
4708
4709bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4710 const APInt &DemandedElts,
4711 bool OrZero, unsigned Depth) const {
4712 if (Depth >= MaxRecursionDepth)
4713 return false; // Limit search depth.
4714
4715 EVT OpVT = Val.getValueType();
4716 unsigned BitWidth = OpVT.getScalarSizeInBits();
4717 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4718 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4719 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4720 assert(
4721 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4722 "Unexpected vector size");
4723
4724 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4725 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4726 return (OrZero && V.isZero()) || V.isPowerOf2();
4727 };
4728
4729 // Is the constant a known power of 2 or zero?
4730 if (ISD::matchUnaryPredicate(Op: Val, Match: IsPowerOfTwoOrZero))
4731 return true;
4732
4733 switch (Val.getOpcode()) {
4734 case ISD::BUILD_VECTOR:
4735 // Are all operands of a build vector constant powers of two or zero?
4736 if (all_of(Range: enumerate(First: Val->ops()), P: [&](auto P) {
4737 auto *C = dyn_cast<ConstantSDNode>(P.value());
4738 return !DemandedElts[P.index()] || (C && IsPowerOfTwoOrZero(C));
4739 }))
4740 return true;
4741 break;
4742
4743 case ISD::SPLAT_VECTOR:
4744 // Is the operand of a splat vector a constant power of two?
4745 if (auto *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4746 if (IsPowerOfTwoOrZero(C))
4747 return true;
4748 break;
4749
4750 case ISD::EXTRACT_VECTOR_ELT: {
4751 SDValue InVec = Val.getOperand(i: 0);
4752 SDValue EltNo = Val.getOperand(i: 1);
4753 EVT VecVT = InVec.getValueType();
4754
4755 // Skip scalable vectors or implicit extensions.
4756 if (VecVT.isScalableVector() ||
4757 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4758 break;
4759
4760 // If we know the element index, just demand that vector element, else for
4761 // an unknown element index, ignore DemandedElts and demand them all.
4762 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4763 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4764 APInt DemandedSrcElts =
4765 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4766 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4767 : APInt::getAllOnes(numBits: NumSrcElts);
4768 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4769 }
4770
4771 case ISD::AND: {
4772 // Looking for `x & -x` pattern:
4773 // If x == 0:
4774 // x & -x -> 0
4775 // If x != 0:
4776 // x & -x -> non-zero pow2
4777 // so if we find the pattern return whether we know `x` is non-zero.
4778 SDValue X, Z;
4779 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))) ||
4780 (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Sub(L: m_Value(N&: Z), R: m_Deferred(V&: X)))) &&
4781 MaskedVectorIsZero(V: Z, DemandedElts, Depth: Depth + 1)))
4782 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4783 break;
4784 }
4785
4786 case ISD::SHL: {
4787 // A left-shift of a constant one will have exactly one bit set because
4788 // shifting the bit off the end is undefined.
4789 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4790 if (C && C->getAPIntValue() == 1)
4791 return true;
4792 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4793 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4794 Depth: Depth + 1);
4795 }
4796
4797 case ISD::SRL: {
4798 // A logical right-shift of a constant sign-bit will have exactly
4799 // one bit set.
4800 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4801 if (C && C->getAPIntValue().isSignMask())
4802 return true;
4803 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4804 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4805 Depth: Depth + 1);
4806 }
4807
4808 case ISD::TRUNCATE:
4809 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4810 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4811 Depth: Depth + 1);
4812
4813 case ISD::ROTL:
4814 case ISD::ROTR:
4815 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4816 Depth: Depth + 1);
4817 case ISD::BSWAP:
4818 case ISD::BITREVERSE:
4819 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4820 Depth: Depth + 1);
4821
4822 case ISD::SMIN:
4823 case ISD::SMAX:
4824 case ISD::UMIN:
4825 case ISD::UMAX:
4826 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4827 Depth: Depth + 1) &&
4828 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4829 Depth: Depth + 1);
4830
4831 case ISD::SELECT:
4832 case ISD::VSELECT:
4833 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4834 Depth: Depth + 1) &&
4835 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4836 Depth: Depth + 1);
4837
4838 case ISD::ZERO_EXTEND:
4839 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4840 Depth: Depth + 1);
4841
4842 case ISD::VSCALE:
4843 // vscale(power-of-two) is a power-of-two
4844 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4845 Depth: Depth + 1);
4846
4847 case ISD::VECTOR_SHUFFLE: {
4848 assert(!Val.getValueType().isScalableVector());
4849 // Demanded elements with undef shuffle mask elements are unknown
4850 // - we cannot guarantee they are a power of two, so return false.
4851 APInt DemandedLHS, DemandedRHS;
4852 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4853 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4854 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4855 DemandedLHS, DemandedRHS))
4856 return false;
4857
4858 // All demanded elements from LHS must be known power of two.
4859 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4860 OrZero, Depth: Depth + 1))
4861 return false;
4862
4863 // All demanded elements from RHS must be known power of two.
4864 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4865 OrZero, Depth: Depth + 1))
4866 return false;
4867
4868 return true;
4869 }
4870 }
4871
4872 // More could be done here, though the above checks are enough
4873 // to handle some common cases.
4874 return false;
4875}
4876
4877bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4878 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4879 return C1->getValueAPF().getExactLog2Abs() >= 0;
4880
4881 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4882 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4883
4884 return false;
4885}
4886
4887unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4888 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4889 return ComputeNumSignBits(Op, DemandedElts, Depth);
4890}
4891
4892unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4893 unsigned Depth) const {
4894 EVT VT = Op.getValueType();
4895 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4896 unsigned VTBits = VT.getScalarSizeInBits();
4897 unsigned NumElts = DemandedElts.getBitWidth();
4898 unsigned Tmp, Tmp2;
4899 unsigned FirstAnswer = 1;
4900
4901 assert((!VT.isScalableVector() || NumElts == 1) &&
4902 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4903
4904 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4905 const APInt &Val = C->getAPIntValue();
4906 return Val.getNumSignBits();
4907 }
4908
4909 if (Depth >= MaxRecursionDepth)
4910 return 1; // Limit search depth.
4911
4912 if (!DemandedElts)
4913 return 1; // No demanded elts, better to assume we don't know anything.
4914
4915 unsigned Opcode = Op.getOpcode();
4916 switch (Opcode) {
4917 default: break;
4918 case ISD::AssertSext:
4919 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4920 return VTBits-Tmp+1;
4921 case ISD::AssertZext:
4922 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4923 return VTBits-Tmp;
4924 case ISD::FREEZE:
4925 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4926 /*PoisonOnly=*/false))
4927 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4928 break;
4929 case ISD::MERGE_VALUES:
4930 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4931 Depth: Depth + 1);
4932 case ISD::SPLAT_VECTOR: {
4933 // Check if the sign bits of source go down as far as the truncated value.
4934 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4935 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4936 if (NumSrcSignBits > (NumSrcBits - VTBits))
4937 return NumSrcSignBits - (NumSrcBits - VTBits);
4938 break;
4939 }
4940 case ISD::BUILD_VECTOR:
4941 assert(!VT.isScalableVector());
4942 Tmp = VTBits;
4943 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4944 if (!DemandedElts[i])
4945 continue;
4946
4947 SDValue SrcOp = Op.getOperand(i);
4948 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4949 // for constant nodes to ensure we only look at the sign bits.
4950 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4951 APInt T = C->getAPIntValue().trunc(width: VTBits);
4952 Tmp2 = T.getNumSignBits();
4953 } else {
4954 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4955
4956 if (SrcOp.getValueSizeInBits() != VTBits) {
4957 assert(SrcOp.getValueSizeInBits() > VTBits &&
4958 "Expected BUILD_VECTOR implicit truncation");
4959 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4960 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4961 }
4962 }
4963 Tmp = std::min(a: Tmp, b: Tmp2);
4964 }
4965 return Tmp;
4966
4967 case ISD::VECTOR_COMPRESS: {
4968 SDValue Vec = Op.getOperand(i: 0);
4969 SDValue PassThru = Op.getOperand(i: 2);
4970 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
4971 if (Tmp == 1)
4972 return 1;
4973 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
4974 Tmp = std::min(a: Tmp, b: Tmp2);
4975 return Tmp;
4976 }
4977
4978 case ISD::VECTOR_SHUFFLE: {
4979 // Collect the minimum number of sign bits that are shared by every vector
4980 // element referenced by the shuffle.
4981 APInt DemandedLHS, DemandedRHS;
4982 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4983 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4984 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4985 DemandedLHS, DemandedRHS))
4986 return 1;
4987
4988 Tmp = std::numeric_limits<unsigned>::max();
4989 if (!!DemandedLHS)
4990 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
4991 if (!!DemandedRHS) {
4992 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
4993 Tmp = std::min(a: Tmp, b: Tmp2);
4994 }
4995 // If we don't know anything, early out and try computeKnownBits fall-back.
4996 if (Tmp == 1)
4997 break;
4998 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4999 return Tmp;
5000 }
5001
5002 case ISD::BITCAST: {
5003 if (VT.isScalableVector())
5004 break;
5005 SDValue N0 = Op.getOperand(i: 0);
5006 EVT SrcVT = N0.getValueType();
5007 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5008
5009 // Ignore bitcasts from unsupported types..
5010 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5011 break;
5012
5013 // Fast handling of 'identity' bitcasts.
5014 if (VTBits == SrcBits)
5015 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
5016
5017 bool IsLE = getDataLayout().isLittleEndian();
5018
5019 // Bitcast 'large element' scalar/vector to 'small element' vector.
5020 if ((SrcBits % VTBits) == 0) {
5021 assert(VT.isVector() && "Expected bitcast to vector");
5022
5023 unsigned Scale = SrcBits / VTBits;
5024 APInt SrcDemandedElts =
5025 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
5026
5027 // Fast case - sign splat can be simply split across the small elements.
5028 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
5029 if (Tmp == SrcBits)
5030 return VTBits;
5031
5032 // Slow case - determine how far the sign extends into each sub-element.
5033 Tmp2 = VTBits;
5034 for (unsigned i = 0; i != NumElts; ++i)
5035 if (DemandedElts[i]) {
5036 unsigned SubOffset = i % Scale;
5037 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5038 SubOffset = SubOffset * VTBits;
5039 if (Tmp <= SubOffset)
5040 return 1;
5041 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
5042 }
5043 return Tmp2;
5044 }
5045 break;
5046 }
5047
5048 case ISD::FP_TO_SINT_SAT:
5049 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5050 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5051 return VTBits - Tmp + 1;
5052 case ISD::SIGN_EXTEND:
5053 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5054 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5055 case ISD::SIGN_EXTEND_INREG:
5056 // Max of the input and what this extends.
5057 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5058 Tmp = VTBits-Tmp+1;
5059 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5060 return std::max(a: Tmp, b: Tmp2);
5061 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5062 if (VT.isScalableVector())
5063 break;
5064 SDValue Src = Op.getOperand(i: 0);
5065 EVT SrcVT = Src.getValueType();
5066 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5067 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5068 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5069 }
5070 case ISD::SRA:
5071 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5072 // SRA X, C -> adds C sign bits.
5073 if (std::optional<unsigned> ShAmt =
5074 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5075 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5076 return Tmp;
5077 case ISD::SHL:
5078 if (std::optional<ConstantRange> ShAmtRange =
5079 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5080 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5081 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5082 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5083 // shifted out, then we can compute the number of sign bits for the
5084 // operand being extended. A future improvement could be to pass along the
5085 // "shifted left by" information in the recursive calls to
5086 // ComputeKnownSignBits. Allowing us to handle this more generically.
5087 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5088 SDValue Ext = Op.getOperand(i: 0);
5089 EVT ExtVT = Ext.getValueType();
5090 SDValue Extendee = Ext.getOperand(i: 0);
5091 EVT ExtendeeVT = Extendee.getValueType();
5092 unsigned SizeDifference =
5093 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5094 if (SizeDifference <= MinShAmt) {
5095 Tmp = SizeDifference +
5096 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5097 if (MaxShAmt < Tmp)
5098 return Tmp - MaxShAmt;
5099 }
5100 }
5101 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5102 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5103 if (MaxShAmt < Tmp)
5104 return Tmp - MaxShAmt;
5105 }
5106 break;
5107 case ISD::AND:
5108 case ISD::OR:
5109 case ISD::XOR: // NOT is handled here.
5110 // Logical binary ops preserve the number of sign bits at the worst.
5111 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5112 if (Tmp != 1) {
5113 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5114 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5115 // We computed what we know about the sign bits as our first
5116 // answer. Now proceed to the generic code that uses
5117 // computeKnownBits, and pick whichever answer is better.
5118 }
5119 break;
5120
5121 case ISD::SELECT:
5122 case ISD::VSELECT:
5123 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5124 if (Tmp == 1) return 1; // Early out.
5125 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5126 return std::min(a: Tmp, b: Tmp2);
5127 case ISD::SELECT_CC:
5128 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5129 if (Tmp == 1) return 1; // Early out.
5130 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5131 return std::min(a: Tmp, b: Tmp2);
5132
5133 case ISD::SMIN:
5134 case ISD::SMAX: {
5135 // If we have a clamp pattern, we know that the number of sign bits will be
5136 // the minimum of the clamp min/max range.
5137 bool IsMax = (Opcode == ISD::SMAX);
5138 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5139 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5140 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5141 CstHigh =
5142 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5143 if (CstLow && CstHigh) {
5144 if (!IsMax)
5145 std::swap(a&: CstLow, b&: CstHigh);
5146 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5147 Tmp = CstLow->getAPIntValue().getNumSignBits();
5148 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5149 return std::min(a: Tmp, b: Tmp2);
5150 }
5151 }
5152
5153 // Fallback - just get the minimum number of sign bits of the operands.
5154 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5155 if (Tmp == 1)
5156 return 1; // Early out.
5157 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5158 return std::min(a: Tmp, b: Tmp2);
5159 }
5160 case ISD::UMIN:
5161 case ISD::UMAX:
5162 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5163 if (Tmp == 1)
5164 return 1; // Early out.
5165 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5166 return std::min(a: Tmp, b: Tmp2);
5167 case ISD::SSUBO_CARRY:
5168 case ISD::USUBO_CARRY:
5169 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5170 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5171 return VTBits;
5172 [[fallthrough]];
5173 case ISD::SADDO:
5174 case ISD::UADDO:
5175 case ISD::SADDO_CARRY:
5176 case ISD::UADDO_CARRY:
5177 case ISD::SSUBO:
5178 case ISD::USUBO:
5179 case ISD::SMULO:
5180 case ISD::UMULO:
5181 if (Op.getResNo() != 1)
5182 break;
5183 // The boolean result conforms to getBooleanContents. Fall through.
5184 // If setcc returns 0/-1, all bits are sign bits.
5185 // We know that we have an integer-based boolean since these operations
5186 // are only available for integer.
5187 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5188 TargetLowering::ZeroOrNegativeOneBooleanContent)
5189 return VTBits;
5190 break;
5191 case ISD::SETCC:
5192 case ISD::SETCCCARRY:
5193 case ISD::STRICT_FSETCC:
5194 case ISD::STRICT_FSETCCS: {
5195 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5196 // If setcc returns 0/-1, all bits are sign bits.
5197 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5198 TargetLowering::ZeroOrNegativeOneBooleanContent)
5199 return VTBits;
5200 break;
5201 }
5202 case ISD::ROTL:
5203 case ISD::ROTR:
5204 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5205
5206 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
5207 if (Tmp == VTBits)
5208 return VTBits;
5209
5210 if (ConstantSDNode *C =
5211 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
5212 unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
5213
5214 // Handle rotate right by N like a rotate left by 32-N.
5215 if (Opcode == ISD::ROTR)
5216 RotAmt = (VTBits - RotAmt) % VTBits;
5217
5218 // If we aren't rotating out all of the known-in sign bits, return the
5219 // number that are left. This handles rotl(sext(x), 1) for example.
5220 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
5221 }
5222 break;
5223 case ISD::ADD:
5224 case ISD::ADDC:
5225 // TODO: Move Operand 1 check before Operand 0 check
5226 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5227 if (Tmp == 1) return 1; // Early out.
5228
5229 // Special case decrementing a value (ADD X, -1):
5230 if (ConstantSDNode *CRHS =
5231 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5232 if (CRHS->isAllOnes()) {
5233 KnownBits Known =
5234 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5235
5236 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5237 // sign bits set.
5238 if ((Known.Zero | 1).isAllOnes())
5239 return VTBits;
5240
5241 // If we are subtracting one from a positive number, there is no carry
5242 // out of the result.
5243 if (Known.isNonNegative())
5244 return Tmp;
5245 }
5246
5247 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5248 if (Tmp2 == 1) return 1; // Early out.
5249
5250 // Add can have at most one carry bit. Thus we know that the output
5251 // is, at worst, one more bit than the inputs.
5252 return std::min(a: Tmp, b: Tmp2) - 1;
5253 case ISD::SUB:
5254 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5255 if (Tmp2 == 1) return 1; // Early out.
5256
5257 // Handle NEG.
5258 if (ConstantSDNode *CLHS =
5259 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5260 if (CLHS->isZero()) {
5261 KnownBits Known =
5262 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5263 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5264 // sign bits set.
5265 if ((Known.Zero | 1).isAllOnes())
5266 return VTBits;
5267
5268 // If the input is known to be positive (the sign bit is known clear),
5269 // the output of the NEG has the same number of sign bits as the input.
5270 if (Known.isNonNegative())
5271 return Tmp2;
5272
5273 // Otherwise, we treat this like a SUB.
5274 }
5275
5276 // Sub can have at most one carry bit. Thus we know that the output
5277 // is, at worst, one more bit than the inputs.
5278 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5279 if (Tmp == 1) return 1; // Early out.
5280 return std::min(a: Tmp, b: Tmp2) - 1;
5281 case ISD::MUL: {
5282 // The output of the Mul can be at most twice the valid bits in the inputs.
5283 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5284 if (SignBitsOp0 == 1)
5285 break;
5286 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5287 if (SignBitsOp1 == 1)
5288 break;
5289 unsigned OutValidBits =
5290 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5291 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5292 }
5293 case ISD::AVGCEILS:
5294 case ISD::AVGFLOORS:
5295 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5296 if (Tmp == 1)
5297 return 1; // Early out.
5298 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5299 return std::min(a: Tmp, b: Tmp2);
5300 case ISD::SREM:
5301 // The sign bit is the LHS's sign bit, except when the result of the
5302 // remainder is zero. The magnitude of the result should be less than or
5303 // equal to the magnitude of the LHS. Therefore, the result should have
5304 // at least as many sign bits as the left hand side.
5305 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5306 case ISD::TRUNCATE: {
5307 // Check if the sign bits of source go down as far as the truncated value.
5308 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5309 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5310 if (NumSrcSignBits > (NumSrcBits - VTBits))
5311 return NumSrcSignBits - (NumSrcBits - VTBits);
5312 break;
5313 }
5314 case ISD::EXTRACT_ELEMENT: {
5315 if (VT.isScalableVector())
5316 break;
5317 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5318 const int BitWidth = Op.getValueSizeInBits();
5319 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5320
5321 // Get reverse index (starting from 1), Op1 value indexes elements from
5322 // little end. Sign starts at big end.
5323 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5324
5325 // If the sign portion ends in our element the subtraction gives correct
5326 // result. Otherwise it gives either negative or > bitwidth result
5327 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5328 }
5329 case ISD::INSERT_VECTOR_ELT: {
5330 if (VT.isScalableVector())
5331 break;
5332 // If we know the element index, split the demand between the
5333 // source vector and the inserted element, otherwise assume we need
5334 // the original demanded vector elements and the value.
5335 SDValue InVec = Op.getOperand(i: 0);
5336 SDValue InVal = Op.getOperand(i: 1);
5337 SDValue EltNo = Op.getOperand(i: 2);
5338 bool DemandedVal = true;
5339 APInt DemandedVecElts = DemandedElts;
5340 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5341 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5342 unsigned EltIdx = CEltNo->getZExtValue();
5343 DemandedVal = !!DemandedElts[EltIdx];
5344 DemandedVecElts.clearBit(BitPosition: EltIdx);
5345 }
5346 Tmp = std::numeric_limits<unsigned>::max();
5347 if (DemandedVal) {
5348 // TODO - handle implicit truncation of inserted elements.
5349 if (InVal.getScalarValueSizeInBits() != VTBits)
5350 break;
5351 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5352 Tmp = std::min(a: Tmp, b: Tmp2);
5353 }
5354 if (!!DemandedVecElts) {
5355 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5356 Tmp = std::min(a: Tmp, b: Tmp2);
5357 }
5358 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5359 return Tmp;
5360 }
5361 case ISD::EXTRACT_VECTOR_ELT: {
5362 SDValue InVec = Op.getOperand(i: 0);
5363 SDValue EltNo = Op.getOperand(i: 1);
5364 EVT VecVT = InVec.getValueType();
5365 // ComputeNumSignBits not yet implemented for scalable vectors.
5366 if (VecVT.isScalableVector())
5367 break;
5368 const unsigned BitWidth = Op.getValueSizeInBits();
5369 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5370 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5371
5372 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5373 // anything about sign bits. But if the sizes match we can derive knowledge
5374 // about sign bits from the vector operand.
5375 if (BitWidth != EltBitWidth)
5376 break;
5377
5378 // If we know the element index, just demand that vector element, else for
5379 // an unknown element index, ignore DemandedElts and demand them all.
5380 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5381 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5382 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5383 DemandedSrcElts =
5384 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5385
5386 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5387 }
5388 case ISD::EXTRACT_SUBVECTOR: {
5389 // Offset the demanded elts by the subvector index.
5390 SDValue Src = Op.getOperand(i: 0);
5391
5392 APInt DemandedSrcElts;
5393 if (Src.getValueType().isScalableVector())
5394 DemandedSrcElts = APInt(1, 1);
5395 else {
5396 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5397 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5398 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5399 }
5400 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5401 }
5402 case ISD::CONCAT_VECTORS: {
5403 if (VT.isScalableVector())
5404 break;
5405 // Determine the minimum number of sign bits across all demanded
5406 // elts of the input vectors. Early out if the result is already 1.
5407 Tmp = std::numeric_limits<unsigned>::max();
5408 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5409 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5410 unsigned NumSubVectors = Op.getNumOperands();
5411 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5412 APInt DemandedSub =
5413 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5414 if (!DemandedSub)
5415 continue;
5416 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5417 Tmp = std::min(a: Tmp, b: Tmp2);
5418 }
5419 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5420 return Tmp;
5421 }
5422 case ISD::INSERT_SUBVECTOR: {
5423 if (VT.isScalableVector())
5424 break;
5425 // Demand any elements from the subvector and the remainder from the src its
5426 // inserted into.
5427 SDValue Src = Op.getOperand(i: 0);
5428 SDValue Sub = Op.getOperand(i: 1);
5429 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5430 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5431 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5432 APInt DemandedSrcElts = DemandedElts;
5433 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5434
5435 Tmp = std::numeric_limits<unsigned>::max();
5436 if (!!DemandedSubElts) {
5437 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5438 if (Tmp == 1)
5439 return 1; // early-out
5440 }
5441 if (!!DemandedSrcElts) {
5442 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5443 Tmp = std::min(a: Tmp, b: Tmp2);
5444 }
5445 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5446 return Tmp;
5447 }
5448 case ISD::LOAD: {
5449 // If we are looking at the loaded value of the SDNode.
5450 if (Op.getResNo() != 0)
5451 break;
5452
5453 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5454 if (const MDNode *Ranges = LD->getRanges()) {
5455 if (DemandedElts != 1)
5456 break;
5457
5458 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5459 if (VTBits > CR.getBitWidth()) {
5460 switch (LD->getExtensionType()) {
5461 case ISD::SEXTLOAD:
5462 CR = CR.signExtend(BitWidth: VTBits);
5463 break;
5464 case ISD::ZEXTLOAD:
5465 CR = CR.zeroExtend(BitWidth: VTBits);
5466 break;
5467 default:
5468 break;
5469 }
5470 }
5471
5472 if (VTBits != CR.getBitWidth())
5473 break;
5474 return std::min(a: CR.getSignedMin().getNumSignBits(),
5475 b: CR.getSignedMax().getNumSignBits());
5476 }
5477
5478 unsigned ExtType = LD->getExtensionType();
5479 switch (ExtType) {
5480 default:
5481 break;
5482 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5483 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5484 return VTBits - Tmp + 1;
5485 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5486 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5487 return VTBits - Tmp;
5488 case ISD::NON_EXTLOAD:
5489 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5490 // We only need to handle vectors - computeKnownBits should handle
5491 // scalar cases.
5492 Type *CstTy = Cst->getType();
5493 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5494 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5495 VTBits == CstTy->getScalarSizeInBits()) {
5496 Tmp = VTBits;
5497 for (unsigned i = 0; i != NumElts; ++i) {
5498 if (!DemandedElts[i])
5499 continue;
5500 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5501 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5502 const APInt &Value = CInt->getValue();
5503 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5504 continue;
5505 }
5506 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5507 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5508 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5509 continue;
5510 }
5511 }
5512 // Unknown type. Conservatively assume no bits match sign bit.
5513 return 1;
5514 }
5515 return Tmp;
5516 }
5517 }
5518 break;
5519 }
5520
5521 break;
5522 }
5523 case ISD::ATOMIC_CMP_SWAP:
5524 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5525 case ISD::ATOMIC_SWAP:
5526 case ISD::ATOMIC_LOAD_ADD:
5527 case ISD::ATOMIC_LOAD_SUB:
5528 case ISD::ATOMIC_LOAD_AND:
5529 case ISD::ATOMIC_LOAD_CLR:
5530 case ISD::ATOMIC_LOAD_OR:
5531 case ISD::ATOMIC_LOAD_XOR:
5532 case ISD::ATOMIC_LOAD_NAND:
5533 case ISD::ATOMIC_LOAD_MIN:
5534 case ISD::ATOMIC_LOAD_MAX:
5535 case ISD::ATOMIC_LOAD_UMIN:
5536 case ISD::ATOMIC_LOAD_UMAX:
5537 case ISD::ATOMIC_LOAD: {
5538 auto *AT = cast<AtomicSDNode>(Val&: Op);
5539 // If we are looking at the loaded value.
5540 if (Op.getResNo() == 0) {
5541 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5542 if (Tmp == VTBits)
5543 return 1; // early-out
5544
5545 // For atomic_load, prefer to use the extension type.
5546 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5547 switch (AT->getExtensionType()) {
5548 default:
5549 break;
5550 case ISD::SEXTLOAD:
5551 return VTBits - Tmp + 1;
5552 case ISD::ZEXTLOAD:
5553 return VTBits - Tmp;
5554 }
5555 }
5556
5557 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5558 return VTBits - Tmp + 1;
5559 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5560 return VTBits - Tmp;
5561 }
5562 break;
5563 }
5564 }
5565
5566 // Allow the target to implement this method for its nodes.
5567 if (Opcode >= ISD::BUILTIN_OP_END ||
5568 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5569 Opcode == ISD::INTRINSIC_W_CHAIN ||
5570 Opcode == ISD::INTRINSIC_VOID) {
5571 // TODO: This can probably be removed once target code is audited. This
5572 // is here purely to reduce patch size and review complexity.
5573 if (!VT.isScalableVector()) {
5574 unsigned NumBits =
5575 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5576 if (NumBits > 1)
5577 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5578 }
5579 }
5580
5581 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5582 // use this information.
5583 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5584 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5585}
5586
5587unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5588 unsigned Depth) const {
5589 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5590 return Op.getScalarValueSizeInBits() - SignBits + 1;
5591}
5592
5593unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5594 const APInt &DemandedElts,
5595 unsigned Depth) const {
5596 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5597 return Op.getScalarValueSizeInBits() - SignBits + 1;
5598}
5599
5600bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
5601 unsigned Depth) const {
5602 // Early out for FREEZE.
5603 if (Op.getOpcode() == ISD::FREEZE)
5604 return true;
5605
5606 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5607 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
5608}
5609
5610bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5611 const APInt &DemandedElts,
5612 bool PoisonOnly,
5613 unsigned Depth) const {
5614 unsigned Opcode = Op.getOpcode();
5615
5616 // Early out for FREEZE.
5617 if (Opcode == ISD::FREEZE)
5618 return true;
5619
5620 if (Depth >= MaxRecursionDepth)
5621 return false; // Limit search depth.
5622
5623 if (isIntOrFPConstant(V: Op))
5624 return true;
5625
5626 switch (Opcode) {
5627 case ISD::CONDCODE:
5628 case ISD::VALUETYPE:
5629 case ISD::FrameIndex:
5630 case ISD::TargetFrameIndex:
5631 case ISD::CopyFromReg:
5632 return true;
5633
5634 case ISD::POISON:
5635 return false;
5636
5637 case ISD::UNDEF:
5638 return PoisonOnly;
5639
5640 case ISD::BUILD_VECTOR:
5641 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5642 // this shouldn't affect the result.
5643 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5644 if (!DemandedElts[i])
5645 continue;
5646 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly,
5647 Depth: Depth + 1))
5648 return false;
5649 }
5650 return true;
5651
5652 case ISD::EXTRACT_SUBVECTOR: {
5653 SDValue Src = Op.getOperand(i: 0);
5654 if (Src.getValueType().isScalableVector())
5655 break;
5656 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5657 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5658 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5659 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5660 Depth: Depth + 1);
5661 }
5662
5663 case ISD::INSERT_SUBVECTOR: {
5664 if (Op.getValueType().isScalableVector())
5665 break;
5666 SDValue Src = Op.getOperand(i: 0);
5667 SDValue Sub = Op.getOperand(i: 1);
5668 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5669 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5670 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5671 APInt DemandedSrcElts = DemandedElts;
5672 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5673
5674 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5675 Op: Sub, DemandedElts: DemandedSubElts, PoisonOnly, Depth: Depth + 1))
5676 return false;
5677 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5678 Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly, Depth: Depth + 1))
5679 return false;
5680 return true;
5681 }
5682
5683 case ISD::EXTRACT_VECTOR_ELT: {
5684 SDValue Src = Op.getOperand(i: 0);
5685 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5686 EVT SrcVT = Src.getValueType();
5687 if (SrcVT.isFixedLengthVector() && IndexC &&
5688 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5689 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5690 BitNo: IndexC->getZExtValue());
5691 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5692 Depth: Depth + 1);
5693 }
5694 break;
5695 }
5696
5697 case ISD::INSERT_VECTOR_ELT: {
5698 SDValue InVec = Op.getOperand(i: 0);
5699 SDValue InVal = Op.getOperand(i: 1);
5700 SDValue EltNo = Op.getOperand(i: 2);
5701 EVT VT = InVec.getValueType();
5702 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5703 if (IndexC && VT.isFixedLengthVector() &&
5704 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5705 if (DemandedElts[IndexC->getZExtValue()] &&
5706 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, PoisonOnly, Depth: Depth + 1))
5707 return false;
5708 APInt InVecDemandedElts = DemandedElts;
5709 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5710 if (!!InVecDemandedElts &&
5711 !isGuaranteedNotToBeUndefOrPoison(
5712 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5713 DemandedElts: InVecDemandedElts, PoisonOnly, Depth: Depth + 1))
5714 return false;
5715 return true;
5716 }
5717 break;
5718 }
5719
5720 case ISD::SCALAR_TO_VECTOR:
5721 // Check upper (known undef) elements.
5722 if (DemandedElts.ugt(RHS: 1) && !PoisonOnly)
5723 return false;
5724 // Check element zero.
5725 if (DemandedElts[0] && !isGuaranteedNotToBeUndefOrPoison(
5726 Op: Op.getOperand(i: 0), PoisonOnly, Depth: Depth + 1))
5727 return false;
5728 return true;
5729
5730 case ISD::SPLAT_VECTOR:
5731 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), PoisonOnly,
5732 Depth: Depth + 1);
5733
5734 case ISD::VECTOR_SHUFFLE: {
5735 APInt DemandedLHS, DemandedRHS;
5736 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5737 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5738 DemandedElts, DemandedLHS, DemandedRHS,
5739 /*AllowUndefElts=*/false))
5740 return false;
5741 if (!DemandedLHS.isZero() &&
5742 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS,
5743 PoisonOnly, Depth: Depth + 1))
5744 return false;
5745 if (!DemandedRHS.isZero() &&
5746 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS,
5747 PoisonOnly, Depth: Depth + 1))
5748 return false;
5749 return true;
5750 }
5751
5752 case ISD::SHL:
5753 case ISD::SRL:
5754 case ISD::SRA:
5755 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5756 // enough to check operand 0 if Op can't create undef/poison.
5757 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5758 /*ConsiderFlags*/ true, Depth) &&
5759 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5760 PoisonOnly, Depth: Depth + 1);
5761
5762 case ISD::BSWAP:
5763 case ISD::CTPOP:
5764 case ISD::BITREVERSE:
5765 case ISD::AND:
5766 case ISD::OR:
5767 case ISD::XOR:
5768 case ISD::ADD:
5769 case ISD::SUB:
5770 case ISD::MUL:
5771 case ISD::SADDSAT:
5772 case ISD::UADDSAT:
5773 case ISD::SSUBSAT:
5774 case ISD::USUBSAT:
5775 case ISD::SSHLSAT:
5776 case ISD::USHLSAT:
5777 case ISD::SMIN:
5778 case ISD::SMAX:
5779 case ISD::UMIN:
5780 case ISD::UMAX:
5781 case ISD::ZERO_EXTEND:
5782 case ISD::SIGN_EXTEND:
5783 case ISD::ANY_EXTEND:
5784 case ISD::TRUNCATE:
5785 case ISD::VSELECT: {
5786 // If Op can't create undef/poison and none of its operands are undef/poison
5787 // then Op is never undef/poison. A difference from the more common check
5788 // below, outside the switch, is that we handle elementwise operations for
5789 // which the DemandedElts mask is valid for all operands here.
5790 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5791 /*ConsiderFlags*/ true, Depth) &&
5792 all_of(Range: Op->ops(), P: [&](SDValue V) {
5793 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts,
5794 PoisonOnly, Depth: Depth + 1);
5795 });
5796 }
5797
5798 // TODO: Search for noundef attributes from library functions.
5799
5800 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5801
5802 default:
5803 // Allow the target to implement this method for its nodes.
5804 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5805 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5806 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5807 Op, DemandedElts, DAG: *this, PoisonOnly, Depth);
5808 break;
5809 }
5810
5811 // If Op can't create undef/poison and none of its operands are undef/poison
5812 // then Op is never undef/poison.
5813 // NOTE: TargetNodes can handle this in themselves in
5814 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5815 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5816 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5817 Depth) &&
5818 all_of(Range: Op->ops(), P: [&](SDValue V) {
5819 return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1);
5820 });
5821}
5822
5823bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
5824 bool ConsiderFlags,
5825 unsigned Depth) const {
5826 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5827 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5828 Depth);
5829}
5830
5831bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5832 bool PoisonOnly, bool ConsiderFlags,
5833 unsigned Depth) const {
5834 if (ConsiderFlags && Op->hasPoisonGeneratingFlags())
5835 return true;
5836
5837 unsigned Opcode = Op.getOpcode();
5838 switch (Opcode) {
5839 case ISD::AssertSext:
5840 case ISD::AssertZext:
5841 case ISD::AssertAlign:
5842 case ISD::AssertNoFPClass:
5843 // Assertion nodes can create poison if the assertion fails.
5844 return true;
5845
5846 case ISD::FREEZE:
5847 case ISD::CONCAT_VECTORS:
5848 case ISD::INSERT_SUBVECTOR:
5849 case ISD::EXTRACT_SUBVECTOR:
5850 case ISD::SADDSAT:
5851 case ISD::UADDSAT:
5852 case ISD::SSUBSAT:
5853 case ISD::USUBSAT:
5854 case ISD::MULHU:
5855 case ISD::MULHS:
5856 case ISD::AVGFLOORS:
5857 case ISD::AVGFLOORU:
5858 case ISD::AVGCEILS:
5859 case ISD::AVGCEILU:
5860 case ISD::ABDU:
5861 case ISD::ABDS:
5862 case ISD::SMIN:
5863 case ISD::SMAX:
5864 case ISD::SCMP:
5865 case ISD::UMIN:
5866 case ISD::UMAX:
5867 case ISD::UCMP:
5868 case ISD::AND:
5869 case ISD::XOR:
5870 case ISD::ROTL:
5871 case ISD::ROTR:
5872 case ISD::FSHL:
5873 case ISD::FSHR:
5874 case ISD::BSWAP:
5875 case ISD::CTTZ:
5876 case ISD::CTLZ:
5877 case ISD::CTLS:
5878 case ISD::CTPOP:
5879 case ISD::BITREVERSE:
5880 case ISD::PARITY:
5881 case ISD::SIGN_EXTEND:
5882 case ISD::TRUNCATE:
5883 case ISD::SIGN_EXTEND_INREG:
5884 case ISD::SIGN_EXTEND_VECTOR_INREG:
5885 case ISD::ZERO_EXTEND_VECTOR_INREG:
5886 case ISD::BITCAST:
5887 case ISD::BUILD_VECTOR:
5888 case ISD::BUILD_PAIR:
5889 case ISD::SPLAT_VECTOR:
5890 case ISD::FABS:
5891 return false;
5892
5893 case ISD::ABS:
5894 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
5895 // Different to Intrinsic::abs.
5896 return false;
5897
5898 case ISD::ADDC:
5899 case ISD::SUBC:
5900 case ISD::ADDE:
5901 case ISD::SUBE:
5902 case ISD::SADDO:
5903 case ISD::SSUBO:
5904 case ISD::SMULO:
5905 case ISD::SADDO_CARRY:
5906 case ISD::SSUBO_CARRY:
5907 case ISD::UADDO:
5908 case ISD::USUBO:
5909 case ISD::UMULO:
5910 case ISD::UADDO_CARRY:
5911 case ISD::USUBO_CARRY:
5912 // No poison on result or overflow flags.
5913 return false;
5914
5915 case ISD::SELECT_CC:
5916 case ISD::SETCC: {
5917 // Integer setcc cannot create undef or poison.
5918 if (Op.getOperand(i: 0).getValueType().isInteger())
5919 return false;
5920
5921 // FP compares are more complicated. They can create poison for nan/infinity
5922 // based on options and flags. The options and flags also cause special
5923 // nonan condition codes to be used. Those condition codes may be preserved
5924 // even if the nonan flag is dropped somewhere.
5925 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
5926 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
5927 return (unsigned)CCCode & 0x10U;
5928 }
5929
5930 case ISD::OR:
5931 case ISD::ZERO_EXTEND:
5932 case ISD::SELECT:
5933 case ISD::VSELECT:
5934 case ISD::ADD:
5935 case ISD::SUB:
5936 case ISD::MUL:
5937 case ISD::FNEG:
5938 case ISD::FADD:
5939 case ISD::FSUB:
5940 case ISD::FMUL:
5941 case ISD::FDIV:
5942 case ISD::FREM:
5943 case ISD::FCOPYSIGN:
5944 case ISD::FMA:
5945 case ISD::FMAD:
5946 case ISD::FMULADD:
5947 case ISD::FP_EXTEND:
5948 case ISD::FP_TO_SINT_SAT:
5949 case ISD::FP_TO_UINT_SAT:
5950 case ISD::TRUNCATE_SSAT_S:
5951 case ISD::TRUNCATE_SSAT_U:
5952 case ISD::TRUNCATE_USAT_U:
5953 // No poison except from flags (which is handled above)
5954 return false;
5955
5956 case ISD::SHL:
5957 case ISD::SRL:
5958 case ISD::SRA:
5959 // If the max shift amount isn't in range, then the shift can
5960 // create poison.
5961 return !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
5962
5963 case ISD::CTTZ_ZERO_UNDEF:
5964 case ISD::CTLZ_ZERO_UNDEF:
5965 // If the amount is zero then the result will be poison.
5966 // TODO: Add isKnownNeverZero DemandedElts handling.
5967 return !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5968
5969 case ISD::SCALAR_TO_VECTOR:
5970 // Check if we demand any upper (undef) elements.
5971 return !PoisonOnly && DemandedElts.ugt(RHS: 1);
5972
5973 case ISD::INSERT_VECTOR_ELT:
5974 case ISD::EXTRACT_VECTOR_ELT: {
5975 // Ensure that the element index is in bounds.
5976 EVT VecVT = Op.getOperand(i: 0).getValueType();
5977 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
5978 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
5979 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5980 }
5981
5982 case ISD::VECTOR_SHUFFLE: {
5983 // Check for any demanded shuffle element that is undef.
5984 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5985 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
5986 if (Elt < 0 && DemandedElts[Idx])
5987 return true;
5988 return false;
5989 }
5990
5991 case ISD::VECTOR_COMPRESS:
5992 return false;
5993
5994 default:
5995 // Allow the target to implement this method for its nodes.
5996 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5997 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5998 return TLI->canCreateUndefOrPoisonForTargetNode(
5999 Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth);
6000 break;
6001 }
6002
6003 // Be conservative and return true.
6004 return true;
6005}
6006
6007bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6008 unsigned Opcode = Op.getOpcode();
6009 if (Opcode == ISD::OR)
6010 return Op->getFlags().hasDisjoint() ||
6011 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
6012 if (Opcode == ISD::XOR)
6013 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
6014 return false;
6015}
6016
6017bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
6018 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
6019 (Op.isAnyAdd() || isADDLike(Op));
6020}
6021
6022KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6023 FPClassTest InterestedClasses,
6024 unsigned Depth) const {
6025 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6026 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6027}
6028
6029KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6030 const APInt &DemandedElts,
6031 FPClassTest InterestedClasses,
6032 unsigned Depth) const {
6033 KnownFPClass Known;
6034
6035 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
6036 return KnownFPClass(CFP->getValueAPF());
6037
6038 if (Depth >= MaxRecursionDepth)
6039 return Known;
6040
6041 if (Op.getOpcode() == ISD::UNDEF)
6042 return Known;
6043
6044 EVT VT = Op.getValueType();
6045 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6046 assert((!VT.isFixedLengthVector() ||
6047 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6048 "Unexpected vector size");
6049
6050 if (!DemandedElts)
6051 return Known;
6052
6053 unsigned Opcode = Op.getOpcode();
6054 switch (Opcode) {
6055 case ISD::POISON: {
6056 Known.KnownFPClasses = fcNone;
6057 Known.SignBit = false;
6058 break;
6059 }
6060 case ISD::BUILD_VECTOR: {
6061 assert(!VT.isScalableVector());
6062 bool First = true;
6063 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6064 if (!DemandedElts[I])
6065 continue;
6066
6067 if (First) {
6068 Known =
6069 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6070 First = false;
6071 } else {
6072 Known |=
6073 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6074 }
6075
6076 if (Known.isUnknown())
6077 break;
6078 }
6079 break;
6080 }
6081 case ISD::SPLAT_VECTOR: {
6082 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), InterestedClasses, Depth: Depth + 1);
6083 break;
6084 }
6085 case ISD::BITCAST: {
6086 // FIXME: It should not be necessary to check for an elementwise bitcast.
6087 // If a bitcast is not elementwise between vector / scalar types,
6088 // computeKnownBits already splices the known bits of the source elements
6089 // appropriately so as to line up with the bits of the result's demanded
6090 // elements.
6091 EVT SrcVT = Op.getOperand(i: 0).getValueType();
6092 if (VT.isScalableVector() || SrcVT.isScalableVector())
6093 break;
6094 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6095 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6096 if (VTNumElts != SrcVTNumElts)
6097 break;
6098
6099 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth: Depth + 1);
6100 Known = KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits);
6101 break;
6102 }
6103 case ISD::FABS: {
6104 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6105 InterestedClasses, Depth: Depth + 1);
6106 Known.fabs();
6107 break;
6108 }
6109 default:
6110 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6111 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6112 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, DAG: *this,
6113 Depth);
6114 }
6115 break;
6116 }
6117
6118 return Known;
6119}
6120
6121bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
6122 unsigned Depth) const {
6123 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6124 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6125}
6126
6127bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
6128 bool SNaN, unsigned Depth) const {
6129 assert(!DemandedElts.isZero() && "No demanded elements");
6130
6131 // If we're told that NaNs won't happen, assume they won't.
6132 if (Op->getFlags().hasNoNaNs())
6133 return true;
6134
6135 if (Depth >= MaxRecursionDepth)
6136 return false; // Limit search depth.
6137
6138 unsigned Opcode = Op.getOpcode();
6139 switch (Opcode) {
6140 case ISD::FADD:
6141 case ISD::FSUB:
6142 case ISD::FMUL:
6143 case ISD::FDIV:
6144 case ISD::FREM:
6145 case ISD::FSIN:
6146 case ISD::FCOS:
6147 case ISD::FTAN:
6148 case ISD::FASIN:
6149 case ISD::FACOS:
6150 case ISD::FATAN:
6151 case ISD::FATAN2:
6152 case ISD::FSINH:
6153 case ISD::FCOSH:
6154 case ISD::FTANH:
6155 case ISD::FMA:
6156 case ISD::FMULADD:
6157 case ISD::FMAD: {
6158 if (SNaN)
6159 return true;
6160 // TODO: Need isKnownNeverInfinity
6161 return false;
6162 }
6163 case ISD::FCANONICALIZE:
6164 case ISD::FEXP:
6165 case ISD::FEXP2:
6166 case ISD::FEXP10:
6167 case ISD::FTRUNC:
6168 case ISD::FFLOOR:
6169 case ISD::FCEIL:
6170 case ISD::FROUND:
6171 case ISD::FROUNDEVEN:
6172 case ISD::LROUND:
6173 case ISD::LLROUND:
6174 case ISD::FRINT:
6175 case ISD::LRINT:
6176 case ISD::LLRINT:
6177 case ISD::FNEARBYINT:
6178 case ISD::FLDEXP: {
6179 if (SNaN)
6180 return true;
6181 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6182 }
6183 case ISD::FABS:
6184 case ISD::FNEG:
6185 case ISD::FCOPYSIGN: {
6186 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6187 }
6188 case ISD::SELECT:
6189 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6190 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6191 case ISD::FP_EXTEND:
6192 case ISD::FP_ROUND: {
6193 if (SNaN)
6194 return true;
6195 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6196 }
6197 case ISD::SINT_TO_FP:
6198 case ISD::UINT_TO_FP:
6199 return true;
6200 case ISD::FSQRT: // Need is known positive
6201 case ISD::FLOG:
6202 case ISD::FLOG2:
6203 case ISD::FLOG10:
6204 case ISD::FPOWI:
6205 case ISD::FPOW: {
6206 if (SNaN)
6207 return true;
6208 // TODO: Refine on operand
6209 return false;
6210 }
6211 case ISD::FMINNUM:
6212 case ISD::FMAXNUM:
6213 case ISD::FMINIMUMNUM:
6214 case ISD::FMAXIMUMNUM: {
6215 // Only one needs to be known not-nan, since it will be returned if the
6216 // other ends up being one.
6217 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6218 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6219 }
6220 case ISD::FMINNUM_IEEE:
6221 case ISD::FMAXNUM_IEEE: {
6222 if (SNaN)
6223 return true;
6224 // This can return a NaN if either operand is an sNaN, or if both operands
6225 // are NaN.
6226 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6227 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6228 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6229 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6230 }
6231 case ISD::FMINIMUM:
6232 case ISD::FMAXIMUM: {
6233 // TODO: Does this quiet or return the origina NaN as-is?
6234 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6235 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6236 }
6237 case ISD::EXTRACT_VECTOR_ELT: {
6238 SDValue Src = Op.getOperand(i: 0);
6239 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6240 EVT SrcVT = Src.getValueType();
6241 if (SrcVT.isFixedLengthVector() && Idx &&
6242 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6243 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6244 BitNo: Idx->getZExtValue());
6245 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6246 }
6247 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6248 }
6249 case ISD::EXTRACT_SUBVECTOR: {
6250 SDValue Src = Op.getOperand(i: 0);
6251 if (Src.getValueType().isFixedLengthVector()) {
6252 unsigned Idx = Op.getConstantOperandVal(i: 1);
6253 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6254 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6255 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6256 }
6257 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6258 }
6259 case ISD::INSERT_SUBVECTOR: {
6260 SDValue BaseVector = Op.getOperand(i: 0);
6261 SDValue SubVector = Op.getOperand(i: 1);
6262 EVT BaseVectorVT = BaseVector.getValueType();
6263 if (BaseVectorVT.isFixedLengthVector()) {
6264 unsigned Idx = Op.getConstantOperandVal(i: 2);
6265 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6266 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6267
6268 // Clear/Extract the bits at the position where the subvector will be
6269 // inserted.
6270 APInt DemandedMask =
6271 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6272 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6273 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6274
6275 bool NeverNaN = true;
6276 if (!DemandedSrcElts.isZero())
6277 NeverNaN &=
6278 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6279 if (NeverNaN && !DemandedSubElts.isZero())
6280 NeverNaN &=
6281 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6282 return NeverNaN;
6283 }
6284 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6285 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6286 }
6287 case ISD::BUILD_VECTOR: {
6288 unsigned NumElts = Op.getNumOperands();
6289 for (unsigned I = 0; I != NumElts; ++I)
6290 if (DemandedElts[I] &&
6291 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6292 return false;
6293 return true;
6294 }
6295 case ISD::SPLAT_VECTOR:
6296 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6297 case ISD::AssertNoFPClass: {
6298 FPClassTest NoFPClass =
6299 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6300 if ((NoFPClass & fcNan) == fcNan)
6301 return true;
6302 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6303 return true;
6304 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6305 }
6306 default:
6307 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6308 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6309 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6310 Depth);
6311 }
6312 break;
6313 }
6314
6315 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6316 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, InterestedClasses: NanMask, Depth);
6317 return Known.isKnownNever(Mask: NanMask);
6318}
6319
6320bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
6321 assert(Op.getValueType().isFloatingPoint() &&
6322 "Floating point type expected");
6323
6324 // If the value is a constant, we can obviously see if it is a zero or not.
6325 return ISD::matchUnaryFpPredicate(
6326 Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); });
6327}
6328
6329bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6330 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6331 return isKnownNeverZero(Op, DemandedElts, Depth);
6332}
6333
6334bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6335 unsigned Depth) const {
6336 if (Depth >= MaxRecursionDepth)
6337 return false; // Limit search depth.
6338
6339 EVT OpVT = Op.getValueType();
6340 unsigned BitWidth = OpVT.getScalarSizeInBits();
6341
6342 assert(!Op.getValueType().isFloatingPoint() &&
6343 "Floating point types unsupported - use isKnownNeverZeroFloat");
6344
6345 // If the value is a constant, we can obviously see if it is a zero or not.
6346 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6347 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6348 return !V.isZero();
6349 };
6350
6351 if (ISD::matchUnaryPredicate(Op, Match: IsNeverZero))
6352 return true;
6353
6354 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6355 // some degree.
6356 switch (Op.getOpcode()) {
6357 default:
6358 break;
6359
6360 case ISD::BUILD_VECTOR:
6361 // Are all operands of a build vector constant non-zero?
6362 if (all_of(Range: enumerate(First: Op->ops()), P: [&](auto P) {
6363 auto *C = dyn_cast<ConstantSDNode>(P.value());
6364 return !DemandedElts[P.index()] || (C && IsNeverZero(C));
6365 }))
6366 return true;
6367 break;
6368
6369 case ISD::SPLAT_VECTOR:
6370 // Is the operand of a splat vector a constant non-zero?
6371 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 0)))
6372 if (IsNeverZero(C))
6373 return true;
6374 break;
6375
6376 case ISD::EXTRACT_VECTOR_ELT: {
6377 SDValue InVec = Op.getOperand(i: 0);
6378 SDValue EltNo = Op.getOperand(i: 1);
6379 EVT VecVT = InVec.getValueType();
6380
6381 // Skip scalable vectors or implicit extensions.
6382 if (VecVT.isScalableVector() ||
6383 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6384 break;
6385
6386 // If we know the element index, just demand that vector element, else for
6387 // an unknown element index, ignore DemandedElts and demand them all.
6388 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6389 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6390 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6391 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6392 DemandedSrcElts =
6393 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6394
6395 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6396 }
6397
6398 case ISD::OR:
6399 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6400 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6401
6402 case ISD::VSELECT:
6403 case ISD::SELECT:
6404 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6405 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6406
6407 case ISD::SHL: {
6408 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6409 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6410 KnownBits ValKnown =
6411 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6412 // 1 << X is never zero.
6413 if (ValKnown.One[0])
6414 return true;
6415 // If max shift cnt of known ones is non-zero, result is non-zero.
6416 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6417 .getMaxValue();
6418 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6419 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6420 return true;
6421 break;
6422 }
6423
6424 case ISD::VECTOR_SHUFFLE: {
6425 if (Op.getValueType().isScalableVector())
6426 return false;
6427
6428 unsigned NumElts = DemandedElts.getBitWidth();
6429
6430 // All demanded elements from LHS and RHS must be known non-zero.
6431 // Demanded elements with undef shuffle mask elements are unknown.
6432
6433 APInt DemandedLHS, DemandedRHS;
6434 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6435 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6436 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
6437 DemandedLHS, DemandedRHS))
6438 return false;
6439
6440 return (!DemandedLHS ||
6441 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1)) &&
6442 (!DemandedRHS ||
6443 isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1));
6444 }
6445
6446 case ISD::UADDSAT:
6447 case ISD::UMAX:
6448 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6449 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6450
6451 case ISD::UMIN:
6452 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6453 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6454
6455 // For smin/smax: If either operand is known negative/positive
6456 // respectively we don't need the other to be known at all.
6457 case ISD::SMAX: {
6458 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6459 if (Op1.isStrictlyPositive())
6460 return true;
6461
6462 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6463 if (Op0.isStrictlyPositive())
6464 return true;
6465
6466 if (Op1.isNonZero() && Op0.isNonZero())
6467 return true;
6468
6469 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6470 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6471 }
6472 case ISD::SMIN: {
6473 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6474 if (Op1.isNegative())
6475 return true;
6476
6477 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6478 if (Op0.isNegative())
6479 return true;
6480
6481 if (Op1.isNonZero() && Op0.isNonZero())
6482 return true;
6483
6484 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6485 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6486 }
6487
6488 case ISD::ROTL:
6489 case ISD::ROTR:
6490 case ISD::BITREVERSE:
6491 case ISD::BSWAP:
6492 case ISD::CTPOP:
6493 case ISD::ABS:
6494 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6495
6496 case ISD::SRA:
6497 case ISD::SRL: {
6498 if (Op->getFlags().hasExact())
6499 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6500 KnownBits ValKnown =
6501 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6502 if (ValKnown.isNegative())
6503 return true;
6504 // If max shift cnt of known ones is non-zero, result is non-zero.
6505 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6506 .getMaxValue();
6507 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6508 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6509 return true;
6510 break;
6511 }
6512 case ISD::UDIV:
6513 case ISD::SDIV:
6514 // div exact can only produce a zero if the dividend is zero.
6515 // TODO: For udiv this is also true if Op1 u<= Op0
6516 if (Op->getFlags().hasExact())
6517 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6518 break;
6519
6520 case ISD::ADD:
6521 if (Op->getFlags().hasNoUnsignedWrap())
6522 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6523 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6524 return true;
6525 // TODO: There are a lot more cases we can prove for add.
6526 break;
6527
6528 case ISD::SUB: {
6529 if (isNullConstant(V: Op.getOperand(i: 0)))
6530 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6531
6532 std::optional<bool> ne = KnownBits::ne(
6533 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6534 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6535 return ne && *ne;
6536 }
6537
6538 case ISD::MUL:
6539 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6540 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6541 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6542 return true;
6543 break;
6544
6545 case ISD::ZERO_EXTEND:
6546 case ISD::SIGN_EXTEND:
6547 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6548 case ISD::VSCALE: {
6549 const Function &F = getMachineFunction().getFunction();
6550 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6551 ConstantRange CR =
6552 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6553 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6554 return true;
6555 break;
6556 }
6557 }
6558
6559 return computeKnownBits(Op, Depth).isNonZero();
6560}
6561
6562bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6563 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6564 return !C1->isNegative();
6565
6566 switch (Op.getOpcode()) {
6567 case ISD::FABS:
6568 case ISD::FEXP:
6569 case ISD::FEXP2:
6570 case ISD::FEXP10:
6571 return true;
6572 default:
6573 return false;
6574 }
6575
6576 llvm_unreachable("covered opcode switch");
6577}
6578
6579bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6580 assert(Use.getValueType().isFloatingPoint());
6581 const SDNode *User = Use.getUser();
6582 if (User->getFlags().hasNoSignedZeros())
6583 return true;
6584
6585 unsigned OperandNo = Use.getOperandNo();
6586 // Check if this use is insensitive to the sign of zero
6587 switch (User->getOpcode()) {
6588 case ISD::SETCC:
6589 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6590 case ISD::FABS:
6591 // fabs always produces +0.0.
6592 return true;
6593 case ISD::FCOPYSIGN:
6594 // copysign overwrites the sign bit of the first operand.
6595 return OperandNo == 0;
6596 case ISD::FADD:
6597 case ISD::FSUB: {
6598 // Arithmetic with non-zero constants fixes the uncertainty around the
6599 // sign bit.
6600 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6601 return isKnownNeverZeroFloat(Op: Other);
6602 }
6603 case ISD::FP_TO_SINT:
6604 case ISD::FP_TO_UINT:
6605 // fp-to-int conversions normalize signed zeros.
6606 return true;
6607 default:
6608 return false;
6609 }
6610}
6611
6612bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6613 if (Op->getFlags().hasNoSignedZeros())
6614 return true;
6615 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6616 // regression. Ideally, this should be implemented as a demanded-bits
6617 // optimization that stems from the users.
6618 if (Op->use_size() > 2)
6619 return false;
6620 return all_of(Range: Op->uses(),
6621 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6622}
6623
6624bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6625 // Check the obvious case.
6626 if (A == B) return true;
6627
6628 // For negative and positive zero.
6629 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6630 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6631 if (CA->isZero() && CB->isZero()) return true;
6632
6633 // Otherwise they may not be equal.
6634 return false;
6635}
6636
6637// Only bits set in Mask must be negated, other bits may be arbitrary.
6638SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6639 if (isBitwiseNot(V, AllowUndefs))
6640 return V.getOperand(i: 0);
6641
6642 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6643 // bits in the non-extended part.
6644 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6645 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6646 return SDValue();
6647 SDValue ExtArg = V.getOperand(i: 0);
6648 if (ExtArg.getScalarValueSizeInBits() >=
6649 MaskC->getAPIntValue().getActiveBits() &&
6650 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6651 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6652 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6653 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6654 return SDValue();
6655}
6656
6657static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6658 // Match masked merge pattern (X & ~M) op (Y & M)
6659 // Including degenerate case (X & ~M) op M
6660 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6661 SDValue Other) {
6662 if (SDValue NotOperand =
6663 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6664 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6665 NotOperand->getOpcode() == ISD::TRUNCATE)
6666 NotOperand = NotOperand->getOperand(Num: 0);
6667
6668 if (Other == NotOperand)
6669 return true;
6670 if (Other->getOpcode() == ISD::AND)
6671 return NotOperand == Other->getOperand(Num: 0) ||
6672 NotOperand == Other->getOperand(Num: 1);
6673 }
6674 return false;
6675 };
6676
6677 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6678 A = A->getOperand(Num: 0);
6679
6680 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6681 B = B->getOperand(Num: 0);
6682
6683 if (A->getOpcode() == ISD::AND)
6684 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6685 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6686 return false;
6687}
6688
6689// FIXME: unify with llvm::haveNoCommonBitsSet.
6690bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6691 assert(A.getValueType() == B.getValueType() &&
6692 "Values must have the same type");
6693 if (haveNoCommonBitsSetCommutative(A, B) ||
6694 haveNoCommonBitsSetCommutative(A: B, B: A))
6695 return true;
6696 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6697 RHS: computeKnownBits(Op: B));
6698}
6699
6700static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6701 SelectionDAG &DAG) {
6702 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6703 return DAG.getConstant(Val: 0, DL, VT);
6704
6705 return SDValue();
6706}
6707
6708static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6709 ArrayRef<SDValue> Ops,
6710 SelectionDAG &DAG) {
6711 int NumOps = Ops.size();
6712 assert(NumOps != 0 && "Can't build an empty vector!");
6713 assert(!VT.isScalableVector() &&
6714 "BUILD_VECTOR cannot be used with scalable types");
6715 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6716 "Incorrect element count in BUILD_VECTOR!");
6717
6718 // BUILD_VECTOR of UNDEFs is UNDEF.
6719 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6720 return DAG.getUNDEF(VT);
6721
6722 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6723 SDValue IdentitySrc;
6724 bool IsIdentity = true;
6725 for (int i = 0; i != NumOps; ++i) {
6726 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6727 Ops[i].getOperand(i: 0).getValueType() != VT ||
6728 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6729 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6730 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6731 IsIdentity = false;
6732 break;
6733 }
6734 IdentitySrc = Ops[i].getOperand(i: 0);
6735 }
6736 if (IsIdentity)
6737 return IdentitySrc;
6738
6739 return SDValue();
6740}
6741
6742/// Try to simplify vector concatenation to an input value, undef, or build
6743/// vector.
6744static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6745 ArrayRef<SDValue> Ops,
6746 SelectionDAG &DAG) {
6747 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6748 assert(llvm::all_of(Ops,
6749 [Ops](SDValue Op) {
6750 return Ops[0].getValueType() == Op.getValueType();
6751 }) &&
6752 "Concatenation of vectors with inconsistent value types!");
6753 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6754 VT.getVectorElementCount() &&
6755 "Incorrect element count in vector concatenation!");
6756
6757 if (Ops.size() == 1)
6758 return Ops[0];
6759
6760 // Concat of UNDEFs is UNDEF.
6761 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6762 return DAG.getUNDEF(VT);
6763
6764 // Scan the operands and look for extract operations from a single source
6765 // that correspond to insertion at the same location via this concatenation:
6766 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6767 SDValue IdentitySrc;
6768 bool IsIdentity = true;
6769 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6770 SDValue Op = Ops[i];
6771 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6772 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6773 Op.getOperand(i: 0).getValueType() != VT ||
6774 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
6775 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
6776 IsIdentity = false;
6777 break;
6778 }
6779 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6780 "Unexpected identity source vector for concat of extracts");
6781 IdentitySrc = Op.getOperand(i: 0);
6782 }
6783 if (IsIdentity) {
6784 assert(IdentitySrc && "Failed to set source vector of extracts");
6785 return IdentitySrc;
6786 }
6787
6788 // The code below this point is only designed to work for fixed width
6789 // vectors, so we bail out for now.
6790 if (VT.isScalableVector())
6791 return SDValue();
6792
6793 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
6794 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
6795 // BUILD_VECTOR.
6796 // FIXME: Add support for SCALAR_TO_VECTOR as well.
6797 EVT SVT = VT.getScalarType();
6798 SmallVector<SDValue, 16> Elts;
6799 for (SDValue Op : Ops) {
6800 EVT OpVT = Op.getValueType();
6801 if (Op.isUndef())
6802 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
6803 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
6804 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
6805 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
6806 OpVT.getVectorNumElements() == 1 &&
6807 isNullConstant(V: Op.getOperand(i: 2)))
6808 Elts.push_back(Elt: Op.getOperand(i: 1));
6809 else
6810 return SDValue();
6811 }
6812
6813 // BUILD_VECTOR requires all inputs to be of the same type, find the
6814 // maximum type and extend them all.
6815 for (SDValue Op : Elts)
6816 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
6817
6818 if (SVT.bitsGT(VT: VT.getScalarType())) {
6819 for (SDValue &Op : Elts) {
6820 if (Op.isUndef())
6821 Op = DAG.getUNDEF(VT: SVT);
6822 else
6823 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
6824 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
6825 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
6826 }
6827 }
6828
6829 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
6830 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
6831 return V;
6832}
6833
6834/// Gets or creates the specified node.
6835SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
6836 SDVTList VTs = getVTList(VT);
6837 FoldingSetNodeID ID;
6838 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: {});
6839 void *IP = nullptr;
6840 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
6841 return SDValue(E, 0);
6842
6843 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6844 CSEMap.InsertNode(N, InsertPos: IP);
6845
6846 InsertNode(N);
6847 SDValue V = SDValue(N, 0);
6848 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6849 return V;
6850}
6851
6852SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6853 SDValue N1) {
6854 SDNodeFlags Flags;
6855 if (Inserter)
6856 Flags = Inserter->getFlags();
6857 return getNode(Opcode, DL, VT, Operand: N1, Flags);
6858}
6859
6860SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6861 SDValue N1, const SDNodeFlags Flags) {
6862 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
6863
6864 // Constant fold unary operations with a vector integer or float operand.
6865 switch (Opcode) {
6866 default:
6867 // FIXME: Entirely reasonable to perform folding of other unary
6868 // operations here as the need arises.
6869 break;
6870 case ISD::FNEG:
6871 case ISD::FABS:
6872 case ISD::FCEIL:
6873 case ISD::FTRUNC:
6874 case ISD::FFLOOR:
6875 case ISD::FP_EXTEND:
6876 case ISD::FP_TO_SINT:
6877 case ISD::FP_TO_UINT:
6878 case ISD::FP_TO_FP16:
6879 case ISD::FP_TO_BF16:
6880 case ISD::TRUNCATE:
6881 case ISD::ANY_EXTEND:
6882 case ISD::ZERO_EXTEND:
6883 case ISD::SIGN_EXTEND:
6884 case ISD::UINT_TO_FP:
6885 case ISD::SINT_TO_FP:
6886 case ISD::FP16_TO_FP:
6887 case ISD::BF16_TO_FP:
6888 case ISD::BITCAST:
6889 case ISD::ABS:
6890 case ISD::BITREVERSE:
6891 case ISD::BSWAP:
6892 case ISD::CTLZ:
6893 case ISD::CTLZ_ZERO_UNDEF:
6894 case ISD::CTTZ:
6895 case ISD::CTTZ_ZERO_UNDEF:
6896 case ISD::CTPOP:
6897 case ISD::CTLS:
6898 case ISD::STEP_VECTOR: {
6899 SDValue Ops = {N1};
6900 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
6901 return Fold;
6902 }
6903 }
6904
6905 unsigned OpOpcode = N1.getNode()->getOpcode();
6906 switch (Opcode) {
6907 case ISD::STEP_VECTOR:
6908 assert(VT.isScalableVector() &&
6909 "STEP_VECTOR can only be used with scalable types");
6910 assert(OpOpcode == ISD::TargetConstant &&
6911 VT.getVectorElementType() == N1.getValueType() &&
6912 "Unexpected step operand");
6913 break;
6914 case ISD::FREEZE:
6915 assert(VT == N1.getValueType() && "Unexpected VT!");
6916 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly=*/false))
6917 return N1;
6918 break;
6919 case ISD::TokenFactor:
6920 case ISD::MERGE_VALUES:
6921 case ISD::CONCAT_VECTORS:
6922 return N1; // Factor, merge or concat of one node? No need.
6923 case ISD::BUILD_VECTOR: {
6924 // Attempt to simplify BUILD_VECTOR.
6925 SDValue Ops[] = {N1};
6926 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
6927 return V;
6928 break;
6929 }
6930 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
6931 case ISD::FP_EXTEND:
6932 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
6933 "Invalid FP cast!");
6934 if (N1.getValueType() == VT) return N1; // noop conversion.
6935 assert((!VT.isVector() || VT.getVectorElementCount() ==
6936 N1.getValueType().getVectorElementCount()) &&
6937 "Vector element count mismatch!");
6938 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
6939 if (N1.isUndef())
6940 return getUNDEF(VT);
6941 break;
6942 case ISD::FP_TO_SINT:
6943 case ISD::FP_TO_UINT:
6944 if (N1.isUndef())
6945 return getUNDEF(VT);
6946 break;
6947 case ISD::SINT_TO_FP:
6948 case ISD::UINT_TO_FP:
6949 // [us]itofp(undef) = 0, because the result value is bounded.
6950 if (N1.isUndef())
6951 return getConstantFP(Val: 0.0, DL, VT);
6952 break;
6953 case ISD::SIGN_EXTEND:
6954 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6955 "Invalid SIGN_EXTEND!");
6956 assert(VT.isVector() == N1.getValueType().isVector() &&
6957 "SIGN_EXTEND result type type should be vector iff the operand "
6958 "type is vector!");
6959 if (N1.getValueType() == VT) return N1; // noop extension
6960 assert((!VT.isVector() || VT.getVectorElementCount() ==
6961 N1.getValueType().getVectorElementCount()) &&
6962 "Vector element count mismatch!");
6963 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
6964 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
6965 SDNodeFlags Flags;
6966 if (OpOpcode == ISD::ZERO_EXTEND)
6967 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6968 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6969 transferDbgValues(From: N1, To: NewVal);
6970 return NewVal;
6971 }
6972
6973 if (OpOpcode == ISD::POISON)
6974 return getPOISON(VT);
6975
6976 if (N1.isUndef())
6977 // sext(undef) = 0, because the top bits will all be the same.
6978 return getConstant(Val: 0, DL, VT);
6979
6980 // Skip unnecessary sext_inreg pattern:
6981 // (sext (trunc x)) -> x iff the upper bits are all signbits.
6982 if (OpOpcode == ISD::TRUNCATE) {
6983 SDValue OpOp = N1.getOperand(i: 0);
6984 if (OpOp.getValueType() == VT) {
6985 unsigned NumSignExtBits =
6986 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
6987 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
6988 transferDbgValues(From: N1, To: OpOp);
6989 return OpOp;
6990 }
6991 }
6992 }
6993 break;
6994 case ISD::ZERO_EXTEND:
6995 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6996 "Invalid ZERO_EXTEND!");
6997 assert(VT.isVector() == N1.getValueType().isVector() &&
6998 "ZERO_EXTEND result type type should be vector iff the operand "
6999 "type is vector!");
7000 if (N1.getValueType() == VT) return N1; // noop extension
7001 assert((!VT.isVector() || VT.getVectorElementCount() ==
7002 N1.getValueType().getVectorElementCount()) &&
7003 "Vector element count mismatch!");
7004 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7005 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7006 SDNodeFlags Flags;
7007 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7008 SDValue NewVal =
7009 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
7010 transferDbgValues(From: N1, To: NewVal);
7011 return NewVal;
7012 }
7013
7014 if (OpOpcode == ISD::POISON)
7015 return getPOISON(VT);
7016
7017 if (N1.isUndef())
7018 // zext(undef) = 0, because the top bits will be zero.
7019 return getConstant(Val: 0, DL, VT);
7020
7021 // Skip unnecessary zext_inreg pattern:
7022 // (zext (trunc x)) -> x iff the upper bits are known zero.
7023 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7024 // use to recognise zext_inreg patterns.
7025 if (OpOpcode == ISD::TRUNCATE) {
7026 SDValue OpOp = N1.getOperand(i: 0);
7027 if (OpOp.getValueType() == VT) {
7028 if (OpOp.getOpcode() != ISD::AND) {
7029 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
7030 loBit: N1.getScalarValueSizeInBits());
7031 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
7032 transferDbgValues(From: N1, To: OpOp);
7033 return OpOp;
7034 }
7035 }
7036 }
7037 }
7038 break;
7039 case ISD::ANY_EXTEND:
7040 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7041 "Invalid ANY_EXTEND!");
7042 assert(VT.isVector() == N1.getValueType().isVector() &&
7043 "ANY_EXTEND result type type should be vector iff the operand "
7044 "type is vector!");
7045 if (N1.getValueType() == VT) return N1; // noop extension
7046 assert((!VT.isVector() || VT.getVectorElementCount() ==
7047 N1.getValueType().getVectorElementCount()) &&
7048 "Vector element count mismatch!");
7049 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7050
7051 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7052 OpOpcode == ISD::ANY_EXTEND) {
7053 SDNodeFlags Flags;
7054 if (OpOpcode == ISD::ZERO_EXTEND)
7055 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7056 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7057 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7058 }
7059 if (N1.isUndef())
7060 return getUNDEF(VT);
7061
7062 // (ext (trunc x)) -> x
7063 if (OpOpcode == ISD::TRUNCATE) {
7064 SDValue OpOp = N1.getOperand(i: 0);
7065 if (OpOp.getValueType() == VT) {
7066 transferDbgValues(From: N1, To: OpOp);
7067 return OpOp;
7068 }
7069 }
7070 break;
7071 case ISD::TRUNCATE:
7072 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7073 "Invalid TRUNCATE!");
7074 assert(VT.isVector() == N1.getValueType().isVector() &&
7075 "TRUNCATE result type type should be vector iff the operand "
7076 "type is vector!");
7077 if (N1.getValueType() == VT) return N1; // noop truncate
7078 assert((!VT.isVector() || VT.getVectorElementCount() ==
7079 N1.getValueType().getVectorElementCount()) &&
7080 "Vector element count mismatch!");
7081 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7082 if (OpOpcode == ISD::TRUNCATE)
7083 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7084 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7085 OpOpcode == ISD::ANY_EXTEND) {
7086 // If the source is smaller than the dest, we still need an extend.
7087 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
7088 VT: VT.getScalarType())) {
7089 SDNodeFlags Flags;
7090 if (OpOpcode == ISD::ZERO_EXTEND)
7091 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7092 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7093 }
7094 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
7095 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7096 return N1.getOperand(i: 0);
7097 }
7098 if (N1.isUndef())
7099 return getUNDEF(VT);
7100 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7101 return getVScale(DL, VT,
7102 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
7103 break;
7104 case ISD::ANY_EXTEND_VECTOR_INREG:
7105 case ISD::ZERO_EXTEND_VECTOR_INREG:
7106 case ISD::SIGN_EXTEND_VECTOR_INREG:
7107 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7108 assert(N1.getValueType().bitsLE(VT) &&
7109 "The input must be the same size or smaller than the result.");
7110 assert(VT.getVectorMinNumElements() <
7111 N1.getValueType().getVectorMinNumElements() &&
7112 "The destination vector type must have fewer lanes than the input.");
7113 break;
7114 case ISD::ABS:
7115 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7116 if (N1.isUndef())
7117 return getConstant(Val: 0, DL, VT);
7118 break;
7119 case ISD::BSWAP:
7120 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7121 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7122 "BSWAP types must be a multiple of 16 bits!");
7123 if (N1.isUndef())
7124 return getUNDEF(VT);
7125 // bswap(bswap(X)) -> X.
7126 if (OpOpcode == ISD::BSWAP)
7127 return N1.getOperand(i: 0);
7128 break;
7129 case ISD::BITREVERSE:
7130 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7131 if (N1.isUndef())
7132 return getUNDEF(VT);
7133 break;
7134 case ISD::BITCAST:
7135 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
7136 "Cannot BITCAST between types of different sizes!");
7137 if (VT == N1.getValueType()) return N1; // noop conversion.
7138 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7139 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
7140 if (N1.isUndef())
7141 return getUNDEF(VT);
7142 break;
7143 case ISD::SCALAR_TO_VECTOR:
7144 assert(VT.isVector() && !N1.getValueType().isVector() &&
7145 (VT.getVectorElementType() == N1.getValueType() ||
7146 (VT.getVectorElementType().isInteger() &&
7147 N1.getValueType().isInteger() &&
7148 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7149 "Illegal SCALAR_TO_VECTOR node!");
7150 if (N1.isUndef())
7151 return getUNDEF(VT);
7152 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7153 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7154 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7155 N1.getConstantOperandVal(i: 1) == 0 &&
7156 N1.getOperand(i: 0).getValueType() == VT)
7157 return N1.getOperand(i: 0);
7158 break;
7159 case ISD::FNEG:
7160 // Negation of an unknown bag of bits is still completely undefined.
7161 if (N1.isUndef())
7162 return getUNDEF(VT);
7163
7164 if (OpOpcode == ISD::FNEG) // --X -> X
7165 return N1.getOperand(i: 0);
7166 break;
7167 case ISD::FABS:
7168 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7169 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7170 break;
7171 case ISD::VSCALE:
7172 assert(VT == N1.getValueType() && "Unexpected VT!");
7173 break;
7174 case ISD::CTPOP:
7175 if (N1.getValueType().getScalarType() == MVT::i1)
7176 return N1;
7177 break;
7178 case ISD::CTLZ:
7179 case ISD::CTTZ:
7180 if (N1.getValueType().getScalarType() == MVT::i1)
7181 return getNOT(DL, Val: N1, VT: N1.getValueType());
7182 break;
7183 case ISD::CTLS:
7184 if (N1.getValueType().getScalarType() == MVT::i1)
7185 return getConstant(Val: 0, DL, VT);
7186 break;
7187 case ISD::VECREDUCE_ADD:
7188 if (N1.getValueType().getScalarType() == MVT::i1)
7189 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7190 break;
7191 case ISD::VECREDUCE_SMIN:
7192 case ISD::VECREDUCE_UMAX:
7193 if (N1.getValueType().getScalarType() == MVT::i1)
7194 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7195 break;
7196 case ISD::VECREDUCE_SMAX:
7197 case ISD::VECREDUCE_UMIN:
7198 if (N1.getValueType().getScalarType() == MVT::i1)
7199 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7200 break;
7201 case ISD::SPLAT_VECTOR:
7202 assert(VT.isVector() && "Wrong return type!");
7203 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7204 // that for now.
7205 assert((VT.getVectorElementType() == N1.getValueType() ||
7206 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7207 (VT.getVectorElementType().isInteger() &&
7208 N1.getValueType().isInteger() &&
7209 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7210 "Wrong operand type!");
7211 break;
7212 }
7213
7214 SDNode *N;
7215 SDVTList VTs = getVTList(VT);
7216 SDValue Ops[] = {N1};
7217 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7218 FoldingSetNodeID ID;
7219 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7220 void *IP = nullptr;
7221 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7222 E->intersectFlagsWith(Flags);
7223 return SDValue(E, 0);
7224 }
7225
7226 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7227 N->setFlags(Flags);
7228 createOperands(Node: N, Vals: Ops);
7229 CSEMap.InsertNode(N, InsertPos: IP);
7230 } else {
7231 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7232 createOperands(Node: N, Vals: Ops);
7233 }
7234
7235 InsertNode(N);
7236 SDValue V = SDValue(N, 0);
7237 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7238 return V;
7239}
7240
7241static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7242 const APInt &C2) {
7243 switch (Opcode) {
7244 case ISD::ADD: return C1 + C2;
7245 case ISD::SUB: return C1 - C2;
7246 case ISD::MUL: return C1 * C2;
7247 case ISD::AND: return C1 & C2;
7248 case ISD::OR: return C1 | C2;
7249 case ISD::XOR: return C1 ^ C2;
7250 case ISD::SHL: return C1 << C2;
7251 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7252 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7253 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7254 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7255 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7256 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7257 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7258 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7259 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7260 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7261 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7262 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7263 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7264 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7265 case ISD::UDIV:
7266 if (!C2.getBoolValue())
7267 break;
7268 return C1.udiv(RHS: C2);
7269 case ISD::UREM:
7270 if (!C2.getBoolValue())
7271 break;
7272 return C1.urem(RHS: C2);
7273 case ISD::SDIV:
7274 if (!C2.getBoolValue())
7275 break;
7276 return C1.sdiv(RHS: C2);
7277 case ISD::SREM:
7278 if (!C2.getBoolValue())
7279 break;
7280 return C1.srem(RHS: C2);
7281 case ISD::AVGFLOORS:
7282 return APIntOps::avgFloorS(C1, C2);
7283 case ISD::AVGFLOORU:
7284 return APIntOps::avgFloorU(C1, C2);
7285 case ISD::AVGCEILS:
7286 return APIntOps::avgCeilS(C1, C2);
7287 case ISD::AVGCEILU:
7288 return APIntOps::avgCeilU(C1, C2);
7289 case ISD::ABDS:
7290 return APIntOps::abds(A: C1, B: C2);
7291 case ISD::ABDU:
7292 return APIntOps::abdu(A: C1, B: C2);
7293 case ISD::MULHS:
7294 return APIntOps::mulhs(C1, C2);
7295 case ISD::MULHU:
7296 return APIntOps::mulhu(C1, C2);
7297 case ISD::CLMUL:
7298 return APIntOps::clmul(LHS: C1, RHS: C2);
7299 case ISD::CLMULR:
7300 return APIntOps::clmulr(LHS: C1, RHS: C2);
7301 case ISD::CLMULH:
7302 return APIntOps::clmulh(LHS: C1, RHS: C2);
7303 }
7304 return std::nullopt;
7305}
7306// Handle constant folding with UNDEF.
7307// TODO: Handle more cases.
7308static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7309 bool IsUndef1, const APInt &C2,
7310 bool IsUndef2) {
7311 if (!(IsUndef1 || IsUndef2))
7312 return FoldValue(Opcode, C1, C2);
7313
7314 // Fold and(x, undef) -> 0
7315 // Fold mul(x, undef) -> 0
7316 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7317 return APInt::getZero(numBits: C1.getBitWidth());
7318
7319 return std::nullopt;
7320}
7321
7322SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7323 const GlobalAddressSDNode *GA,
7324 const SDNode *N2) {
7325 if (GA->getOpcode() != ISD::GlobalAddress)
7326 return SDValue();
7327 if (!TLI->isOffsetFoldingLegal(GA))
7328 return SDValue();
7329 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7330 if (!C2)
7331 return SDValue();
7332 int64_t Offset = C2->getSExtValue();
7333 switch (Opcode) {
7334 case ISD::ADD:
7335 case ISD::PTRADD:
7336 break;
7337 case ISD::SUB: Offset = -uint64_t(Offset); break;
7338 default: return SDValue();
7339 }
7340 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7341 Offset: GA->getOffset() + uint64_t(Offset));
7342}
7343
7344bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7345 switch (Opcode) {
7346 case ISD::SDIV:
7347 case ISD::UDIV:
7348 case ISD::SREM:
7349 case ISD::UREM: {
7350 // If a divisor is zero/undef or any element of a divisor vector is
7351 // zero/undef, the whole op is undef.
7352 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7353 SDValue Divisor = Ops[1];
7354 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7355 return true;
7356
7357 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7358 llvm::any_of(Range: Divisor->op_values(),
7359 P: [](SDValue V) { return V.isUndef() ||
7360 isNullConstant(V); });
7361 // TODO: Handle signed overflow.
7362 }
7363 // TODO: Handle oversized shifts.
7364 default:
7365 return false;
7366 }
7367}
7368
7369SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7370 EVT VT, ArrayRef<SDValue> Ops,
7371 SDNodeFlags Flags) {
7372 // If the opcode is a target-specific ISD node, there's nothing we can
7373 // do here and the operand rules may not line up with the below, so
7374 // bail early.
7375 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7376 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7377 // foldCONCAT_VECTORS in getNode before this is called.
7378 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7379 return SDValue();
7380
7381 unsigned NumOps = Ops.size();
7382 if (NumOps == 0)
7383 return SDValue();
7384
7385 if (isUndef(Opcode, Ops))
7386 return getUNDEF(VT);
7387
7388 // Handle unary special cases.
7389 if (NumOps == 1) {
7390 SDValue N1 = Ops[0];
7391
7392 // Constant fold unary operations with an integer constant operand. Even
7393 // opaque constant will be folded, because the folding of unary operations
7394 // doesn't create new constants with different values. Nevertheless, the
7395 // opaque flag is preserved during folding to prevent future folding with
7396 // other constants.
7397 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7398 const APInt &Val = C->getAPIntValue();
7399 switch (Opcode) {
7400 case ISD::SIGN_EXTEND:
7401 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7402 isT: C->isTargetOpcode(), isO: C->isOpaque());
7403 case ISD::TRUNCATE:
7404 if (C->isOpaque())
7405 break;
7406 [[fallthrough]];
7407 case ISD::ZERO_EXTEND:
7408 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7409 isT: C->isTargetOpcode(), isO: C->isOpaque());
7410 case ISD::ANY_EXTEND:
7411 // Some targets like RISCV prefer to sign extend some types.
7412 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7413 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7414 isT: C->isTargetOpcode(), isO: C->isOpaque());
7415 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7416 isT: C->isTargetOpcode(), isO: C->isOpaque());
7417 case ISD::ABS:
7418 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7419 isO: C->isOpaque());
7420 case ISD::BITREVERSE:
7421 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7422 isO: C->isOpaque());
7423 case ISD::BSWAP:
7424 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7425 isO: C->isOpaque());
7426 case ISD::CTPOP:
7427 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7428 isO: C->isOpaque());
7429 case ISD::CTLZ:
7430 case ISD::CTLZ_ZERO_UNDEF:
7431 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7432 isO: C->isOpaque());
7433 case ISD::CTTZ:
7434 case ISD::CTTZ_ZERO_UNDEF:
7435 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7436 isO: C->isOpaque());
7437 case ISD::CTLS:
7438 // CTLS returns the number of extra sign bits so subtract one.
7439 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7440 isT: C->isTargetOpcode(), isO: C->isOpaque());
7441 case ISD::UINT_TO_FP:
7442 case ISD::SINT_TO_FP: {
7443 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7444 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7445 RM: APFloat::rmNearestTiesToEven);
7446 return getConstantFP(V: FPV, DL, VT);
7447 }
7448 case ISD::FP16_TO_FP:
7449 case ISD::BF16_TO_FP: {
7450 bool Ignored;
7451 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7452 : APFloat::BFloat(),
7453 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7454
7455 // This can return overflow, underflow, or inexact; we don't care.
7456 // FIXME need to be more flexible about rounding mode.
7457 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7458 losesInfo: &Ignored);
7459 return getConstantFP(V: FPV, DL, VT);
7460 }
7461 case ISD::STEP_VECTOR:
7462 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7463 return V;
7464 break;
7465 case ISD::BITCAST:
7466 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7467 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7468 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7469 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7470 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7471 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7472 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7473 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7474 break;
7475 }
7476 }
7477
7478 // Constant fold unary operations with a floating point constant operand.
7479 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7480 APFloat V = C->getValueAPF(); // make copy
7481 switch (Opcode) {
7482 case ISD::FNEG:
7483 V.changeSign();
7484 return getConstantFP(V, DL, VT);
7485 case ISD::FABS:
7486 V.clearSign();
7487 return getConstantFP(V, DL, VT);
7488 case ISD::FCEIL: {
7489 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7490 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7491 return getConstantFP(V, DL, VT);
7492 return SDValue();
7493 }
7494 case ISD::FTRUNC: {
7495 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7496 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7497 return getConstantFP(V, DL, VT);
7498 return SDValue();
7499 }
7500 case ISD::FFLOOR: {
7501 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7502 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7503 return getConstantFP(V, DL, VT);
7504 return SDValue();
7505 }
7506 case ISD::FP_EXTEND: {
7507 bool ignored;
7508 // This can return overflow, underflow, or inexact; we don't care.
7509 // FIXME need to be more flexible about rounding mode.
7510 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7511 losesInfo: &ignored);
7512 return getConstantFP(V, DL, VT);
7513 }
7514 case ISD::FP_TO_SINT:
7515 case ISD::FP_TO_UINT: {
7516 bool ignored;
7517 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7518 // FIXME need to be more flexible about rounding mode.
7519 APFloat::opStatus s =
7520 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7521 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7522 break;
7523 return getConstant(Val: IntVal, DL, VT);
7524 }
7525 case ISD::FP_TO_FP16:
7526 case ISD::FP_TO_BF16: {
7527 bool Ignored;
7528 // This can return overflow, underflow, or inexact; we don't care.
7529 // FIXME need to be more flexible about rounding mode.
7530 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7531 : APFloat::BFloat(),
7532 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7533 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7534 }
7535 case ISD::BITCAST:
7536 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7537 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7538 VT);
7539 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7540 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7541 VT);
7542 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7543 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7544 VT);
7545 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7546 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7547 break;
7548 }
7549 }
7550
7551 // Early-out if we failed to constant fold a bitcast.
7552 if (Opcode == ISD::BITCAST)
7553 return SDValue();
7554 }
7555
7556 // Handle binops special cases.
7557 if (NumOps == 2) {
7558 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7559 return CFP;
7560
7561 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7562 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7563 if (C1->isOpaque() || C2->isOpaque())
7564 return SDValue();
7565
7566 std::optional<APInt> FoldAttempt =
7567 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7568 if (!FoldAttempt)
7569 return SDValue();
7570
7571 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7572 assert((!Folded || !VT.isVector()) &&
7573 "Can't fold vectors ops with scalar operands");
7574 return Folded;
7575 }
7576 }
7577
7578 // fold (add Sym, c) -> Sym+c
7579 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7580 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7581 if (TLI->isCommutativeBinOp(Opcode))
7582 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7583 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7584
7585 // fold (sext_in_reg c1) -> c2
7586 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7587 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7588
7589 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7590 unsigned FromBits = EVT.getScalarSizeInBits();
7591 Val <<= Val.getBitWidth() - FromBits;
7592 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7593 return getConstant(Val, DL, VT: ConstantVT);
7594 };
7595
7596 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7597 const APInt &Val = C1->getAPIntValue();
7598 return SignExtendInReg(Val, VT);
7599 }
7600
7601 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7602 SmallVector<SDValue, 8> ScalarOps;
7603 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7604 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7605 SDValue Op = Ops[0].getOperand(i: I);
7606 if (Op.isUndef()) {
7607 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7608 continue;
7609 }
7610 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7611 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7612 }
7613 return getBuildVector(VT, DL, Ops: ScalarOps);
7614 }
7615
7616 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7617 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7618 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7619 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7620 Ops[0].getOperand(i: 0).getValueType()));
7621 }
7622 }
7623
7624 // Handle fshl/fshr special cases.
7625 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7626 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7627 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7628 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7629
7630 if (C1 && C2 && C3) {
7631 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7632 return SDValue();
7633 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7634 &V3 = C3->getAPIntValue();
7635
7636 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7637 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7638 return getConstant(Val: FoldedVal, DL, VT);
7639 }
7640 }
7641
7642 // Handle fma/fmad special cases.
7643 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7644 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7645 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7646 Ops[2].getValueType() == VT && "FMA types must match!");
7647 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
7648 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
7649 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
7650 if (C1 && C2 && C3) {
7651 APFloat V1 = C1->getValueAPF();
7652 const APFloat &V2 = C2->getValueAPF();
7653 const APFloat &V3 = C3->getValueAPF();
7654 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7655 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7656 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7657 } else
7658 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7659 return getConstantFP(V: V1, DL, VT);
7660 }
7661 }
7662
7663 // This is for vector folding only from here on.
7664 if (!VT.isVector())
7665 return SDValue();
7666
7667 ElementCount NumElts = VT.getVectorElementCount();
7668
7669 // See if we can fold through any bitcasted integer ops.
7670 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
7671 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7672 (Ops[0].getOpcode() == ISD::BITCAST ||
7673 Ops[1].getOpcode() == ISD::BITCAST)) {
7674 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
7675 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
7676 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
7677 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
7678 if (BV1 && BV2 && N1.getValueType().isInteger() &&
7679 N2.getValueType().isInteger()) {
7680 bool IsLE = getDataLayout().isLittleEndian();
7681 unsigned EltBits = VT.getScalarSizeInBits();
7682 SmallVector<APInt> RawBits1, RawBits2;
7683 BitVector UndefElts1, UndefElts2;
7684 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
7685 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
7686 SmallVector<APInt> RawBits;
7687 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
7688 std::optional<APInt> Fold = FoldValueWithUndef(
7689 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
7690 if (!Fold)
7691 break;
7692 RawBits.push_back(Elt: *Fold);
7693 }
7694 if (RawBits.size() == NumElts.getFixedValue()) {
7695 // We have constant folded, but we might need to cast this again back
7696 // to the original (possibly legalized) type.
7697 EVT BVVT, BVEltVT;
7698 if (N1.getValueType() == VT) {
7699 BVVT = N1.getValueType();
7700 BVEltVT = BV1->getOperand(Num: 0).getValueType();
7701 } else {
7702 BVVT = N2.getValueType();
7703 BVEltVT = BV2->getOperand(Num: 0).getValueType();
7704 }
7705 unsigned BVEltBits = BVEltVT.getSizeInBits();
7706 SmallVector<APInt> DstBits;
7707 BitVector DstUndefs;
7708 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
7709 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
7710 SrcUndefElements: BitVector(RawBits.size(), false));
7711 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
7712 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
7713 if (DstUndefs[I])
7714 continue;
7715 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
7716 }
7717 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
7718 }
7719 }
7720 }
7721 }
7722
7723 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
7724 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
7725 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
7726 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
7727 APInt RHSVal;
7728 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
7729 APInt NewStep = Opcode == ISD::MUL
7730 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
7731 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
7732 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
7733 }
7734 }
7735
7736 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
7737 return !Op.getValueType().isVector() ||
7738 Op.getValueType().getVectorElementCount() == NumElts;
7739 };
7740
7741 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
7742 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
7743 Op.getOpcode() == ISD::BUILD_VECTOR ||
7744 Op.getOpcode() == ISD::SPLAT_VECTOR;
7745 };
7746
7747 // All operands must be vector types with the same number of elements as
7748 // the result type and must be either UNDEF or a build/splat vector
7749 // or UNDEF scalars.
7750 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
7751 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
7752 return SDValue();
7753
7754 // If we are comparing vectors, then the result needs to be a i1 boolean that
7755 // is then extended back to the legal result type depending on how booleans
7756 // are represented.
7757 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
7758 ISD::NodeType ExtendCode =
7759 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
7760 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
7761 : ISD::SIGN_EXTEND;
7762
7763 // Find legal integer scalar type for constant promotion and
7764 // ensure that its scalar size is at least as large as source.
7765 EVT LegalSVT = VT.getScalarType();
7766 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
7767 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
7768 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
7769 return SDValue();
7770 }
7771
7772 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
7773 // only have one operand to check. For fixed-length vector types we may have
7774 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
7775 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
7776
7777 // Constant fold each scalar lane separately.
7778 SmallVector<SDValue, 4> ScalarResults;
7779 for (unsigned I = 0; I != NumVectorElts; I++) {
7780 SmallVector<SDValue, 4> ScalarOps;
7781 for (SDValue Op : Ops) {
7782 EVT InSVT = Op.getValueType().getScalarType();
7783 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
7784 Op.getOpcode() != ISD::SPLAT_VECTOR) {
7785 if (Op.isUndef())
7786 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
7787 else
7788 ScalarOps.push_back(Elt: Op);
7789 continue;
7790 }
7791
7792 SDValue ScalarOp =
7793 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
7794 EVT ScalarVT = ScalarOp.getValueType();
7795
7796 // Build vector (integer) scalar operands may need implicit
7797 // truncation - do this before constant folding.
7798 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
7799 // Don't create illegally-typed nodes unless they're constants or undef
7800 // - if we fail to constant fold we can't guarantee the (dead) nodes
7801 // we're creating will be cleaned up before being visited for
7802 // legalization.
7803 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
7804 !isa<ConstantSDNode>(Val: ScalarOp) &&
7805 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
7806 TargetLowering::TypeLegal)
7807 return SDValue();
7808 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
7809 }
7810
7811 ScalarOps.push_back(Elt: ScalarOp);
7812 }
7813
7814 // Constant fold the scalar operands.
7815 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
7816
7817 // Scalar folding only succeeded if the result is a constant or UNDEF.
7818 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
7819 ScalarResult.getOpcode() != ISD::ConstantFP)
7820 return SDValue();
7821
7822 // Legalize the (integer) scalar constant if necessary. We only do
7823 // this once we know the folding succeeded, since otherwise we would
7824 // get a node with illegal type which has a user.
7825 if (LegalSVT != SVT)
7826 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
7827
7828 ScalarResults.push_back(Elt: ScalarResult);
7829 }
7830
7831 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
7832 : getBuildVector(VT, DL, Ops: ScalarResults);
7833 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
7834 return V;
7835}
7836
7837SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
7838 EVT VT, ArrayRef<SDValue> Ops) {
7839 // TODO: Add support for unary/ternary fp opcodes.
7840 if (Ops.size() != 2)
7841 return SDValue();
7842
7843 // TODO: We don't do any constant folding for strict FP opcodes here, but we
7844 // should. That will require dealing with a potentially non-default
7845 // rounding mode, checking the "opStatus" return value from the APFloat
7846 // math calculations, and possibly other variations.
7847 SDValue N1 = Ops[0];
7848 SDValue N2 = Ops[1];
7849 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
7850 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
7851 if (N1CFP && N2CFP) {
7852 APFloat C1 = N1CFP->getValueAPF(); // make copy
7853 const APFloat &C2 = N2CFP->getValueAPF();
7854 switch (Opcode) {
7855 case ISD::FADD:
7856 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7857 return getConstantFP(V: C1, DL, VT);
7858 case ISD::FSUB:
7859 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7860 return getConstantFP(V: C1, DL, VT);
7861 case ISD::FMUL:
7862 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7863 return getConstantFP(V: C1, DL, VT);
7864 case ISD::FDIV:
7865 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7866 return getConstantFP(V: C1, DL, VT);
7867 case ISD::FREM:
7868 C1.mod(RHS: C2);
7869 return getConstantFP(V: C1, DL, VT);
7870 case ISD::FCOPYSIGN:
7871 C1.copySign(RHS: C2);
7872 return getConstantFP(V: C1, DL, VT);
7873 case ISD::FMINNUM:
7874 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
7875 case ISD::FMAXNUM:
7876 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
7877 case ISD::FMINIMUM:
7878 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
7879 case ISD::FMAXIMUM:
7880 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
7881 case ISD::FMINIMUMNUM:
7882 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
7883 case ISD::FMAXIMUMNUM:
7884 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
7885 default: break;
7886 }
7887 }
7888 if (N1CFP && Opcode == ISD::FP_ROUND) {
7889 APFloat C1 = N1CFP->getValueAPF(); // make copy
7890 bool Unused;
7891 // This can return overflow, underflow, or inexact; we don't care.
7892 // FIXME need to be more flexible about rounding mode.
7893 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7894 losesInfo: &Unused);
7895 return getConstantFP(V: C1, DL, VT);
7896 }
7897
7898 switch (Opcode) {
7899 case ISD::FSUB:
7900 // -0.0 - undef --> undef (consistent with "fneg undef")
7901 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
7902 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
7903 return getUNDEF(VT);
7904 [[fallthrough]];
7905
7906 case ISD::FADD:
7907 case ISD::FMUL:
7908 case ISD::FDIV:
7909 case ISD::FREM:
7910 // If both operands are undef, the result is undef. If 1 operand is undef,
7911 // the result is NaN. This should match the behavior of the IR optimizer.
7912 if (N1.isUndef() && N2.isUndef())
7913 return getUNDEF(VT);
7914 if (N1.isUndef() || N2.isUndef())
7915 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
7916 }
7917 return SDValue();
7918}
7919
7920SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
7921 const SDLoc &DL, EVT DstEltVT) {
7922 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
7923
7924 // If this is already the right type, we're done.
7925 if (SrcEltVT == DstEltVT)
7926 return SDValue(BV, 0);
7927
7928 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
7929 unsigned DstBitSize = DstEltVT.getSizeInBits();
7930
7931 // If this is a conversion of N elements of one type to N elements of another
7932 // type, convert each element. This handles FP<->INT cases.
7933 if (SrcBitSize == DstBitSize) {
7934 SmallVector<SDValue, 8> Ops;
7935 for (SDValue Op : BV->op_values()) {
7936 // If the vector element type is not legal, the BUILD_VECTOR operands
7937 // are promoted and implicitly truncated. Make that explicit here.
7938 if (Op.getValueType() != SrcEltVT)
7939 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
7940 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
7941 }
7942 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
7943 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
7944 return getBuildVector(VT, DL, Ops);
7945 }
7946
7947 // Otherwise, we're growing or shrinking the elements. To avoid having to
7948 // handle annoying details of growing/shrinking FP values, we convert them to
7949 // int first.
7950 if (SrcEltVT.isFloatingPoint()) {
7951 // Convert the input float vector to a int vector where the elements are the
7952 // same sizes.
7953 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
7954 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7955 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7956 DstEltVT);
7957 return SDValue();
7958 }
7959
7960 // Now we know the input is an integer vector. If the output is a FP type,
7961 // convert to integer first, then to FP of the right size.
7962 if (DstEltVT.isFloatingPoint()) {
7963 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
7964 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7965 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7966 DstEltVT);
7967 return SDValue();
7968 }
7969
7970 // Okay, we know the src/dst types are both integers of differing types.
7971 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
7972
7973 // Extract the constant raw bit data.
7974 BitVector UndefElements;
7975 SmallVector<APInt> RawBits;
7976 bool IsLE = getDataLayout().isLittleEndian();
7977 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
7978 return SDValue();
7979
7980 SmallVector<SDValue, 8> Ops;
7981 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
7982 if (UndefElements[I])
7983 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
7984 else
7985 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
7986 }
7987
7988 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
7989 return getBuildVector(VT, DL, Ops);
7990}
7991
7992SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
7993 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
7994
7995 // There's no need to assert on a byte-aligned pointer. All pointers are at
7996 // least byte aligned.
7997 if (A == Align(1))
7998 return Val;
7999
8000 SDVTList VTs = getVTList(VT: Val.getValueType());
8001 FoldingSetNodeID ID;
8002 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: VTs, OpList: {Val});
8003 ID.AddInteger(I: A.value());
8004
8005 void *IP = nullptr;
8006 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
8007 return SDValue(E, 0);
8008
8009 auto *N =
8010 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
8011 createOperands(Node: N, Vals: {Val});
8012
8013 CSEMap.InsertNode(N, InsertPos: IP);
8014 InsertNode(N);
8015
8016 SDValue V(N, 0);
8017 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8018 return V;
8019}
8020
8021SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8022 SDValue N1, SDValue N2) {
8023 SDNodeFlags Flags;
8024 if (Inserter)
8025 Flags = Inserter->getFlags();
8026 return getNode(Opcode, DL, VT, N1, N2, Flags);
8027}
8028
8029void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
8030 SDValue &N2) const {
8031 if (!TLI->isCommutativeBinOp(Opcode))
8032 return;
8033
8034 // Canonicalize:
8035 // binop(const, nonconst) -> binop(nonconst, const)
8036 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
8037 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
8038 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
8039 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
8040 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8041 std::swap(a&: N1, b&: N2);
8042
8043 // Canonicalize:
8044 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8045 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8046 N2.getOpcode() == ISD::STEP_VECTOR)
8047 std::swap(a&: N1, b&: N2);
8048}
8049
8050SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8051 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8052 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8053 N2.getOpcode() != ISD::DELETED_NODE &&
8054 "Operand is DELETED_NODE!");
8055
8056 canonicalizeCommutativeBinop(Opcode, N1, N2);
8057
8058 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
8059 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
8060
8061 // Don't allow undefs in vector splats - we might be returning N2 when folding
8062 // to zero etc.
8063 ConstantSDNode *N2CV =
8064 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8065
8066 switch (Opcode) {
8067 default: break;
8068 case ISD::TokenFactor:
8069 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8070 N2.getValueType() == MVT::Other && "Invalid token factor!");
8071 // Fold trivial token factors.
8072 if (N1.getOpcode() == ISD::EntryToken) return N2;
8073 if (N2.getOpcode() == ISD::EntryToken) return N1;
8074 if (N1 == N2) return N1;
8075 break;
8076 case ISD::BUILD_VECTOR: {
8077 // Attempt to simplify BUILD_VECTOR.
8078 SDValue Ops[] = {N1, N2};
8079 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8080 return V;
8081 break;
8082 }
8083 case ISD::CONCAT_VECTORS: {
8084 SDValue Ops[] = {N1, N2};
8085 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8086 return V;
8087 break;
8088 }
8089 case ISD::AND:
8090 assert(VT.isInteger() && "This operator does not apply to FP types!");
8091 assert(N1.getValueType() == N2.getValueType() &&
8092 N1.getValueType() == VT && "Binary operator types must match!");
8093 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8094 // worth handling here.
8095 if (N2CV && N2CV->isZero())
8096 return N2;
8097 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8098 return N1;
8099 break;
8100 case ISD::OR:
8101 case ISD::XOR:
8102 case ISD::ADD:
8103 case ISD::PTRADD:
8104 case ISD::SUB:
8105 assert(VT.isInteger() && "This operator does not apply to FP types!");
8106 assert(N1.getValueType() == N2.getValueType() &&
8107 N1.getValueType() == VT && "Binary operator types must match!");
8108 // The equal operand types requirement is unnecessarily strong for PTRADD.
8109 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8110 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8111 // logic everywhere where PTRADDs may be folded or combined to properly
8112 // support them. If/when we introduce pointer types to the SDAG, we will
8113 // need to relax this constraint.
8114
8115 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8116 // it's worth handling here.
8117 if (N2CV && N2CV->isZero())
8118 return N1;
8119 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8120 VT.getScalarType() == MVT::i1)
8121 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8122 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8123 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8124 N2.getOpcode() == ISD::VSCALE) {
8125 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
8126 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
8127 return getVScale(DL, VT, MulImm: C1 + C2);
8128 }
8129 break;
8130 case ISD::MUL:
8131 assert(VT.isInteger() && "This operator does not apply to FP types!");
8132 assert(N1.getValueType() == N2.getValueType() &&
8133 N1.getValueType() == VT && "Binary operator types must match!");
8134 if (VT.getScalarType() == MVT::i1)
8135 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8136 if (N2CV && N2CV->isZero())
8137 return N2;
8138 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8139 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8140 const APInt &N2CImm = N2C->getAPIntValue();
8141 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
8142 }
8143 break;
8144 case ISD::UDIV:
8145 case ISD::UREM:
8146 case ISD::MULHU:
8147 case ISD::MULHS:
8148 case ISD::SDIV:
8149 case ISD::SREM:
8150 case ISD::SADDSAT:
8151 case ISD::SSUBSAT:
8152 case ISD::UADDSAT:
8153 case ISD::USUBSAT:
8154 assert(VT.isInteger() && "This operator does not apply to FP types!");
8155 assert(N1.getValueType() == N2.getValueType() &&
8156 N1.getValueType() == VT && "Binary operator types must match!");
8157 if (VT.getScalarType() == MVT::i1) {
8158 // fold (add_sat x, y) -> (or x, y) for bool types.
8159 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8160 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8161 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8162 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8163 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8164 }
8165 break;
8166 case ISD::SCMP:
8167 case ISD::UCMP:
8168 assert(N1.getValueType() == N2.getValueType() &&
8169 "Types of operands of UCMP/SCMP must match");
8170 assert(N1.getValueType().isVector() == VT.isVector() &&
8171 "Operands and return type of must both be scalars or vectors");
8172 if (VT.isVector())
8173 assert(VT.getVectorElementCount() ==
8174 N1.getValueType().getVectorElementCount() &&
8175 "Result and operands must have the same number of elements");
8176 break;
8177 case ISD::AVGFLOORS:
8178 case ISD::AVGFLOORU:
8179 case ISD::AVGCEILS:
8180 case ISD::AVGCEILU:
8181 assert(VT.isInteger() && "This operator does not apply to FP types!");
8182 assert(N1.getValueType() == N2.getValueType() &&
8183 N1.getValueType() == VT && "Binary operator types must match!");
8184 break;
8185 case ISD::ABDS:
8186 case ISD::ABDU:
8187 assert(VT.isInteger() && "This operator does not apply to FP types!");
8188 assert(N1.getValueType() == N2.getValueType() &&
8189 N1.getValueType() == VT && "Binary operator types must match!");
8190 if (VT.getScalarType() == MVT::i1)
8191 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8192 break;
8193 case ISD::SMIN:
8194 case ISD::UMAX:
8195 assert(VT.isInteger() && "This operator does not apply to FP types!");
8196 assert(N1.getValueType() == N2.getValueType() &&
8197 N1.getValueType() == VT && "Binary operator types must match!");
8198 if (VT.getScalarType() == MVT::i1)
8199 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8200 break;
8201 case ISD::SMAX:
8202 case ISD::UMIN:
8203 assert(VT.isInteger() && "This operator does not apply to FP types!");
8204 assert(N1.getValueType() == N2.getValueType() &&
8205 N1.getValueType() == VT && "Binary operator types must match!");
8206 if (VT.getScalarType() == MVT::i1)
8207 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8208 break;
8209 case ISD::FADD:
8210 case ISD::FSUB:
8211 case ISD::FMUL:
8212 case ISD::FDIV:
8213 case ISD::FREM:
8214 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8215 assert(N1.getValueType() == N2.getValueType() &&
8216 N1.getValueType() == VT && "Binary operator types must match!");
8217 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8218 return V;
8219 break;
8220 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8221 assert(N1.getValueType() == VT &&
8222 N1.getValueType().isFloatingPoint() &&
8223 N2.getValueType().isFloatingPoint() &&
8224 "Invalid FCOPYSIGN!");
8225 break;
8226 case ISD::SHL:
8227 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8228 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8229 const APInt &ShiftImm = N2C->getAPIntValue();
8230 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8231 }
8232 [[fallthrough]];
8233 case ISD::SRA:
8234 case ISD::SRL:
8235 if (SDValue V = simplifyShift(X: N1, Y: N2))
8236 return V;
8237 [[fallthrough]];
8238 case ISD::ROTL:
8239 case ISD::ROTR:
8240 case ISD::SSHLSAT:
8241 case ISD::USHLSAT:
8242 assert(VT == N1.getValueType() &&
8243 "Shift operators return type must be the same as their first arg");
8244 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8245 "Shifts only work on integers");
8246 assert((!VT.isVector() || VT == N2.getValueType()) &&
8247 "Vector shift amounts must be in the same as their first arg");
8248 // Verify that the shift amount VT is big enough to hold valid shift
8249 // amounts. This catches things like trying to shift an i1024 value by an
8250 // i8, which is easy to fall into in generic code that uses
8251 // TLI.getShiftAmount().
8252 assert(N2.getValueType().getScalarSizeInBits() >=
8253 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8254 "Invalid use of small shift amount with oversized value!");
8255
8256 // Always fold shifts of i1 values so the code generator doesn't need to
8257 // handle them. Since we know the size of the shift has to be less than the
8258 // size of the value, the shift/rotate count is guaranteed to be zero.
8259 if (VT == MVT::i1)
8260 return N1;
8261 if (N2CV && N2CV->isZero())
8262 return N1;
8263 break;
8264 case ISD::FP_ROUND:
8265 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8266 VT.bitsLE(N1.getValueType()) && N2C &&
8267 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8268 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8269 if (N1.getValueType() == VT) return N1; // noop conversion.
8270 break;
8271 case ISD::IS_FPCLASS: {
8272 assert(N1.getValueType().isFloatingPoint() &&
8273 "IS_FPCLASS is used for a non-floating type");
8274 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8275 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8276 // If all tests are made, it doesn't matter what the value is.
8277 if ((Mask & fcAllFlags) == fcAllFlags)
8278 return getBoolConstant(V: true, DL, VT, OpVT: N1.getValueType());
8279 if ((Mask & fcAllFlags) == 0)
8280 return getBoolConstant(V: false, DL, VT, OpVT: N1.getValueType());
8281 break;
8282 }
8283 case ISD::AssertNoFPClass: {
8284 assert(N1.getValueType().isFloatingPoint() &&
8285 "AssertNoFPClass is used for a non-floating type");
8286 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8287 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8288 assert(llvm::to_underlying(NoFPClass) <=
8289 BitmaskEnumDetail::Mask<FPClassTest>() &&
8290 "FPClassTest value too large");
8291 (void)NoFPClass;
8292 break;
8293 }
8294 case ISD::AssertSext:
8295 case ISD::AssertZext: {
8296 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8297 assert(VT == N1.getValueType() && "Not an inreg extend!");
8298 assert(VT.isInteger() && EVT.isInteger() &&
8299 "Cannot *_EXTEND_INREG FP types");
8300 assert(!EVT.isVector() &&
8301 "AssertSExt/AssertZExt type should be the vector element type "
8302 "rather than the vector type!");
8303 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8304 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8305 break;
8306 }
8307 case ISD::SIGN_EXTEND_INREG: {
8308 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8309 assert(VT == N1.getValueType() && "Not an inreg extend!");
8310 assert(VT.isInteger() && EVT.isInteger() &&
8311 "Cannot *_EXTEND_INREG FP types");
8312 assert(EVT.isVector() == VT.isVector() &&
8313 "SIGN_EXTEND_INREG type should be vector iff the operand "
8314 "type is vector!");
8315 assert((!EVT.isVector() ||
8316 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8317 "Vector element counts must match in SIGN_EXTEND_INREG");
8318 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8319 if (EVT == VT) return N1; // Not actually extending
8320 break;
8321 }
8322 case ISD::FP_TO_SINT_SAT:
8323 case ISD::FP_TO_UINT_SAT: {
8324 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8325 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8326 assert(N1.getValueType().isVector() == VT.isVector() &&
8327 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8328 "vector!");
8329 assert((!VT.isVector() || VT.getVectorElementCount() ==
8330 N1.getValueType().getVectorElementCount()) &&
8331 "Vector element counts must match in FP_TO_*INT_SAT");
8332 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8333 "Type to saturate to must be a scalar.");
8334 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8335 "Not extending!");
8336 break;
8337 }
8338 case ISD::EXTRACT_VECTOR_ELT:
8339 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8340 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8341 element type of the vector.");
8342
8343 // Extract from an undefined value or using an undefined index is undefined.
8344 if (N1.isUndef() || N2.isUndef())
8345 return getUNDEF(VT);
8346
8347 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
8348 // vectors. For scalable vectors we will provide appropriate support for
8349 // dealing with arbitrary indices.
8350 if (N2C && N1.getValueType().isFixedLengthVector() &&
8351 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8352 return getUNDEF(VT);
8353
8354 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8355 // expanding copies of large vectors from registers. This only works for
8356 // fixed length vectors, since we need to know the exact number of
8357 // elements.
8358 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8359 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8360 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8361 return getExtractVectorElt(DL, VT,
8362 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8363 Idx: N2C->getZExtValue() % Factor);
8364 }
8365
8366 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8367 // lowering is expanding large vector constants.
8368 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8369 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8370 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8371 N1.getValueType().isFixedLengthVector()) &&
8372 "BUILD_VECTOR used for scalable vectors");
8373 unsigned Index =
8374 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8375 SDValue Elt = N1.getOperand(i: Index);
8376
8377 if (VT != Elt.getValueType())
8378 // If the vector element type is not legal, the BUILD_VECTOR operands
8379 // are promoted and implicitly truncated, and the result implicitly
8380 // extended. Make that explicit here.
8381 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8382
8383 return Elt;
8384 }
8385
8386 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8387 // operations are lowered to scalars.
8388 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8389 // If the indices are the same, return the inserted element else
8390 // if the indices are known different, extract the element from
8391 // the original vector.
8392 SDValue N1Op2 = N1.getOperand(i: 2);
8393 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8394
8395 if (N1Op2C && N2C) {
8396 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8397 if (VT == N1.getOperand(i: 1).getValueType())
8398 return N1.getOperand(i: 1);
8399 if (VT.isFloatingPoint()) {
8400 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8401 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8402 }
8403 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8404 }
8405 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8406 }
8407 }
8408
8409 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8410 // when vector types are scalarized and v1iX is legal.
8411 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8412 // Here we are completely ignoring the extract element index (N2),
8413 // which is fine for fixed width vectors, since any index other than 0
8414 // is undefined anyway. However, this cannot be ignored for scalable
8415 // vectors - in theory we could support this, but we don't want to do this
8416 // without a profitability check.
8417 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8418 N1.getValueType().isFixedLengthVector() &&
8419 N1.getValueType().getVectorNumElements() == 1) {
8420 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8421 N2: N1.getOperand(i: 1));
8422 }
8423 break;
8424 case ISD::EXTRACT_ELEMENT:
8425 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8426 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8427 (N1.getValueType().isInteger() == VT.isInteger()) &&
8428 N1.getValueType() != VT &&
8429 "Wrong types for EXTRACT_ELEMENT!");
8430
8431 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8432 // 64-bit integers into 32-bit parts. Instead of building the extract of
8433 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8434 if (N1.getOpcode() == ISD::BUILD_PAIR)
8435 return N1.getOperand(i: N2C->getZExtValue());
8436
8437 // EXTRACT_ELEMENT of a constant int is also very common.
8438 if (N1C) {
8439 unsigned ElementSize = VT.getSizeInBits();
8440 unsigned Shift = ElementSize * N2C->getZExtValue();
8441 const APInt &Val = N1C->getAPIntValue();
8442 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8443 }
8444 break;
8445 case ISD::EXTRACT_SUBVECTOR: {
8446 EVT N1VT = N1.getValueType();
8447 assert(VT.isVector() && N1VT.isVector() &&
8448 "Extract subvector VTs must be vectors!");
8449 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8450 "Extract subvector VTs must have the same element type!");
8451 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8452 "Cannot extract a scalable vector from a fixed length vector!");
8453 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8454 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8455 "Extract subvector must be from larger vector to smaller vector!");
8456 assert(N2C && "Extract subvector index must be a constant");
8457 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8458 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8459 N1VT.getVectorMinNumElements()) &&
8460 "Extract subvector overflow!");
8461 assert(N2C->getAPIntValue().getBitWidth() ==
8462 TLI->getVectorIdxWidth(getDataLayout()) &&
8463 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8464 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8465 "Extract index is not a multiple of the output vector length");
8466
8467 // Trivial extraction.
8468 if (VT == N1VT)
8469 return N1;
8470
8471 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8472 if (N1.isUndef())
8473 return getUNDEF(VT);
8474
8475 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8476 // the concat have the same type as the extract.
8477 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8478 VT == N1.getOperand(i: 0).getValueType()) {
8479 unsigned Factor = VT.getVectorMinNumElements();
8480 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8481 }
8482
8483 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8484 // during shuffle legalization.
8485 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8486 VT == N1.getOperand(i: 1).getValueType())
8487 return N1.getOperand(i: 1);
8488 break;
8489 }
8490 }
8491
8492 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8493 switch (Opcode) {
8494 case ISD::XOR:
8495 case ISD::ADD:
8496 case ISD::PTRADD:
8497 case ISD::SUB:
8498 case ISD::SIGN_EXTEND_INREG:
8499 case ISD::UDIV:
8500 case ISD::SDIV:
8501 case ISD::UREM:
8502 case ISD::SREM:
8503 case ISD::MUL:
8504 case ISD::AND:
8505 case ISD::SSUBSAT:
8506 case ISD::USUBSAT:
8507 case ISD::UMIN:
8508 case ISD::OR:
8509 case ISD::SADDSAT:
8510 case ISD::UADDSAT:
8511 case ISD::UMAX:
8512 case ISD::SMAX:
8513 case ISD::SMIN:
8514 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8515 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8516 }
8517 }
8518
8519 // Canonicalize an UNDEF to the RHS, even over a constant.
8520 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8521 if (TLI->isCommutativeBinOp(Opcode)) {
8522 std::swap(a&: N1, b&: N2);
8523 } else {
8524 switch (Opcode) {
8525 case ISD::PTRADD:
8526 case ISD::SUB:
8527 // fold op(undef, non_undef_arg2) -> undef.
8528 return N1;
8529 case ISD::SIGN_EXTEND_INREG:
8530 case ISD::UDIV:
8531 case ISD::SDIV:
8532 case ISD::UREM:
8533 case ISD::SREM:
8534 case ISD::SSUBSAT:
8535 case ISD::USUBSAT:
8536 // fold op(undef, non_undef_arg2) -> 0.
8537 return getConstant(Val: 0, DL, VT);
8538 }
8539 }
8540 }
8541
8542 // Fold a bunch of operators when the RHS is undef.
8543 if (N2.getOpcode() == ISD::UNDEF) {
8544 switch (Opcode) {
8545 case ISD::XOR:
8546 if (N1.getOpcode() == ISD::UNDEF)
8547 // Handle undef ^ undef -> 0 special case. This is a common
8548 // idiom (misuse).
8549 return getConstant(Val: 0, DL, VT);
8550 [[fallthrough]];
8551 case ISD::ADD:
8552 case ISD::PTRADD:
8553 case ISD::SUB:
8554 // fold op(arg1, undef) -> undef.
8555 return N2;
8556 case ISD::UDIV:
8557 case ISD::SDIV:
8558 case ISD::UREM:
8559 case ISD::SREM:
8560 // fold op(arg1, undef) -> poison.
8561 return getPOISON(VT);
8562 case ISD::MUL:
8563 case ISD::AND:
8564 case ISD::SSUBSAT:
8565 case ISD::USUBSAT:
8566 case ISD::UMIN:
8567 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
8568 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
8569 case ISD::OR:
8570 case ISD::SADDSAT:
8571 case ISD::UADDSAT:
8572 case ISD::UMAX:
8573 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
8574 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
8575 case ISD::SMAX:
8576 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
8577 return N1.getOpcode() == ISD::UNDEF
8578 ? N2
8579 : getConstant(
8580 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
8581 VT);
8582 case ISD::SMIN:
8583 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
8584 return N1.getOpcode() == ISD::UNDEF
8585 ? N2
8586 : getConstant(
8587 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
8588 VT);
8589 }
8590 }
8591
8592 // Perform trivial constant folding.
8593 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
8594 return SV;
8595
8596 // Memoize this node if possible.
8597 SDNode *N;
8598 SDVTList VTs = getVTList(VT);
8599 SDValue Ops[] = {N1, N2};
8600 if (VT != MVT::Glue) {
8601 FoldingSetNodeID ID;
8602 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8603 void *IP = nullptr;
8604 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8605 E->intersectFlagsWith(Flags);
8606 return SDValue(E, 0);
8607 }
8608
8609 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8610 N->setFlags(Flags);
8611 createOperands(Node: N, Vals: Ops);
8612 CSEMap.InsertNode(N, InsertPos: IP);
8613 } else {
8614 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8615 createOperands(Node: N, Vals: Ops);
8616 }
8617
8618 InsertNode(N);
8619 SDValue V = SDValue(N, 0);
8620 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8621 return V;
8622}
8623
8624SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8625 SDValue N1, SDValue N2, SDValue N3) {
8626 SDNodeFlags Flags;
8627 if (Inserter)
8628 Flags = Inserter->getFlags();
8629 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
8630}
8631
8632SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8633 SDValue N1, SDValue N2, SDValue N3,
8634 const SDNodeFlags Flags) {
8635 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8636 N2.getOpcode() != ISD::DELETED_NODE &&
8637 N3.getOpcode() != ISD::DELETED_NODE &&
8638 "Operand is DELETED_NODE!");
8639 // Perform various simplifications.
8640 switch (Opcode) {
8641 case ISD::BUILD_VECTOR: {
8642 // Attempt to simplify BUILD_VECTOR.
8643 SDValue Ops[] = {N1, N2, N3};
8644 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8645 return V;
8646 break;
8647 }
8648 case ISD::CONCAT_VECTORS: {
8649 SDValue Ops[] = {N1, N2, N3};
8650 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8651 return V;
8652 break;
8653 }
8654 case ISD::SETCC: {
8655 assert(VT.isInteger() && "SETCC result type must be an integer!");
8656 assert(N1.getValueType() == N2.getValueType() &&
8657 "SETCC operands must have the same type!");
8658 assert(VT.isVector() == N1.getValueType().isVector() &&
8659 "SETCC type should be vector iff the operand type is vector!");
8660 assert((!VT.isVector() || VT.getVectorElementCount() ==
8661 N1.getValueType().getVectorElementCount()) &&
8662 "SETCC vector element counts must match!");
8663 // Use FoldSetCC to simplify SETCC's.
8664 if (SDValue V =
8665 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
8666 return V;
8667 break;
8668 }
8669 case ISD::SELECT:
8670 case ISD::VSELECT:
8671 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
8672 return V;
8673 break;
8674 case ISD::VECTOR_SHUFFLE:
8675 llvm_unreachable("should use getVectorShuffle constructor!");
8676 case ISD::VECTOR_SPLICE_LEFT:
8677 if (isNullConstant(V: N3))
8678 return N1;
8679 break;
8680 case ISD::VECTOR_SPLICE_RIGHT:
8681 if (isNullConstant(V: N3))
8682 return N2;
8683 break;
8684 case ISD::INSERT_VECTOR_ELT: {
8685 assert(VT.isVector() && VT == N1.getValueType() &&
8686 "INSERT_VECTOR_ELT vector type mismatch");
8687 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
8688 "INSERT_VECTOR_ELT scalar fp/int mismatch");
8689 assert((!VT.isFloatingPoint() ||
8690 VT.getVectorElementType() == N2.getValueType()) &&
8691 "INSERT_VECTOR_ELT fp scalar type mismatch");
8692 assert((!VT.isInteger() ||
8693 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
8694 "INSERT_VECTOR_ELT int scalar size mismatch");
8695
8696 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
8697 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
8698 // for scalable vectors where we will generate appropriate code to
8699 // deal with out-of-bounds cases correctly.
8700 if (N3C && VT.isFixedLengthVector() &&
8701 N3C->getZExtValue() >= VT.getVectorNumElements())
8702 return getUNDEF(VT);
8703
8704 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
8705 if (N3.isUndef())
8706 return getUNDEF(VT);
8707
8708 // If inserting poison, just use the input vector.
8709 if (N2.getOpcode() == ISD::POISON)
8710 return N1;
8711
8712 // Inserting undef into undef/poison is still undef.
8713 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8714 return getUNDEF(VT);
8715
8716 // If the inserted element is an UNDEF, just use the input vector.
8717 // But not if skipping the insert could make the result more poisonous.
8718 if (N2.isUndef()) {
8719 if (N3C && VT.isFixedLengthVector()) {
8720 APInt EltMask =
8721 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
8722 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8723 return N1;
8724 } else if (isGuaranteedNotToBePoison(Op: N1))
8725 return N1;
8726 }
8727 break;
8728 }
8729 case ISD::INSERT_SUBVECTOR: {
8730 // If inserting poison, just use the input vector,
8731 if (N2.getOpcode() == ISD::POISON)
8732 return N1;
8733
8734 // Inserting undef into undef/poison is still undef.
8735 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8736 return getUNDEF(VT);
8737
8738 EVT N2VT = N2.getValueType();
8739 assert(VT == N1.getValueType() &&
8740 "Dest and insert subvector source types must match!");
8741 assert(VT.isVector() && N2VT.isVector() &&
8742 "Insert subvector VTs must be vectors!");
8743 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
8744 "Insert subvector VTs must have the same element type!");
8745 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
8746 "Cannot insert a scalable vector into a fixed length vector!");
8747 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8748 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
8749 "Insert subvector must be from smaller vector to larger vector!");
8750 assert(isa<ConstantSDNode>(N3) &&
8751 "Insert subvector index must be constant");
8752 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8753 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
8754 VT.getVectorMinNumElements()) &&
8755 "Insert subvector overflow!");
8756 assert(N3->getAsAPIntVal().getBitWidth() ==
8757 TLI->getVectorIdxWidth(getDataLayout()) &&
8758 "Constant index for INSERT_SUBVECTOR has an invalid size");
8759
8760 // Trivial insertion.
8761 if (VT == N2VT)
8762 return N2;
8763
8764 // If this is an insert of an extracted vector into an undef/poison vector,
8765 // we can just use the input to the extract. But not if skipping the
8766 // extract+insert could make the result more poisonous.
8767 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8768 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
8769 if (N1.getOpcode() == ISD::POISON)
8770 return N2.getOperand(i: 0);
8771 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
8772 unsigned LoBit = N3->getAsZExtVal();
8773 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8774 APInt EltMask =
8775 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8776 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
8777 return N2.getOperand(i: 0);
8778 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
8779 return N2.getOperand(i: 0);
8780 }
8781
8782 // If the inserted subvector is UNDEF, just use the input vector.
8783 // But not if skipping the insert could make the result more poisonous.
8784 if (N2.isUndef()) {
8785 if (VT.isFixedLengthVector()) {
8786 unsigned LoBit = N3->getAsZExtVal();
8787 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8788 APInt EltMask =
8789 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8790 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8791 return N1;
8792 } else if (isGuaranteedNotToBePoison(Op: N1))
8793 return N1;
8794 }
8795 break;
8796 }
8797 case ISD::BITCAST:
8798 // Fold bit_convert nodes from a type to themselves.
8799 if (N1.getValueType() == VT)
8800 return N1;
8801 break;
8802 case ISD::VP_TRUNCATE:
8803 case ISD::VP_SIGN_EXTEND:
8804 case ISD::VP_ZERO_EXTEND:
8805 // Don't create noop casts.
8806 if (N1.getValueType() == VT)
8807 return N1;
8808 break;
8809 case ISD::VECTOR_COMPRESS: {
8810 [[maybe_unused]] EVT VecVT = N1.getValueType();
8811 [[maybe_unused]] EVT MaskVT = N2.getValueType();
8812 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
8813 assert(VT == VecVT && "Vector and result type don't match.");
8814 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
8815 "All inputs must be vectors.");
8816 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
8817 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
8818 "Vector and mask must have same number of elements.");
8819
8820 if (N1.isUndef() || N2.isUndef())
8821 return N3;
8822
8823 break;
8824 }
8825 case ISD::PARTIAL_REDUCE_UMLA:
8826 case ISD::PARTIAL_REDUCE_SMLA:
8827 case ISD::PARTIAL_REDUCE_SUMLA:
8828 case ISD::PARTIAL_REDUCE_FMLA: {
8829 [[maybe_unused]] EVT AccVT = N1.getValueType();
8830 [[maybe_unused]] EVT Input1VT = N2.getValueType();
8831 [[maybe_unused]] EVT Input2VT = N3.getValueType();
8832 assert(Input1VT.isVector() && Input1VT == Input2VT &&
8833 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8834 "node to have the same type!");
8835 assert(VT.isVector() && VT == AccVT &&
8836 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
8837 "the same type as its result!");
8838 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
8839 AccVT.getVectorElementCount()) &&
8840 "Expected the element count of the second and third operands of the "
8841 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
8842 "element count of the first operand and the result!");
8843 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
8844 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8845 "node to have an element type which is the same as or smaller than "
8846 "the element type of the first operand and result!");
8847 break;
8848 }
8849 }
8850
8851 // Perform trivial constant folding for arithmetic operators.
8852 switch (Opcode) {
8853 case ISD::FMA:
8854 case ISD::FMAD:
8855 case ISD::SETCC:
8856 case ISD::FSHL:
8857 case ISD::FSHR:
8858 if (SDValue SV =
8859 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
8860 return SV;
8861 break;
8862 }
8863
8864 // Memoize node if it doesn't produce a glue result.
8865 SDNode *N;
8866 SDVTList VTs = getVTList(VT);
8867 SDValue Ops[] = {N1, N2, N3};
8868 if (VT != MVT::Glue) {
8869 FoldingSetNodeID ID;
8870 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8871 void *IP = nullptr;
8872 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8873 E->intersectFlagsWith(Flags);
8874 return SDValue(E, 0);
8875 }
8876
8877 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8878 N->setFlags(Flags);
8879 createOperands(Node: N, Vals: Ops);
8880 CSEMap.InsertNode(N, InsertPos: IP);
8881 } else {
8882 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8883 createOperands(Node: N, Vals: Ops);
8884 }
8885
8886 InsertNode(N);
8887 SDValue V = SDValue(N, 0);
8888 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8889 return V;
8890}
8891
8892SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8893 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8894 const SDNodeFlags Flags) {
8895 SDValue Ops[] = { N1, N2, N3, N4 };
8896 return getNode(Opcode, DL, VT, Ops, Flags);
8897}
8898
8899SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8900 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
8901 SDNodeFlags Flags;
8902 if (Inserter)
8903 Flags = Inserter->getFlags();
8904 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
8905}
8906
8907SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8908 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8909 SDValue N5, const SDNodeFlags Flags) {
8910 SDValue Ops[] = { N1, N2, N3, N4, N5 };
8911 return getNode(Opcode, DL, VT, Ops, Flags);
8912}
8913
8914SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8915 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8916 SDValue N5) {
8917 SDNodeFlags Flags;
8918 if (Inserter)
8919 Flags = Inserter->getFlags();
8920 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
8921}
8922
8923/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
8924/// the incoming stack arguments to be loaded from the stack.
8925SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
8926 SmallVector<SDValue, 8> ArgChains;
8927
8928 // Include the original chain at the beginning of the list. When this is
8929 // used by target LowerCall hooks, this helps legalize find the
8930 // CALLSEQ_BEGIN node.
8931 ArgChains.push_back(Elt: Chain);
8932
8933 // Add a chain value for each stack argument.
8934 for (SDNode *U : getEntryNode().getNode()->users())
8935 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
8936 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
8937 if (FI->getIndex() < 0)
8938 ArgChains.push_back(Elt: SDValue(L, 1));
8939
8940 // Build a tokenfactor for all the chains.
8941 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
8942}
8943
8944/// getMemsetValue - Vectorized representation of the memset value
8945/// operand.
8946static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
8947 const SDLoc &dl) {
8948 assert(!Value.isUndef());
8949
8950 unsigned NumBits = VT.getScalarSizeInBits();
8951 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
8952 assert(C->getAPIntValue().getBitWidth() == 8);
8953 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
8954 if (VT.isInteger()) {
8955 bool IsOpaque = VT.getSizeInBits() > 64 ||
8956 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
8957 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
8958 }
8959 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
8960 }
8961
8962 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
8963 EVT IntVT = VT.getScalarType();
8964 if (!IntVT.isInteger())
8965 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
8966
8967 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
8968 if (NumBits > 8) {
8969 // Use a multiplication with 0x010101... to extend the input to the
8970 // required length.
8971 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
8972 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
8973 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
8974 }
8975
8976 if (VT != Value.getValueType() && !VT.isInteger())
8977 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
8978 if (VT != Value.getValueType())
8979 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
8980
8981 return Value;
8982}
8983
8984/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
8985/// used when a memcpy is turned into a memset when the source is a constant
8986/// string ptr.
8987static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
8988 const TargetLowering &TLI,
8989 const ConstantDataArraySlice &Slice) {
8990 // Handle vector with all elements zero.
8991 if (Slice.Array == nullptr) {
8992 if (VT.isInteger())
8993 return DAG.getConstant(Val: 0, DL: dl, VT);
8994 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
8995 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
8996 }
8997
8998 assert(!VT.isVector() && "Can't handle vector type here!");
8999 unsigned NumVTBits = VT.getSizeInBits();
9000 unsigned NumVTBytes = NumVTBits / 8;
9001 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
9002
9003 APInt Val(NumVTBits, 0);
9004 if (DAG.getDataLayout().isLittleEndian()) {
9005 for (unsigned i = 0; i != NumBytes; ++i)
9006 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9007 } else {
9008 for (unsigned i = 0; i != NumBytes; ++i)
9009 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9010 }
9011
9012 // If the "cost" of materializing the integer immediate is less than the cost
9013 // of a load, then it is cost effective to turn the load into the immediate.
9014 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
9015 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
9016 return DAG.getConstant(Val, DL: dl, VT);
9017 return SDValue();
9018}
9019
9020SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
9021 const SDLoc &DL,
9022 const SDNodeFlags Flags) {
9023 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
9024 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
9025}
9026
9027SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
9028 const SDLoc &DL,
9029 const SDNodeFlags Flags) {
9030 assert(Offset.getValueType().isInteger());
9031 EVT BasePtrVT = Ptr.getValueType();
9032 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
9033 PtrVT: BasePtrVT))
9034 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
9035 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9036 SDNodeFlags AddFlags = Flags;
9037 AddFlags.setInBounds(false);
9038 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
9039}
9040
9041/// Returns true if memcpy source is constant data.
9042static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
9043 uint64_t SrcDelta = 0;
9044 GlobalAddressSDNode *G = nullptr;
9045 if (Src.getOpcode() == ISD::GlobalAddress)
9046 G = cast<GlobalAddressSDNode>(Val&: Src);
9047 else if (Src->isAnyAdd() &&
9048 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
9049 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
9050 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
9051 SrcDelta = Src.getConstantOperandVal(i: 1);
9052 }
9053 if (!G)
9054 return false;
9055
9056 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
9057 Offset: SrcDelta + G->getOffset());
9058}
9059
9060static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
9061 SelectionDAG &DAG) {
9062 // On Darwin, -Os means optimize for size without hurting performance, so
9063 // only really optimize for size when -Oz (MinSize) is used.
9064 if (MF.getTarget().getTargetTriple().isOSDarwin())
9065 return MF.getFunction().hasMinSize();
9066 return DAG.shouldOptForSize();
9067}
9068
9069static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
9070 SmallVector<SDValue, 32> &OutChains, unsigned From,
9071 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9072 SmallVector<SDValue, 16> &OutStoreChains) {
9073 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9074 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9075 SmallVector<SDValue, 16> GluedLoadChains;
9076 for (unsigned i = From; i < To; ++i) {
9077 OutChains.push_back(Elt: OutLoadChains[i]);
9078 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
9079 }
9080
9081 // Chain for all loads.
9082 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
9083 Ops: GluedLoadChains);
9084
9085 for (unsigned i = From; i < To; ++i) {
9086 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
9087 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
9088 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
9089 MMO: ST->getMemOperand());
9090 OutChains.push_back(Elt: NewStore);
9091 }
9092}
9093
9094static SDValue getMemcpyLoadsAndStores(
9095 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9096 uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline,
9097 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
9098 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
9099 // Turn a memcpy of undef to nop.
9100 // FIXME: We need to honor volatile even is Src is undef.
9101 if (Src.isUndef())
9102 return Chain;
9103
9104 // Expand memcpy to a series of load and store ops if the size operand falls
9105 // below a certain threshold.
9106 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9107 // rather than maybe a humongous number of loads and stores.
9108 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9109 const DataLayout &DL = DAG.getDataLayout();
9110 LLVMContext &C = *DAG.getContext();
9111 std::vector<EVT> MemOps;
9112 bool DstAlignCanChange = false;
9113 MachineFunction &MF = DAG.getMachineFunction();
9114 MachineFrameInfo &MFI = MF.getFrameInfo();
9115 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9116 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9117 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9118 DstAlignCanChange = true;
9119 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
9120 if (!SrcAlign || Alignment > *SrcAlign)
9121 SrcAlign = Alignment;
9122 assert(SrcAlign && "SrcAlign must be set");
9123 ConstantDataArraySlice Slice;
9124 // If marked as volatile, perform a copy even when marked as constant.
9125 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9126 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9127 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9128 const MemOp Op = isZeroConstant
9129 ? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment,
9130 /*IsZeroMemset*/ true, IsVolatile: isVol)
9131 : MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment,
9132 SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
9133 if (!TLI.findOptimalMemOpLowering(
9134 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
9135 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9136 return SDValue();
9137
9138 if (DstAlignCanChange) {
9139 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9140 Align NewAlign = DL.getABITypeAlign(Ty);
9141
9142 // Don't promote to an alignment that would require dynamic stack
9143 // realignment which may conflict with optimizations such as tail call
9144 // optimization.
9145 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9146 if (!TRI->hasStackRealignment(MF))
9147 if (MaybeAlign StackAlign = DL.getStackAlignment())
9148 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9149
9150 if (NewAlign > Alignment) {
9151 // Give the stack frame object a larger alignment if needed.
9152 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9153 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9154 Alignment = NewAlign;
9155 }
9156 }
9157
9158 // Prepare AAInfo for loads/stores after lowering this memcpy.
9159 AAMDNodes NewAAInfo = AAInfo;
9160 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9161
9162 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9163 bool isConstant =
9164 BatchAA && SrcVal &&
9165 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9166
9167 MachineMemOperand::Flags MMOFlags =
9168 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9169 SmallVector<SDValue, 16> OutLoadChains;
9170 SmallVector<SDValue, 16> OutStoreChains;
9171 SmallVector<SDValue, 32> OutChains;
9172 unsigned NumMemOps = MemOps.size();
9173 uint64_t SrcOff = 0, DstOff = 0;
9174 for (unsigned i = 0; i != NumMemOps; ++i) {
9175 EVT VT = MemOps[i];
9176 unsigned VTSize = VT.getSizeInBits() / 8;
9177 SDValue Value, Store;
9178
9179 if (VTSize > Size) {
9180 // Issuing an unaligned load / store pair that overlaps with the previous
9181 // pair. Adjust the offset accordingly.
9182 assert(i == NumMemOps-1 && i != 0);
9183 SrcOff -= VTSize - Size;
9184 DstOff -= VTSize - Size;
9185 }
9186
9187 if (CopyFromConstant &&
9188 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9189 // It's unlikely a store of a vector immediate can be done in a single
9190 // instruction. It would require a load from a constantpool first.
9191 // We only handle zero vectors here.
9192 // FIXME: Handle other cases where store of vector immediate is done in
9193 // a single instruction.
9194 ConstantDataArraySlice SubSlice;
9195 if (SrcOff < Slice.Length) {
9196 SubSlice = Slice;
9197 SubSlice.move(Delta: SrcOff);
9198 } else {
9199 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9200 SubSlice.Array = nullptr;
9201 SubSlice.Offset = 0;
9202 SubSlice.Length = VTSize;
9203 }
9204 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9205 if (Value.getNode()) {
9206 Store = DAG.getStore(
9207 Chain, dl, Val: Value,
9208 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9209 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
9210 OutChains.push_back(Elt: Store);
9211 }
9212 }
9213
9214 if (!Store.getNode()) {
9215 // The type might not be legal for the target. This should only happen
9216 // if the type is smaller than a legal type, as on PPC, so the right
9217 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9218 // to Load/Store if NVT==VT.
9219 // FIXME does the case above also need this?
9220 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9221 assert(NVT.bitsGE(VT));
9222
9223 bool isDereferenceable =
9224 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9225 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9226 if (isDereferenceable)
9227 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9228 if (isConstant)
9229 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9230
9231 Value = DAG.getExtLoad(
9232 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9233 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9234 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9235 Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9236 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9237
9238 Store = DAG.getTruncStore(
9239 Chain, dl, Val: Value,
9240 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9241 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo);
9242 OutStoreChains.push_back(Elt: Store);
9243 }
9244 SrcOff += VTSize;
9245 DstOff += VTSize;
9246 Size -= VTSize;
9247 }
9248
9249 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9250 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9251 unsigned NumLdStInMemcpy = OutStoreChains.size();
9252
9253 if (NumLdStInMemcpy) {
9254 // It may be that memcpy might be converted to memset if it's memcpy
9255 // of constants. In such a case, we won't have loads and stores, but
9256 // just stores. In the absence of loads, there is nothing to gang up.
9257 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9258 // If target does not care, just leave as it.
9259 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9260 OutChains.push_back(Elt: OutLoadChains[i]);
9261 OutChains.push_back(Elt: OutStoreChains[i]);
9262 }
9263 } else {
9264 // Ld/St less than/equal limit set by target.
9265 if (NumLdStInMemcpy <= GluedLdStLimit) {
9266 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9267 To: NumLdStInMemcpy, OutLoadChains,
9268 OutStoreChains);
9269 } else {
9270 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9271 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9272 unsigned GlueIter = 0;
9273
9274 // Residual ld/st.
9275 if (RemainingLdStInMemcpy) {
9276 chainLoadsAndStoresForMemcpy(
9277 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9278 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9279 }
9280
9281 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9282 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9283 GlueIter - GluedLdStLimit;
9284 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9285 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9286 OutLoadChains, OutStoreChains);
9287 GlueIter += GluedLdStLimit;
9288 }
9289 }
9290 }
9291 }
9292 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9293}
9294
9295static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
9296 SDValue Chain, SDValue Dst, SDValue Src,
9297 uint64_t Size, Align Alignment,
9298 bool isVol, bool AlwaysInline,
9299 MachinePointerInfo DstPtrInfo,
9300 MachinePointerInfo SrcPtrInfo,
9301 const AAMDNodes &AAInfo) {
9302 // Turn a memmove of undef to nop.
9303 // FIXME: We need to honor volatile even is Src is undef.
9304 if (Src.isUndef())
9305 return Chain;
9306
9307 // Expand memmove to a series of load and store ops if the size operand falls
9308 // below a certain threshold.
9309 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9310 const DataLayout &DL = DAG.getDataLayout();
9311 LLVMContext &C = *DAG.getContext();
9312 std::vector<EVT> MemOps;
9313 bool DstAlignCanChange = false;
9314 MachineFunction &MF = DAG.getMachineFunction();
9315 MachineFrameInfo &MFI = MF.getFrameInfo();
9316 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9317 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9318 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9319 DstAlignCanChange = true;
9320 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
9321 if (!SrcAlign || Alignment > *SrcAlign)
9322 SrcAlign = Alignment;
9323 assert(SrcAlign && "SrcAlign must be set");
9324 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9325 if (!TLI.findOptimalMemOpLowering(
9326 Context&: C, MemOps, Limit,
9327 Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign, IsVolatile: isVol),
9328 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9329 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9330 return SDValue();
9331
9332 if (DstAlignCanChange) {
9333 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9334 Align NewAlign = DL.getABITypeAlign(Ty);
9335
9336 // Don't promote to an alignment that would require dynamic stack
9337 // realignment which may conflict with optimizations such as tail call
9338 // optimization.
9339 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9340 if (!TRI->hasStackRealignment(MF))
9341 if (MaybeAlign StackAlign = DL.getStackAlignment())
9342 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9343
9344 if (NewAlign > Alignment) {
9345 // Give the stack frame object a larger alignment if needed.
9346 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9347 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9348 Alignment = NewAlign;
9349 }
9350 }
9351
9352 // Prepare AAInfo for loads/stores after lowering this memmove.
9353 AAMDNodes NewAAInfo = AAInfo;
9354 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9355
9356 MachineMemOperand::Flags MMOFlags =
9357 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9358 uint64_t SrcOff = 0;
9359 SmallVector<SDValue, 8> LoadValues;
9360 SmallVector<SDValue, 8> LoadChains;
9361 SmallVector<SDValue, 8> OutChains;
9362 unsigned NumMemOps = MemOps.size();
9363 for (unsigned i = 0; i < NumMemOps; i++) {
9364 EVT VT = MemOps[i];
9365 unsigned VTSize = VT.getSizeInBits() / 8;
9366 SDValue Value;
9367 bool IsOverlapping = false;
9368
9369 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
9370 // Issuing an unaligned load / store pair that overlaps with the previous
9371 // pair. Adjust the offset accordingly.
9372 SrcOff = Size - VTSize;
9373 IsOverlapping = true;
9374 }
9375
9376 // Calculate the actual alignment at the current offset. The alignment at
9377 // SrcOff may be lower than the base alignment, especially when using
9378 // overlapping loads.
9379 Align SrcAlignAtOffset = commonAlignment(A: *SrcAlign, Offset: SrcOff);
9380 if (IsOverlapping) {
9381 // Verify that the target allows misaligned memory accesses at the
9382 // adjusted offset when using overlapping loads.
9383 unsigned Fast;
9384 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcPtrInfo.getAddrSpace(),
9385 Alignment: SrcAlignAtOffset, Flags: MMOFlags,
9386 &Fast) ||
9387 !Fast) {
9388 // This should have been caught by findOptimalMemOpLowering, but verify
9389 // here for safety.
9390 return SDValue();
9391 }
9392 }
9393
9394 bool isDereferenceable =
9395 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9396 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9397 if (isDereferenceable)
9398 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9399 Value =
9400 DAG.getLoad(VT, dl, Chain,
9401 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9402 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: SrcAlignAtOffset,
9403 MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9404 LoadValues.push_back(Elt: Value);
9405 LoadChains.push_back(Elt: Value.getValue(R: 1));
9406 SrcOff += VTSize;
9407 }
9408 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
9409 OutChains.clear();
9410 uint64_t DstOff = 0;
9411 for (unsigned i = 0; i < NumMemOps; i++) {
9412 EVT VT = MemOps[i];
9413 unsigned VTSize = VT.getSizeInBits() / 8;
9414 SDValue Store;
9415 bool IsOverlapping = false;
9416
9417 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
9418 // Issuing an unaligned load / store pair that overlaps with the previous
9419 // pair. Adjust the offset accordingly.
9420 DstOff = Size - VTSize;
9421 IsOverlapping = true;
9422 }
9423
9424 // Calculate the actual alignment at the current offset. The alignment at
9425 // DstOff may be lower than the base alignment, especially when using
9426 // overlapping stores.
9427 Align DstAlignAtOffset = commonAlignment(A: Alignment, Offset: DstOff);
9428 if (IsOverlapping) {
9429 // Verify that the target allows misaligned memory accesses at the
9430 // adjusted offset when using overlapping stores.
9431 unsigned Fast;
9432 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: DstPtrInfo.getAddrSpace(),
9433 Alignment: DstAlignAtOffset, Flags: MMOFlags,
9434 &Fast) ||
9435 !Fast) {
9436 // This should have been caught by findOptimalMemOpLowering, but verify
9437 // here for safety.
9438 return SDValue();
9439 }
9440 }
9441 Store = DAG.getStore(
9442 Chain, dl, Val: LoadValues[i],
9443 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9444 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlignAtOffset, MMOFlags,
9445 AAInfo: NewAAInfo);
9446 OutChains.push_back(Elt: Store);
9447 DstOff += VTSize;
9448 }
9449
9450 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9451}
9452
9453/// Lower the call to 'memset' intrinsic function into a series of store
9454/// operations.
9455///
9456/// \param DAG Selection DAG where lowered code is placed.
9457/// \param dl Link to corresponding IR location.
9458/// \param Chain Control flow dependency.
9459/// \param Dst Pointer to destination memory location.
9460/// \param Src Value of byte to write into the memory.
9461/// \param Size Number of bytes to write.
9462/// \param Alignment Alignment of the destination in bytes.
9463/// \param isVol True if destination is volatile.
9464/// \param AlwaysInline Makes sure no function call is generated.
9465/// \param DstPtrInfo IR information on the memory pointer.
9466/// \returns New head in the control flow, if lowering was successful, empty
9467/// SDValue otherwise.
9468///
9469/// The function tries to replace 'llvm.memset' intrinsic with several store
9470/// operations and value calculation code. This is usually profitable for small
9471/// memory size or when the semantic requires inlining.
9472static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9473 SDValue Chain, SDValue Dst, SDValue Src,
9474 uint64_t Size, Align Alignment, bool isVol,
9475 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9476 const AAMDNodes &AAInfo) {
9477 // Turn a memset of undef to nop.
9478 // FIXME: We need to honor volatile even is Src is undef.
9479 if (Src.isUndef())
9480 return Chain;
9481
9482 // Expand memset to a series of load/store ops if the size operand
9483 // falls below a certain threshold.
9484 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9485 std::vector<EVT> MemOps;
9486 bool DstAlignCanChange = false;
9487 LLVMContext &C = *DAG.getContext();
9488 MachineFunction &MF = DAG.getMachineFunction();
9489 MachineFrameInfo &MFI = MF.getFrameInfo();
9490 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9491 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9492 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9493 DstAlignCanChange = true;
9494 bool IsZeroVal = isNullConstant(V: Src);
9495 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9496
9497 EVT LargestVT;
9498 if (!TLI.findOptimalMemOpLowering(
9499 Context&: C, MemOps, Limit,
9500 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9501 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9502 LargestVT: &LargestVT))
9503 return SDValue();
9504
9505 if (DstAlignCanChange) {
9506 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9507 const DataLayout &DL = DAG.getDataLayout();
9508 Align NewAlign = DL.getABITypeAlign(Ty);
9509
9510 // Don't promote to an alignment that would require dynamic stack
9511 // realignment which may conflict with optimizations such as tail call
9512 // optimization.
9513 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9514 if (!TRI->hasStackRealignment(MF))
9515 if (MaybeAlign StackAlign = DL.getStackAlignment())
9516 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9517
9518 if (NewAlign > Alignment) {
9519 // Give the stack frame object a larger alignment if needed.
9520 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9521 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9522 Alignment = NewAlign;
9523 }
9524 }
9525
9526 SmallVector<SDValue, 8> OutChains;
9527 uint64_t DstOff = 0;
9528 unsigned NumMemOps = MemOps.size();
9529
9530 // Find the largest store and generate the bit pattern for it.
9531 // If target didn't set LargestVT, compute it from MemOps.
9532 if (!LargestVT.isSimple()) {
9533 LargestVT = MemOps[0];
9534 for (unsigned i = 1; i < NumMemOps; i++)
9535 if (MemOps[i].bitsGT(VT: LargestVT))
9536 LargestVT = MemOps[i];
9537 }
9538 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9539
9540 // Prepare AAInfo for loads/stores after lowering this memset.
9541 AAMDNodes NewAAInfo = AAInfo;
9542 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9543
9544 for (unsigned i = 0; i < NumMemOps; i++) {
9545 EVT VT = MemOps[i];
9546 unsigned VTSize = VT.getSizeInBits() / 8;
9547 // The target should specify store types that exactly cover the memset size
9548 // (with the last store potentially being oversized for overlapping stores).
9549 assert(Size > 0 && "Target specified more stores than needed in "
9550 "findOptimalMemOpLowering");
9551 if (VTSize > Size) {
9552 // Issuing an unaligned load / store pair that overlaps with the previous
9553 // pair. Adjust the offset accordingly.
9554 assert(i == NumMemOps-1 && i != 0);
9555 DstOff -= VTSize - Size;
9556 }
9557
9558 // If this store is smaller than the largest store see whether we can get
9559 // the smaller value for free with a truncate or extract vector element and
9560 // then store.
9561 SDValue Value = MemSetValue;
9562 if (VT.bitsLT(VT: LargestVT)) {
9563 unsigned Index;
9564 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9565 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
9566 if (!LargestVT.isVector() && !VT.isVector() &&
9567 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
9568 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
9569 else if (LargestVT.isVector() && !VT.isVector() &&
9570 TLI.shallExtractConstSplatVectorElementToStore(
9571 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
9572 ElemSizeInBits: VT.getSizeInBits(), Index) &&
9573 TLI.isTypeLegal(VT: SVT) &&
9574 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
9575 // Target which can combine store(extractelement VectorTy, Idx) can get
9576 // the smaller value for free.
9577 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
9578 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
9579 } else
9580 Value = getMemsetValue(Value: Src, VT, DAG, dl);
9581 }
9582 assert(Value.getValueType() == VT && "Value with wrong type.");
9583 SDValue Store = DAG.getStore(
9584 Chain, dl, Val: Value,
9585 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9586 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
9587 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
9588 AAInfo: NewAAInfo);
9589 OutChains.push_back(Elt: Store);
9590 DstOff += VT.getSizeInBits() / 8;
9591 // For oversized overlapping stores, only subtract the remaining bytes.
9592 // For normal stores, subtract the full store size.
9593 if (VTSize > Size) {
9594 Size = 0;
9595 } else {
9596 Size -= VTSize;
9597 }
9598 }
9599
9600 // After processing all stores, Size should be exactly 0. Any remaining bytes
9601 // indicate a bug in the target's findOptimalMemOpLowering implementation.
9602 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
9603 "stores that exactly cover the memset size");
9604
9605 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9606}
9607
9608static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
9609 unsigned AS) {
9610 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
9611 // pointer operands can be losslessly bitcasted to pointers of address space 0
9612 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
9613 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
9614 Twine(AS));
9615 }
9616}
9617
9618static bool isInTailCallPositionWrapper(const CallInst *CI,
9619 const SelectionDAG *SelDAG,
9620 bool AllowReturnsFirstArg) {
9621 if (!CI || !CI->isTailCall())
9622 return false;
9623 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
9624 // helper symbol we lower to.
9625 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
9626 ReturnsFirstArg: AllowReturnsFirstArg &&
9627 funcReturnsFirstArgOfCall(CI: *CI));
9628}
9629
9630static std::pair<SDValue, SDValue>
9631getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
9632 TargetLowering::ArgListTy &&Args,
9633 const CallInst *CI, RTLIB::Libcall Call,
9634 SelectionDAG *DAG, const TargetLowering *TLI) {
9635 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
9636
9637 if (LCImpl == RTLIB::Unsupported)
9638 return {};
9639
9640 TargetLowering::CallLoweringInfo CLI(*DAG);
9641 bool IsTailCall =
9642 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true);
9643 SDValue Callee =
9644 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
9645
9646 CLI.setDebugLoc(dl)
9647 .setChain(Chain)
9648 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9649 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
9650 .setTailCall(IsTailCall);
9651
9652 return TLI->LowerCallTo(CLI);
9653}
9654
9655std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
9656 const SDLoc &dl, SDValue S1,
9657 SDValue S2,
9658 const CallInst *CI) {
9659 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9660 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9661 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9662 Call: RTLIB::STRCMP, DAG: this, TLI);
9663}
9664
9665std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
9666 const SDLoc &dl, SDValue S1,
9667 SDValue S2,
9668 const CallInst *CI) {
9669 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9670 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9671 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9672 Call: RTLIB::STRSTR, DAG: this, TLI);
9673}
9674
9675std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
9676 const SDLoc &dl,
9677 SDValue Dst, SDValue Src,
9678 SDValue C, SDValue Size,
9679 const CallInst *CI) {
9680 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9681
9682 TargetLowering::ArgListTy Args = {
9683 {Dst, PT},
9684 {Src, PT},
9685 {C, Type::getInt32Ty(C&: *getContext())},
9686 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9687 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9688 Call: RTLIB::MEMCCPY, DAG: this, TLI);
9689}
9690
9691std::pair<SDValue, SDValue>
9692SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
9693 SDValue Mem1, SDValue Size, const CallInst *CI) {
9694 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9695 TargetLowering::ArgListTy Args = {
9696 {Mem0, PT},
9697 {Mem1, PT},
9698 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9699 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9700 Call: RTLIB::MEMCMP, DAG: this, TLI);
9701}
9702
9703std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
9704 const SDLoc &dl,
9705 SDValue Dst, SDValue Src,
9706 const CallInst *CI) {
9707 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9708 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
9709 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9710 Call: RTLIB::STRCPY, DAG: this, TLI);
9711}
9712
9713std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
9714 const SDLoc &dl,
9715 SDValue Src,
9716 const CallInst *CI) {
9717 // Emit a library call.
9718 TargetLowering::ArgListTy Args = {
9719 {Src, PointerType::getUnqual(C&: *getContext())}};
9720 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9721 Call: RTLIB::STRLEN, DAG: this, TLI);
9722}
9723
9724SDValue SelectionDAG::getMemcpy(
9725 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
9726 Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
9727 std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
9728 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9729 BatchAAResults *BatchAA) {
9730 // Check to see if we should lower the memcpy to loads and stores first.
9731 // For cases within the target-specified limits, this is the best choice.
9732 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9733 if (ConstantSize) {
9734 // Memcpy with size zero? Just return the original chain.
9735 if (ConstantSize->isZero())
9736 return Chain;
9737
9738 SDValue Result = getMemcpyLoadsAndStores(
9739 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9740 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9741 if (Result.getNode())
9742 return Result;
9743 }
9744
9745 // Then check to see if we should lower the memcpy with target-specific
9746 // code. If the target chooses to do this, this is the next best.
9747 if (TSI) {
9748 SDValue Result = TSI->EmitTargetCodeForMemcpy(
9749 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
9750 DstPtrInfo, SrcPtrInfo);
9751 if (Result.getNode())
9752 return Result;
9753 }
9754
9755 // If we really need inline code and the target declined to provide it,
9756 // use a (potentially long) sequence of loads and stores.
9757 if (AlwaysInline) {
9758 assert(ConstantSize && "AlwaysInline requires a constant size!");
9759 return getMemcpyLoadsAndStores(
9760 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9761 isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9762 }
9763
9764 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9765 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9766
9767 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
9768 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
9769 // respect volatile, so they may do things like read or write memory
9770 // beyond the given memory regions. But fixing this isn't easy, and most
9771 // people don't care.
9772
9773 // Emit a library call.
9774 TargetLowering::ArgListTy Args;
9775 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9776 Args.emplace_back(args&: Dst, args&: PtrTy);
9777 Args.emplace_back(args&: Src, args&: PtrTy);
9778 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9779 // FIXME: pass in SDLoc
9780 TargetLowering::CallLoweringInfo CLI(*this);
9781 bool IsTailCall = false;
9782 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
9783
9784 if (OverrideTailCall.has_value()) {
9785 IsTailCall = *OverrideTailCall;
9786 } else {
9787 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
9788 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
9789 }
9790
9791 CLI.setDebugLoc(dl)
9792 .setChain(Chain)
9793 .setLibCallee(
9794 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
9795 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9796 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9797 ArgsList: std::move(Args))
9798 .setDiscardResult()
9799 .setTailCall(IsTailCall);
9800
9801 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9802 return CallResult.second;
9803}
9804
9805SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
9806 SDValue Dst, SDValue Src, SDValue Size,
9807 Type *SizeTy, unsigned ElemSz,
9808 bool isTailCall,
9809 MachinePointerInfo DstPtrInfo,
9810 MachinePointerInfo SrcPtrInfo) {
9811 // Emit a library call.
9812 TargetLowering::ArgListTy Args;
9813 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
9814 Args.emplace_back(args&: Dst, args&: ArgTy);
9815 Args.emplace_back(args&: Src, args&: ArgTy);
9816 Args.emplace_back(args&: Size, args&: SizeTy);
9817
9818 RTLIB::Libcall LibraryCall =
9819 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9820 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9821 if (LibcallImpl == RTLIB::Unsupported)
9822 report_fatal_error(reason: "Unsupported element size");
9823
9824 TargetLowering::CallLoweringInfo CLI(*this);
9825 CLI.setDebugLoc(dl)
9826 .setChain(Chain)
9827 .setLibCallee(
9828 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9829 ResultType: Type::getVoidTy(C&: *getContext()),
9830 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9831 ArgsList: std::move(Args))
9832 .setDiscardResult()
9833 .setTailCall(isTailCall);
9834
9835 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9836 return CallResult.second;
9837}
9838
9839SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
9840 SDValue Src, SDValue Size, Align Alignment,
9841 bool isVol, const CallInst *CI,
9842 std::optional<bool> OverrideTailCall,
9843 MachinePointerInfo DstPtrInfo,
9844 MachinePointerInfo SrcPtrInfo,
9845 const AAMDNodes &AAInfo,
9846 BatchAAResults *BatchAA) {
9847 // Check to see if we should lower the memmove to loads and stores first.
9848 // For cases within the target-specified limits, this is the best choice.
9849 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9850 if (ConstantSize) {
9851 // Memmove with size zero? Just return the original chain.
9852 if (ConstantSize->isZero())
9853 return Chain;
9854
9855 SDValue Result = getMemmoveLoadsAndStores(
9856 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9857 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
9858 if (Result.getNode())
9859 return Result;
9860 }
9861
9862 // Then check to see if we should lower the memmove with target-specific
9863 // code. If the target chooses to do this, this is the next best.
9864 if (TSI) {
9865 SDValue Result =
9866 TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size,
9867 Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo);
9868 if (Result.getNode())
9869 return Result;
9870 }
9871
9872 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9873 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9874
9875 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
9876 // not be safe. See memcpy above for more details.
9877
9878 // Emit a library call.
9879 TargetLowering::ArgListTy Args;
9880 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9881 Args.emplace_back(args&: Dst, args&: PtrTy);
9882 Args.emplace_back(args&: Src, args&: PtrTy);
9883 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9884 // FIXME: pass in SDLoc
9885 TargetLowering::CallLoweringInfo CLI(*this);
9886
9887 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
9888
9889 bool IsTailCall = false;
9890 if (OverrideTailCall.has_value()) {
9891 IsTailCall = *OverrideTailCall;
9892 } else {
9893 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
9894 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
9895 }
9896
9897 CLI.setDebugLoc(dl)
9898 .setChain(Chain)
9899 .setLibCallee(
9900 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
9901 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9902 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9903 ArgsList: std::move(Args))
9904 .setDiscardResult()
9905 .setTailCall(IsTailCall);
9906
9907 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9908 return CallResult.second;
9909}
9910
9911SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
9912 SDValue Dst, SDValue Src, SDValue Size,
9913 Type *SizeTy, unsigned ElemSz,
9914 bool isTailCall,
9915 MachinePointerInfo DstPtrInfo,
9916 MachinePointerInfo SrcPtrInfo) {
9917 // Emit a library call.
9918 TargetLowering::ArgListTy Args;
9919 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
9920 Args.emplace_back(args&: Dst, args&: IntPtrTy);
9921 Args.emplace_back(args&: Src, args&: IntPtrTy);
9922 Args.emplace_back(args&: Size, args&: SizeTy);
9923
9924 RTLIB::Libcall LibraryCall =
9925 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9926 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9927 if (LibcallImpl == RTLIB::Unsupported)
9928 report_fatal_error(reason: "Unsupported element size");
9929
9930 TargetLowering::CallLoweringInfo CLI(*this);
9931 CLI.setDebugLoc(dl)
9932 .setChain(Chain)
9933 .setLibCallee(
9934 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9935 ResultType: Type::getVoidTy(C&: *getContext()),
9936 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9937 ArgsList: std::move(Args))
9938 .setDiscardResult()
9939 .setTailCall(isTailCall);
9940
9941 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9942 return CallResult.second;
9943}
9944
9945SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
9946 SDValue Src, SDValue Size, Align Alignment,
9947 bool isVol, bool AlwaysInline,
9948 const CallInst *CI,
9949 MachinePointerInfo DstPtrInfo,
9950 const AAMDNodes &AAInfo) {
9951 // Check to see if we should lower the memset to stores first.
9952 // For cases within the target-specified limits, this is the best choice.
9953 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9954 if (ConstantSize) {
9955 // Memset with size zero? Just return the original chain.
9956 if (ConstantSize->isZero())
9957 return Chain;
9958
9959 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9960 Size: ConstantSize->getZExtValue(), Alignment,
9961 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
9962
9963 if (Result.getNode())
9964 return Result;
9965 }
9966
9967 // Then check to see if we should lower the memset with target-specific
9968 // code. If the target chooses to do this, this is the next best.
9969 if (TSI) {
9970 SDValue Result = TSI->EmitTargetCodeForMemset(
9971 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
9972 if (Result.getNode())
9973 return Result;
9974 }
9975
9976 // If we really need inline code and the target declined to provide it,
9977 // use a (potentially long) sequence of loads and stores.
9978 if (AlwaysInline) {
9979 assert(ConstantSize && "AlwaysInline requires a constant size!");
9980 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9981 Size: ConstantSize->getZExtValue(), Alignment,
9982 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
9983 assert(Result &&
9984 "getMemsetStores must return a valid sequence when AlwaysInline");
9985 return Result;
9986 }
9987
9988 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9989
9990 // Emit a library call.
9991 auto &Ctx = *getContext();
9992 const auto& DL = getDataLayout();
9993
9994 TargetLowering::CallLoweringInfo CLI(*this);
9995 // FIXME: pass in SDLoc
9996 CLI.setDebugLoc(dl).setChain(Chain);
9997
9998 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
9999 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
10000
10001 // If zeroing out and bzero is present, use it.
10002 if (UseBZero) {
10003 TargetLowering::ArgListTy Args;
10004 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10005 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10006 CLI.setLibCallee(
10007 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
10008 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
10009 } else {
10010 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10011
10012 TargetLowering::ArgListTy Args;
10013 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10014 Args.emplace_back(args&: Src, args: Src.getValueType().getTypeForEVT(Context&: Ctx));
10015 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10016 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
10017 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
10018 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
10019 ArgsList: std::move(Args));
10020 }
10021
10022 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10023 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10024
10025 // If we're going to use bzero, make sure not to tail call unless the
10026 // subsequent return doesn't need a value, as bzero doesn't return the first
10027 // arg unlike memset.
10028 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
10029 bool IsTailCall =
10030 CI && CI->isTailCall() &&
10031 isInTailCallPosition(Call: *CI, TM: getTarget(), ReturnsFirstArg: ReturnsFirstArg && LowersToMemset);
10032 CLI.setDiscardResult().setTailCall(IsTailCall);
10033
10034 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10035 return CallResult.second;
10036}
10037
10038SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
10039 SDValue Dst, SDValue Value, SDValue Size,
10040 Type *SizeTy, unsigned ElemSz,
10041 bool isTailCall,
10042 MachinePointerInfo DstPtrInfo) {
10043 // Emit a library call.
10044 TargetLowering::ArgListTy Args;
10045 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
10046 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
10047 Args.emplace_back(args&: Size, args&: SizeTy);
10048
10049 RTLIB::Libcall LibraryCall =
10050 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10051 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10052 if (LibcallImpl == RTLIB::Unsupported)
10053 report_fatal_error(reason: "Unsupported element size");
10054
10055 TargetLowering::CallLoweringInfo CLI(*this);
10056 CLI.setDebugLoc(dl)
10057 .setChain(Chain)
10058 .setLibCallee(
10059 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10060 ResultType: Type::getVoidTy(C&: *getContext()),
10061 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10062 ArgsList: std::move(Args))
10063 .setDiscardResult()
10064 .setTailCall(isTailCall);
10065
10066 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10067 return CallResult.second;
10068}
10069
10070SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10071 SDVTList VTList, ArrayRef<SDValue> Ops,
10072 MachineMemOperand *MMO,
10073 ISD::LoadExtType ExtType) {
10074 FoldingSetNodeID ID;
10075 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10076 ID.AddInteger(I: MemVT.getRawBits());
10077 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
10078 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
10079 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10080 ID.AddInteger(I: MMO->getFlags());
10081 void* IP = nullptr;
10082 if (auto *E = cast_or_null<AtomicSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10083 E->refineAlignment(NewMMO: MMO);
10084 E->refineRanges(NewMMO: MMO);
10085 return SDValue(E, 0);
10086 }
10087
10088 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
10089 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
10090 createOperands(Node: N, Vals: Ops);
10091
10092 CSEMap.InsertNode(N, InsertPos: IP);
10093 InsertNode(N);
10094 SDValue V(N, 0);
10095 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10096 return V;
10097}
10098
10099SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
10100 EVT MemVT, SDVTList VTs, SDValue Chain,
10101 SDValue Ptr, SDValue Cmp, SDValue Swp,
10102 MachineMemOperand *MMO) {
10103 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10104 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
10105 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10106
10107 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10108 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10109}
10110
10111SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10112 SDValue Chain, SDValue Ptr, SDValue Val,
10113 MachineMemOperand *MMO) {
10114 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10115 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10116 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10117 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10118 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10119 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10120 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10121 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10122 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10123 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10124 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10125 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10126 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10127 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10128 Opcode == ISD::ATOMIC_STORE) &&
10129 "Invalid Atomic Op");
10130
10131 EVT VT = Val.getValueType();
10132
10133 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
10134 getVTList(VT1: VT, VT2: MVT::Other);
10135 SDValue Ops[] = {Chain, Ptr, Val};
10136 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10137}
10138
10139SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10140 EVT MemVT, EVT VT, SDValue Chain,
10141 SDValue Ptr, MachineMemOperand *MMO) {
10142 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
10143 SDValue Ops[] = {Chain, Ptr};
10144 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
10145}
10146
10147/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10148SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
10149 if (Ops.size() == 1)
10150 return Ops[0];
10151
10152 SmallVector<EVT, 4> VTs;
10153 VTs.reserve(N: Ops.size());
10154 for (const SDValue &Op : Ops)
10155 VTs.push_back(Elt: Op.getValueType());
10156 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
10157}
10158
10159SDValue SelectionDAG::getMemIntrinsicNode(
10160 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10161 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10162 MachineMemOperand::Flags Flags, LocationSize Size,
10163 const AAMDNodes &AAInfo) {
10164 if (Size.hasValue() && !Size.getValue())
10165 Size = LocationSize::precise(Value: MemVT.getStoreSize());
10166
10167 MachineFunction &MF = getMachineFunction();
10168 MachineMemOperand *MMO =
10169 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, AAInfo);
10170
10171 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10172}
10173
10174SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10175 SDVTList VTList,
10176 ArrayRef<SDValue> Ops, EVT MemVT,
10177 MachineMemOperand *MMO) {
10178 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
10179}
10180
10181SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10182 SDVTList VTList,
10183 ArrayRef<SDValue> Ops, EVT MemVT,
10184 ArrayRef<MachineMemOperand *> MMOs) {
10185 assert(!MMOs.empty() && "Must have at least one MMO");
10186 assert(
10187 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10188 Opcode == ISD::PREFETCH ||
10189 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10190 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10191 "Opcode is not a memory-accessing opcode!");
10192
10193 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
10194 if (MMOs.size() == 1) {
10195 MemRefs = MMOs[0];
10196 } else {
10197 // Allocate: [size_t count][MMO*][MMO*]...
10198 size_t AllocSize =
10199 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10200 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
10201 size_t *CountPtr = static_cast<size_t *>(Buffer);
10202 *CountPtr = MMOs.size();
10203 MachineMemOperand **Array =
10204 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10205 llvm::copy(Range&: MMOs, Out: Array);
10206 MemRefs = Array;
10207 }
10208
10209 // Memoize the node unless it returns a glue result.
10210 MemIntrinsicSDNode *N;
10211 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10212 FoldingSetNodeID ID;
10213 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10214 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10215 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10216 ID.AddInteger(I: MemVT.getRawBits());
10217 for (const MachineMemOperand *MMO : MMOs) {
10218 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10219 ID.AddInteger(I: MMO->getFlags());
10220 }
10221 void *IP = nullptr;
10222 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10223 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10224 return SDValue(E, 0);
10225 }
10226
10227 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10228 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10229 createOperands(Node: N, Vals: Ops);
10230 CSEMap.InsertNode(N, InsertPos: IP);
10231 } else {
10232 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10233 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10234 createOperands(Node: N, Vals: Ops);
10235 }
10236 InsertNode(N);
10237 SDValue V(N, 0);
10238 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10239 return V;
10240}
10241
10242SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
10243 SDValue Chain, int FrameIndex) {
10244 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
10245 const auto VTs = getVTList(VT: MVT::Other);
10246 SDValue Ops[2] = {
10247 Chain,
10248 getFrameIndex(FI: FrameIndex,
10249 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10250 isTarget: true)};
10251
10252 FoldingSetNodeID ID;
10253 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10254 ID.AddInteger(I: FrameIndex);
10255 void *IP = nullptr;
10256 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10257 return SDValue(E, 0);
10258
10259 LifetimeSDNode *N =
10260 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10261 createOperands(Node: N, Vals: Ops);
10262 CSEMap.InsertNode(N, InsertPos: IP);
10263 InsertNode(N);
10264 SDValue V(N, 0);
10265 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10266 return V;
10267}
10268
10269SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10270 uint64_t Guid, uint64_t Index,
10271 uint32_t Attr) {
10272 const unsigned Opcode = ISD::PSEUDO_PROBE;
10273 const auto VTs = getVTList(VT: MVT::Other);
10274 SDValue Ops[] = {Chain};
10275 FoldingSetNodeID ID;
10276 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10277 ID.AddInteger(I: Guid);
10278 ID.AddInteger(I: Index);
10279 void *IP = nullptr;
10280 if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
10281 return SDValue(E, 0);
10282
10283 auto *N = newSDNode<PseudoProbeSDNode>(
10284 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10285 createOperands(Node: N, Vals: Ops);
10286 CSEMap.InsertNode(N, InsertPos: IP);
10287 InsertNode(N);
10288 SDValue V(N, 0);
10289 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10290 return V;
10291}
10292
10293/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10294/// MachinePointerInfo record from it. This is particularly useful because the
10295/// code generator has many cases where it doesn't bother passing in a
10296/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10297static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10298 SelectionDAG &DAG, SDValue Ptr,
10299 int64_t Offset = 0) {
10300 // If this is FI+Offset, we can model it.
10301 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10302 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10303 FI: FI->getIndex(), Offset);
10304
10305 // If this is (FI+Offset1)+Offset2, we can model it.
10306 if (Ptr.getOpcode() != ISD::ADD ||
10307 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10308 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10309 return Info;
10310
10311 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10312 return MachinePointerInfo::getFixedStack(
10313 MF&: DAG.getMachineFunction(), FI,
10314 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10315}
10316
10317/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10318/// MachinePointerInfo record from it. This is particularly useful because the
10319/// code generator has many cases where it doesn't bother passing in a
10320/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10321static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10322 SelectionDAG &DAG, SDValue Ptr,
10323 SDValue OffsetOp) {
10324 // If the 'Offset' value isn't a constant, we can't handle this.
10325 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10326 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10327 if (OffsetOp.isUndef())
10328 return InferPointerInfo(Info, DAG, Ptr);
10329 return Info;
10330}
10331
10332SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10333 EVT VT, const SDLoc &dl, SDValue Chain,
10334 SDValue Ptr, SDValue Offset,
10335 MachinePointerInfo PtrInfo, EVT MemVT,
10336 Align Alignment,
10337 MachineMemOperand::Flags MMOFlags,
10338 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10339 assert(Chain.getValueType() == MVT::Other &&
10340 "Invalid chain type");
10341
10342 MMOFlags |= MachineMemOperand::MOLoad;
10343 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10344 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10345 // clients.
10346 if (PtrInfo.V.isNull())
10347 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10348
10349 TypeSize Size = MemVT.getStoreSize();
10350 MachineFunction &MF = getMachineFunction();
10351 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10352 BaseAlignment: Alignment, AAInfo, Ranges);
10353 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10354}
10355
10356SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10357 EVT VT, const SDLoc &dl, SDValue Chain,
10358 SDValue Ptr, SDValue Offset, EVT MemVT,
10359 MachineMemOperand *MMO) {
10360 if (VT == MemVT) {
10361 ExtType = ISD::NON_EXTLOAD;
10362 } else if (ExtType == ISD::NON_EXTLOAD) {
10363 assert(VT == MemVT && "Non-extending load from different memory type!");
10364 } else {
10365 // Extending load.
10366 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10367 "Should only be an extending load, not truncating!");
10368 assert(VT.isInteger() == MemVT.isInteger() &&
10369 "Cannot convert from FP to Int or Int -> FP!");
10370 assert(VT.isVector() == MemVT.isVector() &&
10371 "Cannot use an ext load to convert to or from a vector!");
10372 assert((!VT.isVector() ||
10373 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10374 "Cannot use an ext load to change the number of vector elements!");
10375 }
10376
10377 assert((!MMO->getRanges() ||
10378 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10379 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10380 MemVT.isInteger())) &&
10381 "Range metadata and load type must match!");
10382
10383 bool Indexed = AM != ISD::UNINDEXED;
10384 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10385
10386 SDVTList VTs = Indexed ?
10387 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10388 SDValue Ops[] = { Chain, Ptr, Offset };
10389 FoldingSetNodeID ID;
10390 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
10391 ID.AddInteger(I: MemVT.getRawBits());
10392 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10393 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10394 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10395 ID.AddInteger(I: MMO->getFlags());
10396 void *IP = nullptr;
10397 if (auto *E = cast_or_null<LoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10398 E->refineAlignment(NewMMO: MMO);
10399 E->refineRanges(NewMMO: MMO);
10400 return SDValue(E, 0);
10401 }
10402 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10403 Args&: ExtType, Args&: MemVT, Args&: MMO);
10404 createOperands(Node: N, Vals: Ops);
10405
10406 CSEMap.InsertNode(N, InsertPos: IP);
10407 InsertNode(N);
10408 SDValue V(N, 0);
10409 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10410 return V;
10411}
10412
10413SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10414 SDValue Ptr, MachinePointerInfo PtrInfo,
10415 MaybeAlign Alignment,
10416 MachineMemOperand::Flags MMOFlags,
10417 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10418 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10419 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10420 PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
10421}
10422
10423SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10424 SDValue Ptr, MachineMemOperand *MMO) {
10425 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10426 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10427 MemVT: VT, MMO);
10428}
10429
10430SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10431 EVT VT, SDValue Chain, SDValue Ptr,
10432 MachinePointerInfo PtrInfo, EVT MemVT,
10433 MaybeAlign Alignment,
10434 MachineMemOperand::Flags MMOFlags,
10435 const AAMDNodes &AAInfo) {
10436 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10437 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10438 MemVT, Alignment, MMOFlags, AAInfo);
10439}
10440
10441SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10442 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10443 MachineMemOperand *MMO) {
10444 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10445 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10446 MemVT, MMO);
10447}
10448
10449SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10450 SDValue Base, SDValue Offset,
10451 ISD::MemIndexedMode AM) {
10452 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10453 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10454 // Don't propagate the invariant or dereferenceable flags.
10455 auto MMOFlags =
10456 LD->getMemOperand()->getFlags() &
10457 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10458 return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10459 Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
10460 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
10461}
10462
10463SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10464 SDValue Ptr, MachinePointerInfo PtrInfo,
10465 Align Alignment,
10466 MachineMemOperand::Flags MMOFlags,
10467 const AAMDNodes &AAInfo) {
10468 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10469
10470 MMOFlags |= MachineMemOperand::MOStore;
10471 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10472
10473 if (PtrInfo.V.isNull())
10474 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10475
10476 MachineFunction &MF = getMachineFunction();
10477 TypeSize Size = Val.getValueType().getStoreSize();
10478 MachineMemOperand *MMO =
10479 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, AAInfo);
10480 return getStore(Chain, dl, Val, Ptr, MMO);
10481}
10482
10483SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10484 SDValue Ptr, MachineMemOperand *MMO) {
10485 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10486 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10487 AM: ISD::UNINDEXED);
10488}
10489
10490SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10491 SDValue Ptr, SDValue Offset, EVT SVT,
10492 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
10493 bool IsTruncating) {
10494 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10495 EVT VT = Val.getValueType();
10496 if (VT == SVT) {
10497 IsTruncating = false;
10498 } else if (!IsTruncating) {
10499 assert(VT == SVT && "No-truncating store from different memory type!");
10500 } else {
10501 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10502 "Should only be a truncating store, not extending!");
10503 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10504 assert(VT.isVector() == SVT.isVector() &&
10505 "Cannot use trunc store to convert to or from a vector!");
10506 assert((!VT.isVector() ||
10507 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10508 "Cannot use trunc store to change the number of vector elements!");
10509 }
10510
10511 bool Indexed = AM != ISD::UNINDEXED;
10512 assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");
10513 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10514 : getVTList(VT: MVT::Other);
10515 SDValue Ops[] = {Chain, Val, Ptr, Offset};
10516 FoldingSetNodeID ID;
10517 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
10518 ID.AddInteger(I: SVT.getRawBits());
10519 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
10520 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
10521 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10522 ID.AddInteger(I: MMO->getFlags());
10523 void *IP = nullptr;
10524 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10525 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10526 return SDValue(E, 0);
10527 }
10528 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10529 Args&: IsTruncating, Args&: SVT, Args&: MMO);
10530 createOperands(Node: N, Vals: Ops);
10531
10532 CSEMap.InsertNode(N, InsertPos: IP);
10533 InsertNode(N);
10534 SDValue V(N, 0);
10535 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10536 return V;
10537}
10538
10539SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10540 SDValue Ptr, MachinePointerInfo PtrInfo,
10541 EVT SVT, Align Alignment,
10542 MachineMemOperand::Flags MMOFlags,
10543 const AAMDNodes &AAInfo) {
10544 assert(Chain.getValueType() == MVT::Other &&
10545 "Invalid chain type");
10546
10547 MMOFlags |= MachineMemOperand::MOStore;
10548 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10549
10550 if (PtrInfo.V.isNull())
10551 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10552
10553 MachineFunction &MF = getMachineFunction();
10554 MachineMemOperand *MMO = MF.getMachineMemOperand(
10555 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10556 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
10557}
10558
10559SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10560 SDValue Ptr, EVT SVT,
10561 MachineMemOperand *MMO) {
10562 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10563 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
10564}
10565
10566SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
10567 SDValue Base, SDValue Offset,
10568 ISD::MemIndexedMode AM) {
10569 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
10570 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
10571 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
10572 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
10573 IsTruncating: ST->isTruncatingStore());
10574}
10575
10576SDValue SelectionDAG::getLoadVP(
10577 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
10578 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
10579 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
10580 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
10581 const MDNode *Ranges, bool IsExpanding) {
10582 MMOFlags |= MachineMemOperand::MOLoad;
10583 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10584 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10585 // clients.
10586 if (PtrInfo.V.isNull())
10587 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10588
10589 TypeSize Size = MemVT.getStoreSize();
10590 MachineFunction &MF = getMachineFunction();
10591 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10592 BaseAlignment: Alignment, AAInfo, Ranges);
10593 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
10594 MMO, IsExpanding);
10595}
10596
10597SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
10598 ISD::LoadExtType ExtType, EVT VT,
10599 const SDLoc &dl, SDValue Chain, SDValue Ptr,
10600 SDValue Offset, SDValue Mask, SDValue EVL,
10601 EVT MemVT, MachineMemOperand *MMO,
10602 bool IsExpanding) {
10603 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10604 assert(Mask.getValueType().getVectorElementCount() ==
10605 VT.getVectorElementCount() &&
10606 "Vector width mismatch between mask and data");
10607
10608 bool Indexed = AM != ISD::UNINDEXED;
10609 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10610
10611 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10612 : getVTList(VT1: VT, VT2: MVT::Other);
10613 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
10614 FoldingSetNodeID ID;
10615 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
10616 ID.AddInteger(I: MemVT.getRawBits());
10617 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
10618 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10619 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10620 ID.AddInteger(I: MMO->getFlags());
10621 void *IP = nullptr;
10622 if (auto *E = cast_or_null<VPLoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10623 E->refineAlignment(NewMMO: MMO);
10624 E->refineRanges(NewMMO: MMO);
10625 return SDValue(E, 0);
10626 }
10627 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10628 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10629 createOperands(Node: N, Vals: Ops);
10630
10631 CSEMap.InsertNode(N, InsertPos: IP);
10632 InsertNode(N);
10633 SDValue V(N, 0);
10634 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10635 return V;
10636}
10637
10638SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10639 SDValue Ptr, SDValue Mask, SDValue EVL,
10640 MachinePointerInfo PtrInfo,
10641 MaybeAlign Alignment,
10642 MachineMemOperand::Flags MMOFlags,
10643 const AAMDNodes &AAInfo, const MDNode *Ranges,
10644 bool IsExpanding) {
10645 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10646 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10647 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
10648 IsExpanding);
10649}
10650
10651SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10652 SDValue Ptr, SDValue Mask, SDValue EVL,
10653 MachineMemOperand *MMO, bool IsExpanding) {
10654 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10655 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10656 Mask, EVL, MemVT: VT, MMO, IsExpanding);
10657}
10658
10659SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10660 EVT VT, SDValue Chain, SDValue Ptr,
10661 SDValue Mask, SDValue EVL,
10662 MachinePointerInfo PtrInfo, EVT MemVT,
10663 MaybeAlign Alignment,
10664 MachineMemOperand::Flags MMOFlags,
10665 const AAMDNodes &AAInfo, bool IsExpanding) {
10666 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10667 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10668 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
10669 IsExpanding);
10670}
10671
10672SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10673 EVT VT, SDValue Chain, SDValue Ptr,
10674 SDValue Mask, SDValue EVL, EVT MemVT,
10675 MachineMemOperand *MMO, bool IsExpanding) {
10676 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10677 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10678 EVL, MemVT, MMO, IsExpanding);
10679}
10680
10681SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
10682 SDValue Base, SDValue Offset,
10683 ISD::MemIndexedMode AM) {
10684 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
10685 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10686 // Don't propagate the invariant or dereferenceable flags.
10687 auto MMOFlags =
10688 LD->getMemOperand()->getFlags() &
10689 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10690 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10691 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
10692 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
10693 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
10694 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
10695}
10696
10697SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
10698 SDValue Ptr, SDValue Offset, SDValue Mask,
10699 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
10700 ISD::MemIndexedMode AM, bool IsTruncating,
10701 bool IsCompressing) {
10702 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10703 assert(Mask.getValueType().getVectorElementCount() ==
10704 Val.getValueType().getVectorElementCount() &&
10705 "Vector width mismatch between mask and data");
10706
10707 bool Indexed = AM != ISD::UNINDEXED;
10708 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10709 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10710 : getVTList(VT: MVT::Other);
10711 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
10712 FoldingSetNodeID ID;
10713 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10714 ID.AddInteger(I: MemVT.getRawBits());
10715 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10716 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10717 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10718 ID.AddInteger(I: MMO->getFlags());
10719 void *IP = nullptr;
10720 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10721 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10722 return SDValue(E, 0);
10723 }
10724 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10725 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10726 createOperands(Node: N, Vals: Ops);
10727
10728 CSEMap.InsertNode(N, InsertPos: IP);
10729 InsertNode(N);
10730 SDValue V(N, 0);
10731 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10732 return V;
10733}
10734
10735SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10736 SDValue Val, SDValue Ptr, SDValue Mask,
10737 SDValue EVL, MachinePointerInfo PtrInfo,
10738 EVT SVT, Align Alignment,
10739 MachineMemOperand::Flags MMOFlags,
10740 const AAMDNodes &AAInfo,
10741 bool IsCompressing) {
10742 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10743
10744 MMOFlags |= MachineMemOperand::MOStore;
10745 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10746
10747 if (PtrInfo.V.isNull())
10748 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10749
10750 MachineFunction &MF = getMachineFunction();
10751 MachineMemOperand *MMO = MF.getMachineMemOperand(
10752 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10753 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
10754 IsCompressing);
10755}
10756
10757SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10758 SDValue Val, SDValue Ptr, SDValue Mask,
10759 SDValue EVL, EVT SVT,
10760 MachineMemOperand *MMO,
10761 bool IsCompressing) {
10762 EVT VT = Val.getValueType();
10763
10764 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10765 if (VT == SVT)
10766 return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
10767 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10768 /*IsTruncating*/ false, IsCompressing);
10769
10770 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10771 "Should only be a truncating store, not extending!");
10772 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10773 assert(VT.isVector() == SVT.isVector() &&
10774 "Cannot use trunc store to convert to or from a vector!");
10775 assert((!VT.isVector() ||
10776 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10777 "Cannot use trunc store to change the number of vector elements!");
10778
10779 SDVTList VTs = getVTList(VT: MVT::Other);
10780 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10781 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
10782 FoldingSetNodeID ID;
10783 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10784 ID.AddInteger(I: SVT.getRawBits());
10785 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10786 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10787 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10788 ID.AddInteger(I: MMO->getFlags());
10789 void *IP = nullptr;
10790 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10791 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10792 return SDValue(E, 0);
10793 }
10794 auto *N =
10795 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10796 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
10797 createOperands(Node: N, Vals: Ops);
10798
10799 CSEMap.InsertNode(N, InsertPos: IP);
10800 InsertNode(N);
10801 SDValue V(N, 0);
10802 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10803 return V;
10804}
10805
10806SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
10807 SDValue Base, SDValue Offset,
10808 ISD::MemIndexedMode AM) {
10809 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
10810 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
10811 SDVTList VTs = getVTList(VT1: Base.getValueType(), VT2: MVT::Other);
10812 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
10813 Offset, ST->getMask(), ST->getVectorLength()};
10814 FoldingSetNodeID ID;
10815 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10816 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
10817 ID.AddInteger(I: ST->getRawSubclassData());
10818 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
10819 ID.AddInteger(I: ST->getMemOperand()->getFlags());
10820 void *IP = nullptr;
10821 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10822 return SDValue(E, 0);
10823
10824 auto *N = newSDNode<VPStoreSDNode>(
10825 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
10826 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
10827 createOperands(Node: N, Vals: Ops);
10828
10829 CSEMap.InsertNode(N, InsertPos: IP);
10830 InsertNode(N);
10831 SDValue V(N, 0);
10832 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10833 return V;
10834}
10835
10836SDValue SelectionDAG::getStridedLoadVP(
10837 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
10838 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
10839 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
10840 bool Indexed = AM != ISD::UNINDEXED;
10841 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10842
10843 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
10844 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10845 : getVTList(VT1: VT, VT2: MVT::Other);
10846 FoldingSetNodeID ID;
10847 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
10848 ID.AddInteger(I: VT.getRawBits());
10849 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
10850 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10851 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10852
10853 void *IP = nullptr;
10854 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10855 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10856 return SDValue(E, 0);
10857 }
10858
10859 auto *N =
10860 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
10861 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10862 createOperands(Node: N, Vals: Ops);
10863 CSEMap.InsertNode(N, InsertPos: IP);
10864 InsertNode(N);
10865 SDValue V(N, 0);
10866 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10867 return V;
10868}
10869
10870SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
10871 SDValue Ptr, SDValue Stride,
10872 SDValue Mask, SDValue EVL,
10873 MachineMemOperand *MMO,
10874 bool IsExpanding) {
10875 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10876 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
10877 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
10878}
10879
10880SDValue SelectionDAG::getExtStridedLoadVP(
10881 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
10882 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
10883 MachineMemOperand *MMO, bool IsExpanding) {
10884 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10885 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
10886 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
10887}
10888
10889SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
10890 SDValue Val, SDValue Ptr,
10891 SDValue Offset, SDValue Stride,
10892 SDValue Mask, SDValue EVL, EVT MemVT,
10893 MachineMemOperand *MMO,
10894 ISD::MemIndexedMode AM,
10895 bool IsTruncating, bool IsCompressing) {
10896 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10897 bool Indexed = AM != ISD::UNINDEXED;
10898 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10899 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10900 : getVTList(VT: MVT::Other);
10901 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
10902 FoldingSetNodeID ID;
10903 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10904 ID.AddInteger(I: MemVT.getRawBits());
10905 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10906 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10907 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10908 void *IP = nullptr;
10909 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10910 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10911 return SDValue(E, 0);
10912 }
10913 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10914 Args&: VTs, Args&: AM, Args&: IsTruncating,
10915 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10916 createOperands(Node: N, Vals: Ops);
10917
10918 CSEMap.InsertNode(N, InsertPos: IP);
10919 InsertNode(N);
10920 SDValue V(N, 0);
10921 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10922 return V;
10923}
10924
10925SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
10926 SDValue Val, SDValue Ptr,
10927 SDValue Stride, SDValue Mask,
10928 SDValue EVL, EVT SVT,
10929 MachineMemOperand *MMO,
10930 bool IsCompressing) {
10931 EVT VT = Val.getValueType();
10932
10933 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10934 if (VT == SVT)
10935 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
10936 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10937 /*IsTruncating*/ false, IsCompressing);
10938
10939 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10940 "Should only be a truncating store, not extending!");
10941 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10942 assert(VT.isVector() == SVT.isVector() &&
10943 "Cannot use trunc store to convert to or from a vector!");
10944 assert((!VT.isVector() ||
10945 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10946 "Cannot use trunc store to change the number of vector elements!");
10947
10948 SDVTList VTs = getVTList(VT: MVT::Other);
10949 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10950 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
10951 FoldingSetNodeID ID;
10952 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10953 ID.AddInteger(I: SVT.getRawBits());
10954 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10955 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10956 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10957 void *IP = nullptr;
10958 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10959 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10960 return SDValue(E, 0);
10961 }
10962 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10963 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
10964 Args&: IsCompressing, Args&: SVT, Args&: MMO);
10965 createOperands(Node: N, Vals: Ops);
10966
10967 CSEMap.InsertNode(N, InsertPos: IP);
10968 InsertNode(N);
10969 SDValue V(N, 0);
10970 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10971 return V;
10972}
10973
10974SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10975 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
10976 ISD::MemIndexType IndexType) {
10977 assert(Ops.size() == 6 && "Incompatible number of operands");
10978
10979 FoldingSetNodeID ID;
10980 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
10981 ID.AddInteger(I: VT.getRawBits());
10982 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
10983 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10984 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10985 ID.AddInteger(I: MMO->getFlags());
10986 void *IP = nullptr;
10987 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10988 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10989 return SDValue(E, 0);
10990 }
10991
10992 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10993 Args&: VT, Args&: MMO, Args&: IndexType);
10994 createOperands(Node: N, Vals: Ops);
10995
10996 assert(N->getMask().getValueType().getVectorElementCount() ==
10997 N->getValueType(0).getVectorElementCount() &&
10998 "Vector width mismatch between mask and data");
10999 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11000 N->getValueType(0).getVectorElementCount().isScalable() &&
11001 "Scalable flags of index and data do not match");
11002 assert(ElementCount::isKnownGE(
11003 N->getIndex().getValueType().getVectorElementCount(),
11004 N->getValueType(0).getVectorElementCount()) &&
11005 "Vector width mismatch between index and data");
11006 assert(isa<ConstantSDNode>(N->getScale()) &&
11007 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11008 "Scale should be a constant power of 2");
11009
11010 CSEMap.InsertNode(N, InsertPos: IP);
11011 InsertNode(N);
11012 SDValue V(N, 0);
11013 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11014 return V;
11015}
11016
11017SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11018 ArrayRef<SDValue> Ops,
11019 MachineMemOperand *MMO,
11020 ISD::MemIndexType IndexType) {
11021 assert(Ops.size() == 7 && "Incompatible number of operands");
11022
11023 FoldingSetNodeID ID;
11024 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
11025 ID.AddInteger(I: VT.getRawBits());
11026 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
11027 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11028 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11029 ID.AddInteger(I: MMO->getFlags());
11030 void *IP = nullptr;
11031 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11032 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11033 return SDValue(E, 0);
11034 }
11035 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11036 Args&: VT, Args&: MMO, Args&: IndexType);
11037 createOperands(Node: N, Vals: Ops);
11038
11039 assert(N->getMask().getValueType().getVectorElementCount() ==
11040 N->getValue().getValueType().getVectorElementCount() &&
11041 "Vector width mismatch between mask and data");
11042 assert(
11043 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11044 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11045 "Scalable flags of index and data do not match");
11046 assert(ElementCount::isKnownGE(
11047 N->getIndex().getValueType().getVectorElementCount(),
11048 N->getValue().getValueType().getVectorElementCount()) &&
11049 "Vector width mismatch between index and data");
11050 assert(isa<ConstantSDNode>(N->getScale()) &&
11051 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11052 "Scale should be a constant power of 2");
11053
11054 CSEMap.InsertNode(N, InsertPos: IP);
11055 InsertNode(N);
11056 SDValue V(N, 0);
11057 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11058 return V;
11059}
11060
11061SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11062 SDValue Base, SDValue Offset, SDValue Mask,
11063 SDValue PassThru, EVT MemVT,
11064 MachineMemOperand *MMO,
11065 ISD::MemIndexedMode AM,
11066 ISD::LoadExtType ExtTy, bool isExpanding) {
11067 bool Indexed = AM != ISD::UNINDEXED;
11068 assert((Indexed || Offset.isUndef()) &&
11069 "Unindexed masked load with an offset!");
11070 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
11071 : getVTList(VT1: VT, VT2: MVT::Other);
11072 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11073 FoldingSetNodeID ID;
11074 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
11075 ID.AddInteger(I: MemVT.getRawBits());
11076 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11077 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
11078 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11079 ID.AddInteger(I: MMO->getFlags());
11080 void *IP = nullptr;
11081 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11082 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11083 return SDValue(E, 0);
11084 }
11085 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11086 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
11087 createOperands(Node: N, Vals: Ops);
11088
11089 CSEMap.InsertNode(N, InsertPos: IP);
11090 InsertNode(N);
11091 SDValue V(N, 0);
11092 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11093 return V;
11094}
11095
11096SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
11097 SDValue Base, SDValue Offset,
11098 ISD::MemIndexedMode AM) {
11099 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
11100 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
11101 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
11102 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
11103 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
11104 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
11105}
11106
11107SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
11108 SDValue Val, SDValue Base, SDValue Offset,
11109 SDValue Mask, EVT MemVT,
11110 MachineMemOperand *MMO,
11111 ISD::MemIndexedMode AM, bool IsTruncating,
11112 bool IsCompressing) {
11113 assert(Chain.getValueType() == MVT::Other &&
11114 "Invalid chain type");
11115 bool Indexed = AM != ISD::UNINDEXED;
11116 assert((Indexed || Offset.isUndef()) &&
11117 "Unindexed masked store with an offset!");
11118 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
11119 : getVTList(VT: MVT::Other);
11120 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11121 FoldingSetNodeID ID;
11122 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
11123 ID.AddInteger(I: MemVT.getRawBits());
11124 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11125 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11126 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11127 ID.AddInteger(I: MMO->getFlags());
11128 void *IP = nullptr;
11129 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11130 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11131 return SDValue(E, 0);
11132 }
11133 auto *N =
11134 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11135 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11136 createOperands(Node: N, Vals: Ops);
11137
11138 CSEMap.InsertNode(N, InsertPos: IP);
11139 InsertNode(N);
11140 SDValue V(N, 0);
11141 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11142 return V;
11143}
11144
11145SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
11146 SDValue Base, SDValue Offset,
11147 ISD::MemIndexedMode AM) {
11148 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
11149 assert(ST->getOffset().isUndef() &&
11150 "Masked store is already a indexed store!");
11151 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
11152 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
11153 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
11154}
11155
11156SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11157 ArrayRef<SDValue> Ops,
11158 MachineMemOperand *MMO,
11159 ISD::MemIndexType IndexType,
11160 ISD::LoadExtType ExtTy) {
11161 assert(Ops.size() == 6 && "Incompatible number of operands");
11162
11163 FoldingSetNodeID ID;
11164 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
11165 ID.AddInteger(I: MemVT.getRawBits());
11166 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11167 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
11168 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11169 ID.AddInteger(I: MMO->getFlags());
11170 void *IP = nullptr;
11171 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11172 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11173 return SDValue(E, 0);
11174 }
11175
11176 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11177 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
11178 createOperands(Node: N, Vals: Ops);
11179
11180 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11181 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11182 assert(N->getMask().getValueType().getVectorElementCount() ==
11183 N->getValueType(0).getVectorElementCount() &&
11184 "Vector width mismatch between mask and data");
11185 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11186 N->getValueType(0).getVectorElementCount().isScalable() &&
11187 "Scalable flags of index and data do not match");
11188 assert(ElementCount::isKnownGE(
11189 N->getIndex().getValueType().getVectorElementCount(),
11190 N->getValueType(0).getVectorElementCount()) &&
11191 "Vector width mismatch between index and data");
11192 assert(isa<ConstantSDNode>(N->getScale()) &&
11193 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11194 "Scale should be a constant power of 2");
11195
11196 CSEMap.InsertNode(N, InsertPos: IP);
11197 InsertNode(N);
11198 SDValue V(N, 0);
11199 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11200 return V;
11201}
11202
11203SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11204 ArrayRef<SDValue> Ops,
11205 MachineMemOperand *MMO,
11206 ISD::MemIndexType IndexType,
11207 bool IsTrunc) {
11208 assert(Ops.size() == 6 && "Incompatible number of operands");
11209
11210 FoldingSetNodeID ID;
11211 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
11212 ID.AddInteger(I: MemVT.getRawBits());
11213 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11214 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11215 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11216 ID.AddInteger(I: MMO->getFlags());
11217 void *IP = nullptr;
11218 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11219 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11220 return SDValue(E, 0);
11221 }
11222
11223 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11224 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11225 createOperands(Node: N, Vals: Ops);
11226
11227 assert(N->getMask().getValueType().getVectorElementCount() ==
11228 N->getValue().getValueType().getVectorElementCount() &&
11229 "Vector width mismatch between mask and data");
11230 assert(
11231 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11232 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11233 "Scalable flags of index and data do not match");
11234 assert(ElementCount::isKnownGE(
11235 N->getIndex().getValueType().getVectorElementCount(),
11236 N->getValue().getValueType().getVectorElementCount()) &&
11237 "Vector width mismatch between index and data");
11238 assert(isa<ConstantSDNode>(N->getScale()) &&
11239 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11240 "Scale should be a constant power of 2");
11241
11242 CSEMap.InsertNode(N, InsertPos: IP);
11243 InsertNode(N);
11244 SDValue V(N, 0);
11245 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11246 return V;
11247}
11248
11249SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11250 const SDLoc &dl, ArrayRef<SDValue> Ops,
11251 MachineMemOperand *MMO,
11252 ISD::MemIndexType IndexType) {
11253 assert(Ops.size() == 7 && "Incompatible number of operands");
11254
11255 FoldingSetNodeID ID;
11256 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTList: VTs, OpList: Ops);
11257 ID.AddInteger(I: MemVT.getRawBits());
11258 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11259 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11260 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11261 ID.AddInteger(I: MMO->getFlags());
11262 void *IP = nullptr;
11263 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11264 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11265 return SDValue(E, 0);
11266 }
11267
11268 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11269 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11270 createOperands(Node: N, Vals: Ops);
11271
11272 assert(N->getMask().getValueType().getVectorElementCount() ==
11273 N->getIndex().getValueType().getVectorElementCount() &&
11274 "Vector width mismatch between mask and data");
11275 assert(isa<ConstantSDNode>(N->getScale()) &&
11276 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11277 "Scale should be a constant power of 2");
11278 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11279
11280 CSEMap.InsertNode(N, InsertPos: IP);
11281 InsertNode(N);
11282 SDValue V(N, 0);
11283 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11284 return V;
11285}
11286
11287SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11288 SDValue Ptr, SDValue Mask, SDValue EVL,
11289 MachineMemOperand *MMO) {
11290 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11291 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11292 FoldingSetNodeID ID;
11293 AddNodeIDNode(ID, OpC: ISD::VP_LOAD_FF, VTList: VTs, OpList: Ops);
11294 ID.AddInteger(I: VT.getRawBits());
11295 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11296 Args&: VTs, Args&: VT, Args&: MMO));
11297 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11298 ID.AddInteger(I: MMO->getFlags());
11299 void *IP = nullptr;
11300 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11301 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11302 return SDValue(E, 0);
11303 }
11304 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11305 Args&: VT, Args&: MMO);
11306 createOperands(Node: N, Vals: Ops);
11307
11308 CSEMap.InsertNode(N, InsertPos: IP);
11309 InsertNode(N);
11310 SDValue V(N, 0);
11311 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11312 return V;
11313}
11314
11315SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11316 EVT MemVT, MachineMemOperand *MMO) {
11317 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11318 SDVTList VTs = getVTList(VT: MVT::Other);
11319 SDValue Ops[] = {Chain, Ptr};
11320 FoldingSetNodeID ID;
11321 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
11322 ID.AddInteger(I: MemVT.getRawBits());
11323 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11324 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11325 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11326 ID.AddInteger(I: MMO->getFlags());
11327 void *IP = nullptr;
11328 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11329 return SDValue(E, 0);
11330
11331 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11332 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11333 createOperands(Node: N, Vals: Ops);
11334
11335 CSEMap.InsertNode(N, InsertPos: IP);
11336 InsertNode(N);
11337 SDValue V(N, 0);
11338 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11339 return V;
11340}
11341
11342SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11343 EVT MemVT, MachineMemOperand *MMO) {
11344 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11345 SDVTList VTs = getVTList(VT: MVT::Other);
11346 SDValue Ops[] = {Chain, Ptr};
11347 FoldingSetNodeID ID;
11348 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
11349 ID.AddInteger(I: MemVT.getRawBits());
11350 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11351 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11352 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11353 ID.AddInteger(I: MMO->getFlags());
11354 void *IP = nullptr;
11355 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11356 return SDValue(E, 0);
11357
11358 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11359 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11360 createOperands(Node: N, Vals: Ops);
11361
11362 CSEMap.InsertNode(N, InsertPos: IP);
11363 InsertNode(N);
11364 SDValue V(N, 0);
11365 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11366 return V;
11367}
11368
11369SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11370 // select undef, T, F --> T (if T is a constant), otherwise F
11371 // select, ?, undef, F --> F
11372 // select, ?, T, undef --> T
11373 if (Cond.isUndef())
11374 return isConstantValueOfAnyType(N: T) ? T : F;
11375 if (T.isUndef())
11376 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11377 if (F.isUndef())
11378 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11379
11380 // select true, T, F --> T
11381 // select false, T, F --> F
11382 if (auto C = isBoolConstant(N: Cond))
11383 return *C ? T : F;
11384
11385 // select ?, T, T --> T
11386 if (T == F)
11387 return T;
11388
11389 return SDValue();
11390}
11391
11392SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11393 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11394 if (X.isUndef())
11395 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11396 // shift X, undef --> undef (because it may shift by the bitwidth)
11397 if (Y.isUndef())
11398 return getUNDEF(VT: X.getValueType());
11399
11400 // shift 0, Y --> 0
11401 // shift X, 0 --> X
11402 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11403 return X;
11404
11405 // shift X, C >= bitwidth(X) --> undef
11406 // All vector elements must be too big (or undef) to avoid partial undefs.
11407 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11408 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11409 };
11410 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11411 return getUNDEF(VT: X.getValueType());
11412
11413 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11414 if (X.getValueType().getScalarType() == MVT::i1)
11415 return X;
11416
11417 return SDValue();
11418}
11419
11420SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11421 SDNodeFlags Flags) {
11422 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11423 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11424 // operation is poison. That result can be relaxed to undef.
11425 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11426 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11427 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11428 (YC && YC->getValueAPF().isNaN());
11429 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11430 (YC && YC->getValueAPF().isInfinity());
11431
11432 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11433 return getUNDEF(VT: X.getValueType());
11434
11435 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11436 return getUNDEF(VT: X.getValueType());
11437
11438 if (!YC)
11439 return SDValue();
11440
11441 // X + -0.0 --> X
11442 if (Opcode == ISD::FADD)
11443 if (YC->getValueAPF().isNegZero())
11444 return X;
11445
11446 // X - +0.0 --> X
11447 if (Opcode == ISD::FSUB)
11448 if (YC->getValueAPF().isPosZero())
11449 return X;
11450
11451 // X * 1.0 --> X
11452 // X / 1.0 --> X
11453 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11454 if (YC->getValueAPF().isExactlyValue(V: 1.0))
11455 return X;
11456
11457 // X * 0.0 --> 0.0
11458 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11459 if (YC->getValueAPF().isZero())
11460 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11461
11462 return SDValue();
11463}
11464
11465SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11466 SDValue Ptr, SDValue SV, unsigned Align) {
11467 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11468 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11469}
11470
11471SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11472 ArrayRef<SDUse> Ops) {
11473 switch (Ops.size()) {
11474 case 0: return getNode(Opcode, DL, VT);
11475 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11476 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11477 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11478 default: break;
11479 }
11480
11481 // Copy from an SDUse array into an SDValue array for use with
11482 // the regular getNode logic.
11483 SmallVector<SDValue, 8> NewOps(Ops);
11484 return getNode(Opcode, DL, VT, Ops: NewOps);
11485}
11486
11487SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11488 ArrayRef<SDValue> Ops) {
11489 SDNodeFlags Flags;
11490 if (Inserter)
11491 Flags = Inserter->getFlags();
11492 return getNode(Opcode, DL, VT, Ops, Flags);
11493}
11494
11495SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11496 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11497 unsigned NumOps = Ops.size();
11498 switch (NumOps) {
11499 case 0: return getNode(Opcode, DL, VT);
11500 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11501 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11502 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11503 default: break;
11504 }
11505
11506#ifndef NDEBUG
11507 for (const auto &Op : Ops)
11508 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11509 "Operand is DELETED_NODE!");
11510#endif
11511
11512 switch (Opcode) {
11513 default: break;
11514 case ISD::BUILD_VECTOR:
11515 // Attempt to simplify BUILD_VECTOR.
11516 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11517 return V;
11518 break;
11519 case ISD::CONCAT_VECTORS:
11520 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11521 return V;
11522 break;
11523 case ISD::SELECT_CC:
11524 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11525 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11526 "LHS and RHS of condition must have same type!");
11527 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11528 "True and False arms of SelectCC must have same type!");
11529 assert(Ops[2].getValueType() == VT &&
11530 "select_cc node must be of same type as true and false value!");
11531 assert((!Ops[0].getValueType().isVector() ||
11532 Ops[0].getValueType().getVectorElementCount() ==
11533 VT.getVectorElementCount()) &&
11534 "Expected select_cc with vector result to have the same sized "
11535 "comparison type!");
11536 break;
11537 case ISD::BR_CC:
11538 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11539 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11540 "LHS/RHS of comparison should match types!");
11541 break;
11542 case ISD::VP_ADD:
11543 case ISD::VP_SUB:
11544 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
11545 if (VT.getScalarType() == MVT::i1)
11546 Opcode = ISD::VP_XOR;
11547 break;
11548 case ISD::VP_MUL:
11549 // If it is VP_MUL mask operation then turn it to VP_AND
11550 if (VT.getScalarType() == MVT::i1)
11551 Opcode = ISD::VP_AND;
11552 break;
11553 case ISD::VP_REDUCE_MUL:
11554 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11555 if (VT == MVT::i1)
11556 Opcode = ISD::VP_REDUCE_AND;
11557 break;
11558 case ISD::VP_REDUCE_ADD:
11559 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11560 if (VT == MVT::i1)
11561 Opcode = ISD::VP_REDUCE_XOR;
11562 break;
11563 case ISD::VP_REDUCE_SMAX:
11564 case ISD::VP_REDUCE_UMIN:
11565 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11566 // VP_REDUCE_AND.
11567 if (VT == MVT::i1)
11568 Opcode = ISD::VP_REDUCE_AND;
11569 break;
11570 case ISD::VP_REDUCE_SMIN:
11571 case ISD::VP_REDUCE_UMAX:
11572 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11573 // VP_REDUCE_OR.
11574 if (VT == MVT::i1)
11575 Opcode = ISD::VP_REDUCE_OR;
11576 break;
11577 }
11578
11579 // Memoize nodes.
11580 SDNode *N;
11581 SDVTList VTs = getVTList(VT);
11582
11583 if (VT != MVT::Glue) {
11584 FoldingSetNodeID ID;
11585 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
11586 void *IP = nullptr;
11587
11588 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11589 E->intersectFlagsWith(Flags);
11590 return SDValue(E, 0);
11591 }
11592
11593 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11594 createOperands(Node: N, Vals: Ops);
11595
11596 CSEMap.InsertNode(N, InsertPos: IP);
11597 } else {
11598 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11599 createOperands(Node: N, Vals: Ops);
11600 }
11601
11602 N->setFlags(Flags);
11603 InsertNode(N);
11604 SDValue V(N, 0);
11605 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11606 return V;
11607}
11608
11609SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11610 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
11611 SDNodeFlags Flags;
11612 if (Inserter)
11613 Flags = Inserter->getFlags();
11614 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11615}
11616
11617SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11618 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
11619 const SDNodeFlags Flags) {
11620 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11621}
11622
11623SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11624 ArrayRef<SDValue> Ops) {
11625 SDNodeFlags Flags;
11626 if (Inserter)
11627 Flags = Inserter->getFlags();
11628 return getNode(Opcode, DL, VTList, Ops, Flags);
11629}
11630
11631SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11632 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11633 if (VTList.NumVTs == 1)
11634 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
11635
11636#ifndef NDEBUG
11637 for (const auto &Op : Ops)
11638 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11639 "Operand is DELETED_NODE!");
11640#endif
11641
11642 switch (Opcode) {
11643 case ISD::SADDO:
11644 case ISD::UADDO:
11645 case ISD::SSUBO:
11646 case ISD::USUBO: {
11647 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11648 "Invalid add/sub overflow op!");
11649 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11650 Ops[0].getValueType() == Ops[1].getValueType() &&
11651 Ops[0].getValueType() == VTList.VTs[0] &&
11652 "Binary operator types must match!");
11653 SDValue N1 = Ops[0], N2 = Ops[1];
11654 canonicalizeCommutativeBinop(Opcode, N1, N2);
11655
11656 // (X +- 0) -> X with zero-overflow.
11657 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
11658 /*AllowTruncation*/ true);
11659 if (N2CV && N2CV->isZero()) {
11660 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
11661 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
11662 }
11663
11664 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
11665 VTList.VTs[1].getScalarType() == MVT::i1) {
11666 SDValue F1 = getFreeze(V: N1);
11667 SDValue F2 = getFreeze(V: N2);
11668 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
11669 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
11670 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11671 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11672 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
11673 Flags);
11674 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
11675 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
11676 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
11677 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11678 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11679 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
11680 Flags);
11681 }
11682 }
11683 break;
11684 }
11685 case ISD::SADDO_CARRY:
11686 case ISD::UADDO_CARRY:
11687 case ISD::SSUBO_CARRY:
11688 case ISD::USUBO_CARRY:
11689 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
11690 "Invalid add/sub overflow op!");
11691 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11692 Ops[0].getValueType() == Ops[1].getValueType() &&
11693 Ops[0].getValueType() == VTList.VTs[0] &&
11694 Ops[2].getValueType() == VTList.VTs[1] &&
11695 "Binary operator types must match!");
11696 break;
11697 case ISD::SMUL_LOHI:
11698 case ISD::UMUL_LOHI: {
11699 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
11700 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
11701 VTList.VTs[0] == Ops[0].getValueType() &&
11702 VTList.VTs[0] == Ops[1].getValueType() &&
11703 "Binary operator types must match!");
11704 // Constant fold.
11705 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
11706 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
11707 if (LHS && RHS) {
11708 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
11709 unsigned OutWidth = Width * 2;
11710 APInt Val = LHS->getAPIntValue();
11711 APInt Mul = RHS->getAPIntValue();
11712 if (Opcode == ISD::SMUL_LOHI) {
11713 Val = Val.sext(width: OutWidth);
11714 Mul = Mul.sext(width: OutWidth);
11715 } else {
11716 Val = Val.zext(width: OutWidth);
11717 Mul = Mul.zext(width: OutWidth);
11718 }
11719 Val *= Mul;
11720
11721 SDValue Hi =
11722 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
11723 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
11724 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
11725 }
11726 break;
11727 }
11728 case ISD::FFREXP: {
11729 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
11730 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
11731 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
11732
11733 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
11734 int FrexpExp;
11735 APFloat FrexpMant =
11736 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
11737 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
11738 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
11739 DL, VT: VTList.VTs[1]);
11740 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
11741 }
11742
11743 break;
11744 }
11745 case ISD::STRICT_FP_EXTEND:
11746 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11747 "Invalid STRICT_FP_EXTEND!");
11748 assert(VTList.VTs[0].isFloatingPoint() &&
11749 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
11750 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11751 "STRICT_FP_EXTEND result type should be vector iff the operand "
11752 "type is vector!");
11753 assert((!VTList.VTs[0].isVector() ||
11754 VTList.VTs[0].getVectorElementCount() ==
11755 Ops[1].getValueType().getVectorElementCount()) &&
11756 "Vector element count mismatch!");
11757 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
11758 "Invalid fpext node, dst <= src!");
11759 break;
11760 case ISD::STRICT_FP_ROUND:
11761 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
11762 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11763 "STRICT_FP_ROUND result type should be vector iff the operand "
11764 "type is vector!");
11765 assert((!VTList.VTs[0].isVector() ||
11766 VTList.VTs[0].getVectorElementCount() ==
11767 Ops[1].getValueType().getVectorElementCount()) &&
11768 "Vector element count mismatch!");
11769 assert(VTList.VTs[0].isFloatingPoint() &&
11770 Ops[1].getValueType().isFloatingPoint() &&
11771 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
11772 Ops[2].getOpcode() == ISD::TargetConstant &&
11773 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
11774 "Invalid STRICT_FP_ROUND!");
11775 break;
11776 }
11777
11778 // Memoize the node unless it returns a glue result.
11779 SDNode *N;
11780 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
11781 FoldingSetNodeID ID;
11782 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
11783 void *IP = nullptr;
11784 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11785 E->intersectFlagsWith(Flags);
11786 return SDValue(E, 0);
11787 }
11788
11789 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11790 createOperands(Node: N, Vals: Ops);
11791 CSEMap.InsertNode(N, InsertPos: IP);
11792 } else {
11793 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11794 createOperands(Node: N, Vals: Ops);
11795 }
11796
11797 N->setFlags(Flags);
11798 InsertNode(N);
11799 SDValue V(N, 0);
11800 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11801 return V;
11802}
11803
11804SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11805 SDVTList VTList) {
11806 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
11807}
11808
11809SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11810 SDValue N1) {
11811 SDValue Ops[] = { N1 };
11812 return getNode(Opcode, DL, VTList, Ops);
11813}
11814
11815SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11816 SDValue N1, SDValue N2) {
11817 SDValue Ops[] = { N1, N2 };
11818 return getNode(Opcode, DL, VTList, Ops);
11819}
11820
11821SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11822 SDValue N1, SDValue N2, SDValue N3) {
11823 SDValue Ops[] = { N1, N2, N3 };
11824 return getNode(Opcode, DL, VTList, Ops);
11825}
11826
11827SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11828 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
11829 SDValue Ops[] = { N1, N2, N3, N4 };
11830 return getNode(Opcode, DL, VTList, Ops);
11831}
11832
11833SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11834 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
11835 SDValue N5) {
11836 SDValue Ops[] = { N1, N2, N3, N4, N5 };
11837 return getNode(Opcode, DL, VTList, Ops);
11838}
11839
11840SDVTList SelectionDAG::getVTList(EVT VT) {
11841 if (!VT.isExtended())
11842 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
11843
11844 return makeVTList(VTs: &(*EVTs.insert(x: VT).first), NumVTs: 1);
11845}
11846
11847SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
11848 FoldingSetNodeID ID;
11849 ID.AddInteger(I: 2U);
11850 ID.AddInteger(I: VT1.getRawBits());
11851 ID.AddInteger(I: VT2.getRawBits());
11852
11853 void *IP = nullptr;
11854 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11855 if (!Result) {
11856 EVT *Array = Allocator.Allocate<EVT>(Num: 2);
11857 Array[0] = VT1;
11858 Array[1] = VT2;
11859 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
11860 VTListMap.InsertNode(N: Result, InsertPos: IP);
11861 }
11862 return Result->getSDVTList();
11863}
11864
11865SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
11866 FoldingSetNodeID ID;
11867 ID.AddInteger(I: 3U);
11868 ID.AddInteger(I: VT1.getRawBits());
11869 ID.AddInteger(I: VT2.getRawBits());
11870 ID.AddInteger(I: VT3.getRawBits());
11871
11872 void *IP = nullptr;
11873 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11874 if (!Result) {
11875 EVT *Array = Allocator.Allocate<EVT>(Num: 3);
11876 Array[0] = VT1;
11877 Array[1] = VT2;
11878 Array[2] = VT3;
11879 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
11880 VTListMap.InsertNode(N: Result, InsertPos: IP);
11881 }
11882 return Result->getSDVTList();
11883}
11884
11885SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
11886 FoldingSetNodeID ID;
11887 ID.AddInteger(I: 4U);
11888 ID.AddInteger(I: VT1.getRawBits());
11889 ID.AddInteger(I: VT2.getRawBits());
11890 ID.AddInteger(I: VT3.getRawBits());
11891 ID.AddInteger(I: VT4.getRawBits());
11892
11893 void *IP = nullptr;
11894 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11895 if (!Result) {
11896 EVT *Array = Allocator.Allocate<EVT>(Num: 4);
11897 Array[0] = VT1;
11898 Array[1] = VT2;
11899 Array[2] = VT3;
11900 Array[3] = VT4;
11901 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
11902 VTListMap.InsertNode(N: Result, InsertPos: IP);
11903 }
11904 return Result->getSDVTList();
11905}
11906
11907SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
11908 unsigned NumVTs = VTs.size();
11909 FoldingSetNodeID ID;
11910 ID.AddInteger(I: NumVTs);
11911 for (unsigned index = 0; index < NumVTs; index++) {
11912 ID.AddInteger(I: VTs[index].getRawBits());
11913 }
11914
11915 void *IP = nullptr;
11916 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11917 if (!Result) {
11918 EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
11919 llvm::copy(Range&: VTs, Out: Array);
11920 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
11921 VTListMap.InsertNode(N: Result, InsertPos: IP);
11922 }
11923 return Result->getSDVTList();
11924}
11925
11926
11927/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
11928/// specified operands. If the resultant node already exists in the DAG,
11929/// this does not modify the specified node, instead it returns the node that
11930/// already exists. If the resultant node does not exist in the DAG, the
11931/// input node is returned. As a degenerate case, if you specify the same
11932/// input operands as the node already has, the input node is returned.
11933SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
11934 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
11935
11936 // Check to see if there is no change.
11937 if (Op == N->getOperand(Num: 0)) return N;
11938
11939 // See if the modified node already exists.
11940 void *InsertPos = nullptr;
11941 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
11942 return Existing;
11943
11944 // Nope it doesn't. Remove the node from its current place in the maps.
11945 if (InsertPos)
11946 if (!RemoveNodeFromCSEMaps(N))
11947 InsertPos = nullptr;
11948
11949 // Now we update the operands.
11950 N->OperandList[0].set(Op);
11951
11952 updateDivergence(N);
11953 // If this gets put into a CSE map, add it.
11954 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11955 return N;
11956}
11957
11958SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
11959 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
11960
11961 // Check to see if there is no change.
11962 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
11963 return N; // No operands changed, just return the input node.
11964
11965 // See if the modified node already exists.
11966 void *InsertPos = nullptr;
11967 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
11968 return Existing;
11969
11970 // Nope it doesn't. Remove the node from its current place in the maps.
11971 if (InsertPos)
11972 if (!RemoveNodeFromCSEMaps(N))
11973 InsertPos = nullptr;
11974
11975 // Now we update the operands.
11976 if (N->OperandList[0] != Op1)
11977 N->OperandList[0].set(Op1);
11978 if (N->OperandList[1] != Op2)
11979 N->OperandList[1].set(Op2);
11980
11981 updateDivergence(N);
11982 // If this gets put into a CSE map, add it.
11983 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11984 return N;
11985}
11986
11987SDNode *SelectionDAG::
11988UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
11989 SDValue Ops[] = { Op1, Op2, Op3 };
11990 return UpdateNodeOperands(N, Ops);
11991}
11992
11993SDNode *SelectionDAG::
11994UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11995 SDValue Op3, SDValue Op4) {
11996 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
11997 return UpdateNodeOperands(N, Ops);
11998}
11999
12000SDNode *SelectionDAG::
12001UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12002 SDValue Op3, SDValue Op4, SDValue Op5) {
12003 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12004 return UpdateNodeOperands(N, Ops);
12005}
12006
12007SDNode *SelectionDAG::
12008UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
12009 unsigned NumOps = Ops.size();
12010 assert(N->getNumOperands() == NumOps &&
12011 "Update with wrong number of operands");
12012
12013 // If no operands changed just return the input node.
12014 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
12015 return N;
12016
12017 // See if the modified node already exists.
12018 void *InsertPos = nullptr;
12019 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
12020 return Existing;
12021
12022 // Nope it doesn't. Remove the node from its current place in the maps.
12023 if (InsertPos)
12024 if (!RemoveNodeFromCSEMaps(N))
12025 InsertPos = nullptr;
12026
12027 // Now we update the operands.
12028 for (unsigned i = 0; i != NumOps; ++i)
12029 if (N->OperandList[i] != Ops[i])
12030 N->OperandList[i].set(Ops[i]);
12031
12032 updateDivergence(N);
12033 // If this gets put into a CSE map, add it.
12034 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
12035 return N;
12036}
12037
12038/// DropOperands - Release the operands and set this node to have
12039/// zero operands.
12040void SDNode::DropOperands() {
12041 // Unlike the code in MorphNodeTo that does this, we don't need to
12042 // watch for dead nodes here.
12043 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12044 SDUse &Use = *I++;
12045 Use.set(SDValue());
12046 }
12047}
12048
12049void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
12050 ArrayRef<MachineMemOperand *> NewMemRefs) {
12051 if (NewMemRefs.empty()) {
12052 N->clearMemRefs();
12053 return;
12054 }
12055
12056 // Check if we can avoid allocating by storing a single reference directly.
12057 if (NewMemRefs.size() == 1) {
12058 N->MemRefs = NewMemRefs[0];
12059 N->NumMemRefs = 1;
12060 return;
12061 }
12062
12063 MachineMemOperand **MemRefsBuffer =
12064 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
12065 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
12066 N->MemRefs = MemRefsBuffer;
12067 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12068}
12069
12070/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12071/// machine opcode.
12072///
12073SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12074 EVT VT) {
12075 SDVTList VTs = getVTList(VT);
12076 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12077}
12078
12079SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12080 EVT VT, SDValue Op1) {
12081 SDVTList VTs = getVTList(VT);
12082 SDValue Ops[] = { Op1 };
12083 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12084}
12085
12086SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12087 EVT VT, SDValue Op1,
12088 SDValue Op2) {
12089 SDVTList VTs = getVTList(VT);
12090 SDValue Ops[] = { Op1, Op2 };
12091 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12092}
12093
12094SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12095 EVT VT, SDValue Op1,
12096 SDValue Op2, SDValue Op3) {
12097 SDVTList VTs = getVTList(VT);
12098 SDValue Ops[] = { Op1, Op2, Op3 };
12099 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12100}
12101
12102SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12103 EVT VT, ArrayRef<SDValue> Ops) {
12104 SDVTList VTs = getVTList(VT);
12105 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12106}
12107
12108SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12109 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12110 SDVTList VTs = getVTList(VT1, VT2);
12111 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12112}
12113
12114SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12115 EVT VT1, EVT VT2) {
12116 SDVTList VTs = getVTList(VT1, VT2);
12117 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12118}
12119
12120SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12121 EVT VT1, EVT VT2, EVT VT3,
12122 ArrayRef<SDValue> Ops) {
12123 SDVTList VTs = getVTList(VT1, VT2, VT3);
12124 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12125}
12126
12127SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12128 EVT VT1, EVT VT2,
12129 SDValue Op1, SDValue Op2) {
12130 SDVTList VTs = getVTList(VT1, VT2);
12131 SDValue Ops[] = { Op1, Op2 };
12132 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12133}
12134
12135SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12136 SDVTList VTs,ArrayRef<SDValue> Ops) {
12137 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
12138 // Reset the NodeID to -1.
12139 New->setNodeId(-1);
12140 if (New != N) {
12141 ReplaceAllUsesWith(From: N, To: New);
12142 RemoveDeadNode(N);
12143 }
12144 return New;
12145}
12146
12147/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12148/// the line number information on the merged node since it is not possible to
12149/// preserve the information that operation is associated with multiple lines.
12150/// This will make the debugger working better at -O0, were there is a higher
12151/// probability having other instructions associated with that line.
12152///
12153/// For IROrder, we keep the smaller of the two
12154SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12155 DebugLoc NLoc = N->getDebugLoc();
12156 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12157 N->setDebugLoc(DebugLoc());
12158 }
12159 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
12160 N->setIROrder(Order);
12161 return N;
12162}
12163
12164/// MorphNodeTo - This *mutates* the specified node to have the specified
12165/// return type, opcode, and operands.
12166///
12167/// Note that MorphNodeTo returns the resultant node. If there is already a
12168/// node of the specified opcode and operands, it returns that node instead of
12169/// the current one. Note that the SDLoc need not be the same.
12170///
12171/// Using MorphNodeTo is faster than creating a new node and swapping it in
12172/// with ReplaceAllUsesWith both because it often avoids allocating a new
12173/// node, and because it doesn't require CSE recalculation for any of
12174/// the node's users.
12175///
12176/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12177/// As a consequence it isn't appropriate to use from within the DAG combiner or
12178/// the legalizer which maintain worklists that would need to be updated when
12179/// deleting things.
12180SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
12181 SDVTList VTs, ArrayRef<SDValue> Ops) {
12182 // If an identical node already exists, use it.
12183 void *IP = nullptr;
12184 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12185 FoldingSetNodeID ID;
12186 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
12187 if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
12188 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
12189 }
12190
12191 if (!RemoveNodeFromCSEMaps(N))
12192 IP = nullptr;
12193
12194 // Start the morphing.
12195 N->NodeType = Opc;
12196 N->ValueList = VTs.VTs;
12197 N->NumValues = VTs.NumVTs;
12198
12199 // Clear the operands list, updating used nodes to remove this from their
12200 // use list. Keep track of any operands that become dead as a result.
12201 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12202 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12203 SDUse &Use = *I++;
12204 SDNode *Used = Use.getNode();
12205 Use.set(SDValue());
12206 if (Used->use_empty())
12207 DeadNodeSet.insert(Ptr: Used);
12208 }
12209
12210 // For MachineNode, initialize the memory references information.
12211 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12212 MN->clearMemRefs();
12213
12214 // Swap for an appropriately sized array from the recycler.
12215 removeOperands(Node: N);
12216 createOperands(Node: N, Vals: Ops);
12217
12218 // Delete any nodes that are still dead after adding the uses for the
12219 // new operands.
12220 if (!DeadNodeSet.empty()) {
12221 SmallVector<SDNode *, 16> DeadNodes;
12222 for (SDNode *N : DeadNodeSet)
12223 if (N->use_empty())
12224 DeadNodes.push_back(Elt: N);
12225 RemoveDeadNodes(DeadNodes);
12226 }
12227
12228 if (IP)
12229 CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
12230 return N;
12231}
12232
12233SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12234 unsigned OrigOpc = Node->getOpcode();
12235 unsigned NewOpc;
12236 switch (OrigOpc) {
12237 default:
12238 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12239#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12240 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12241#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12242 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12243#include "llvm/IR/ConstrainedOps.def"
12244 }
12245
12246 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12247
12248 // We're taking this node out of the chain, so we need to re-link things.
12249 SDValue InputChain = Node->getOperand(Num: 0);
12250 SDValue OutputChain = SDValue(Node, 1);
12251 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12252
12253 SmallVector<SDValue, 3> Ops;
12254 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12255 Ops.push_back(Elt: Node->getOperand(Num: i));
12256
12257 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12258 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12259
12260 // MorphNodeTo can operate in two ways: if an existing node with the
12261 // specified operands exists, it can just return it. Otherwise, it
12262 // updates the node in place to have the requested operands.
12263 if (Res == Node) {
12264 // If we updated the node in place, reset the node ID. To the isel,
12265 // this should be just like a newly allocated machine node.
12266 Res->setNodeId(-1);
12267 } else {
12268 ReplaceAllUsesWith(From: Node, To: Res);
12269 RemoveDeadNode(N: Node);
12270 }
12271
12272 return Res;
12273}
12274
12275/// getMachineNode - These are used for target selectors to create a new node
12276/// with specified return type(s), MachineInstr opcode, and operands.
12277///
12278/// Note that getMachineNode returns the resultant node. If there is already a
12279/// node of the specified opcode and operands, it returns that node instead of
12280/// the current one.
12281MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12282 EVT VT) {
12283 SDVTList VTs = getVTList(VT);
12284 return getMachineNode(Opcode, dl, VTs, Ops: {});
12285}
12286
12287MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12288 EVT VT, SDValue Op1) {
12289 SDVTList VTs = getVTList(VT);
12290 SDValue Ops[] = { Op1 };
12291 return getMachineNode(Opcode, dl, VTs, Ops);
12292}
12293
12294MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12295 EVT VT, SDValue Op1, SDValue Op2) {
12296 SDVTList VTs = getVTList(VT);
12297 SDValue Ops[] = { Op1, Op2 };
12298 return getMachineNode(Opcode, dl, VTs, Ops);
12299}
12300
12301MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12302 EVT VT, SDValue Op1, SDValue Op2,
12303 SDValue Op3) {
12304 SDVTList VTs = getVTList(VT);
12305 SDValue Ops[] = { Op1, Op2, Op3 };
12306 return getMachineNode(Opcode, dl, VTs, Ops);
12307}
12308
12309MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12310 EVT VT, ArrayRef<SDValue> Ops) {
12311 SDVTList VTs = getVTList(VT);
12312 return getMachineNode(Opcode, dl, VTs, Ops);
12313}
12314
12315MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12316 EVT VT1, EVT VT2, SDValue Op1,
12317 SDValue Op2) {
12318 SDVTList VTs = getVTList(VT1, VT2);
12319 SDValue Ops[] = { Op1, Op2 };
12320 return getMachineNode(Opcode, dl, VTs, Ops);
12321}
12322
12323MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12324 EVT VT1, EVT VT2, SDValue Op1,
12325 SDValue Op2, SDValue Op3) {
12326 SDVTList VTs = getVTList(VT1, VT2);
12327 SDValue Ops[] = { Op1, Op2, Op3 };
12328 return getMachineNode(Opcode, dl, VTs, Ops);
12329}
12330
12331MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12332 EVT VT1, EVT VT2,
12333 ArrayRef<SDValue> Ops) {
12334 SDVTList VTs = getVTList(VT1, VT2);
12335 return getMachineNode(Opcode, dl, VTs, Ops);
12336}
12337
12338MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12339 EVT VT1, EVT VT2, EVT VT3,
12340 SDValue Op1, SDValue Op2) {
12341 SDVTList VTs = getVTList(VT1, VT2, VT3);
12342 SDValue Ops[] = { Op1, Op2 };
12343 return getMachineNode(Opcode, dl, VTs, Ops);
12344}
12345
12346MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12347 EVT VT1, EVT VT2, EVT VT3,
12348 SDValue Op1, SDValue Op2,
12349 SDValue Op3) {
12350 SDVTList VTs = getVTList(VT1, VT2, VT3);
12351 SDValue Ops[] = { Op1, Op2, Op3 };
12352 return getMachineNode(Opcode, dl, VTs, Ops);
12353}
12354
12355MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12356 EVT VT1, EVT VT2, EVT VT3,
12357 ArrayRef<SDValue> Ops) {
12358 SDVTList VTs = getVTList(VT1, VT2, VT3);
12359 return getMachineNode(Opcode, dl, VTs, Ops);
12360}
12361
12362MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12363 ArrayRef<EVT> ResultTys,
12364 ArrayRef<SDValue> Ops) {
12365 SDVTList VTs = getVTList(VTs: ResultTys);
12366 return getMachineNode(Opcode, dl, VTs, Ops);
12367}
12368
12369MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12370 SDVTList VTs,
12371 ArrayRef<SDValue> Ops) {
12372 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12373 MachineSDNode *N;
12374 void *IP = nullptr;
12375
12376 if (DoCSE) {
12377 FoldingSetNodeID ID;
12378 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
12379 IP = nullptr;
12380 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
12381 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12382 }
12383 }
12384
12385 // Allocate a new MachineSDNode.
12386 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12387 createOperands(Node: N, Vals: Ops);
12388
12389 if (DoCSE)
12390 CSEMap.InsertNode(N, InsertPos: IP);
12391
12392 InsertNode(N);
12393 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12394 return N;
12395}
12396
12397/// getTargetExtractSubreg - A convenience function for creating
12398/// TargetOpcode::EXTRACT_SUBREG nodes.
12399SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12400 SDValue Operand) {
12401 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12402 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12403 VT, Op1: Operand, Op2: SRIdxVal);
12404 return SDValue(Subreg, 0);
12405}
12406
12407/// getTargetInsertSubreg - A convenience function for creating
12408/// TargetOpcode::INSERT_SUBREG nodes.
12409SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12410 SDValue Operand, SDValue Subreg) {
12411 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12412 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12413 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12414 return SDValue(Result, 0);
12415}
12416
12417/// getNodeIfExists - Get the specified node if it's already available, or
12418/// else return NULL.
12419SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12420 ArrayRef<SDValue> Ops,
12421 bool AllowCommute) {
12422 SDNodeFlags Flags;
12423 if (Inserter)
12424 Flags = Inserter->getFlags();
12425 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12426}
12427
12428SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12429 ArrayRef<SDValue> Ops,
12430 const SDNodeFlags Flags,
12431 bool AllowCommute) {
12432 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12433 return nullptr;
12434
12435 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12436 FoldingSetNodeID ID;
12437 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: LookupOps);
12438 void *IP = nullptr;
12439 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) {
12440 E->intersectFlagsWith(Flags);
12441 return E;
12442 }
12443 return nullptr;
12444 };
12445
12446 if (SDNode *Existing = Lookup(Ops))
12447 return Existing;
12448
12449 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12450 return Lookup({Ops[1], Ops[0]});
12451
12452 return nullptr;
12453}
12454
12455/// doesNodeExist - Check if a node exists without modifying its flags.
12456bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12457 ArrayRef<SDValue> Ops) {
12458 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12459 FoldingSetNodeID ID;
12460 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12461 void *IP = nullptr;
12462 if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
12463 return true;
12464 }
12465 return false;
12466}
12467
12468/// getDbgValue - Creates a SDDbgValue node.
12469///
12470/// SDNode
12471SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12472 SDNode *N, unsigned R, bool IsIndirect,
12473 const DebugLoc &DL, unsigned O) {
12474 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12475 "Expected inlined-at fields to agree");
12476 return new (DbgInfo->getAlloc())
12477 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12478 {}, IsIndirect, DL, O,
12479 /*IsVariadic=*/false);
12480}
12481
12482/// Constant
12483SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12484 DIExpression *Expr,
12485 const Value *C,
12486 const DebugLoc &DL, unsigned O) {
12487 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12488 "Expected inlined-at fields to agree");
12489 return new (DbgInfo->getAlloc())
12490 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12491 /*IsIndirect=*/false, DL, O,
12492 /*IsVariadic=*/false);
12493}
12494
12495/// FrameIndex
12496SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12497 DIExpression *Expr, unsigned FI,
12498 bool IsIndirect,
12499 const DebugLoc &DL,
12500 unsigned O) {
12501 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12502 "Expected inlined-at fields to agree");
12503 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12504}
12505
12506/// FrameIndex with dependencies
12507SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12508 DIExpression *Expr, unsigned FI,
12509 ArrayRef<SDNode *> Dependencies,
12510 bool IsIndirect,
12511 const DebugLoc &DL,
12512 unsigned O) {
12513 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12514 "Expected inlined-at fields to agree");
12515 return new (DbgInfo->getAlloc())
12516 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12517 Dependencies, IsIndirect, DL, O,
12518 /*IsVariadic=*/false);
12519}
12520
12521/// VReg
12522SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12523 Register VReg, bool IsIndirect,
12524 const DebugLoc &DL, unsigned O) {
12525 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12526 "Expected inlined-at fields to agree");
12527 return new (DbgInfo->getAlloc())
12528 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12529 {}, IsIndirect, DL, O,
12530 /*IsVariadic=*/false);
12531}
12532
12533SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12534 ArrayRef<SDDbgOperand> Locs,
12535 ArrayRef<SDNode *> Dependencies,
12536 bool IsIndirect, const DebugLoc &DL,
12537 unsigned O, bool IsVariadic) {
12538 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12539 "Expected inlined-at fields to agree");
12540 return new (DbgInfo->getAlloc())
12541 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12542 DL, O, IsVariadic);
12543}
12544
12545void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12546 unsigned OffsetInBits, unsigned SizeInBits,
12547 bool InvalidateDbg) {
12548 SDNode *FromNode = From.getNode();
12549 SDNode *ToNode = To.getNode();
12550 assert(FromNode && ToNode && "Can't modify dbg values");
12551
12552 // PR35338
12553 // TODO: assert(From != To && "Redundant dbg value transfer");
12554 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12555 if (From == To || FromNode == ToNode)
12556 return;
12557
12558 if (!FromNode->getHasDebugValue())
12559 return;
12560
12561 SDDbgOperand FromLocOp =
12562 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12563 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12564
12565 SmallVector<SDDbgValue *, 2> ClonedDVs;
12566 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12567 if (Dbg->isInvalidated())
12568 continue;
12569
12570 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12571
12572 // Create a new location ops vector that is equal to the old vector, but
12573 // with each instance of FromLocOp replaced with ToLocOp.
12574 bool Changed = false;
12575 auto NewLocOps = Dbg->copyLocationOps();
12576 std::replace_if(
12577 first: NewLocOps.begin(), last: NewLocOps.end(),
12578 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12579 bool Match = Op == FromLocOp;
12580 Changed |= Match;
12581 return Match;
12582 },
12583 new_value: ToLocOp);
12584 // Ignore this SDDbgValue if we didn't find a matching location.
12585 if (!Changed)
12586 continue;
12587
12588 DIVariable *Var = Dbg->getVariable();
12589 auto *Expr = Dbg->getExpression();
12590 // If a fragment is requested, update the expression.
12591 if (SizeInBits) {
12592 // When splitting a larger (e.g., sign-extended) value whose
12593 // lower bits are described with an SDDbgValue, do not attempt
12594 // to transfer the SDDbgValue to the upper bits.
12595 if (auto FI = Expr->getFragmentInfo())
12596 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12597 continue;
12598 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12599 SizeInBits);
12600 if (!Fragment)
12601 continue;
12602 Expr = *Fragment;
12603 }
12604
12605 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12606 // Clone the SDDbgValue and move it to To.
12607 SDDbgValue *Clone = getDbgValueList(
12608 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12609 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12610 IsVariadic: Dbg->isVariadic());
12611 ClonedDVs.push_back(Elt: Clone);
12612
12613 if (InvalidateDbg) {
12614 // Invalidate value and indicate the SDDbgValue should not be emitted.
12615 Dbg->setIsInvalidated();
12616 Dbg->setIsEmitted();
12617 }
12618 }
12619
12620 for (SDDbgValue *Dbg : ClonedDVs) {
12621 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12622 "Transferred DbgValues should depend on the new SDNode");
12623 AddDbgValue(DB: Dbg, isParameter: false);
12624 }
12625}
12626
12627void SelectionDAG::salvageDebugInfo(SDNode &N) {
12628 if (!N.getHasDebugValue())
12629 return;
12630
12631 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
12632 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
12633 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
12634 return SDDbgOperand::fromNode(Node, ResNo);
12635 };
12636
12637 SmallVector<SDDbgValue *, 2> ClonedDVs;
12638 for (auto *DV : GetDbgValues(SD: &N)) {
12639 if (DV->isInvalidated())
12640 continue;
12641 switch (N.getOpcode()) {
12642 default:
12643 break;
12644 case ISD::ADD: {
12645 SDValue N0 = N.getOperand(Num: 0);
12646 SDValue N1 = N.getOperand(Num: 1);
12647 if (!isa<ConstantSDNode>(Val: N0)) {
12648 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
12649 uint64_t Offset;
12650 if (RHSConstant)
12651 Offset = N.getConstantOperandVal(Num: 1);
12652 // We are not allowed to turn indirect debug values variadic, so
12653 // don't salvage those.
12654 if (!RHSConstant && DV->isIndirect())
12655 continue;
12656
12657 // Rewrite an ADD constant node into a DIExpression. Since we are
12658 // performing arithmetic to compute the variable's *value* in the
12659 // DIExpression, we need to mark the expression with a
12660 // DW_OP_stack_value.
12661 auto *DIExpr = DV->getExpression();
12662 auto NewLocOps = DV->copyLocationOps();
12663 bool Changed = false;
12664 size_t OrigLocOpsSize = NewLocOps.size();
12665 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
12666 // We're not given a ResNo to compare against because the whole
12667 // node is going away. We know that any ISD::ADD only has one
12668 // result, so we can assume any node match is using the result.
12669 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12670 NewLocOps[i].getSDNode() != &N)
12671 continue;
12672 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12673 if (RHSConstant) {
12674 SmallVector<uint64_t, 3> ExprOps;
12675 DIExpression::appendOffset(Ops&: ExprOps, Offset);
12676 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12677 } else {
12678 // Convert to a variadic expression (if not already).
12679 // convertToVariadicExpression() returns a const pointer, so we use
12680 // a temporary const variable here.
12681 const auto *TmpDIExpr =
12682 DIExpression::convertToVariadicExpression(Expr: DIExpr);
12683 SmallVector<uint64_t, 3> ExprOps;
12684 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
12685 ExprOps.push_back(Elt: NewLocOps.size());
12686 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
12687 SDDbgOperand RHS =
12688 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
12689 NewLocOps.push_back(Elt: RHS);
12690 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12691 }
12692 Changed = true;
12693 }
12694 (void)Changed;
12695 assert(Changed && "Salvage target doesn't use N");
12696
12697 bool IsVariadic =
12698 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
12699
12700 auto AdditionalDependencies = DV->getAdditionalDependencies();
12701 SDDbgValue *Clone = getDbgValueList(
12702 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
12703 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
12704 ClonedDVs.push_back(Elt: Clone);
12705 DV->setIsInvalidated();
12706 DV->setIsEmitted();
12707 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
12708 N0.getNode()->dumprFull(this);
12709 dbgs() << " into " << *DIExpr << '\n');
12710 }
12711 break;
12712 }
12713 case ISD::TRUNCATE: {
12714 SDValue N0 = N.getOperand(Num: 0);
12715 TypeSize FromSize = N0.getValueSizeInBits();
12716 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
12717
12718 DIExpression *DbgExpression = DV->getExpression();
12719 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
12720 auto NewLocOps = DV->copyLocationOps();
12721 bool Changed = false;
12722 for (size_t i = 0; i < NewLocOps.size(); ++i) {
12723 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12724 NewLocOps[i].getSDNode() != &N)
12725 continue;
12726
12727 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12728 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
12729 Changed = true;
12730 }
12731 assert(Changed && "Salvage target doesn't use N");
12732 (void)Changed;
12733
12734 SDDbgValue *Clone =
12735 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
12736 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
12737 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
12738
12739 ClonedDVs.push_back(Elt: Clone);
12740 DV->setIsInvalidated();
12741 DV->setIsEmitted();
12742 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
12743 dbgs() << " into " << *DbgExpression << '\n');
12744 break;
12745 }
12746 }
12747 }
12748
12749 for (SDDbgValue *Dbg : ClonedDVs) {
12750 assert((!Dbg->getSDNodes().empty() ||
12751 llvm::any_of(Dbg->getLocationOps(),
12752 [&](const SDDbgOperand &Op) {
12753 return Op.getKind() == SDDbgOperand::FRAMEIX;
12754 })) &&
12755 "Salvaged DbgValue should depend on a new SDNode");
12756 AddDbgValue(DB: Dbg, isParameter: false);
12757 }
12758}
12759
12760/// Creates a SDDbgLabel node.
12761SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
12762 const DebugLoc &DL, unsigned O) {
12763 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
12764 "Expected inlined-at fields to agree");
12765 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
12766}
12767
12768namespace {
12769
12770/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
12771/// pointed to by a use iterator is deleted, increment the use iterator
12772/// so that it doesn't dangle.
12773///
12774class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
12775 SDNode::use_iterator &UI;
12776 SDNode::use_iterator &UE;
12777
12778 void NodeDeleted(SDNode *N, SDNode *E) override {
12779 // Increment the iterator as needed.
12780 while (UI != UE && N == UI->getUser())
12781 ++UI;
12782 }
12783
12784public:
12785 RAUWUpdateListener(SelectionDAG &d,
12786 SDNode::use_iterator &ui,
12787 SDNode::use_iterator &ue)
12788 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
12789};
12790
12791} // end anonymous namespace
12792
12793/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12794/// This can cause recursive merging of nodes in the DAG.
12795///
12796/// This version assumes From has a single result value.
12797///
12798void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
12799 SDNode *From = FromN.getNode();
12800 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
12801 "Cannot replace with this method!");
12802 assert(From != To.getNode() && "Cannot replace uses of with self");
12803
12804 // Preserve Debug Values
12805 transferDbgValues(From: FromN, To);
12806 // Preserve extra info.
12807 copyExtraInfo(From, To: To.getNode());
12808
12809 // Iterate over all the existing uses of From. New uses will be added
12810 // to the beginning of the use list, which we avoid visiting.
12811 // This specifically avoids visiting uses of From that arise while the
12812 // replacement is happening, because any such uses would be the result
12813 // of CSE: If an existing node looks like From after one of its operands
12814 // is replaced by To, we don't want to replace of all its users with To
12815 // too. See PR3018 for more info.
12816 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12817 RAUWUpdateListener Listener(*this, UI, UE);
12818 while (UI != UE) {
12819 SDNode *User = UI->getUser();
12820
12821 // This node is about to morph, remove its old self from the CSE maps.
12822 RemoveNodeFromCSEMaps(N: User);
12823
12824 // A user can appear in a use list multiple times, and when this
12825 // happens the uses are usually next to each other in the list.
12826 // To help reduce the number of CSE recomputations, process all
12827 // the uses of this user that we can find this way.
12828 do {
12829 SDUse &Use = *UI;
12830 ++UI;
12831 Use.set(To);
12832 if (To->isDivergent() != From->isDivergent())
12833 updateDivergence(N: User);
12834 } while (UI != UE && UI->getUser() == User);
12835 // Now that we have modified User, add it back to the CSE maps. If it
12836 // already exists there, recursively merge the results together.
12837 AddModifiedNodeToCSEMaps(N: User);
12838 }
12839
12840 // If we just RAUW'd the root, take note.
12841 if (FromN == getRoot())
12842 setRoot(To);
12843}
12844
12845/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12846/// This can cause recursive merging of nodes in the DAG.
12847///
12848/// This version assumes that for each value of From, there is a
12849/// corresponding value in To in the same position with the same type.
12850///
12851void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
12852#ifndef NDEBUG
12853 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12854 assert((!From->hasAnyUseOfValue(i) ||
12855 From->getValueType(i) == To->getValueType(i)) &&
12856 "Cannot use this version of ReplaceAllUsesWith!");
12857#endif
12858
12859 // Handle the trivial case.
12860 if (From == To)
12861 return;
12862
12863 // Preserve Debug Info. Only do this if there's a use.
12864 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12865 if (From->hasAnyUseOfValue(Value: i)) {
12866 assert((i < To->getNumValues()) && "Invalid To location");
12867 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
12868 }
12869 // Preserve extra info.
12870 copyExtraInfo(From, To);
12871
12872 // Iterate over just the existing users of From. See the comments in
12873 // the ReplaceAllUsesWith above.
12874 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12875 RAUWUpdateListener Listener(*this, UI, UE);
12876 while (UI != UE) {
12877 SDNode *User = UI->getUser();
12878
12879 // This node is about to morph, remove its old self from the CSE maps.
12880 RemoveNodeFromCSEMaps(N: User);
12881
12882 // A user can appear in a use list multiple times, and when this
12883 // happens the uses are usually next to each other in the list.
12884 // To help reduce the number of CSE recomputations, process all
12885 // the uses of this user that we can find this way.
12886 do {
12887 SDUse &Use = *UI;
12888 ++UI;
12889 Use.setNode(To);
12890 if (To->isDivergent() != From->isDivergent())
12891 updateDivergence(N: User);
12892 } while (UI != UE && UI->getUser() == User);
12893
12894 // Now that we have modified User, add it back to the CSE maps. If it
12895 // already exists there, recursively merge the results together.
12896 AddModifiedNodeToCSEMaps(N: User);
12897 }
12898
12899 // If we just RAUW'd the root, take note.
12900 if (From == getRoot().getNode())
12901 setRoot(SDValue(To, getRoot().getResNo()));
12902}
12903
12904/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12905/// This can cause recursive merging of nodes in the DAG.
12906///
12907/// This version can replace From with any result values. To must match the
12908/// number and types of values returned by From.
12909void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
12910 if (From->getNumValues() == 1) // Handle the simple case efficiently.
12911 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
12912
12913 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
12914 // Preserve Debug Info.
12915 transferDbgValues(From: SDValue(From, i), To: To[i]);
12916 // Preserve extra info.
12917 copyExtraInfo(From, To: To[i].getNode());
12918 }
12919
12920 // Iterate over just the existing users of From. See the comments in
12921 // the ReplaceAllUsesWith above.
12922 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12923 RAUWUpdateListener Listener(*this, UI, UE);
12924 while (UI != UE) {
12925 SDNode *User = UI->getUser();
12926
12927 // This node is about to morph, remove its old self from the CSE maps.
12928 RemoveNodeFromCSEMaps(N: User);
12929
12930 // A user can appear in a use list multiple times, and when this happens the
12931 // uses are usually next to each other in the list. To help reduce the
12932 // number of CSE and divergence recomputations, process all the uses of this
12933 // user that we can find this way.
12934 bool To_IsDivergent = false;
12935 do {
12936 SDUse &Use = *UI;
12937 const SDValue &ToOp = To[Use.getResNo()];
12938 ++UI;
12939 Use.set(ToOp);
12940 if (ToOp.getValueType() != MVT::Other)
12941 To_IsDivergent |= ToOp->isDivergent();
12942 } while (UI != UE && UI->getUser() == User);
12943
12944 if (To_IsDivergent != From->isDivergent())
12945 updateDivergence(N: User);
12946
12947 // Now that we have modified User, add it back to the CSE maps. If it
12948 // already exists there, recursively merge the results together.
12949 AddModifiedNodeToCSEMaps(N: User);
12950 }
12951
12952 // If we just RAUW'd the root, take note.
12953 if (From == getRoot().getNode())
12954 setRoot(SDValue(To[getRoot().getResNo()]));
12955}
12956
12957/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
12958/// uses of other values produced by From.getNode() alone. The Deleted
12959/// vector is handled the same way as for ReplaceAllUsesWith.
12960void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
12961 // Handle the really simple, really trivial case efficiently.
12962 if (From == To) return;
12963
12964 // Handle the simple, trivial, case efficiently.
12965 if (From.getNode()->getNumValues() == 1) {
12966 ReplaceAllUsesWith(FromN: From, To);
12967 return;
12968 }
12969
12970 // Preserve Debug Info.
12971 transferDbgValues(From, To);
12972 copyExtraInfo(From: From.getNode(), To: To.getNode());
12973
12974 // Iterate over just the existing users of From. See the comments in
12975 // the ReplaceAllUsesWith above.
12976 SDNode::use_iterator UI = From.getNode()->use_begin(),
12977 UE = From.getNode()->use_end();
12978 RAUWUpdateListener Listener(*this, UI, UE);
12979 while (UI != UE) {
12980 SDNode *User = UI->getUser();
12981 bool UserRemovedFromCSEMaps = false;
12982
12983 // A user can appear in a use list multiple times, and when this
12984 // happens the uses are usually next to each other in the list.
12985 // To help reduce the number of CSE recomputations, process all
12986 // the uses of this user that we can find this way.
12987 do {
12988 SDUse &Use = *UI;
12989
12990 // Skip uses of different values from the same node.
12991 if (Use.getResNo() != From.getResNo()) {
12992 ++UI;
12993 continue;
12994 }
12995
12996 // If this node hasn't been modified yet, it's still in the CSE maps,
12997 // so remove its old self from the CSE maps.
12998 if (!UserRemovedFromCSEMaps) {
12999 RemoveNodeFromCSEMaps(N: User);
13000 UserRemovedFromCSEMaps = true;
13001 }
13002
13003 ++UI;
13004 Use.set(To);
13005 if (To->isDivergent() != From->isDivergent())
13006 updateDivergence(N: User);
13007 } while (UI != UE && UI->getUser() == User);
13008 // We are iterating over all uses of the From node, so if a use
13009 // doesn't use the specific value, no changes are made.
13010 if (!UserRemovedFromCSEMaps)
13011 continue;
13012
13013 // Now that we have modified User, add it back to the CSE maps. If it
13014 // already exists there, recursively merge the results together.
13015 AddModifiedNodeToCSEMaps(N: User);
13016 }
13017
13018 // If we just RAUW'd the root, take note.
13019 if (From == getRoot())
13020 setRoot(To);
13021}
13022
13023namespace {
13024
13025/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13026/// to record information about a use.
13027struct UseMemo {
13028 SDNode *User;
13029 unsigned Index;
13030 SDUse *Use;
13031};
13032
13033/// operator< - Sort Memos by User.
13034bool operator<(const UseMemo &L, const UseMemo &R) {
13035 return (intptr_t)L.User < (intptr_t)R.User;
13036}
13037
13038/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13039/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13040/// the node already has been taken care of recursively.
13041class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13042 SmallVectorImpl<UseMemo> &Uses;
13043
13044 void NodeDeleted(SDNode *N, SDNode *E) override {
13045 for (UseMemo &Memo : Uses)
13046 if (Memo.User == N)
13047 Memo.User = nullptr;
13048 }
13049
13050public:
13051 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13052 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13053};
13054
13055} // end anonymous namespace
13056
13057/// Return true if a glue output should propagate divergence information.
13058static bool gluePropagatesDivergence(const SDNode *Node) {
13059 switch (Node->getOpcode()) {
13060 case ISD::CopyFromReg:
13061 case ISD::CopyToReg:
13062 return false;
13063 default:
13064 return true;
13065 }
13066
13067 llvm_unreachable("covered opcode switch");
13068}
13069
13070bool SelectionDAG::calculateDivergence(SDNode *N) {
13071 if (TLI->isSDNodeAlwaysUniform(N)) {
13072 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13073 "Conflicting divergence information!");
13074 return false;
13075 }
13076 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13077 return true;
13078 for (const auto &Op : N->ops()) {
13079 EVT VT = Op.getValueType();
13080
13081 // Skip Chain. It does not carry divergence.
13082 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13083 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
13084 return true;
13085 }
13086 return false;
13087}
13088
13089void SelectionDAG::updateDivergence(SDNode *N) {
13090 SmallVector<SDNode *, 16> Worklist(1, N);
13091 do {
13092 N = Worklist.pop_back_val();
13093 bool IsDivergent = calculateDivergence(N);
13094 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13095 N->SDNodeBits.IsDivergent = IsDivergent;
13096 llvm::append_range(C&: Worklist, R: N->users());
13097 }
13098 } while (!Worklist.empty());
13099}
13100
13101void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13102 DenseMap<SDNode *, unsigned> Degree;
13103 Order.reserve(n: AllNodes.size());
13104 for (auto &N : allnodes()) {
13105 unsigned NOps = N.getNumOperands();
13106 Degree[&N] = NOps;
13107 if (0 == NOps)
13108 Order.push_back(x: &N);
13109 }
13110 for (size_t I = 0; I != Order.size(); ++I) {
13111 SDNode *N = Order[I];
13112 for (auto *U : N->users()) {
13113 unsigned &UnsortedOps = Degree[U];
13114 if (0 == --UnsortedOps)
13115 Order.push_back(x: U);
13116 }
13117 }
13118}
13119
13120#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13121void SelectionDAG::VerifyDAGDivergence() {
13122 std::vector<SDNode *> TopoOrder;
13123 CreateTopologicalOrder(TopoOrder);
13124 for (auto *N : TopoOrder) {
13125 assert(calculateDivergence(N) == N->isDivergent() &&
13126 "Divergence bit inconsistency detected");
13127 }
13128}
13129#endif
13130
13131/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13132/// uses of other values produced by From.getNode() alone. The same value
13133/// may appear in both the From and To list. The Deleted vector is
13134/// handled the same way as for ReplaceAllUsesWith.
13135void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
13136 const SDValue *To,
13137 unsigned Num){
13138 // Handle the simple, trivial case efficiently.
13139 if (Num == 1)
13140 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
13141
13142 transferDbgValues(From: *From, To: *To);
13143 copyExtraInfo(From: From->getNode(), To: To->getNode());
13144
13145 // Read up all the uses and make records of them. This helps
13146 // processing new uses that are introduced during the
13147 // replacement process.
13148 SmallVector<UseMemo, 4> Uses;
13149 for (unsigned i = 0; i != Num; ++i) {
13150 unsigned FromResNo = From[i].getResNo();
13151 SDNode *FromNode = From[i].getNode();
13152 for (SDUse &Use : FromNode->uses()) {
13153 if (Use.getResNo() == FromResNo) {
13154 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
13155 Uses.push_back(Elt: Memo);
13156 }
13157 }
13158 }
13159
13160 // Sort the uses, so that all the uses from a given User are together.
13161 llvm::sort(C&: Uses);
13162 RAUOVWUpdateListener Listener(*this, Uses);
13163
13164 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13165 UseIndex != UseIndexEnd; ) {
13166 // We know that this user uses some value of From. If it is the right
13167 // value, update it.
13168 SDNode *User = Uses[UseIndex].User;
13169 // If the node has been deleted by recursive CSE updates when updating
13170 // another node, then just skip this entry.
13171 if (User == nullptr) {
13172 ++UseIndex;
13173 continue;
13174 }
13175
13176 // This node is about to morph, remove its old self from the CSE maps.
13177 RemoveNodeFromCSEMaps(N: User);
13178
13179 // The Uses array is sorted, so all the uses for a given User
13180 // are next to each other in the list.
13181 // To help reduce the number of CSE recomputations, process all
13182 // the uses of this user that we can find this way.
13183 do {
13184 unsigned i = Uses[UseIndex].Index;
13185 SDUse &Use = *Uses[UseIndex].Use;
13186 ++UseIndex;
13187
13188 Use.set(To[i]);
13189 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13190
13191 // Now that we have modified User, add it back to the CSE maps. If it
13192 // already exists there, recursively merge the results together.
13193 AddModifiedNodeToCSEMaps(N: User);
13194 }
13195}
13196
13197/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13198/// based on their topological order. It returns the maximum id and a vector
13199/// of the SDNodes* in assigned order by reference.
13200unsigned SelectionDAG::AssignTopologicalOrder() {
13201 unsigned DAGSize = 0;
13202
13203 // SortedPos tracks the progress of the algorithm. Nodes before it are
13204 // sorted, nodes after it are unsorted. When the algorithm completes
13205 // it is at the end of the list.
13206 allnodes_iterator SortedPos = allnodes_begin();
13207
13208 // Visit all the nodes. Move nodes with no operands to the front of
13209 // the list immediately. Annotate nodes that do have operands with their
13210 // operand count. Before we do this, the Node Id fields of the nodes
13211 // may contain arbitrary values. After, the Node Id fields for nodes
13212 // before SortedPos will contain the topological sort index, and the
13213 // Node Id fields for nodes At SortedPos and after will contain the
13214 // count of outstanding operands.
13215 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13216 checkForCycles(N: &N, DAG: this);
13217 unsigned Degree = N.getNumOperands();
13218 if (Degree == 0) {
13219 // A node with no uses, add it to the result array immediately.
13220 N.setNodeId(DAGSize++);
13221 allnodes_iterator Q(&N);
13222 if (Q != SortedPos)
13223 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13224 assert(SortedPos != AllNodes.end() && "Overran node list");
13225 ++SortedPos;
13226 } else {
13227 // Temporarily use the Node Id as scratch space for the degree count.
13228 N.setNodeId(Degree);
13229 }
13230 }
13231
13232 // Visit all the nodes. As we iterate, move nodes into sorted order,
13233 // such that by the time the end is reached all nodes will be sorted.
13234 for (SDNode &Node : allnodes()) {
13235 SDNode *N = &Node;
13236 checkForCycles(N, DAG: this);
13237 // N is in sorted position, so all its uses have one less operand
13238 // that needs to be sorted.
13239 for (SDNode *P : N->users()) {
13240 unsigned Degree = P->getNodeId();
13241 assert(Degree != 0 && "Invalid node degree");
13242 --Degree;
13243 if (Degree == 0) {
13244 // All of P's operands are sorted, so P may sorted now.
13245 P->setNodeId(DAGSize++);
13246 if (P->getIterator() != SortedPos)
13247 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13248 assert(SortedPos != AllNodes.end() && "Overran node list");
13249 ++SortedPos;
13250 } else {
13251 // Update P's outstanding operand count.
13252 P->setNodeId(Degree);
13253 }
13254 }
13255 if (Node.getIterator() == SortedPos) {
13256#ifndef NDEBUG
13257 allnodes_iterator I(N);
13258 SDNode *S = &*++I;
13259 dbgs() << "Overran sorted position:\n";
13260 S->dumprFull(this); dbgs() << "\n";
13261 dbgs() << "Checking if this is due to cycles\n";
13262 checkForCycles(this, true);
13263#endif
13264 llvm_unreachable(nullptr);
13265 }
13266 }
13267
13268 assert(SortedPos == AllNodes.end() &&
13269 "Topological sort incomplete!");
13270 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13271 "First node in topological sort is not the entry token!");
13272 assert(AllNodes.front().getNodeId() == 0 &&
13273 "First node in topological sort has non-zero id!");
13274 assert(AllNodes.front().getNumOperands() == 0 &&
13275 "First node in topological sort has operands!");
13276 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13277 "Last node in topologic sort has unexpected id!");
13278 assert(AllNodes.back().use_empty() &&
13279 "Last node in topologic sort has users!");
13280 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13281 return DAGSize;
13282}
13283
13284void SelectionDAG::getTopologicallyOrderedNodes(
13285 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13286 SortedNodes.clear();
13287 // Node -> remaining number of outstanding operands.
13288 DenseMap<const SDNode *, unsigned> RemainingOperands;
13289
13290 // Put nodes without any operands into SortedNodes first.
13291 for (const SDNode &N : allnodes()) {
13292 checkForCycles(N: &N, DAG: this);
13293 unsigned NumOperands = N.getNumOperands();
13294 if (NumOperands == 0)
13295 SortedNodes.push_back(Elt: &N);
13296 else
13297 // Record their total number of outstanding operands.
13298 RemainingOperands[&N] = NumOperands;
13299 }
13300
13301 // A node is pushed into SortedNodes when all of its operands (predecessors in
13302 // the graph) are also in SortedNodes.
13303 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13304 const SDNode *N = SortedNodes[i];
13305 for (const SDNode *U : N->users()) {
13306 // HandleSDNode is never part of a DAG and therefore has no entry in
13307 // RemainingOperands.
13308 if (U->getOpcode() == ISD::HANDLENODE)
13309 continue;
13310 unsigned &NumRemOperands = RemainingOperands[U];
13311 assert(NumRemOperands && "Invalid number of remaining operands");
13312 --NumRemOperands;
13313 if (!NumRemOperands)
13314 SortedNodes.push_back(Elt: U);
13315 }
13316 }
13317
13318 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13319 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13320 "First node in topological sort is not the entry token");
13321 assert(SortedNodes.front()->getNumOperands() == 0 &&
13322 "First node in topological sort has operands");
13323}
13324
13325/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13326/// value is produced by SD.
13327void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13328 for (SDNode *SD : DB->getSDNodes()) {
13329 if (!SD)
13330 continue;
13331 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13332 SD->setHasDebugValue(true);
13333 }
13334 DbgInfo->add(V: DB, isParameter);
13335}
13336
13337void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13338
13339SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13340 SDValue NewMemOpChain) {
13341 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13342 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13343 // The new memory operation must have the same position as the old load in
13344 // terms of memory dependency. Create a TokenFactor for the old load and new
13345 // memory operation and update uses of the old load's output chain to use that
13346 // TokenFactor.
13347 if (OldChain == NewMemOpChain || OldChain.use_empty())
13348 return NewMemOpChain;
13349
13350 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13351 N1: OldChain, N2: NewMemOpChain);
13352 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13353 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13354 return TokenFactor;
13355}
13356
13357SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13358 SDValue NewMemOp) {
13359 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13360 SDValue OldChain = SDValue(OldLoad, 1);
13361 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13362 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13363}
13364
13365SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13366 Function **OutFunction) {
13367 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13368
13369 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13370 auto *Module = MF->getFunction().getParent();
13371 auto *Function = Module->getFunction(Name: Symbol);
13372
13373 if (OutFunction != nullptr)
13374 *OutFunction = Function;
13375
13376 if (Function != nullptr) {
13377 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13378 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13379 }
13380
13381 std::string ErrorStr;
13382 raw_string_ostream ErrorFormatter(ErrorStr);
13383 ErrorFormatter << "Undefined external symbol ";
13384 ErrorFormatter << '"' << Symbol << '"';
13385 report_fatal_error(reason: Twine(ErrorStr));
13386}
13387
13388//===----------------------------------------------------------------------===//
13389// SDNode Class
13390//===----------------------------------------------------------------------===//
13391
13392bool llvm::isNullConstant(SDValue V) {
13393 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13394 return Const != nullptr && Const->isZero();
13395}
13396
13397bool llvm::isNullConstantOrUndef(SDValue V) {
13398 return V.isUndef() || isNullConstant(V);
13399}
13400
13401bool llvm::isNullFPConstant(SDValue V) {
13402 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13403 return Const != nullptr && Const->isZero() && !Const->isNegative();
13404}
13405
13406bool llvm::isAllOnesConstant(SDValue V) {
13407 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13408 return Const != nullptr && Const->isAllOnes();
13409}
13410
13411bool llvm::isOneConstant(SDValue V) {
13412 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13413 return Const != nullptr && Const->isOne();
13414}
13415
13416bool llvm::isMinSignedConstant(SDValue V) {
13417 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13418 return Const != nullptr && Const->isMinSignedValue();
13419}
13420
13421bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
13422 unsigned OperandNo) {
13423 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13424 // TODO: Target-specific opcodes could be added.
13425 if (auto *ConstV = isConstOrConstSplat(N: V, /*AllowUndefs*/ false,
13426 /*AllowTruncation*/ true)) {
13427 APInt Const = ConstV->getAPIntValue().trunc(width: V.getScalarValueSizeInBits());
13428 switch (Opcode) {
13429 case ISD::ADD:
13430 case ISD::OR:
13431 case ISD::XOR:
13432 case ISD::UMAX:
13433 return Const.isZero();
13434 case ISD::MUL:
13435 return Const.isOne();
13436 case ISD::AND:
13437 case ISD::UMIN:
13438 return Const.isAllOnes();
13439 case ISD::SMAX:
13440 return Const.isMinSignedValue();
13441 case ISD::SMIN:
13442 return Const.isMaxSignedValue();
13443 case ISD::SUB:
13444 case ISD::SHL:
13445 case ISD::SRA:
13446 case ISD::SRL:
13447 return OperandNo == 1 && Const.isZero();
13448 case ISD::UDIV:
13449 case ISD::SDIV:
13450 return OperandNo == 1 && Const.isOne();
13451 }
13452 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) {
13453 switch (Opcode) {
13454 case ISD::FADD:
13455 return ConstFP->isZero() &&
13456 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13457 case ISD::FSUB:
13458 return OperandNo == 1 && ConstFP->isZero() &&
13459 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13460 case ISD::FMUL:
13461 return ConstFP->isExactlyValue(V: 1.0);
13462 case ISD::FDIV:
13463 return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0);
13464 case ISD::FMINNUM:
13465 case ISD::FMAXNUM: {
13466 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
13467 EVT VT = V.getValueType();
13468 const fltSemantics &Semantics = VT.getFltSemantics();
13469 APFloat NeutralAF = !Flags.hasNoNaNs()
13470 ? APFloat::getQNaN(Sem: Semantics)
13471 : !Flags.hasNoInfs()
13472 ? APFloat::getInf(Sem: Semantics)
13473 : APFloat::getLargest(Sem: Semantics);
13474 if (Opcode == ISD::FMAXNUM)
13475 NeutralAF.changeSign();
13476
13477 return ConstFP->isExactlyValue(V: NeutralAF);
13478 }
13479 }
13480 }
13481 return false;
13482}
13483
13484SDValue llvm::peekThroughBitcasts(SDValue V) {
13485 while (V.getOpcode() == ISD::BITCAST)
13486 V = V.getOperand(i: 0);
13487 return V;
13488}
13489
13490SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13491 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13492 V = V.getOperand(i: 0);
13493 return V;
13494}
13495
13496SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13497 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13498 V = V.getOperand(i: 0);
13499 return V;
13500}
13501
13502SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13503 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13504 SDValue InVec = V.getOperand(i: 0);
13505 SDValue EltNo = V.getOperand(i: 2);
13506 EVT VT = InVec.getValueType();
13507 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13508 if (IndexC && VT.isFixedLengthVector() &&
13509 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13510 !DemandedElts[IndexC->getZExtValue()]) {
13511 V = InVec;
13512 continue;
13513 }
13514 break;
13515 }
13516 return V;
13517}
13518
13519SDValue llvm::peekThroughTruncates(SDValue V) {
13520 while (V.getOpcode() == ISD::TRUNCATE)
13521 V = V.getOperand(i: 0);
13522 return V;
13523}
13524
13525bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13526 if (V.getOpcode() != ISD::XOR)
13527 return false;
13528 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13529 unsigned NumBits = V.getScalarValueSizeInBits();
13530 ConstantSDNode *C =
13531 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13532 return C && (C->getAPIntValue().countr_one() >= NumBits);
13533}
13534
13535ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13536 bool AllowTruncation) {
13537 APInt DemandedElts = getDemandAllEltsMask(V: N);
13538 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13539}
13540
13541ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13542 bool AllowUndefs,
13543 bool AllowTruncation) {
13544 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13545 return CN;
13546
13547 // SplatVectors can truncate their operands. Ignore that case here unless
13548 // AllowTruncation is set.
13549 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13550 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13551 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13552 EVT CVT = CN->getValueType(ResNo: 0);
13553 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13554 if (AllowTruncation || CVT == VecEltVT)
13555 return CN;
13556 }
13557 }
13558
13559 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13560 BitVector UndefElements;
13561 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13562
13563 // BuildVectors can truncate their operands. Ignore that case here unless
13564 // AllowTruncation is set.
13565 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13566 if (CN && (UndefElements.none() || AllowUndefs)) {
13567 EVT CVT = CN->getValueType(ResNo: 0);
13568 EVT NSVT = N.getValueType().getScalarType();
13569 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13570 if (AllowTruncation || (CVT == NSVT))
13571 return CN;
13572 }
13573 }
13574
13575 return nullptr;
13576}
13577
13578ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13579 APInt DemandedElts = getDemandAllEltsMask(V: N);
13580 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13581}
13582
13583ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13584 const APInt &DemandedElts,
13585 bool AllowUndefs) {
13586 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13587 return CN;
13588
13589 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13590 BitVector UndefElements;
13591 ConstantFPSDNode *CN =
13592 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13593 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13594 if (CN && (UndefElements.none() || AllowUndefs))
13595 return CN;
13596 }
13597
13598 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13599 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13600 return CN;
13601
13602 return nullptr;
13603}
13604
13605bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13606 // TODO: may want to use peekThroughBitcast() here.
13607 ConstantSDNode *C =
13608 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13609 return C && C->isZero();
13610}
13611
13612bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
13613 ConstantSDNode *C =
13614 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
13615 return C && C->isOne();
13616}
13617
13618bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
13619 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13620 return C && C->isExactlyValue(V: 1.0);
13621}
13622
13623bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
13624 N = peekThroughBitcasts(V: N);
13625 unsigned BitWidth = N.getScalarValueSizeInBits();
13626 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13627 return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth;
13628}
13629
13630bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
13631 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13632 return C && APInt::isSameValue(I1: C->getAPIntValue(),
13633 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
13634}
13635
13636bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
13637 N = peekThroughBitcasts(V: N);
13638 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
13639 return C && C->isZero();
13640}
13641
13642bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
13643 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13644 return C && C->isZero();
13645}
13646
13647HandleSDNode::~HandleSDNode() {
13648 DropOperands();
13649}
13650
13651MemSDNode::MemSDNode(
13652 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
13653 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
13654 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
13655 bool IsVolatile = false;
13656 bool IsNonTemporal = false;
13657 bool IsDereferenceable = true;
13658 bool IsInvariant = true;
13659 for (const MachineMemOperand *MMO : memoperands()) {
13660 IsVolatile |= MMO->isVolatile();
13661 IsNonTemporal |= MMO->isNonTemporal();
13662 IsDereferenceable &= MMO->isDereferenceable();
13663 IsInvariant &= MMO->isInvariant();
13664 }
13665 MemSDNodeBits.IsVolatile = IsVolatile;
13666 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
13667 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
13668 MemSDNodeBits.IsInvariant = IsInvariant;
13669
13670 // For the single-MMO case, we check here that the size of the memory operand
13671 // fits within the size of the MMO. This is because the MMO might indicate
13672 // only a possible address range instead of specifying the affected memory
13673 // addresses precisely.
13674 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
13675 TypeSize::isKnownLE(memvt.getStoreSize(),
13676 getMemOperand()->getSize().getValue())) &&
13677 "Size mismatch!");
13678}
13679
13680/// Profile - Gather unique data for the node.
13681///
13682void SDNode::Profile(FoldingSetNodeID &ID) const {
13683 AddNodeIDNode(ID, N: this);
13684}
13685
13686namespace {
13687
13688 struct EVTArray {
13689 std::vector<EVT> VTs;
13690
13691 EVTArray() {
13692 VTs.reserve(n: MVT::VALUETYPE_SIZE);
13693 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
13694 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
13695 }
13696 };
13697
13698} // end anonymous namespace
13699
13700/// getValueTypeList - Return a pointer to the specified value type.
13701///
13702const EVT *SDNode::getValueTypeList(MVT VT) {
13703 static EVTArray SimpleVTArray;
13704
13705 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
13706 return &SimpleVTArray.VTs[VT.SimpleTy];
13707}
13708
13709/// hasAnyUseOfValue - Return true if there are any use of the indicated
13710/// value. This method ignores uses of other values defined by this operation.
13711bool SDNode::hasAnyUseOfValue(unsigned Value) const {
13712 assert(Value < getNumValues() && "Bad value!");
13713
13714 for (SDUse &U : uses())
13715 if (U.getResNo() == Value)
13716 return true;
13717
13718 return false;
13719}
13720
13721/// isOnlyUserOf - Return true if this node is the only use of N.
13722bool SDNode::isOnlyUserOf(const SDNode *N) const {
13723 bool Seen = false;
13724 for (const SDNode *User : N->users()) {
13725 if (User == this)
13726 Seen = true;
13727 else
13728 return false;
13729 }
13730
13731 return Seen;
13732}
13733
13734/// Return true if the only users of N are contained in Nodes.
13735bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
13736 bool Seen = false;
13737 for (const SDNode *User : N->users()) {
13738 if (llvm::is_contained(Range&: Nodes, Element: User))
13739 Seen = true;
13740 else
13741 return false;
13742 }
13743
13744 return Seen;
13745}
13746
13747/// Return true if the referenced return value is an operand of N.
13748bool SDValue::isOperandOf(const SDNode *N) const {
13749 return is_contained(Range: N->op_values(), Element: *this);
13750}
13751
13752bool SDNode::isOperandOf(const SDNode *N) const {
13753 return any_of(Range: N->op_values(),
13754 P: [this](SDValue Op) { return this == Op.getNode(); });
13755}
13756
13757/// reachesChainWithoutSideEffects - Return true if this operand (which must
13758/// be a chain) reaches the specified operand without crossing any
13759/// side-effecting instructions on any chain path. In practice, this looks
13760/// through token factors and non-volatile loads. In order to remain efficient,
13761/// this only looks a couple of nodes in, it does not do an exhaustive search.
13762///
13763/// Note that we only need to examine chains when we're searching for
13764/// side-effects; SelectionDAG requires that all side-effects are represented
13765/// by chains, even if another operand would force a specific ordering. This
13766/// constraint is necessary to allow transformations like splitting loads.
13767bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
13768 unsigned Depth) const {
13769 if (*this == Dest) return true;
13770
13771 // Don't search too deeply, we just want to be able to see through
13772 // TokenFactor's etc.
13773 if (Depth == 0) return false;
13774
13775 // If this is a token factor, all inputs to the TF happen in parallel.
13776 if (getOpcode() == ISD::TokenFactor) {
13777 // First, try a shallow search.
13778 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
13779 // We found the chain we want as an operand of this TokenFactor.
13780 // Essentially, we reach the chain without side-effects if we could
13781 // serialize the TokenFactor into a simple chain of operations with
13782 // Dest as the last operation. This is automatically true if the
13783 // chain has one use: there are no other ordering constraints.
13784 // If the chain has more than one use, we give up: some other
13785 // use of Dest might force a side-effect between Dest and the current
13786 // node.
13787 if (Dest.hasOneUse())
13788 return true;
13789 }
13790 // Next, try a deep search: check whether every operand of the TokenFactor
13791 // reaches Dest.
13792 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
13793 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
13794 });
13795 }
13796
13797 // Loads don't have side effects, look through them.
13798 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
13799 if (Ld->isUnordered())
13800 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
13801 }
13802 return false;
13803}
13804
13805bool SDNode::hasPredecessor(const SDNode *N) const {
13806 SmallPtrSet<const SDNode *, 32> Visited;
13807 SmallVector<const SDNode *, 16> Worklist;
13808 Worklist.push_back(Elt: this);
13809 return hasPredecessorHelper(N, Visited, Worklist);
13810}
13811
13812void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
13813 this->Flags &= Flags;
13814}
13815
13816SDValue
13817SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
13818 ArrayRef<ISD::NodeType> CandidateBinOps,
13819 bool AllowPartials) {
13820 // The pattern must end in an extract from index 0.
13821 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
13822 !isNullConstant(V: Extract->getOperand(Num: 1)))
13823 return SDValue();
13824
13825 // Match against one of the candidate binary ops.
13826 SDValue Op = Extract->getOperand(Num: 0);
13827 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
13828 return Op.getOpcode() == unsigned(BinOp);
13829 }))
13830 return SDValue();
13831
13832 // Floating-point reductions may require relaxed constraints on the final step
13833 // of the reduction because they may reorder intermediate operations.
13834 unsigned CandidateBinOp = Op.getOpcode();
13835 if (Op.getValueType().isFloatingPoint()) {
13836 SDNodeFlags Flags = Op->getFlags();
13837 switch (CandidateBinOp) {
13838 case ISD::FADD:
13839 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
13840 return SDValue();
13841 break;
13842 default:
13843 llvm_unreachable("Unhandled FP opcode for binop reduction");
13844 }
13845 }
13846
13847 // Matching failed - attempt to see if we did enough stages that a partial
13848 // reduction from a subvector is possible.
13849 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
13850 if (!AllowPartials || !Op)
13851 return SDValue();
13852 EVT OpVT = Op.getValueType();
13853 EVT OpSVT = OpVT.getScalarType();
13854 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
13855 if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
13856 return SDValue();
13857 BinOp = (ISD::NodeType)CandidateBinOp;
13858 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
13859 };
13860
13861 // At each stage, we're looking for something that looks like:
13862 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
13863 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
13864 // i32 undef, i32 undef, i32 undef, i32 undef>
13865 // %a = binop <8 x i32> %op, %s
13866 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
13867 // we expect something like:
13868 // <4,5,6,7,u,u,u,u>
13869 // <2,3,u,u,u,u,u,u>
13870 // <1,u,u,u,u,u,u,u>
13871 // While a partial reduction match would be:
13872 // <2,3,u,u,u,u,u,u>
13873 // <1,u,u,u,u,u,u,u>
13874 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
13875 SDValue PrevOp;
13876 for (unsigned i = 0; i < Stages; ++i) {
13877 unsigned MaskEnd = (1 << i);
13878
13879 if (Op.getOpcode() != CandidateBinOp)
13880 return PartialReduction(PrevOp, MaskEnd);
13881
13882 SDValue Op0 = Op.getOperand(i: 0);
13883 SDValue Op1 = Op.getOperand(i: 1);
13884
13885 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
13886 if (Shuffle) {
13887 Op = Op1;
13888 } else {
13889 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
13890 Op = Op0;
13891 }
13892
13893 // The first operand of the shuffle should be the same as the other operand
13894 // of the binop.
13895 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
13896 return PartialReduction(PrevOp, MaskEnd);
13897
13898 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
13899 for (int Index = 0; Index < (int)MaskEnd; ++Index)
13900 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
13901 return PartialReduction(PrevOp, MaskEnd);
13902
13903 PrevOp = Op;
13904 }
13905
13906 // Handle subvector reductions, which tend to appear after the shuffle
13907 // reduction stages.
13908 while (Op.getOpcode() == CandidateBinOp) {
13909 unsigned NumElts = Op.getValueType().getVectorNumElements();
13910 SDValue Op0 = Op.getOperand(i: 0);
13911 SDValue Op1 = Op.getOperand(i: 1);
13912 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13913 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13914 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
13915 break;
13916 SDValue Src = Op0.getOperand(i: 0);
13917 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
13918 if (NumSrcElts != (2 * NumElts))
13919 break;
13920 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
13921 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
13922 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
13923 Op0.getConstantOperandAPInt(i: 1) == NumElts))
13924 break;
13925 Op = Src;
13926 }
13927
13928 BinOp = (ISD::NodeType)CandidateBinOp;
13929 return Op;
13930}
13931
13932SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
13933 EVT VT = N->getValueType(ResNo: 0);
13934 EVT EltVT = VT.getVectorElementType();
13935 unsigned NE = VT.getVectorNumElements();
13936
13937 SDLoc dl(N);
13938
13939 // If ResNE is 0, fully unroll the vector op.
13940 if (ResNE == 0)
13941 ResNE = NE;
13942 else if (NE > ResNE)
13943 NE = ResNE;
13944
13945 if (N->getNumValues() == 2) {
13946 SmallVector<SDValue, 8> Scalars0, Scalars1;
13947 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13948 EVT VT1 = N->getValueType(ResNo: 1);
13949 EVT EltVT1 = VT1.getVectorElementType();
13950
13951 unsigned i;
13952 for (i = 0; i != NE; ++i) {
13953 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13954 SDValue Operand = N->getOperand(Num: j);
13955 EVT OperandVT = Operand.getValueType();
13956
13957 // A vector operand; extract a single element.
13958 EVT OperandEltVT = OperandVT.getVectorElementType();
13959 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13960 }
13961
13962 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
13963 Scalars0.push_back(Elt: EltOp);
13964 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
13965 }
13966
13967 for (; i < ResNE; ++i) {
13968 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
13969 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
13970 }
13971
13972 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
13973 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, NumElements: ResNE);
13974 SDValue Vec0 = getBuildVector(VT: VecVT, DL: dl, Ops: Scalars0);
13975 SDValue Vec1 = getBuildVector(VT: VecVT1, DL: dl, Ops: Scalars1);
13976 return getMergeValues(Ops: {Vec0, Vec1}, dl);
13977 }
13978
13979 assert(N->getNumValues() == 1 &&
13980 "Can't unroll a vector with multiple results!");
13981
13982 SmallVector<SDValue, 8> Scalars;
13983 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13984
13985 unsigned i;
13986 for (i= 0; i != NE; ++i) {
13987 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13988 SDValue Operand = N->getOperand(Num: j);
13989 EVT OperandVT = Operand.getValueType();
13990 if (OperandVT.isVector()) {
13991 // A vector operand; extract a single element.
13992 EVT OperandEltVT = OperandVT.getVectorElementType();
13993 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13994 } else {
13995 // A scalar operand; just use it as is.
13996 Operands[j] = Operand;
13997 }
13998 }
13999
14000 switch (N->getOpcode()) {
14001 default: {
14002 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
14003 Flags: N->getFlags()));
14004 break;
14005 }
14006 case ISD::VSELECT:
14007 Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
14008 break;
14009 case ISD::SHL:
14010 case ISD::SRA:
14011 case ISD::SRL:
14012 case ISD::ROTL:
14013 case ISD::ROTR:
14014 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
14015 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
14016 Op: Operands[1])));
14017 break;
14018 case ISD::SIGN_EXTEND_INREG: {
14019 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
14020 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
14021 N1: Operands[0],
14022 N2: getValueType(VT: ExtVT)));
14023 break;
14024 }
14025 case ISD::ADDRSPACECAST: {
14026 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
14027 Scalars.push_back(Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0],
14028 SrcAS: ASC->getSrcAddressSpace(),
14029 DestAS: ASC->getDestAddressSpace()));
14030 break;
14031 }
14032 }
14033 }
14034
14035 for (; i < ResNE; ++i)
14036 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
14037
14038 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14039 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
14040}
14041
14042std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14043 SDNode *N, unsigned ResNE) {
14044 unsigned Opcode = N->getOpcode();
14045 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14046 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14047 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14048 "Expected an overflow opcode");
14049
14050 EVT ResVT = N->getValueType(ResNo: 0);
14051 EVT OvVT = N->getValueType(ResNo: 1);
14052 EVT ResEltVT = ResVT.getVectorElementType();
14053 EVT OvEltVT = OvVT.getVectorElementType();
14054 SDLoc dl(N);
14055
14056 // If ResNE is 0, fully unroll the vector op.
14057 unsigned NE = ResVT.getVectorNumElements();
14058 if (ResNE == 0)
14059 ResNE = NE;
14060 else if (NE > ResNE)
14061 NE = ResNE;
14062
14063 SmallVector<SDValue, 8> LHSScalars;
14064 SmallVector<SDValue, 8> RHSScalars;
14065 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
14066 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
14067
14068 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
14069 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
14070 SmallVector<SDValue, 8> ResScalars;
14071 SmallVector<SDValue, 8> OvScalars;
14072 for (unsigned i = 0; i < NE; ++i) {
14073 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
14074 SDValue Ov =
14075 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
14076 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
14077 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
14078
14079 ResScalars.push_back(Elt: Res);
14080 OvScalars.push_back(Elt: Ov);
14081 }
14082
14083 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
14084 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
14085
14086 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
14087 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
14088 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
14089 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
14090}
14091
14092bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
14093 LoadSDNode *Base,
14094 unsigned Bytes,
14095 int Dist) const {
14096 if (LD->isVolatile() || Base->isVolatile())
14097 return false;
14098 // TODO: probably too restrictive for atomics, revisit
14099 if (!LD->isSimple())
14100 return false;
14101 if (LD->isIndexed() || Base->isIndexed())
14102 return false;
14103 if (LD->getChain() != Base->getChain())
14104 return false;
14105 EVT VT = LD->getMemoryVT();
14106 if (VT.getSizeInBits() / 8 != Bytes)
14107 return false;
14108
14109 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
14110 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
14111
14112 int64_t Offset = 0;
14113 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
14114 return (Dist * (int64_t)Bytes == Offset);
14115 return false;
14116}
14117
14118/// InferPtrAlignment - Infer alignment of a load / store address. Return
14119/// std::nullopt if it cannot be inferred.
14120MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
14121 // If this is a GlobalAddress + cst, return the alignment.
14122 const GlobalValue *GV = nullptr;
14123 int64_t GVOffset = 0;
14124 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
14125 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14126 KnownBits Known(PtrWidth);
14127 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
14128 unsigned AlignBits = Known.countMinTrailingZeros();
14129 if (AlignBits)
14130 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
14131 }
14132
14133 // If this is a direct reference to a stack slot, use information about the
14134 // stack slot's alignment.
14135 int FrameIdx = INT_MIN;
14136 int64_t FrameOffset = 0;
14137 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
14138 FrameIdx = FI->getIndex();
14139 } else if (isBaseWithConstantOffset(Op: Ptr) &&
14140 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
14141 // Handle FI+Cst
14142 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
14143 FrameOffset = Ptr.getConstantOperandVal(i: 1);
14144 }
14145
14146 if (FrameIdx != INT_MIN) {
14147 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
14148 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
14149 }
14150
14151 return std::nullopt;
14152}
14153
14154/// Split the scalar node with EXTRACT_ELEMENT using the provided
14155/// VTs and return the low/high part.
14156std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14157 const SDLoc &DL,
14158 const EVT &LoVT,
14159 const EVT &HiVT) {
14160 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14161 "Split node must be a scalar type");
14162 SDValue Lo =
14163 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
14164 SDValue Hi =
14165 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
14166 return std::make_pair(x&: Lo, y&: Hi);
14167}
14168
14169/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14170/// which is split (or expanded) into two not necessarily identical pieces.
14171std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14172 // Currently all types are split in half.
14173 EVT LoVT, HiVT;
14174 if (!VT.isVector())
14175 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
14176 else
14177 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
14178
14179 return std::make_pair(x&: LoVT, y&: HiVT);
14180}
14181
14182/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14183/// type, dependent on an enveloping VT that has been split into two identical
14184/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14185std::pair<EVT, EVT>
14186SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
14187 bool *HiIsEmpty) const {
14188 EVT EltTp = VT.getVectorElementType();
14189 // Examples:
14190 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14191 // custom VL=9 with enveloping VL=8/8 yields 8/1
14192 // custom VL=10 with enveloping VL=8/8 yields 8/2
14193 // etc.
14194 ElementCount VTNumElts = VT.getVectorElementCount();
14195 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14196 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14197 "Mixing fixed width and scalable vectors when enveloping a type");
14198 EVT LoVT, HiVT;
14199 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14200 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14201 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
14202 *HiIsEmpty = false;
14203 } else {
14204 // Flag that hi type has zero storage size, but return split envelop type
14205 // (this would be easier if vector types with zero elements were allowed).
14206 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14207 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14208 *HiIsEmpty = true;
14209 }
14210 return std::make_pair(x&: LoVT, y&: HiVT);
14211}
14212
14213/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14214/// low/high part.
14215std::pair<SDValue, SDValue>
14216SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14217 const EVT &HiVT) {
14218 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14219 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14220 "Splitting vector with an invalid mixture of fixed and scalable "
14221 "vector types");
14222 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14223 N.getValueType().getVectorMinNumElements() &&
14224 "More vector elements requested than available!");
14225 SDValue Lo, Hi;
14226 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14227 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14228 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14229 // IDX with the runtime scaling factor of the result vector type. For
14230 // fixed-width result vectors, that runtime scaling factor is 1.
14231 Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
14232 N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
14233 return std::make_pair(x&: Lo, y&: Hi);
14234}
14235
14236std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14237 const SDLoc &DL) {
14238 // Split the vector length parameter.
14239 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14240 EVT VT = N.getValueType();
14241 assert(VecVT.getVectorElementCount().isKnownEven() &&
14242 "Expecting the mask to be an evenly-sized vector");
14243 SDValue HalfNumElts = getElementCount(
14244 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14245 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14246 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14247 return std::make_pair(x&: Lo, y&: Hi);
14248}
14249
14250/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14251SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14252 EVT VT = N.getValueType();
14253 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14254 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14255 return getInsertSubvector(DL, Vec: getUNDEF(VT: WideVT), SubVec: N, Idx: 0);
14256}
14257
14258void SelectionDAG::ExtractVectorElements(SDValue Op,
14259 SmallVectorImpl<SDValue> &Args,
14260 unsigned Start, unsigned Count,
14261 EVT EltVT) {
14262 EVT VT = Op.getValueType();
14263 if (Count == 0)
14264 Count = VT.getVectorNumElements();
14265 if (EltVT == EVT())
14266 EltVT = VT.getVectorElementType();
14267 SDLoc SL(Op);
14268 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14269 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14270 }
14271}
14272
14273// getAddressSpace - Return the address space this GlobalAddress belongs to.
14274unsigned GlobalAddressSDNode::getAddressSpace() const {
14275 return getGlobal()->getType()->getAddressSpace();
14276}
14277
14278Type *ConstantPoolSDNode::getType() const {
14279 if (isMachineConstantPoolEntry())
14280 return Val.MachineCPVal->getType();
14281 return Val.ConstVal->getType();
14282}
14283
14284bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14285 unsigned &SplatBitSize,
14286 bool &HasAnyUndefs,
14287 unsigned MinSplatBits,
14288 bool IsBigEndian) const {
14289 EVT VT = getValueType(ResNo: 0);
14290 assert(VT.isVector() && "Expected a vector type");
14291 unsigned VecWidth = VT.getSizeInBits();
14292 if (MinSplatBits > VecWidth)
14293 return false;
14294
14295 // FIXME: The widths are based on this node's type, but build vectors can
14296 // truncate their operands.
14297 SplatValue = APInt(VecWidth, 0);
14298 SplatUndef = APInt(VecWidth, 0);
14299
14300 // Get the bits. Bits with undefined values (when the corresponding element
14301 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14302 // in SplatValue. If any of the values are not constant, give up and return
14303 // false.
14304 unsigned int NumOps = getNumOperands();
14305 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14306 unsigned EltWidth = VT.getScalarSizeInBits();
14307
14308 for (unsigned j = 0; j < NumOps; ++j) {
14309 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14310 SDValue OpVal = getOperand(Num: i);
14311 unsigned BitPos = j * EltWidth;
14312
14313 if (OpVal.isUndef())
14314 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14315 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14316 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14317 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14318 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14319 else
14320 return false;
14321 }
14322
14323 // The build_vector is all constants or undefs. Find the smallest element
14324 // size that splats the vector.
14325 HasAnyUndefs = (SplatUndef != 0);
14326
14327 // FIXME: This does not work for vectors with elements less than 8 bits.
14328 while (VecWidth > 8) {
14329 // If we can't split in half, stop here.
14330 if (VecWidth & 1)
14331 break;
14332
14333 unsigned HalfSize = VecWidth / 2;
14334 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14335 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14336 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14337 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14338
14339 // If the two halves do not match (ignoring undef bits), stop here.
14340 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14341 MinSplatBits > HalfSize)
14342 break;
14343
14344 SplatValue = HighValue | LowValue;
14345 SplatUndef = HighUndef & LowUndef;
14346
14347 VecWidth = HalfSize;
14348 }
14349
14350 // FIXME: The loop above only tries to split in halves. But if the input
14351 // vector for example is <3 x i16> it wouldn't be able to detect a
14352 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14353 // optimizations. I guess that back in the days when this helper was created
14354 // vectors normally was power-of-2 sized.
14355
14356 SplatBitSize = VecWidth;
14357 return true;
14358}
14359
14360SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14361 BitVector *UndefElements) const {
14362 unsigned NumOps = getNumOperands();
14363 if (UndefElements) {
14364 UndefElements->clear();
14365 UndefElements->resize(N: NumOps);
14366 }
14367 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14368 if (!DemandedElts)
14369 return SDValue();
14370 SDValue Splatted;
14371 for (unsigned i = 0; i != NumOps; ++i) {
14372 if (!DemandedElts[i])
14373 continue;
14374 SDValue Op = getOperand(Num: i);
14375 if (Op.isUndef()) {
14376 if (UndefElements)
14377 (*UndefElements)[i] = true;
14378 } else if (!Splatted) {
14379 Splatted = Op;
14380 } else if (Splatted != Op) {
14381 return SDValue();
14382 }
14383 }
14384
14385 if (!Splatted) {
14386 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14387 assert(getOperand(FirstDemandedIdx).isUndef() &&
14388 "Can only have a splat without a constant for all undefs.");
14389 return getOperand(Num: FirstDemandedIdx);
14390 }
14391
14392 return Splatted;
14393}
14394
14395SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14396 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14397 return getSplatValue(DemandedElts, UndefElements);
14398}
14399
14400bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14401 SmallVectorImpl<SDValue> &Sequence,
14402 BitVector *UndefElements) const {
14403 unsigned NumOps = getNumOperands();
14404 Sequence.clear();
14405 if (UndefElements) {
14406 UndefElements->clear();
14407 UndefElements->resize(N: NumOps);
14408 }
14409 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14410 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14411 return false;
14412
14413 // Set the undefs even if we don't find a sequence (like getSplatValue).
14414 if (UndefElements)
14415 for (unsigned I = 0; I != NumOps; ++I)
14416 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14417 (*UndefElements)[I] = true;
14418
14419 // Iteratively widen the sequence length looking for repetitions.
14420 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14421 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14422 for (unsigned I = 0; I != NumOps; ++I) {
14423 if (!DemandedElts[I])
14424 continue;
14425 SDValue &SeqOp = Sequence[I % SeqLen];
14426 SDValue Op = getOperand(Num: I);
14427 if (Op.isUndef()) {
14428 if (!SeqOp)
14429 SeqOp = Op;
14430 continue;
14431 }
14432 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14433 Sequence.clear();
14434 break;
14435 }
14436 SeqOp = Op;
14437 }
14438 if (!Sequence.empty())
14439 return true;
14440 }
14441
14442 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14443 return false;
14444}
14445
14446bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14447 BitVector *UndefElements) const {
14448 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14449 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14450}
14451
14452ConstantSDNode *
14453BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14454 BitVector *UndefElements) const {
14455 return dyn_cast_or_null<ConstantSDNode>(
14456 Val: getSplatValue(DemandedElts, UndefElements));
14457}
14458
14459ConstantSDNode *
14460BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14461 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14462}
14463
14464ConstantFPSDNode *
14465BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14466 BitVector *UndefElements) const {
14467 return dyn_cast_or_null<ConstantFPSDNode>(
14468 Val: getSplatValue(DemandedElts, UndefElements));
14469}
14470
14471ConstantFPSDNode *
14472BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14473 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14474}
14475
14476int32_t
14477BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14478 uint32_t BitWidth) const {
14479 if (ConstantFPSDNode *CN =
14480 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14481 bool IsExact;
14482 APSInt IntVal(BitWidth);
14483 const APFloat &APF = CN->getValueAPF();
14484 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14485 APFloat::opOK ||
14486 !IsExact)
14487 return -1;
14488
14489 return IntVal.exactLogBase2();
14490 }
14491 return -1;
14492}
14493
14494bool BuildVectorSDNode::getConstantRawBits(
14495 bool IsLittleEndian, unsigned DstEltSizeInBits,
14496 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14497 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14498 if (!isConstant())
14499 return false;
14500
14501 unsigned NumSrcOps = getNumOperands();
14502 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14503 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14504 "Invalid bitcast scale");
14505
14506 // Extract raw src bits.
14507 SmallVector<APInt> SrcBitElements(NumSrcOps,
14508 APInt::getZero(numBits: SrcEltSizeInBits));
14509 BitVector SrcUndeElements(NumSrcOps, false);
14510
14511 for (unsigned I = 0; I != NumSrcOps; ++I) {
14512 SDValue Op = getOperand(Num: I);
14513 if (Op.isUndef()) {
14514 SrcUndeElements.set(I);
14515 continue;
14516 }
14517 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14518 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14519 assert((CInt || CFP) && "Unknown constant");
14520 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14521 : CFP->getValueAPF().bitcastToAPInt();
14522 }
14523
14524 // Recast to dst width.
14525 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14526 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14527 return true;
14528}
14529
14530void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14531 unsigned DstEltSizeInBits,
14532 SmallVectorImpl<APInt> &DstBitElements,
14533 ArrayRef<APInt> SrcBitElements,
14534 BitVector &DstUndefElements,
14535 const BitVector &SrcUndefElements) {
14536 unsigned NumSrcOps = SrcBitElements.size();
14537 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14538 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14539 "Invalid bitcast scale");
14540 assert(NumSrcOps == SrcUndefElements.size() &&
14541 "Vector size mismatch");
14542
14543 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14544 DstUndefElements.clear();
14545 DstUndefElements.resize(N: NumDstOps, t: false);
14546 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14547
14548 // Concatenate src elements constant bits together into dst element.
14549 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14550 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14551 for (unsigned I = 0; I != NumDstOps; ++I) {
14552 DstUndefElements.set(I);
14553 APInt &DstBits = DstBitElements[I];
14554 for (unsigned J = 0; J != Scale; ++J) {
14555 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14556 if (SrcUndefElements[Idx])
14557 continue;
14558 DstUndefElements.reset(Idx: I);
14559 const APInt &SrcBits = SrcBitElements[Idx];
14560 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14561 "Illegal constant bitwidths");
14562 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14563 }
14564 }
14565 return;
14566 }
14567
14568 // Split src element constant bits into dst elements.
14569 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14570 for (unsigned I = 0; I != NumSrcOps; ++I) {
14571 if (SrcUndefElements[I]) {
14572 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14573 continue;
14574 }
14575 const APInt &SrcBits = SrcBitElements[I];
14576 for (unsigned J = 0; J != Scale; ++J) {
14577 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14578 APInt &DstBits = DstBitElements[Idx];
14579 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14580 }
14581 }
14582}
14583
14584bool BuildVectorSDNode::isConstant() const {
14585 for (const SDValue &Op : op_values()) {
14586 unsigned Opc = Op.getOpcode();
14587 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14588 return false;
14589 }
14590 return true;
14591}
14592
14593std::optional<std::pair<APInt, APInt>>
14594BuildVectorSDNode::isArithmeticSequence() const {
14595 unsigned NumOps = getNumOperands();
14596 if (NumOps < 2)
14597 return std::nullopt;
14598
14599 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14600 APInt Start, Stride;
14601 int FirstIdx = -1, SecondIdx = -1;
14602
14603 // Find the first two non-undef constant elements to determine Start and
14604 // Stride, then verify all remaining elements match the sequence.
14605 for (unsigned I = 0; I < NumOps; ++I) {
14606 SDValue Op = getOperand(Num: I);
14607 if (Op->isUndef())
14608 continue;
14609 if (!isa<ConstantSDNode>(Val: Op))
14610 return std::nullopt;
14611
14612 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
14613 if (FirstIdx < 0) {
14614 FirstIdx = I;
14615 Start = Val;
14616 } else if (SecondIdx < 0) {
14617 SecondIdx = I;
14618 // Compute stride using modular arithmetic. Simple division would handle
14619 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
14620 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
14621 // Note that modular arithmetic is agnostic to signed/unsigned.
14622 unsigned IdxDiff = I - FirstIdx;
14623 APInt ValDiff = Val - Start;
14624
14625 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
14626 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
14627 if (ValDiff.countr_zero() < CommonPow2Bits)
14628 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
14629 IdxDiff >>= CommonPow2Bits;
14630 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
14631
14632 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
14633 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
14634 // one, but we could try all candidates to handle more cases.
14635 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
14636 if (Stride.isZero())
14637 return std::nullopt;
14638
14639 // Step 3: Adjust Start based on the first defined element's index.
14640 Start -= Stride * FirstIdx;
14641 } else {
14642 // Verify this element matches the sequence.
14643 if (Val != Start + Stride * I)
14644 return std::nullopt;
14645 }
14646 }
14647
14648 // Need at least two defined elements.
14649 if (SecondIdx < 0)
14650 return std::nullopt;
14651
14652 return std::make_pair(x&: Start, y&: Stride);
14653}
14654
14655bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
14656 // Find the first non-undef value in the shuffle mask.
14657 unsigned i, e;
14658 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
14659 /* search */;
14660
14661 // If all elements are undefined, this shuffle can be considered a splat
14662 // (although it should eventually get simplified away completely).
14663 if (i == e)
14664 return true;
14665
14666 // Make sure all remaining elements are either undef or the same as the first
14667 // non-undef value.
14668 for (int Idx = Mask[i]; i != e; ++i)
14669 if (Mask[i] >= 0 && Mask[i] != Idx)
14670 return false;
14671 return true;
14672}
14673
14674// Returns true if it is a constant integer BuildVector or constant integer,
14675// possibly hidden by a bitcast.
14676bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
14677 SDValue N, bool AllowOpaques) const {
14678 N = peekThroughBitcasts(V: N);
14679
14680 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
14681 return AllowOpaques || !C->isOpaque();
14682
14683 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
14684 return true;
14685
14686 // Treat a GlobalAddress supporting constant offset folding as a
14687 // constant integer.
14688 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
14689 if (GA->getOpcode() == ISD::GlobalAddress &&
14690 TLI->isOffsetFoldingLegal(GA))
14691 return true;
14692
14693 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14694 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
14695 return true;
14696 return false;
14697}
14698
14699// Returns true if it is a constant float BuildVector or constant float.
14700bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
14701 if (isa<ConstantFPSDNode>(Val: N))
14702 return true;
14703
14704 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
14705 return true;
14706
14707 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14708 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
14709 return true;
14710
14711 return false;
14712}
14713
14714std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
14715 ConstantSDNode *Const =
14716 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
14717 if (!Const)
14718 return std::nullopt;
14719
14720 EVT VT = N->getValueType(ResNo: 0);
14721 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
14722 switch (TLI->getBooleanContents(Type: N.getValueType())) {
14723 case TargetLowering::ZeroOrOneBooleanContent:
14724 if (CVal.isOne())
14725 return true;
14726 if (CVal.isZero())
14727 return false;
14728 return std::nullopt;
14729 case TargetLowering::ZeroOrNegativeOneBooleanContent:
14730 if (CVal.isAllOnes())
14731 return true;
14732 if (CVal.isZero())
14733 return false;
14734 return std::nullopt;
14735 case TargetLowering::UndefinedBooleanContent:
14736 return CVal[0];
14737 }
14738 llvm_unreachable("Unknown BooleanContent enum");
14739}
14740
14741void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
14742 assert(!Node->OperandList && "Node already has operands");
14743 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
14744 "too many operands to fit into SDNode");
14745 SDUse *Ops = OperandRecycler.allocate(
14746 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
14747
14748 bool IsDivergent = false;
14749 for (unsigned I = 0; I != Vals.size(); ++I) {
14750 Ops[I].setUser(Node);
14751 Ops[I].setInitial(Vals[I]);
14752 EVT VT = Ops[I].getValueType();
14753
14754 // Skip Chain. It does not carry divergence.
14755 if (VT != MVT::Other &&
14756 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
14757 Ops[I].getNode()->isDivergent()) {
14758 IsDivergent = true;
14759 }
14760 }
14761 Node->NumOperands = Vals.size();
14762 Node->OperandList = Ops;
14763 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
14764 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
14765 Node->SDNodeBits.IsDivergent = IsDivergent;
14766 }
14767 checkForCycles(N: Node);
14768}
14769
14770SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
14771 SmallVectorImpl<SDValue> &Vals) {
14772 size_t Limit = SDNode::getMaxNumOperands();
14773 while (Vals.size() > Limit) {
14774 unsigned SliceIdx = Vals.size() - Limit;
14775 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
14776 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
14777 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
14778 Vals.emplace_back(Args&: NewTF);
14779 }
14780 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
14781}
14782
14783SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
14784 EVT VT, SDNodeFlags Flags) {
14785 switch (Opcode) {
14786 default:
14787 return SDValue();
14788 case ISD::ADD:
14789 case ISD::OR:
14790 case ISD::XOR:
14791 case ISD::UMAX:
14792 return getConstant(Val: 0, DL, VT);
14793 case ISD::MUL:
14794 return getConstant(Val: 1, DL, VT);
14795 case ISD::AND:
14796 case ISD::UMIN:
14797 return getAllOnesConstant(DL, VT);
14798 case ISD::SMAX:
14799 return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
14800 case ISD::SMIN:
14801 return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
14802 case ISD::FADD:
14803 // If flags allow, prefer positive zero since it's generally cheaper
14804 // to materialize on most targets.
14805 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
14806 case ISD::FMUL:
14807 return getConstantFP(Val: 1.0, DL, VT);
14808 case ISD::FMINNUM:
14809 case ISD::FMAXNUM: {
14810 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
14811 const fltSemantics &Semantics = VT.getFltSemantics();
14812 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
14813 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
14814 APFloat::getLargest(Sem: Semantics);
14815 if (Opcode == ISD::FMAXNUM)
14816 NeutralAF.changeSign();
14817
14818 return getConstantFP(V: NeutralAF, DL, VT);
14819 }
14820 case ISD::FMINIMUM:
14821 case ISD::FMAXIMUM: {
14822 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
14823 const fltSemantics &Semantics = VT.getFltSemantics();
14824 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
14825 : APFloat::getLargest(Sem: Semantics);
14826 if (Opcode == ISD::FMAXIMUM)
14827 NeutralAF.changeSign();
14828
14829 return getConstantFP(V: NeutralAF, DL, VT);
14830 }
14831
14832 }
14833}
14834
14835/// Helper used to make a call to a library function that has one argument of
14836/// pointer type.
14837///
14838/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
14839/// used to get or set floating-point state. They have one argument of pointer
14840/// type, which points to the memory region containing bits of the
14841/// floating-point state. The value returned by such function is ignored in the
14842/// created call.
14843///
14844/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
14845/// \param Ptr Pointer used to save/load state.
14846/// \param InChain Ingoing token chain.
14847/// \returns Outgoing chain token.
14848SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
14849 SDValue InChain,
14850 const SDLoc &DLoc) {
14851 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
14852 TargetLowering::ArgListTy Args;
14853 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
14854 RTLIB::LibcallImpl LibcallImpl =
14855 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
14856 if (LibcallImpl == RTLIB::Unsupported)
14857 reportFatalUsageError(reason: "emitting call to unsupported libcall");
14858
14859 SDValue Callee =
14860 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
14861 TargetLowering::CallLoweringInfo CLI(*this);
14862 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
14863 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
14864 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
14865 return TLI->LowerCallTo(CLI).second;
14866}
14867
14868void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
14869 assert(From && To && "Invalid SDNode; empty source SDValue?");
14870 auto I = SDEI.find(Val: From);
14871 if (I == SDEI.end())
14872 return;
14873
14874 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
14875 // the iterator, hence the need to make a copy to prevent a use-after-free.
14876 NodeExtraInfo NEI = I->second;
14877 if (LLVM_LIKELY(!NEI.PCSections)) {
14878 // No deep copy required for the types of extra info set.
14879 //
14880 // FIXME: Investigate if other types of extra info also need deep copy. This
14881 // depends on the types of nodes they can be attached to: if some extra info
14882 // is only ever attached to nodes where a replacement To node is always the
14883 // node where later use and propagation of the extra info has the intended
14884 // semantics, no deep copy is required.
14885 SDEI[To] = std::move(NEI);
14886 return;
14887 }
14888
14889 const SDNode *EntrySDN = getEntryNode().getNode();
14890
14891 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
14892 // through the replacement of From with To. Otherwise, replacements of a node
14893 // (From) with more complex nodes (To and its operands) may result in lost
14894 // extra info where the root node (To) is insignificant in further propagating
14895 // and using extra info when further lowering to MIR.
14896 //
14897 // In the first step pre-populate the visited set with the nodes reachable
14898 // from the old From node. This avoids copying NodeExtraInfo to parts of the
14899 // DAG that is not new and should be left untouched.
14900 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
14901 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
14902 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
14903 if (MaxDepth == 0) {
14904 // Remember this node in case we need to increase MaxDepth and continue
14905 // populating FromReach from this node.
14906 Leafs.emplace_back(Args&: N);
14907 return;
14908 }
14909 if (!FromReach.insert(V: N).second)
14910 return;
14911 for (const SDValue &Op : N->op_values())
14912 Self(Self, Op.getNode(), MaxDepth - 1);
14913 };
14914
14915 // Copy extra info to To and all its transitive operands (that are new).
14916 SmallPtrSet<const SDNode *, 8> Visited;
14917 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
14918 if (FromReach.contains(V: N))
14919 return true;
14920 if (!Visited.insert(Ptr: N).second)
14921 return true;
14922 if (EntrySDN == N)
14923 return false;
14924 for (const SDValue &Op : N->op_values()) {
14925 if (N == To && Op.getNode() == EntrySDN) {
14926 // Special case: New node's operand is the entry node; just need to
14927 // copy extra info to new node.
14928 break;
14929 }
14930 if (!Self(Self, Op.getNode()))
14931 return false;
14932 }
14933 // Copy only if entry node was not reached.
14934 SDEI[N] = std::move(NEI);
14935 return true;
14936 };
14937
14938 // We first try with a lower MaxDepth, assuming that the path to common
14939 // operands between From and To is relatively short. This significantly
14940 // improves performance in the common case. The initial MaxDepth is big
14941 // enough to avoid retry in the common case; the last MaxDepth is large
14942 // enough to avoid having to use the fallback below (and protects from
14943 // potential stack exhaustion from recursion).
14944 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
14945 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
14946 // StartFrom is the previous (or initial) set of leafs reachable at the
14947 // previous maximum depth.
14948 SmallVector<const SDNode *> StartFrom;
14949 std::swap(LHS&: StartFrom, RHS&: Leafs);
14950 for (const SDNode *N : StartFrom)
14951 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
14952 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
14953 return;
14954 // This should happen very rarely (reached the entry node).
14955 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
14956 assert(!Leafs.empty());
14957 }
14958
14959 // This should not happen - but if it did, that means the subgraph reachable
14960 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
14961 // could not visit all reachable common operands. Consequently, we were able
14962 // to reach the entry node.
14963 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
14964 assert(false && "From subgraph too complex - increase max. MaxDepth?");
14965 // Best-effort fallback if assertions disabled.
14966 SDEI[To] = std::move(NEI);
14967}
14968
14969#ifndef NDEBUG
14970static void checkForCyclesHelper(const SDNode *N,
14971 SmallPtrSetImpl<const SDNode*> &Visited,
14972 SmallPtrSetImpl<const SDNode*> &Checked,
14973 const llvm::SelectionDAG *DAG) {
14974 // If this node has already been checked, don't check it again.
14975 if (Checked.count(N))
14976 return;
14977
14978 // If a node has already been visited on this depth-first walk, reject it as
14979 // a cycle.
14980 if (!Visited.insert(N).second) {
14981 errs() << "Detected cycle in SelectionDAG\n";
14982 dbgs() << "Offending node:\n";
14983 N->dumprFull(DAG); dbgs() << "\n";
14984 abort();
14985 }
14986
14987 for (const SDValue &Op : N->op_values())
14988 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
14989
14990 Checked.insert(N);
14991 Visited.erase(N);
14992}
14993#endif
14994
14995void llvm::checkForCycles(const llvm::SDNode *N,
14996 const llvm::SelectionDAG *DAG,
14997 bool force) {
14998#ifndef NDEBUG
14999 bool check = force;
15000#ifdef EXPENSIVE_CHECKS
15001 check = true;
15002#endif // EXPENSIVE_CHECKS
15003 if (check) {
15004 assert(N && "Checking nonexistent SDNode");
15005 SmallPtrSet<const SDNode*, 32> visited;
15006 SmallPtrSet<const SDNode*, 32> checked;
15007 checkForCyclesHelper(N, visited, checked, DAG);
15008 }
15009#endif // !NDEBUG
15010}
15011
15012void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15013 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
15014}
15015