1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/FunctionLoweringInfo.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineConstantPool.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/RuntimeLibcallUtil.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetFrameLowering.h"
46#include "llvm/CodeGen/TargetLowering.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGen/ValueTypes.h"
50#include "llvm/CodeGenTypes/MachineValueType.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfoMetadata.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
61#include "llvm/Support/Casting.h"
62#include "llvm/Support/CodeGen.h"
63#include "llvm/Support/Compiler.h"
64#include "llvm/Support/Debug.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/MathExtras.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/TargetParser/Triple.h"
72#include "llvm/Transforms/Utils/SizeOpts.h"
73#include <algorithm>
74#include <cassert>
75#include <cstdint>
76#include <cstdlib>
77#include <limits>
78#include <optional>
79#include <string>
80#include <utility>
81#include <vector>
82
83using namespace llvm;
84using namespace llvm::SDPatternMatch;
85
86/// makeVTList - Return an instance of the SDVTList struct initialized with the
87/// specified members.
88static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
89 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
90 return Res;
91}
92
93// Default null implementations of the callbacks.
94void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
95void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
96void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
97
98void SelectionDAG::DAGNodeDeletedListener::anchor() {}
99void SelectionDAG::DAGNodeInsertedListener::anchor() {}
100
101#define DEBUG_TYPE "selectiondag"
102
103static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
104 cl::Hidden, cl::init(Val: true),
105 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
106
107static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
108 cl::desc("Number limit for gluing ld/st of memcpy."),
109 cl::Hidden, cl::init(Val: 0));
110
111static cl::opt<unsigned>
112 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
113 cl::desc("DAG combiner limit number of steps when searching DAG "
114 "for predecessor nodes"));
115
116static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
117 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
118}
119
120unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
121
122//===----------------------------------------------------------------------===//
123// ConstantFPSDNode Class
124//===----------------------------------------------------------------------===//
125
126/// isExactlyValue - We don't rely on operator== working on double values, as
127/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
128/// As such, this method can be used to do an exact bit-for-bit comparison of
129/// two floating point values.
130bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
131 return getValueAPF().bitwiseIsEqual(RHS: V);
132}
133
134bool ConstantFPSDNode::isValueValidForType(EVT VT,
135 const APFloat& Val) {
136 assert(VT.isFloatingPoint() && "Can only convert between FP types");
137
138 // convert modifies in place, so make a copy.
139 APFloat Val2 = APFloat(Val);
140 bool losesInfo;
141 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
142 losesInfo: &losesInfo);
143 return !losesInfo;
144}
145
146//===----------------------------------------------------------------------===//
147// ISD Namespace
148//===----------------------------------------------------------------------===//
149
150bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
151 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
152 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
153 unsigned EltSize =
154 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
155 SplatVal = OptAPInt->trunc(width: EltSize);
156 return true;
157 }
158 }
159
160 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
161 if (!BV)
162 return false;
163
164 APInt SplatUndef;
165 unsigned SplatBitSize;
166 bool HasUndefs;
167 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
168 // Endianness does not matter here. We are checking for a splat given the
169 // element size of the vector, and if we find such a splat for little endian
170 // layout, then that should be valid also for big endian (as the full vector
171 // size is known to be a multiple of the element size).
172 const bool IsBigEndian = false;
173 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
174 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
175 EltSize == SplatBitSize;
176}
177
178// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
179// specializations of the more general isConstantSplatVector()?
180
181bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
182 // Look through a bit convert.
183 while (N->getOpcode() == ISD::BITCAST)
184 N = N->getOperand(Num: 0).getNode();
185
186 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
187 APInt SplatVal;
188 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
189 }
190
191 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
192
193 unsigned i = 0, e = N->getNumOperands();
194
195 // Skip over all of the undef values.
196 while (i != e && N->getOperand(Num: i).isUndef())
197 ++i;
198
199 // Do not accept an all-undef vector.
200 if (i == e) return false;
201
202 // Do not accept build_vectors that aren't all constants or which have non-~0
203 // elements. We have to be a bit careful here, as the type of the constant
204 // may not be the same as the type of the vector elements due to type
205 // legalization (the elements are promoted to a legal type for the target and
206 // a vector of a type may be legal when the base element type is not).
207 // We only want to check enough bits to cover the vector elements, because
208 // we care if the resultant vector is all ones, not whether the individual
209 // constants are.
210 SDValue NotZero = N->getOperand(Num: i);
211 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
212 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
213 if (OptAPInt->countr_one() < EltSize)
214 return false;
215 } else
216 return false;
217
218 // Okay, we have at least one ~0 value, check to see if the rest match or are
219 // undefs. Even with the above element type twiddling, this should be OK, as
220 // the same type legalization should have applied to all the elements.
221 for (++i; i != e; ++i)
222 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
223 return false;
224 return true;
225}
226
227bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
228 // Look through a bit convert.
229 while (N->getOpcode() == ISD::BITCAST)
230 N = N->getOperand(Num: 0).getNode();
231
232 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
233 APInt SplatVal;
234 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
235 }
236
237 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
238
239 bool IsAllUndef = true;
240 for (const SDValue &Op : N->op_values()) {
241 if (Op.isUndef())
242 continue;
243 IsAllUndef = false;
244 // Do not accept build_vectors that aren't all constants or which have non-0
245 // elements. We have to be a bit careful here, as the type of the constant
246 // may not be the same as the type of the vector elements due to type
247 // legalization (the elements are promoted to a legal type for the target
248 // and a vector of a type may be legal when the base element type is not).
249 // We only want to check enough bits to cover the vector elements, because
250 // we care if the resultant vector is all zeros, not whether the individual
251 // constants are.
252 if (auto OptAPInt = Op->bitcastToAPInt()) {
253 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
254 if (OptAPInt->countr_zero() < EltSize)
255 return false;
256 } else
257 return false;
258 }
259
260 // Do not accept an all-undef vector.
261 if (IsAllUndef)
262 return false;
263 return true;
264}
265
266bool ISD::isBuildVectorAllOnes(const SDNode *N) {
267 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
268}
269
270bool ISD::isBuildVectorAllZeros(const SDNode *N) {
271 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
272}
273
274bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
275 if (N->getOpcode() != ISD::BUILD_VECTOR)
276 return false;
277
278 for (const SDValue &Op : N->op_values()) {
279 if (Op.isUndef())
280 continue;
281 if (!isa<ConstantSDNode>(Val: Op))
282 return false;
283 }
284 return true;
285}
286
287bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
288 if (N->getOpcode() != ISD::BUILD_VECTOR)
289 return false;
290
291 for (const SDValue &Op : N->op_values()) {
292 if (Op.isUndef())
293 continue;
294 if (!isa<ConstantFPSDNode>(Val: Op))
295 return false;
296 }
297 return true;
298}
299
300bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
301 bool Signed) {
302 assert(N->getValueType(0).isVector() && "Expected a vector!");
303
304 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
305 if (EltSize <= NewEltSize)
306 return false;
307
308 if (N->getOpcode() == ISD::ZERO_EXTEND) {
309 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
310 NewEltSize) &&
311 !Signed;
312 }
313 if (N->getOpcode() == ISD::SIGN_EXTEND) {
314 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
315 NewEltSize) &&
316 Signed;
317 }
318 if (N->getOpcode() != ISD::BUILD_VECTOR)
319 return false;
320
321 for (const SDValue &Op : N->op_values()) {
322 if (Op.isUndef())
323 continue;
324 if (!isa<ConstantSDNode>(Val: Op))
325 return false;
326
327 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
328 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
329 return false;
330 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
331 return false;
332 }
333
334 return true;
335}
336
337bool ISD::allOperandsUndef(const SDNode *N) {
338 // Return false if the node has no operands.
339 // This is "logically inconsistent" with the definition of "all" but
340 // is probably the desired behavior.
341 if (N->getNumOperands() == 0)
342 return false;
343 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
344}
345
346bool ISD::isFreezeUndef(const SDNode *N) {
347 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
348}
349
350template <typename ConstNodeType>
351bool ISD::matchUnaryPredicateImpl(SDValue Op,
352 std::function<bool(ConstNodeType *)> Match,
353 bool AllowUndefs, bool AllowTruncation) {
354 // FIXME: Add support for scalar UNDEF cases?
355 if (auto *C = dyn_cast<ConstNodeType>(Op))
356 return Match(C);
357
358 // FIXME: Add support for vector UNDEF cases?
359 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
360 ISD::SPLAT_VECTOR != Op.getOpcode())
361 return false;
362
363 EVT SVT = Op.getValueType().getScalarType();
364 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
365 if (AllowUndefs && Op.getOperand(i).isUndef()) {
366 if (!Match(nullptr))
367 return false;
368 continue;
369 }
370
371 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
372 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
373 !Match(Cst))
374 return false;
375 }
376 return true;
377}
378// Build used template types.
379template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
380 SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);
381template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
382 SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);
383
384bool ISD::matchBinaryPredicate(
385 SDValue LHS, SDValue RHS,
386 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
387 bool AllowUndefs, bool AllowTypeMismatch) {
388 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
389 return false;
390
391 // TODO: Add support for scalar UNDEF cases?
392 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
393 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
394 return Match(LHSCst, RHSCst);
395
396 // TODO: Add support for vector UNDEF cases?
397 if (LHS.getOpcode() != RHS.getOpcode() ||
398 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
399 LHS.getOpcode() != ISD::SPLAT_VECTOR))
400 return false;
401
402 EVT SVT = LHS.getValueType().getScalarType();
403 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
404 SDValue LHSOp = LHS.getOperand(i);
405 SDValue RHSOp = RHS.getOperand(i);
406 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
407 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
408 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
409 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
410 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
411 return false;
412 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
413 LHSOp.getValueType() != RHSOp.getValueType()))
414 return false;
415 if (!Match(LHSCst, RHSCst))
416 return false;
417 }
418 return true;
419}
420
421ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
422 switch (MinMaxOpc) {
423 default:
424 llvm_unreachable("unrecognized opcode");
425 case ISD::UMIN:
426 return ISD::UMAX;
427 case ISD::UMAX:
428 return ISD::UMIN;
429 case ISD::SMIN:
430 return ISD::SMAX;
431 case ISD::SMAX:
432 return ISD::SMIN;
433 }
434}
435
436ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
437 switch (MinMaxOpc) {
438 default:
439 llvm_unreachable("unrecognized min/max opcode");
440 case ISD::SMIN:
441 return ISD::UMIN;
442 case ISD::SMAX:
443 return ISD::UMAX;
444 case ISD::UMIN:
445 return ISD::SMIN;
446 case ISD::UMAX:
447 return ISD::SMAX;
448 }
449}
450
451ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
452 switch (VecReduceOpcode) {
453 default:
454 llvm_unreachable("Expected VECREDUCE opcode");
455 case ISD::VECREDUCE_FADD:
456 case ISD::VECREDUCE_SEQ_FADD:
457 case ISD::VP_REDUCE_FADD:
458 case ISD::VP_REDUCE_SEQ_FADD:
459 return ISD::FADD;
460 case ISD::VECREDUCE_FMUL:
461 case ISD::VECREDUCE_SEQ_FMUL:
462 case ISD::VP_REDUCE_FMUL:
463 case ISD::VP_REDUCE_SEQ_FMUL:
464 return ISD::FMUL;
465 case ISD::VECREDUCE_ADD:
466 case ISD::VP_REDUCE_ADD:
467 return ISD::ADD;
468 case ISD::VECREDUCE_MUL:
469 case ISD::VP_REDUCE_MUL:
470 return ISD::MUL;
471 case ISD::VECREDUCE_AND:
472 case ISD::VP_REDUCE_AND:
473 return ISD::AND;
474 case ISD::VECREDUCE_OR:
475 case ISD::VP_REDUCE_OR:
476 return ISD::OR;
477 case ISD::VECREDUCE_XOR:
478 case ISD::VP_REDUCE_XOR:
479 return ISD::XOR;
480 case ISD::VECREDUCE_SMAX:
481 case ISD::VP_REDUCE_SMAX:
482 return ISD::SMAX;
483 case ISD::VECREDUCE_SMIN:
484 case ISD::VP_REDUCE_SMIN:
485 return ISD::SMIN;
486 case ISD::VECREDUCE_UMAX:
487 case ISD::VP_REDUCE_UMAX:
488 return ISD::UMAX;
489 case ISD::VECREDUCE_UMIN:
490 case ISD::VP_REDUCE_UMIN:
491 return ISD::UMIN;
492 case ISD::VECREDUCE_FMAX:
493 case ISD::VP_REDUCE_FMAX:
494 return ISD::FMAXNUM;
495 case ISD::VECREDUCE_FMIN:
496 case ISD::VP_REDUCE_FMIN:
497 return ISD::FMINNUM;
498 case ISD::VECREDUCE_FMAXIMUM:
499 case ISD::VP_REDUCE_FMAXIMUM:
500 return ISD::FMAXIMUM;
501 case ISD::VECREDUCE_FMINIMUM:
502 case ISD::VP_REDUCE_FMINIMUM:
503 return ISD::FMINIMUM;
504 }
505}
506
507bool ISD::isVPOpcode(unsigned Opcode) {
508 switch (Opcode) {
509 default:
510 return false;
511#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
512 case ISD::VPSD: \
513 return true;
514#include "llvm/IR/VPIntrinsics.def"
515 }
516}
517
518bool ISD::isVPBinaryOp(unsigned Opcode) {
519 switch (Opcode) {
520 default:
521 break;
522#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
523#define VP_PROPERTY_BINARYOP return true;
524#define END_REGISTER_VP_SDNODE(VPSD) break;
525#include "llvm/IR/VPIntrinsics.def"
526 }
527 return false;
528}
529
530bool ISD::isVPReduction(unsigned Opcode) {
531 switch (Opcode) {
532 default:
533 return false;
534 case ISD::VP_REDUCE_ADD:
535 case ISD::VP_REDUCE_MUL:
536 case ISD::VP_REDUCE_AND:
537 case ISD::VP_REDUCE_OR:
538 case ISD::VP_REDUCE_XOR:
539 case ISD::VP_REDUCE_SMAX:
540 case ISD::VP_REDUCE_SMIN:
541 case ISD::VP_REDUCE_UMAX:
542 case ISD::VP_REDUCE_UMIN:
543 case ISD::VP_REDUCE_FMAX:
544 case ISD::VP_REDUCE_FMIN:
545 case ISD::VP_REDUCE_FMAXIMUM:
546 case ISD::VP_REDUCE_FMINIMUM:
547 case ISD::VP_REDUCE_FADD:
548 case ISD::VP_REDUCE_FMUL:
549 case ISD::VP_REDUCE_SEQ_FADD:
550 case ISD::VP_REDUCE_SEQ_FMUL:
551 return true;
552 }
553}
554
555/// The operand position of the vector mask.
556std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
557 switch (Opcode) {
558 default:
559 return std::nullopt;
560#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
561 case ISD::VPSD: \
562 return MASKPOS;
563#include "llvm/IR/VPIntrinsics.def"
564 }
565}
566
567/// The operand position of the explicit vector length parameter.
568std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
569 switch (Opcode) {
570 default:
571 return std::nullopt;
572#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
573 case ISD::VPSD: \
574 return EVLPOS;
575#include "llvm/IR/VPIntrinsics.def"
576 }
577}
578
579std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
580 bool hasFPExcept) {
581 // FIXME: Return strict opcodes in case of fp exceptions.
582 switch (VPOpcode) {
583 default:
584 return std::nullopt;
585#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
586#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
587#define END_REGISTER_VP_SDNODE(VPOPC) break;
588#include "llvm/IR/VPIntrinsics.def"
589 }
590 return std::nullopt;
591}
592
593std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
594 switch (Opcode) {
595 default:
596 return std::nullopt;
597#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
598#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
599#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
600#include "llvm/IR/VPIntrinsics.def"
601 }
602}
603
604ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
605 switch (ExtType) {
606 case ISD::EXTLOAD:
607 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
608 case ISD::SEXTLOAD:
609 return ISD::SIGN_EXTEND;
610 case ISD::ZEXTLOAD:
611 return ISD::ZERO_EXTEND;
612 default:
613 break;
614 }
615
616 llvm_unreachable("Invalid LoadExtType");
617}
618
619ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
620 // To perform this operation, we just need to swap the L and G bits of the
621 // operation.
622 unsigned OldL = (Operation >> 2) & 1;
623 unsigned OldG = (Operation >> 1) & 1;
624 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
625 (OldL << 1) | // New G bit
626 (OldG << 2)); // New L bit.
627}
628
629static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
630 unsigned Operation = Op;
631 if (isIntegerLike)
632 Operation ^= 7; // Flip L, G, E bits, but not U.
633 else
634 Operation ^= 15; // Flip all of the condition bits.
635
636 if (Operation > ISD::SETTRUE2)
637 Operation &= ~8; // Don't let N and U bits get set.
638
639 return ISD::CondCode(Operation);
640}
641
642ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
643 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
644}
645
646ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
647 bool isIntegerLike) {
648 return getSetCCInverseImpl(Op, isIntegerLike);
649}
650
651/// For an integer comparison, return 1 if the comparison is a signed operation
652/// and 2 if the result is an unsigned comparison. Return zero if the operation
653/// does not depend on the sign of the input (setne and seteq).
654static int isSignedOp(ISD::CondCode Opcode) {
655 switch (Opcode) {
656 default: llvm_unreachable("Illegal integer setcc operation!");
657 case ISD::SETEQ:
658 case ISD::SETNE: return 0;
659 case ISD::SETLT:
660 case ISD::SETLE:
661 case ISD::SETGT:
662 case ISD::SETGE: return 1;
663 case ISD::SETULT:
664 case ISD::SETULE:
665 case ISD::SETUGT:
666 case ISD::SETUGE: return 2;
667 }
668}
669
670ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
671 EVT Type) {
672 bool IsInteger = Type.isInteger();
673 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
674 // Cannot fold a signed integer setcc with an unsigned integer setcc.
675 return ISD::SETCC_INVALID;
676
677 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
678
679 // If the N and U bits get set, then the resultant comparison DOES suddenly
680 // care about orderedness, and it is true when ordered.
681 if (Op > ISD::SETTRUE2)
682 Op &= ~16; // Clear the U bit if the N bit is set.
683
684 // Canonicalize illegal integer setcc's.
685 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
686 Op = ISD::SETNE;
687
688 return ISD::CondCode(Op);
689}
690
691ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
692 EVT Type) {
693 bool IsInteger = Type.isInteger();
694 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
695 // Cannot fold a signed setcc with an unsigned setcc.
696 return ISD::SETCC_INVALID;
697
698 // Combine all of the condition bits.
699 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
700
701 // Canonicalize illegal integer setcc's.
702 if (IsInteger) {
703 switch (Result) {
704 default: break;
705 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
706 case ISD::SETOEQ: // SETEQ & SETU[LG]E
707 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
708 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
709 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
710 }
711 }
712
713 return Result;
714}
715
716//===----------------------------------------------------------------------===//
717// SDNode Profile Support
718//===----------------------------------------------------------------------===//
719
720/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
721static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
722 ID.AddInteger(I: OpC);
723}
724
725/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
726/// solely with their pointer.
727static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
728 ID.AddPointer(Ptr: VTList.VTs);
729}
730
731/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
732static void AddNodeIDOperands(FoldingSetNodeID &ID,
733 ArrayRef<SDValue> Ops) {
734 for (const auto &Op : Ops) {
735 ID.AddPointer(Ptr: Op.getNode());
736 ID.AddInteger(I: Op.getResNo());
737 }
738}
739
740/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
741static void AddNodeIDOperands(FoldingSetNodeID &ID,
742 ArrayRef<SDUse> Ops) {
743 for (const auto &Op : Ops) {
744 ID.AddPointer(Ptr: Op.getNode());
745 ID.AddInteger(I: Op.getResNo());
746 }
747}
748
749static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
750 SDVTList VTList, ArrayRef<SDValue> OpList) {
751 AddNodeIDOpcode(ID, OpC);
752 AddNodeIDValueTypes(ID, VTList);
753 AddNodeIDOperands(ID, Ops: OpList);
754}
755
756/// If this is an SDNode with special info, add this info to the NodeID data.
757static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
758 switch (N->getOpcode()) {
759 case ISD::TargetExternalSymbol:
760 case ISD::ExternalSymbol:
761 case ISD::MCSymbol:
762 llvm_unreachable("Should only be used on nodes with operands");
763 default: break; // Normal nodes don't need extra info.
764 case ISD::TargetConstant:
765 case ISD::Constant: {
766 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
767 ID.AddPointer(Ptr: C->getConstantIntValue());
768 ID.AddBoolean(B: C->isOpaque());
769 break;
770 }
771 case ISD::TargetConstantFP:
772 case ISD::ConstantFP:
773 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
774 break;
775 case ISD::TargetGlobalAddress:
776 case ISD::GlobalAddress:
777 case ISD::TargetGlobalTLSAddress:
778 case ISD::GlobalTLSAddress: {
779 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
780 ID.AddPointer(Ptr: GA->getGlobal());
781 ID.AddInteger(I: GA->getOffset());
782 ID.AddInteger(I: GA->getTargetFlags());
783 break;
784 }
785 case ISD::BasicBlock:
786 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
787 break;
788 case ISD::Register:
789 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
790 break;
791 case ISD::RegisterMask:
792 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
793 break;
794 case ISD::SRCVALUE:
795 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
796 break;
797 case ISD::FrameIndex:
798 case ISD::TargetFrameIndex:
799 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
800 break;
801 case ISD::PSEUDO_PROBE:
802 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
803 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
804 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
805 break;
806 case ISD::JumpTable:
807 case ISD::TargetJumpTable:
808 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
809 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
810 break;
811 case ISD::ConstantPool:
812 case ISD::TargetConstantPool: {
813 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
814 ID.AddInteger(I: CP->getAlign().value());
815 ID.AddInteger(I: CP->getOffset());
816 if (CP->isMachineConstantPoolEntry())
817 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
818 else
819 ID.AddPointer(Ptr: CP->getConstVal());
820 ID.AddInteger(I: CP->getTargetFlags());
821 break;
822 }
823 case ISD::TargetIndex: {
824 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
825 ID.AddInteger(I: TI->getIndex());
826 ID.AddInteger(I: TI->getOffset());
827 ID.AddInteger(I: TI->getTargetFlags());
828 break;
829 }
830 case ISD::LOAD: {
831 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
832 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
833 ID.AddInteger(I: LD->getRawSubclassData());
834 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
835 ID.AddInteger(I: LD->getMemOperand()->getFlags());
836 break;
837 }
838 case ISD::STORE: {
839 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
840 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
841 ID.AddInteger(I: ST->getRawSubclassData());
842 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
843 ID.AddInteger(I: ST->getMemOperand()->getFlags());
844 break;
845 }
846 case ISD::VP_LOAD: {
847 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
848 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
849 ID.AddInteger(I: ELD->getRawSubclassData());
850 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
851 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
852 break;
853 }
854 case ISD::VP_LOAD_FF: {
855 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
856 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
857 ID.AddInteger(I: LD->getRawSubclassData());
858 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
859 ID.AddInteger(I: LD->getMemOperand()->getFlags());
860 break;
861 }
862 case ISD::VP_STORE: {
863 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
864 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
865 ID.AddInteger(I: EST->getRawSubclassData());
866 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
867 ID.AddInteger(I: EST->getMemOperand()->getFlags());
868 break;
869 }
870 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
871 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
872 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
873 ID.AddInteger(I: SLD->getRawSubclassData());
874 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
875 break;
876 }
877 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
878 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
879 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
880 ID.AddInteger(I: SST->getRawSubclassData());
881 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
882 break;
883 }
884 case ISD::VP_GATHER: {
885 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
886 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
887 ID.AddInteger(I: EG->getRawSubclassData());
888 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
889 ID.AddInteger(I: EG->getMemOperand()->getFlags());
890 break;
891 }
892 case ISD::VP_SCATTER: {
893 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
894 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
895 ID.AddInteger(I: ES->getRawSubclassData());
896 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
897 ID.AddInteger(I: ES->getMemOperand()->getFlags());
898 break;
899 }
900 case ISD::MLOAD: {
901 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
902 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
903 ID.AddInteger(I: MLD->getRawSubclassData());
904 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
905 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
906 break;
907 }
908 case ISD::MSTORE: {
909 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
910 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
911 ID.AddInteger(I: MST->getRawSubclassData());
912 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
913 ID.AddInteger(I: MST->getMemOperand()->getFlags());
914 break;
915 }
916 case ISD::MGATHER: {
917 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
918 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
919 ID.AddInteger(I: MG->getRawSubclassData());
920 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
921 ID.AddInteger(I: MG->getMemOperand()->getFlags());
922 break;
923 }
924 case ISD::MSCATTER: {
925 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
926 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
927 ID.AddInteger(I: MS->getRawSubclassData());
928 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
929 ID.AddInteger(I: MS->getMemOperand()->getFlags());
930 break;
931 }
932 case ISD::ATOMIC_CMP_SWAP:
933 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
934 case ISD::ATOMIC_SWAP:
935 case ISD::ATOMIC_LOAD_ADD:
936 case ISD::ATOMIC_LOAD_SUB:
937 case ISD::ATOMIC_LOAD_AND:
938 case ISD::ATOMIC_LOAD_CLR:
939 case ISD::ATOMIC_LOAD_OR:
940 case ISD::ATOMIC_LOAD_XOR:
941 case ISD::ATOMIC_LOAD_NAND:
942 case ISD::ATOMIC_LOAD_MIN:
943 case ISD::ATOMIC_LOAD_MAX:
944 case ISD::ATOMIC_LOAD_UMIN:
945 case ISD::ATOMIC_LOAD_UMAX:
946 case ISD::ATOMIC_LOAD:
947 case ISD::ATOMIC_STORE: {
948 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
949 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
950 ID.AddInteger(I: AT->getRawSubclassData());
951 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
952 ID.AddInteger(I: AT->getMemOperand()->getFlags());
953 break;
954 }
955 case ISD::VECTOR_SHUFFLE: {
956 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
957 for (int M : Mask)
958 ID.AddInteger(I: M);
959 break;
960 }
961 case ISD::ADDRSPACECAST: {
962 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
963 ID.AddInteger(I: ASC->getSrcAddressSpace());
964 ID.AddInteger(I: ASC->getDestAddressSpace());
965 break;
966 }
967 case ISD::TargetBlockAddress:
968 case ISD::BlockAddress: {
969 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
970 ID.AddPointer(Ptr: BA->getBlockAddress());
971 ID.AddInteger(I: BA->getOffset());
972 ID.AddInteger(I: BA->getTargetFlags());
973 break;
974 }
975 case ISD::AssertAlign:
976 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
977 break;
978 case ISD::PREFETCH:
979 case ISD::INTRINSIC_VOID:
980 case ISD::INTRINSIC_W_CHAIN:
981 // Handled by MemIntrinsicSDNode check after the switch.
982 break;
983 case ISD::MDNODE_SDNODE:
984 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
985 break;
986 } // end switch (N->getOpcode())
987
988 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
989 // to check.
990 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
991 ID.AddInteger(I: MN->getRawSubclassData());
992 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
993 for (const MachineMemOperand *MMO : MN->memoperands()) {
994 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
995 ID.AddInteger(I: MMO->getFlags());
996 }
997 }
998}
999
1000/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
1001/// data.
1002static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1003 AddNodeIDOpcode(ID, OpC: N->getOpcode());
1004 // Add the return value info.
1005 AddNodeIDValueTypes(ID, VTList: N->getVTList());
1006 // Add the operand info.
1007 AddNodeIDOperands(ID, Ops: N->ops());
1008
1009 // Handle SDNode leafs with special info.
1010 AddNodeIDCustom(ID, N);
1011}
1012
1013//===----------------------------------------------------------------------===//
1014// SelectionDAG Class
1015//===----------------------------------------------------------------------===//
1016
1017/// doNotCSE - Return true if CSE should not be performed for this node.
1018static bool doNotCSE(SDNode *N) {
1019 if (N->getValueType(ResNo: 0) == MVT::Glue)
1020 return true; // Never CSE anything that produces a glue result.
1021
1022 switch (N->getOpcode()) {
1023 default: break;
1024 case ISD::HANDLENODE:
1025 case ISD::EH_LABEL:
1026 return true; // Never CSE these nodes.
1027 }
1028
1029 // Check that remaining values produced are not flags.
1030 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1031 if (N->getValueType(ResNo: i) == MVT::Glue)
1032 return true; // Never CSE anything that produces a glue result.
1033
1034 return false;
1035}
1036
1037/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1038/// SelectionDAG.
1039void SelectionDAG::RemoveDeadNodes() {
1040 // Create a dummy node (which is not added to allnodes), that adds a reference
1041 // to the root node, preventing it from being deleted.
1042 HandleSDNode Dummy(getRoot());
1043
1044 SmallVector<SDNode*, 128> DeadNodes;
1045
1046 // Add all obviously-dead nodes to the DeadNodes worklist.
1047 for (SDNode &Node : allnodes())
1048 if (Node.use_empty())
1049 DeadNodes.push_back(Elt: &Node);
1050
1051 RemoveDeadNodes(DeadNodes);
1052
1053 // If the root changed (e.g. it was a dead load, update the root).
1054 setRoot(Dummy.getValue());
1055}
1056
1057/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1058/// given list, and any nodes that become unreachable as a result.
1059void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1060
1061 // Process the worklist, deleting the nodes and adding their uses to the
1062 // worklist.
1063 while (!DeadNodes.empty()) {
1064 SDNode *N = DeadNodes.pop_back_val();
1065 // Skip to next node if we've already managed to delete the node. This could
1066 // happen if replacing a node causes a node previously added to the node to
1067 // be deleted.
1068 if (N->getOpcode() == ISD::DELETED_NODE)
1069 continue;
1070
1071 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1072 DUL->NodeDeleted(N, nullptr);
1073
1074 // Take the node out of the appropriate CSE map.
1075 RemoveNodeFromCSEMaps(N);
1076
1077 // Next, brutally remove the operand list. This is safe to do, as there are
1078 // no cycles in the graph.
1079 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1080 SDUse &Use = *I++;
1081 SDNode *Operand = Use.getNode();
1082 Use.set(SDValue());
1083
1084 // Now that we removed this operand, see if there are no uses of it left.
1085 if (Operand->use_empty())
1086 DeadNodes.push_back(Elt: Operand);
1087 }
1088
1089 DeallocateNode(N);
1090 }
1091}
1092
1093void SelectionDAG::RemoveDeadNode(SDNode *N){
1094 SmallVector<SDNode*, 16> DeadNodes(1, N);
1095
1096 // Create a dummy node that adds a reference to the root node, preventing
1097 // it from being deleted. (This matters if the root is an operand of the
1098 // dead node.)
1099 HandleSDNode Dummy(getRoot());
1100
1101 RemoveDeadNodes(DeadNodes);
1102}
1103
1104void SelectionDAG::DeleteNode(SDNode *N) {
1105 // First take this out of the appropriate CSE map.
1106 RemoveNodeFromCSEMaps(N);
1107
1108 // Finally, remove uses due to operands of this node, remove from the
1109 // AllNodes list, and delete the node.
1110 DeleteNodeNotInCSEMaps(N);
1111}
1112
1113void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1114 assert(N->getIterator() != AllNodes.begin() &&
1115 "Cannot delete the entry node!");
1116 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1117
1118 // Drop all of the operands and decrement used node's use counts.
1119 N->DropOperands();
1120
1121 DeallocateNode(N);
1122}
1123
1124void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1125 assert(!(V->isVariadic() && isParameter));
1126 if (isParameter)
1127 ByvalParmDbgValues.push_back(Elt: V);
1128 else
1129 DbgValues.push_back(Elt: V);
1130 for (const SDNode *Node : V->getSDNodes())
1131 if (Node)
1132 DbgValMap[Node].push_back(Elt: V);
1133}
1134
1135void SDDbgInfo::erase(const SDNode *Node) {
1136 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1137 if (I == DbgValMap.end())
1138 return;
1139 for (auto &Val: I->second)
1140 Val->setIsInvalidated();
1141 DbgValMap.erase(I);
1142}
1143
1144void SelectionDAG::DeallocateNode(SDNode *N) {
1145 // If we have operands, deallocate them.
1146 removeOperands(Node: N);
1147
1148 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1149
1150 // Set the opcode to DELETED_NODE to help catch bugs when node
1151 // memory is reallocated.
1152 // FIXME: There are places in SDag that have grown a dependency on the opcode
1153 // value in the released node.
1154 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1155 N->NodeType = ISD::DELETED_NODE;
1156
1157 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1158 // them and forget about that node.
1159 DbgInfo->erase(Node: N);
1160
1161 // Invalidate extra info.
1162 SDEI.erase(Val: N);
1163}
1164
1165#ifndef NDEBUG
1166/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1167void SelectionDAG::verifyNode(SDNode *N) const {
1168 switch (N->getOpcode()) {
1169 default:
1170 if (N->isTargetOpcode())
1171 getSelectionDAGInfo().verifyTargetNode(*this, N);
1172 break;
1173 case ISD::BUILD_PAIR: {
1174 EVT VT = N->getValueType(0);
1175 assert(N->getNumValues() == 1 && "Too many results!");
1176 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1177 "Wrong return type!");
1178 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1179 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1180 "Mismatched operand types!");
1181 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1182 "Wrong operand type!");
1183 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1184 "Wrong return type size");
1185 break;
1186 }
1187 case ISD::BUILD_VECTOR: {
1188 assert(N->getNumValues() == 1 && "Too many results!");
1189 assert(N->getValueType(0).isVector() && "Wrong return type!");
1190 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1191 "Wrong number of operands!");
1192 EVT EltVT = N->getValueType(0).getVectorElementType();
1193 for (const SDUse &Op : N->ops()) {
1194 assert((Op.getValueType() == EltVT ||
1195 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1196 EltVT.bitsLE(Op.getValueType()))) &&
1197 "Wrong operand type!");
1198 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1199 "Operands must all have the same type");
1200 }
1201 break;
1202 }
1203 case ISD::SADDO:
1204 case ISD::UADDO:
1205 case ISD::SSUBO:
1206 case ISD::USUBO:
1207 assert(N->getNumValues() == 2 && "Wrong number of results!");
1208 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1209 "Invalid add/sub overflow op!");
1210 assert(N->getVTList().VTs[0].isInteger() &&
1211 N->getVTList().VTs[1].isInteger() &&
1212 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1213 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1214 "Binary operator types must match!");
1215 break;
1216 }
1217}
1218#endif // NDEBUG
1219
1220/// Insert a newly allocated node into the DAG.
1221///
1222/// Handles insertion into the all nodes list and CSE map, as well as
1223/// verification and other common operations when a new node is allocated.
1224void SelectionDAG::InsertNode(SDNode *N) {
1225 AllNodes.push_back(val: N);
1226#ifndef NDEBUG
1227 N->PersistentId = NextPersistentId++;
1228 verifyNode(N);
1229#endif
1230 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1231 DUL->NodeInserted(N);
1232}
1233
1234/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1235/// correspond to it. This is useful when we're about to delete or repurpose
1236/// the node. We don't want future request for structurally identical nodes
1237/// to return N anymore.
1238bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1239 bool Erased = false;
1240 switch (N->getOpcode()) {
1241 case ISD::HANDLENODE: return false; // noop.
1242 case ISD::CONDCODE:
1243 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1244 "Cond code doesn't exist!");
1245 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1246 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1247 break;
1248 case ISD::ExternalSymbol:
1249 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1250 break;
1251 case ISD::TargetExternalSymbol: {
1252 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1253 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1254 ESN->getSymbol(), ESN->getTargetFlags()));
1255 break;
1256 }
1257 case ISD::MCSymbol: {
1258 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1259 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1260 break;
1261 }
1262 case ISD::VALUETYPE: {
1263 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1264 if (VT.isExtended()) {
1265 Erased = ExtendedValueTypeNodes.erase(x: VT);
1266 } else {
1267 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1268 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1269 }
1270 break;
1271 }
1272 default:
1273 // Remove it from the CSE Map.
1274 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1275 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1276 Erased = CSEMap.RemoveNode(N);
1277 break;
1278 }
1279#ifndef NDEBUG
1280 // Verify that the node was actually in one of the CSE maps, unless it has a
1281 // glue result (which cannot be CSE'd) or is one of the special cases that are
1282 // not subject to CSE.
1283 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1284 !N->isMachineOpcode() && !doNotCSE(N)) {
1285 N->dump(this);
1286 dbgs() << "\n";
1287 llvm_unreachable("Node is not in map!");
1288 }
1289#endif
1290 return Erased;
1291}
1292
1293/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1294/// maps and modified in place. Add it back to the CSE maps, unless an identical
1295/// node already exists, in which case transfer all its users to the existing
1296/// node. This transfer can potentially trigger recursive merging.
1297void
1298SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1299 // For node types that aren't CSE'd, just act as if no identical node
1300 // already exists.
1301 if (!doNotCSE(N)) {
1302 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1303 if (Existing != N) {
1304 // If there was already an existing matching node, use ReplaceAllUsesWith
1305 // to replace the dead one with the existing one. This can cause
1306 // recursive merging of other unrelated nodes down the line.
1307 Existing->intersectFlagsWith(Flags: N->getFlags());
1308 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing))
1309 MemNode->refineRanges(NewMMOs: cast<MemSDNode>(Val: N)->memoperands());
1310 ReplaceAllUsesWith(From: N, To: Existing);
1311
1312 // N is now dead. Inform the listeners and delete it.
1313 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1314 DUL->NodeDeleted(N, Existing);
1315 DeleteNodeNotInCSEMaps(N);
1316 return;
1317 }
1318 }
1319
1320 // If the node doesn't already exist, we updated it. Inform listeners.
1321 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1322 DUL->NodeUpdated(N);
1323}
1324
1325/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1326/// were replaced with those specified. If this node is never memoized,
1327/// return null, otherwise return a pointer to the slot it would take. If a
1328/// node already exists with these operands, the slot will be non-null.
1329SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1330 void *&InsertPos) {
1331 if (doNotCSE(N))
1332 return nullptr;
1333
1334 SDValue Ops[] = { Op };
1335 FoldingSetNodeID ID;
1336 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1337 AddNodeIDCustom(ID, N);
1338 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1339 if (Node)
1340 Node->intersectFlagsWith(Flags: N->getFlags());
1341 return Node;
1342}
1343
1344/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1345/// were replaced with those specified. If this node is never memoized,
1346/// return null, otherwise return a pointer to the slot it would take. If a
1347/// node already exists with these operands, the slot will be non-null.
1348SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1349 SDValue Op1, SDValue Op2,
1350 void *&InsertPos) {
1351 if (doNotCSE(N))
1352 return nullptr;
1353
1354 SDValue Ops[] = { Op1, Op2 };
1355 FoldingSetNodeID ID;
1356 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1357 AddNodeIDCustom(ID, N);
1358 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1359 if (Node)
1360 Node->intersectFlagsWith(Flags: N->getFlags());
1361 return Node;
1362}
1363
1364/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1365/// were replaced with those specified. If this node is never memoized,
1366/// return null, otherwise return a pointer to the slot it would take. If a
1367/// node already exists with these operands, the slot will be non-null.
1368SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1369 void *&InsertPos) {
1370 if (doNotCSE(N))
1371 return nullptr;
1372
1373 FoldingSetNodeID ID;
1374 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1375 AddNodeIDCustom(ID, N);
1376 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1377 if (Node)
1378 Node->intersectFlagsWith(Flags: N->getFlags());
1379 return Node;
1380}
1381
1382Align SelectionDAG::getEVTAlign(EVT VT) const {
1383 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1384 : VT.getTypeForEVT(Context&: *getContext());
1385
1386 return getDataLayout().getABITypeAlign(Ty);
1387}
1388
1389// EntryNode could meaningfully have debug info if we can find it...
1390SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1391 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1392 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1393 Root(getEntryNode()) {
1394 InsertNode(N: &EntryNode);
1395 DbgInfo = new SDDbgInfo();
1396}
1397
1398void SelectionDAG::init(MachineFunction &NewMF,
1399 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1400 const TargetLibraryInfo *LibraryInfo,
1401 const LibcallLoweringInfo *LibcallsInfo,
1402 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1403 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
1404 FunctionVarLocs const *VarLocs) {
1405 MF = &NewMF;
1406 SDAGISelPass = PassPtr;
1407 ORE = &NewORE;
1408 TLI = getSubtarget().getTargetLowering();
1409 TSI = getSubtarget().getSelectionDAGInfo();
1410 LibInfo = LibraryInfo;
1411 Libcalls = LibcallsInfo;
1412 Context = &MF->getFunction().getContext();
1413 UA = NewUA;
1414 PSI = PSIin;
1415 BFI = BFIin;
1416 MMI = &MMIin;
1417 FnVarLocs = VarLocs;
1418}
1419
1420SelectionDAG::~SelectionDAG() {
1421 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1422 allnodes_clear();
1423 OperandRecycler.clear(OperandAllocator);
1424 delete DbgInfo;
1425}
1426
1427bool SelectionDAG::shouldOptForSize() const {
1428 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1429}
1430
1431void SelectionDAG::allnodes_clear() {
1432 assert(&*AllNodes.begin() == &EntryNode);
1433 AllNodes.remove(IT: AllNodes.begin());
1434 while (!AllNodes.empty())
1435 DeallocateNode(N: &AllNodes.front());
1436#ifndef NDEBUG
1437 NextPersistentId = 0;
1438#endif
1439}
1440
1441SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1442 void *&InsertPos) {
1443 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1444 if (N) {
1445 switch (N->getOpcode()) {
1446 default: break;
1447 case ISD::Constant:
1448 case ISD::ConstantFP:
1449 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1450 "debug location. Use another overload.");
1451 }
1452 }
1453 return N;
1454}
1455
1456SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1457 const SDLoc &DL, void *&InsertPos) {
1458 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1459 if (N) {
1460 switch (N->getOpcode()) {
1461 case ISD::Constant:
1462 case ISD::ConstantFP:
1463 // Erase debug location from the node if the node is used at several
1464 // different places. Do not propagate one location to all uses as it
1465 // will cause a worse single stepping debugging experience.
1466 if (N->getDebugLoc() != DL.getDebugLoc())
1467 N->setDebugLoc(DebugLoc());
1468 break;
1469 default:
1470 // When the node's point of use is located earlier in the instruction
1471 // sequence than its prior point of use, update its debug info to the
1472 // earlier location.
1473 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1474 N->setDebugLoc(DL.getDebugLoc());
1475 break;
1476 }
1477 }
1478 return N;
1479}
1480
1481void SelectionDAG::clear() {
1482 allnodes_clear();
1483 OperandRecycler.clear(OperandAllocator);
1484 OperandAllocator.Reset();
1485 CSEMap.clear();
1486
1487 ExtendedValueTypeNodes.clear();
1488 ExternalSymbols.clear();
1489 TargetExternalSymbols.clear();
1490 MCSymbols.clear();
1491 SDEI.clear();
1492 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1493 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1494
1495 EntryNode.UseList = nullptr;
1496 InsertNode(N: &EntryNode);
1497 Root = getEntryNode();
1498 DbgInfo->clear();
1499}
1500
1501SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1502 return VT.bitsGT(VT: Op.getValueType())
1503 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1504 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1505 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1506}
1507
1508std::pair<SDValue, SDValue>
1509SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1510 const SDLoc &DL, EVT VT) {
1511 assert(!VT.bitsEq(Op.getValueType()) &&
1512 "Strict no-op FP extend/round not allowed.");
1513 SDValue Res =
1514 VT.bitsGT(VT: Op.getValueType())
1515 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1516 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1517 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1518
1519 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1520}
1521
1522SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1523 return VT.bitsGT(VT: Op.getValueType()) ?
1524 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1525 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1526}
1527
1528SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1529 return VT.bitsGT(VT: Op.getValueType()) ?
1530 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1531 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1532}
1533
1534SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1535 return VT.bitsGT(VT: Op.getValueType()) ?
1536 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1537 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1538}
1539
1540SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1541 EVT VT) {
1542 assert(!VT.isVector());
1543 auto Type = Op.getValueType();
1544 SDValue DestOp;
1545 if (Type == VT)
1546 return Op;
1547 auto Size = Op.getValueSizeInBits();
1548 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1549 if (DestOp.getValueType() == VT)
1550 return DestOp;
1551
1552 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1553}
1554
1555SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1556 EVT VT) {
1557 assert(!VT.isVector());
1558 auto Type = Op.getValueType();
1559 SDValue DestOp;
1560 if (Type == VT)
1561 return Op;
1562 auto Size = Op.getValueSizeInBits();
1563 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1564 if (DestOp.getValueType() == VT)
1565 return DestOp;
1566
1567 return getSExtOrTrunc(Op: DestOp, DL, VT);
1568}
1569
1570SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1571 EVT VT) {
1572 assert(!VT.isVector());
1573 auto Type = Op.getValueType();
1574 SDValue DestOp;
1575 if (Type == VT)
1576 return Op;
1577 auto Size = Op.getValueSizeInBits();
1578 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1579 if (DestOp.getValueType() == VT)
1580 return DestOp;
1581
1582 return getZExtOrTrunc(Op: DestOp, DL, VT);
1583}
1584
1585SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1586 EVT OpVT) {
1587 if (VT.bitsLE(VT: Op.getValueType()))
1588 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1589
1590 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1591 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1592}
1593
1594SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1595 EVT OpVT = Op.getValueType();
1596 assert(VT.isInteger() && OpVT.isInteger() &&
1597 "Cannot getZeroExtendInReg FP types");
1598 assert(VT.isVector() == OpVT.isVector() &&
1599 "getZeroExtendInReg type should be vector iff the operand "
1600 "type is vector!");
1601 assert((!VT.isVector() ||
1602 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1603 "Vector element counts must match in getZeroExtendInReg");
1604 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1605 if (OpVT == VT)
1606 return Op;
1607 // TODO: Use computeKnownBits instead of AssertZext.
1608 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1609 ->getVT()
1610 .getScalarType()
1611 .bitsLE(VT: VT.getScalarType()))
1612 return Op;
1613 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1614 loBitsSet: VT.getScalarSizeInBits());
1615 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1616}
1617
1618SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,
1619 SDValue EVL, const SDLoc &DL,
1620 EVT VT) {
1621 EVT OpVT = Op.getValueType();
1622 assert(VT.isInteger() && OpVT.isInteger() &&
1623 "Cannot getVPZeroExtendInReg FP types");
1624 assert(VT.isVector() && OpVT.isVector() &&
1625 "getVPZeroExtendInReg type and operand type should be vector!");
1626 assert(VT.getVectorElementCount() == OpVT.getVectorElementCount() &&
1627 "Vector element counts must match in getZeroExtendInReg");
1628 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1629 if (OpVT == VT)
1630 return Op;
1631 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1632 loBitsSet: VT.getScalarSizeInBits());
1633 return getNode(Opcode: ISD::VP_AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT), N3: Mask,
1634 N4: EVL);
1635}
1636
1637SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1638 // Only unsigned pointer semantics are supported right now. In the future this
1639 // might delegate to TLI to check pointer signedness.
1640 return getZExtOrTrunc(Op, DL, VT);
1641}
1642
1643SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1644 // Only unsigned pointer semantics are supported right now. In the future this
1645 // might delegate to TLI to check pointer signedness.
1646 return getZeroExtendInReg(Op, DL, VT);
1647}
1648
1649SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1650 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1651}
1652
1653/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1654SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1655 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1656}
1657
1658SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1659 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1660 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1661}
1662
1663SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1664 SDValue Mask, SDValue EVL, EVT VT) {
1665 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1666 return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1667}
1668
1669SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1670 SDValue Mask, SDValue EVL) {
1671 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1672}
1673
1674SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1675 SDValue Mask, SDValue EVL) {
1676 if (VT.bitsGT(VT: Op.getValueType()))
1677 return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1678 if (VT.bitsLT(VT: Op.getValueType()))
1679 return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1680 return Op;
1681}
1682
1683SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1684 EVT OpVT) {
1685 if (!V)
1686 return getConstant(Val: 0, DL, VT);
1687
1688 switch (TLI->getBooleanContents(Type: OpVT)) {
1689 case TargetLowering::ZeroOrOneBooleanContent:
1690 case TargetLowering::UndefinedBooleanContent:
1691 return getConstant(Val: 1, DL, VT);
1692 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1693 return getAllOnesConstant(DL, VT);
1694 }
1695 llvm_unreachable("Unexpected boolean content enum!");
1696}
1697
1698SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1699 bool isT, bool isO) {
1700 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1701 DL, VT, isTarget: isT, isOpaque: isO);
1702}
1703
1704SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1705 bool isT, bool isO) {
1706 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1707}
1708
1709SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1710 EVT VT, bool isT, bool isO) {
1711 assert(VT.isInteger() && "Cannot create FP integer constant!");
1712
1713 EVT EltVT = VT.getScalarType();
1714 const ConstantInt *Elt = &Val;
1715
1716 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1717 // to-be-splatted scalar ConstantInt.
1718 if (isa<VectorType>(Val: Elt->getType()))
1719 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1720
1721 // In some cases the vector type is legal but the element type is illegal and
1722 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1723 // inserted value (the type does not need to match the vector element type).
1724 // Any extra bits introduced will be truncated away.
1725 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1726 TargetLowering::TypePromoteInteger) {
1727 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1728 APInt NewVal;
1729 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1730 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1731 else
1732 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1733 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1734 }
1735 // In other cases the element type is illegal and needs to be expanded, for
1736 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1737 // the value into n parts and use a vector type with n-times the elements.
1738 // Then bitcast to the type requested.
1739 // Legalizing constants too early makes the DAGCombiner's job harder so we
1740 // only legalize if the DAG tells us we must produce legal types.
1741 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1742 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1743 TargetLowering::TypeExpandInteger) {
1744 const APInt &NewVal = Elt->getValue();
1745 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1746 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1747
1748 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1749 if (VT.isScalableVector() ||
1750 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1751 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1752 "Can only handle an even split!");
1753 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1754
1755 SmallVector<SDValue, 2> ScalarParts;
1756 for (unsigned i = 0; i != Parts; ++i)
1757 ScalarParts.push_back(Elt: getConstant(
1758 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1759 VT: ViaEltVT, isT, isO));
1760
1761 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1762 }
1763
1764 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1765 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1766
1767 // Check the temporary vector is the correct size. If this fails then
1768 // getTypeToTransformTo() probably returned a type whose size (in bits)
1769 // isn't a power-of-2 factor of the requested type size.
1770 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1771
1772 SmallVector<SDValue, 2> EltParts;
1773 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1774 EltParts.push_back(Elt: getConstant(
1775 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1776 VT: ViaEltVT, isT, isO));
1777
1778 // EltParts is currently in little endian order. If we actually want
1779 // big-endian order then reverse it now.
1780 if (getDataLayout().isBigEndian())
1781 std::reverse(first: EltParts.begin(), last: EltParts.end());
1782
1783 // The elements must be reversed when the element order is different
1784 // to the endianness of the elements (because the BITCAST is itself a
1785 // vector shuffle in this situation). However, we do not need any code to
1786 // perform this reversal because getConstant() is producing a vector
1787 // splat.
1788 // This situation occurs in MIPS MSA.
1789
1790 SmallVector<SDValue, 8> Ops;
1791 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1792 llvm::append_range(C&: Ops, R&: EltParts);
1793
1794 SDValue V =
1795 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1796 return V;
1797 }
1798
1799 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1800 "APInt size does not match type size!");
1801 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1802 SDVTList VTs = getVTList(VT: EltVT);
1803 FoldingSetNodeID ID;
1804 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1805 ID.AddPointer(Ptr: Elt);
1806 ID.AddBoolean(B: isO);
1807 void *IP = nullptr;
1808 SDNode *N = nullptr;
1809 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1810 if (!VT.isVector())
1811 return SDValue(N, 0);
1812
1813 if (!N) {
1814 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1815 CSEMap.InsertNode(N, InsertPos: IP);
1816 InsertNode(N);
1817 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1818 }
1819
1820 SDValue Result(N, 0);
1821 if (VT.isVector())
1822 Result = getSplat(VT, DL, Op: Result);
1823 return Result;
1824}
1825
1826SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1827 bool isT, bool isO) {
1828 unsigned Size = VT.getScalarSizeInBits();
1829 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1830}
1831
1832SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1833 bool IsOpaque) {
1834 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1835 isT: IsTarget, isO: IsOpaque);
1836}
1837
1838SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1839 bool isTarget) {
1840 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1841}
1842
1843SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1844 const SDLoc &DL) {
1845 assert(VT.isInteger() && "Shift amount is not an integer type!");
1846 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1847 return getConstant(Val, DL, VT: ShiftVT);
1848}
1849
1850SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1851 const SDLoc &DL) {
1852 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1853 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1854}
1855
1856SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1857 bool isTarget) {
1858 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1859}
1860
1861SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1862 bool isTarget) {
1863 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1864}
1865
1866SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1867 EVT VT, bool isTarget) {
1868 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1869
1870 EVT EltVT = VT.getScalarType();
1871 const ConstantFP *Elt = &V;
1872
1873 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1874 // the to-be-splatted scalar ConstantFP.
1875 if (isa<VectorType>(Val: Elt->getType()))
1876 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1877
1878 // Do the map lookup using the actual bit pattern for the floating point
1879 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1880 // we don't have issues with SNANs.
1881 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1882 SDVTList VTs = getVTList(VT: EltVT);
1883 FoldingSetNodeID ID;
1884 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1885 ID.AddPointer(Ptr: Elt);
1886 void *IP = nullptr;
1887 SDNode *N = nullptr;
1888 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1889 if (!VT.isVector())
1890 return SDValue(N, 0);
1891
1892 if (!N) {
1893 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1894 CSEMap.InsertNode(N, InsertPos: IP);
1895 InsertNode(N);
1896 }
1897
1898 SDValue Result(N, 0);
1899 if (VT.isVector())
1900 Result = getSplat(VT, DL, Op: Result);
1901 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1902 return Result;
1903}
1904
1905SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1906 bool isTarget) {
1907 EVT EltVT = VT.getScalarType();
1908 if (EltVT == MVT::f32)
1909 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1910 if (EltVT == MVT::f64)
1911 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1912 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1913 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1914 bool Ignored;
1915 APFloat APF = APFloat(Val);
1916 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1917 losesInfo: &Ignored);
1918 return getConstantFP(V: APF, DL, VT, isTarget);
1919 }
1920 llvm_unreachable("Unsupported type in getConstantFP");
1921}
1922
1923SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1924 EVT VT, int64_t Offset, bool isTargetGA,
1925 unsigned TargetFlags) {
1926 assert((TargetFlags == 0 || isTargetGA) &&
1927 "Cannot set target flags on target-independent globals");
1928
1929 // Truncate (with sign-extension) the offset value to the pointer size.
1930 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1931 if (BitWidth < 64)
1932 Offset = SignExtend64(X: Offset, B: BitWidth);
1933
1934 unsigned Opc;
1935 if (GV->isThreadLocal())
1936 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1937 else
1938 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1939
1940 SDVTList VTs = getVTList(VT);
1941 FoldingSetNodeID ID;
1942 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1943 ID.AddPointer(Ptr: GV);
1944 ID.AddInteger(I: Offset);
1945 ID.AddInteger(I: TargetFlags);
1946 void *IP = nullptr;
1947 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1948 return SDValue(E, 0);
1949
1950 auto *N = newSDNode<GlobalAddressSDNode>(
1951 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1952 CSEMap.InsertNode(N, InsertPos: IP);
1953 InsertNode(N);
1954 return SDValue(N, 0);
1955}
1956
1957SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1958 SDVTList VTs = getVTList(VT: MVT::Untyped);
1959 FoldingSetNodeID ID;
1960 AddNodeIDNode(ID, OpC: ISD::DEACTIVATION_SYMBOL, VTList: VTs, OpList: {});
1961 ID.AddPointer(Ptr: GV);
1962 void *IP = nullptr;
1963 if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
1964 return SDValue(E, 0);
1965
1966 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1967 CSEMap.InsertNode(N, InsertPos: IP);
1968 InsertNode(N);
1969 return SDValue(N, 0);
1970}
1971
1972SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1973 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1974 SDVTList VTs = getVTList(VT);
1975 FoldingSetNodeID ID;
1976 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1977 ID.AddInteger(I: FI);
1978 void *IP = nullptr;
1979 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1980 return SDValue(E, 0);
1981
1982 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1983 CSEMap.InsertNode(N, InsertPos: IP);
1984 InsertNode(N);
1985 return SDValue(N, 0);
1986}
1987
1988SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1989 unsigned TargetFlags) {
1990 assert((TargetFlags == 0 || isTarget) &&
1991 "Cannot set target flags on target-independent jump tables");
1992 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1993 SDVTList VTs = getVTList(VT);
1994 FoldingSetNodeID ID;
1995 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1996 ID.AddInteger(I: JTI);
1997 ID.AddInteger(I: TargetFlags);
1998 void *IP = nullptr;
1999 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2000 return SDValue(E, 0);
2001
2002 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
2003 CSEMap.InsertNode(N, InsertPos: IP);
2004 InsertNode(N);
2005 return SDValue(N, 0);
2006}
2007
2008SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
2009 const SDLoc &DL) {
2010 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
2011 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
2012 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
2013}
2014
2015SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2016 MaybeAlign Alignment, int Offset,
2017 bool isTarget, unsigned TargetFlags) {
2018 assert((TargetFlags == 0 || isTarget) &&
2019 "Cannot set target flags on target-independent globals");
2020 if (!Alignment)
2021 Alignment = shouldOptForSize()
2022 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2023 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2024 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2025 SDVTList VTs = getVTList(VT);
2026 FoldingSetNodeID ID;
2027 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2028 ID.AddInteger(I: Alignment->value());
2029 ID.AddInteger(I: Offset);
2030 ID.AddPointer(Ptr: C);
2031 ID.AddInteger(I: TargetFlags);
2032 void *IP = nullptr;
2033 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2034 return SDValue(E, 0);
2035
2036 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2037 Args&: TargetFlags);
2038 CSEMap.InsertNode(N, InsertPos: IP);
2039 InsertNode(N);
2040 SDValue V = SDValue(N, 0);
2041 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2042 return V;
2043}
2044
2045SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2046 MaybeAlign Alignment, int Offset,
2047 bool isTarget, unsigned TargetFlags) {
2048 assert((TargetFlags == 0 || isTarget) &&
2049 "Cannot set target flags on target-independent globals");
2050 if (!Alignment)
2051 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2052 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2053 SDVTList VTs = getVTList(VT);
2054 FoldingSetNodeID ID;
2055 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2056 ID.AddInteger(I: Alignment->value());
2057 ID.AddInteger(I: Offset);
2058 C->addSelectionDAGCSEId(ID);
2059 ID.AddInteger(I: TargetFlags);
2060 void *IP = nullptr;
2061 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2062 return SDValue(E, 0);
2063
2064 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2065 Args&: TargetFlags);
2066 CSEMap.InsertNode(N, InsertPos: IP);
2067 InsertNode(N);
2068 return SDValue(N, 0);
2069}
2070
2071SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2072 FoldingSetNodeID ID;
2073 AddNodeIDNode(ID, OpC: ISD::BasicBlock, VTList: getVTList(VT: MVT::Other), OpList: {});
2074 ID.AddPointer(Ptr: MBB);
2075 void *IP = nullptr;
2076 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2077 return SDValue(E, 0);
2078
2079 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2080 CSEMap.InsertNode(N, InsertPos: IP);
2081 InsertNode(N);
2082 return SDValue(N, 0);
2083}
2084
2085SDValue SelectionDAG::getValueType(EVT VT) {
2086 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2087 ValueTypeNodes.size())
2088 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2089
2090 SDNode *&N = VT.isExtended() ?
2091 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2092
2093 if (N) return SDValue(N, 0);
2094 N = newSDNode<VTSDNode>(Args&: VT);
2095 InsertNode(N);
2096 return SDValue(N, 0);
2097}
2098
2099SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2100 SDNode *&N = ExternalSymbols[Sym];
2101 if (N) return SDValue(N, 0);
2102 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2103 InsertNode(N);
2104 return SDValue(N, 0);
2105}
2106
2107SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2108 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2109 return getExternalSymbol(Sym: SymName.data(), VT);
2110}
2111
2112SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2113 SDNode *&N = MCSymbols[Sym];
2114 if (N)
2115 return SDValue(N, 0);
2116 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2117 InsertNode(N);
2118 return SDValue(N, 0);
2119}
2120
2121SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2122 unsigned TargetFlags) {
2123 SDNode *&N =
2124 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2125 if (N) return SDValue(N, 0);
2126 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2127 InsertNode(N);
2128 return SDValue(N, 0);
2129}
2130
2131SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2132 EVT VT, unsigned TargetFlags) {
2133 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2134 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2135}
2136
2137SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2138 if ((unsigned)Cond >= CondCodeNodes.size())
2139 CondCodeNodes.resize(new_size: Cond+1);
2140
2141 if (!CondCodeNodes[Cond]) {
2142 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2143 CondCodeNodes[Cond] = N;
2144 InsertNode(N);
2145 }
2146
2147 return SDValue(CondCodeNodes[Cond], 0);
2148}
2149
2150SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2151 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2152 "APInt size does not match type size!");
2153
2154 if (MulImm == 0)
2155 return getConstant(Val: 0, DL, VT);
2156
2157 const MachineFunction &MF = getMachineFunction();
2158 const Function &F = MF.getFunction();
2159 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2160 if (const APInt *C = CR.getSingleElement())
2161 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2162
2163 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2164}
2165
2166/// \returns a value of type \p VT that represents the runtime value of \p
2167/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2168/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2169/// or TypeSize.
2170template <typename Ty>
2171static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2172 EVT VT, Ty Quantity) {
2173 if (Quantity.isScalable())
2174 return DAG.getVScale(
2175 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2176
2177 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2178}
2179
2180SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2181 ElementCount EC) {
2182 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2183}
2184
2185SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2186 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2187}
2188
2189SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2190 ElementCount EC) {
2191 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2192 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2193 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2194 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2195}
2196
2197SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2198 APInt One(ResVT.getScalarSizeInBits(), 1);
2199 return getStepVector(DL, ResVT, StepVal: One);
2200}
2201
2202SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2203 const APInt &StepVal) {
2204 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2205 if (ResVT.isScalableVector())
2206 return getNode(
2207 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2208 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2209
2210 SmallVector<SDValue, 16> OpsStepConstants;
2211 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2212 OpsStepConstants.push_back(
2213 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2214 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2215}
2216
2217/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2218/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2219static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2220 std::swap(a&: N1, b&: N2);
2221 ShuffleVectorSDNode::commuteMask(Mask: M);
2222}
2223
2224SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2225 SDValue N2, ArrayRef<int> Mask) {
2226 assert(VT.getVectorNumElements() == Mask.size() &&
2227 "Must have the same number of vector elements as mask elements!");
2228 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2229 "Invalid VECTOR_SHUFFLE");
2230
2231 // Canonicalize shuffle undef, undef -> undef
2232 if (N1.isUndef() && N2.isUndef())
2233 return getUNDEF(VT);
2234
2235 // Validate that all indices in Mask are within the range of the elements
2236 // input to the shuffle.
2237 int NElts = Mask.size();
2238 assert(llvm::all_of(Mask,
2239 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2240 "Index out of range");
2241
2242 // Copy the mask so we can do any needed cleanup.
2243 SmallVector<int, 8> MaskVec(Mask);
2244
2245 // Canonicalize shuffle v, v -> v, undef
2246 if (N1 == N2) {
2247 N2 = getUNDEF(VT);
2248 for (int i = 0; i != NElts; ++i)
2249 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2250 }
2251
2252 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2253 if (N1.isUndef())
2254 commuteShuffle(N1, N2, M: MaskVec);
2255
2256 if (TLI->hasVectorBlend()) {
2257 // If shuffling a splat, try to blend the splat instead. We do this here so
2258 // that even when this arises during lowering we don't have to re-handle it.
2259 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2260 BitVector UndefElements;
2261 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2262 if (!Splat)
2263 return;
2264
2265 for (int i = 0; i < NElts; ++i) {
2266 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2267 continue;
2268
2269 // If this input comes from undef, mark it as such.
2270 if (UndefElements[MaskVec[i] - Offset]) {
2271 MaskVec[i] = -1;
2272 continue;
2273 }
2274
2275 // If we can blend a non-undef lane, use that instead.
2276 if (!UndefElements[i])
2277 MaskVec[i] = i + Offset;
2278 }
2279 };
2280 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2281 BlendSplat(N1BV, 0);
2282 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2283 BlendSplat(N2BV, NElts);
2284 }
2285
2286 // Canonicalize all index into lhs, -> shuffle lhs, undef
2287 // Canonicalize all index into rhs, -> shuffle rhs, undef
2288 bool AllLHS = true, AllRHS = true;
2289 bool N2Undef = N2.isUndef();
2290 for (int i = 0; i != NElts; ++i) {
2291 if (MaskVec[i] >= NElts) {
2292 if (N2Undef)
2293 MaskVec[i] = -1;
2294 else
2295 AllLHS = false;
2296 } else if (MaskVec[i] >= 0) {
2297 AllRHS = false;
2298 }
2299 }
2300 if (AllLHS && AllRHS)
2301 return getUNDEF(VT);
2302 if (AllLHS && !N2Undef)
2303 N2 = getUNDEF(VT);
2304 if (AllRHS) {
2305 N1 = getUNDEF(VT);
2306 commuteShuffle(N1, N2, M: MaskVec);
2307 }
2308 // Reset our undef status after accounting for the mask.
2309 N2Undef = N2.isUndef();
2310 // Re-check whether both sides ended up undef.
2311 if (N1.isUndef() && N2Undef)
2312 return getUNDEF(VT);
2313
2314 // If Identity shuffle return that node.
2315 bool Identity = true, AllSame = true;
2316 for (int i = 0; i != NElts; ++i) {
2317 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2318 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2319 }
2320 if (Identity && NElts)
2321 return N1;
2322
2323 // Shuffling a constant splat doesn't change the result.
2324 if (N2Undef) {
2325 SDValue V = N1;
2326
2327 // Look through any bitcasts. We check that these don't change the number
2328 // (and size) of elements and just changes their types.
2329 while (V.getOpcode() == ISD::BITCAST)
2330 V = V->getOperand(Num: 0);
2331
2332 // A splat should always show up as a build vector node.
2333 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2334 BitVector UndefElements;
2335 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2336 // If this is a splat of an undef, shuffling it is also undef.
2337 if (Splat && Splat.isUndef())
2338 return getUNDEF(VT);
2339
2340 bool SameNumElts =
2341 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2342
2343 // We only have a splat which can skip shuffles if there is a splatted
2344 // value and no undef lanes rearranged by the shuffle.
2345 if (Splat && UndefElements.none()) {
2346 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2347 // number of elements match or the value splatted is a zero constant.
2348 if (SameNumElts || isNullConstant(V: Splat))
2349 return N1;
2350 }
2351
2352 // If the shuffle itself creates a splat, build the vector directly.
2353 if (AllSame && SameNumElts) {
2354 EVT BuildVT = BV->getValueType(ResNo: 0);
2355 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2356 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2357
2358 // We may have jumped through bitcasts, so the type of the
2359 // BUILD_VECTOR may not match the type of the shuffle.
2360 if (BuildVT != VT)
2361 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2362 return NewBV;
2363 }
2364 }
2365 }
2366
2367 SDVTList VTs = getVTList(VT);
2368 FoldingSetNodeID ID;
2369 SDValue Ops[2] = { N1, N2 };
2370 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: VTs, OpList: Ops);
2371 for (int i = 0; i != NElts; ++i)
2372 ID.AddInteger(I: MaskVec[i]);
2373
2374 void* IP = nullptr;
2375 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2376 return SDValue(E, 0);
2377
2378 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2379 // SDNode doesn't have access to it. This memory will be "leaked" when
2380 // the node is deallocated, but recovered when the NodeAllocator is released.
2381 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2382 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2383
2384 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2385 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2386 createOperands(Node: N, Vals: Ops);
2387
2388 CSEMap.InsertNode(N, InsertPos: IP);
2389 InsertNode(N);
2390 SDValue V = SDValue(N, 0);
2391 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2392 return V;
2393}
2394
2395SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2396 EVT VT = SV.getValueType(ResNo: 0);
2397 SmallVector<int, 8> MaskVec(SV.getMask());
2398 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2399
2400 SDValue Op0 = SV.getOperand(Num: 0);
2401 SDValue Op1 = SV.getOperand(Num: 1);
2402 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2403}
2404
2405SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2406 SDVTList VTs = getVTList(VT);
2407 FoldingSetNodeID ID;
2408 AddNodeIDNode(ID, OpC: ISD::Register, VTList: VTs, OpList: {});
2409 ID.AddInteger(I: Reg.id());
2410 void *IP = nullptr;
2411 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2412 return SDValue(E, 0);
2413
2414 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2415 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2416 CSEMap.InsertNode(N, InsertPos: IP);
2417 InsertNode(N);
2418 return SDValue(N, 0);
2419}
2420
2421SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2422 FoldingSetNodeID ID;
2423 AddNodeIDNode(ID, OpC: ISD::RegisterMask, VTList: getVTList(VT: MVT::Untyped), OpList: {});
2424 ID.AddPointer(Ptr: RegMask);
2425 void *IP = nullptr;
2426 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2427 return SDValue(E, 0);
2428
2429 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2430 CSEMap.InsertNode(N, InsertPos: IP);
2431 InsertNode(N);
2432 return SDValue(N, 0);
2433}
2434
2435SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2436 MCSymbol *Label) {
2437 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2438}
2439
2440SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2441 SDValue Root, MCSymbol *Label) {
2442 FoldingSetNodeID ID;
2443 SDValue Ops[] = { Root };
2444 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT: MVT::Other), OpList: Ops);
2445 ID.AddPointer(Ptr: Label);
2446 void *IP = nullptr;
2447 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2448 return SDValue(E, 0);
2449
2450 auto *N =
2451 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2452 createOperands(Node: N, Vals: Ops);
2453
2454 CSEMap.InsertNode(N, InsertPos: IP);
2455 InsertNode(N);
2456 return SDValue(N, 0);
2457}
2458
2459SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2460 int64_t Offset, bool isTarget,
2461 unsigned TargetFlags) {
2462 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2463 SDVTList VTs = getVTList(VT);
2464
2465 FoldingSetNodeID ID;
2466 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2467 ID.AddPointer(Ptr: BA);
2468 ID.AddInteger(I: Offset);
2469 ID.AddInteger(I: TargetFlags);
2470 void *IP = nullptr;
2471 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2472 return SDValue(E, 0);
2473
2474 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2475 CSEMap.InsertNode(N, InsertPos: IP);
2476 InsertNode(N);
2477 return SDValue(N, 0);
2478}
2479
2480SDValue SelectionDAG::getSrcValue(const Value *V) {
2481 FoldingSetNodeID ID;
2482 AddNodeIDNode(ID, OpC: ISD::SRCVALUE, VTList: getVTList(VT: MVT::Other), OpList: {});
2483 ID.AddPointer(Ptr: V);
2484
2485 void *IP = nullptr;
2486 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2487 return SDValue(E, 0);
2488
2489 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2490 CSEMap.InsertNode(N, InsertPos: IP);
2491 InsertNode(N);
2492 return SDValue(N, 0);
2493}
2494
2495SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2496 FoldingSetNodeID ID;
2497 AddNodeIDNode(ID, OpC: ISD::MDNODE_SDNODE, VTList: getVTList(VT: MVT::Other), OpList: {});
2498 ID.AddPointer(Ptr: MD);
2499
2500 void *IP = nullptr;
2501 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2502 return SDValue(E, 0);
2503
2504 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2505 CSEMap.InsertNode(N, InsertPos: IP);
2506 InsertNode(N);
2507 return SDValue(N, 0);
2508}
2509
2510SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2511 if (VT == V.getValueType())
2512 return V;
2513
2514 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2515}
2516
2517SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2518 unsigned SrcAS, unsigned DestAS) {
2519 SDVTList VTs = getVTList(VT);
2520 SDValue Ops[] = {Ptr};
2521 FoldingSetNodeID ID;
2522 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: VTs, OpList: Ops);
2523 ID.AddInteger(I: SrcAS);
2524 ID.AddInteger(I: DestAS);
2525
2526 void *IP = nullptr;
2527 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2528 return SDValue(E, 0);
2529
2530 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2531 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2532 createOperands(Node: N, Vals: Ops);
2533
2534 CSEMap.InsertNode(N, InsertPos: IP);
2535 InsertNode(N);
2536 return SDValue(N, 0);
2537}
2538
2539SDValue SelectionDAG::getFreeze(SDValue V) {
2540 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2541}
2542
2543/// getShiftAmountOperand - Return the specified value casted to
2544/// the target's desired shift amount type.
2545SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2546 EVT OpTy = Op.getValueType();
2547 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2548 if (OpTy == ShTy || OpTy.isVector()) return Op;
2549
2550 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2551}
2552
2553SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2554 SDLoc dl(Node);
2555 const TargetLowering &TLI = getTargetLoweringInfo();
2556 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2557 EVT VT = Node->getValueType(ResNo: 0);
2558 SDValue Tmp1 = Node->getOperand(Num: 0);
2559 SDValue Tmp2 = Node->getOperand(Num: 1);
2560 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2561
2562 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2563 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2564 SDValue VAList = VAListLoad;
2565
2566 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2567 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2568 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2569
2570 VAList = getNode(
2571 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2572 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2573 }
2574
2575 // Increment the pointer, VAList, to the next vaarg
2576 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2577 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2578 Ty: VT.getTypeForEVT(Context&: *getContext())),
2579 DL: dl, VT: VAList.getValueType()));
2580 // Store the incremented VAList to the legalized pointer
2581 Tmp1 =
2582 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2583 // Load the actual argument out of the pointer VAList
2584 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2585}
2586
2587SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2588 SDLoc dl(Node);
2589 const TargetLowering &TLI = getTargetLoweringInfo();
2590 // This defaults to loading a pointer from the input and storing it to the
2591 // output, returning the chain.
2592 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2593 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2594 SDValue Tmp1 =
2595 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2596 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2597 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2598 PtrInfo: MachinePointerInfo(VD));
2599}
2600
2601Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2602 const DataLayout &DL = getDataLayout();
2603 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2604 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2605
2606 if (TLI->isTypeLegal(VT) || !VT.isVector())
2607 return RedAlign;
2608
2609 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2610 const Align StackAlign = TFI->getStackAlign();
2611
2612 // See if we can choose a smaller ABI alignment in cases where it's an
2613 // illegal vector type that will get broken down.
2614 if (RedAlign > StackAlign) {
2615 EVT IntermediateVT;
2616 MVT RegisterVT;
2617 unsigned NumIntermediates;
2618 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2619 NumIntermediates, RegisterVT);
2620 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2621 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2622 if (RedAlign2 < RedAlign)
2623 RedAlign = RedAlign2;
2624
2625 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2626 // If the stack is not realignable, the alignment should be limited to the
2627 // StackAlignment
2628 RedAlign = std::min(a: RedAlign, b: StackAlign);
2629 }
2630
2631 return RedAlign;
2632}
2633
2634SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2635 MachineFrameInfo &MFI = MF->getFrameInfo();
2636 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2637 int StackID = 0;
2638 if (Bytes.isScalable())
2639 StackID = TFI->getStackIDForScalableVectors();
2640 // The stack id gives an indication of whether the object is scalable or
2641 // not, so it's safe to pass in the minimum size here.
2642 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2643 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2644 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2645}
2646
2647SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2648 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2649 Align StackAlign =
2650 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2651 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2652}
2653
2654SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2655 TypeSize VT1Size = VT1.getStoreSize();
2656 TypeSize VT2Size = VT2.getStoreSize();
2657 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2658 "Don't know how to choose the maximum size when creating a stack "
2659 "temporary");
2660 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2661 ? VT1Size
2662 : VT2Size;
2663
2664 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2665 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2666 const DataLayout &DL = getDataLayout();
2667 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2668 return CreateStackTemporary(Bytes, Alignment: Align);
2669}
2670
2671SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2672 ISD::CondCode Cond, const SDLoc &dl) {
2673 EVT OpVT = N1.getValueType();
2674
2675 auto GetUndefBooleanConstant = [&]() {
2676 if (VT.getScalarType() == MVT::i1 ||
2677 TLI->getBooleanContents(Type: OpVT) ==
2678 TargetLowering::UndefinedBooleanContent)
2679 return getUNDEF(VT);
2680 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2681 // so we cannot use getUNDEF(). Return zero instead.
2682 return getConstant(Val: 0, DL: dl, VT);
2683 };
2684
2685 // These setcc operations always fold.
2686 switch (Cond) {
2687 default: break;
2688 case ISD::SETFALSE:
2689 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2690 case ISD::SETTRUE:
2691 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2692
2693 case ISD::SETOEQ:
2694 case ISD::SETOGT:
2695 case ISD::SETOGE:
2696 case ISD::SETOLT:
2697 case ISD::SETOLE:
2698 case ISD::SETONE:
2699 case ISD::SETO:
2700 case ISD::SETUO:
2701 case ISD::SETUEQ:
2702 case ISD::SETUNE:
2703 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2704 break;
2705 }
2706
2707 if (OpVT.isInteger()) {
2708 // For EQ and NE, we can always pick a value for the undef to make the
2709 // predicate pass or fail, so we can return undef.
2710 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2711 // icmp eq/ne X, undef -> undef.
2712 if ((N1.isUndef() || N2.isUndef()) &&
2713 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2714 return GetUndefBooleanConstant();
2715
2716 // If both operands are undef, we can return undef for int comparison.
2717 // icmp undef, undef -> undef.
2718 if (N1.isUndef() && N2.isUndef())
2719 return GetUndefBooleanConstant();
2720
2721 // icmp X, X -> true/false
2722 // icmp X, undef -> true/false because undef could be X.
2723 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2724 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2725 }
2726
2727 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2728 const APInt &C2 = N2C->getAPIntValue();
2729 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2730 const APInt &C1 = N1C->getAPIntValue();
2731
2732 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2733 DL: dl, VT, OpVT);
2734 }
2735 }
2736
2737 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2738 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2739
2740 if (N1CFP && N2CFP) {
2741 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2742 switch (Cond) {
2743 default: break;
2744 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2745 return GetUndefBooleanConstant();
2746 [[fallthrough]];
2747 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2748 OpVT);
2749 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2750 return GetUndefBooleanConstant();
2751 [[fallthrough]];
2752 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2753 R==APFloat::cmpLessThan, DL: dl, VT,
2754 OpVT);
2755 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2756 return GetUndefBooleanConstant();
2757 [[fallthrough]];
2758 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2759 OpVT);
2760 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2761 return GetUndefBooleanConstant();
2762 [[fallthrough]];
2763 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2764 VT, OpVT);
2765 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2766 return GetUndefBooleanConstant();
2767 [[fallthrough]];
2768 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2769 R==APFloat::cmpEqual, DL: dl, VT,
2770 OpVT);
2771 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2772 return GetUndefBooleanConstant();
2773 [[fallthrough]];
2774 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2775 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2776 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2777 OpVT);
2778 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2779 OpVT);
2780 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2781 R==APFloat::cmpEqual, DL: dl, VT,
2782 OpVT);
2783 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2784 OpVT);
2785 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2786 R==APFloat::cmpLessThan, DL: dl, VT,
2787 OpVT);
2788 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2789 R==APFloat::cmpUnordered, DL: dl, VT,
2790 OpVT);
2791 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2792 VT, OpVT);
2793 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2794 OpVT);
2795 }
2796 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2797 // Ensure that the constant occurs on the RHS.
2798 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2799 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2800 return SDValue();
2801 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond);
2802 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2803 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2804 // If an operand is known to be a nan (or undef that could be a nan), we can
2805 // fold it.
2806 // Choosing NaN for the undef will always make unordered comparison succeed
2807 // and ordered comparison fails.
2808 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2809 switch (ISD::getUnorderedFlavor(Cond)) {
2810 default:
2811 llvm_unreachable("Unknown flavor!");
2812 case 0: // Known false.
2813 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2814 case 1: // Known true.
2815 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2816 case 2: // Undefined.
2817 return GetUndefBooleanConstant();
2818 }
2819 }
2820
2821 // Could not fold it.
2822 return SDValue();
2823}
2824
2825/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2826/// use this predicate to simplify operations downstream.
2827bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2828 unsigned BitWidth = Op.getScalarValueSizeInBits();
2829 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2830}
2831
2832// TODO: Should have argument to specify if sign bit of nan is ignorable.
2833bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2834 if (Depth >= MaxRecursionDepth)
2835 return false; // Limit search depth.
2836
2837 unsigned Opc = Op.getOpcode();
2838 switch (Opc) {
2839 case ISD::FABS:
2840 return true;
2841 case ISD::AssertNoFPClass: {
2842 FPClassTest NoFPClass =
2843 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2844
2845 const FPClassTest TestMask = fcNan | fcNegative;
2846 return (NoFPClass & TestMask) == TestMask;
2847 }
2848 case ISD::ARITH_FENCE:
2849 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2850 case ISD::FEXP:
2851 case ISD::FEXP2:
2852 case ISD::FEXP10:
2853 return Op->getFlags().hasNoNaNs();
2854 case ISD::FMINNUM:
2855 case ISD::FMINNUM_IEEE:
2856 case ISD::FMINIMUM:
2857 case ISD::FMINIMUMNUM:
2858 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2859 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2860 case ISD::FMAXNUM:
2861 case ISD::FMAXNUM_IEEE:
2862 case ISD::FMAXIMUM:
2863 case ISD::FMAXIMUMNUM:
2864 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2865 // is sufficient.
2866 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2867 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2868 default:
2869 return false;
2870 }
2871
2872 llvm_unreachable("covered opcode switch");
2873}
2874
2875/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2876/// this predicate to simplify operations downstream. Mask is known to be zero
2877/// for bits that V cannot have.
2878bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2879 unsigned Depth) const {
2880 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2881}
2882
2883/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2884/// DemandedElts. We use this predicate to simplify operations downstream.
2885/// Mask is known to be zero for bits that V cannot have.
2886bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2887 const APInt &DemandedElts,
2888 unsigned Depth) const {
2889 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2890}
2891
2892/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2893/// DemandedElts. We use this predicate to simplify operations downstream.
2894bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2895 unsigned Depth /* = 0 */) const {
2896 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2897}
2898
2899/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2900bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2901 unsigned Depth) const {
2902 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2903}
2904
2905APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2906 const APInt &DemandedElts,
2907 unsigned Depth) const {
2908 EVT VT = Op.getValueType();
2909 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2910
2911 unsigned NumElts = VT.getVectorNumElements();
2912 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2913
2914 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2915 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2916 if (!DemandedElts[EltIdx])
2917 continue; // Don't query elements that are not demanded.
2918 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2919 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2920 KnownZeroElements.setBit(EltIdx);
2921 }
2922 return KnownZeroElements;
2923}
2924
2925/// isSplatValue - Return true if the vector V has the same value
2926/// across all DemandedElts. For scalable vectors, we don't know the
2927/// number of lanes at compile time. Instead, we use a 1 bit APInt
2928/// to represent a conservative value for all lanes; that is, that
2929/// one bit value is implicitly splatted across all lanes.
2930bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2931 APInt &UndefElts, unsigned Depth) const {
2932 unsigned Opcode = V.getOpcode();
2933 EVT VT = V.getValueType();
2934 assert(VT.isVector() && "Vector type expected");
2935 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2936 "scalable demanded bits are ignored");
2937
2938 if (!DemandedElts)
2939 return false; // No demanded elts, better to assume we don't know anything.
2940
2941 if (Depth >= MaxRecursionDepth)
2942 return false; // Limit search depth.
2943
2944 // Deal with some common cases here that work for both fixed and scalable
2945 // vector types.
2946 switch (Opcode) {
2947 case ISD::SPLAT_VECTOR:
2948 UndefElts = V.getOperand(i: 0).isUndef()
2949 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2950 : APInt(DemandedElts.getBitWidth(), 0);
2951 return true;
2952 case ISD::ADD:
2953 case ISD::SUB:
2954 case ISD::AND:
2955 case ISD::XOR:
2956 case ISD::OR: {
2957 APInt UndefLHS, UndefRHS;
2958 SDValue LHS = V.getOperand(i: 0);
2959 SDValue RHS = V.getOperand(i: 1);
2960 // Only recognize splats with the same demanded undef elements for both
2961 // operands, otherwise we might fail to handle binop-specific undef
2962 // handling.
2963 // e.g. (and undef, 0) -> 0 etc.
2964 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2965 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2966 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2967 UndefElts = UndefLHS | UndefRHS;
2968 return true;
2969 }
2970 return false;
2971 }
2972 case ISD::ABS:
2973 case ISD::TRUNCATE:
2974 case ISD::SIGN_EXTEND:
2975 case ISD::ZERO_EXTEND:
2976 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
2977 default:
2978 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2979 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2980 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
2981 Depth);
2982 break;
2983 }
2984
2985 // We don't support other cases than those above for scalable vectors at
2986 // the moment.
2987 if (VT.isScalableVector())
2988 return false;
2989
2990 unsigned NumElts = VT.getVectorNumElements();
2991 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2992 UndefElts = APInt::getZero(numBits: NumElts);
2993
2994 switch (Opcode) {
2995 case ISD::BUILD_VECTOR: {
2996 SDValue Scl;
2997 for (unsigned i = 0; i != NumElts; ++i) {
2998 SDValue Op = V.getOperand(i);
2999 if (Op.isUndef()) {
3000 UndefElts.setBit(i);
3001 continue;
3002 }
3003 if (!DemandedElts[i])
3004 continue;
3005 if (Scl && Scl != Op)
3006 return false;
3007 Scl = Op;
3008 }
3009 return true;
3010 }
3011 case ISD::VECTOR_SHUFFLE: {
3012 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3013 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3014 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3015 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3016 for (int i = 0; i != (int)NumElts; ++i) {
3017 int M = Mask[i];
3018 if (M < 0) {
3019 UndefElts.setBit(i);
3020 continue;
3021 }
3022 if (!DemandedElts[i])
3023 continue;
3024 if (M < (int)NumElts)
3025 DemandedLHS.setBit(M);
3026 else
3027 DemandedRHS.setBit(M - NumElts);
3028 }
3029
3030 // If we aren't demanding either op, assume there's no splat.
3031 // If we are demanding both ops, assume there's no splat.
3032 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3033 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3034 return false;
3035
3036 // See if the demanded elts of the source op is a splat or we only demand
3037 // one element, which should always be a splat.
3038 // TODO: Handle source ops splats with undefs.
3039 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3040 APInt SrcUndefs;
3041 return (SrcElts.popcount() == 1) ||
3042 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3043 (SrcElts & SrcUndefs).isZero());
3044 };
3045 if (!DemandedLHS.isZero())
3046 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3047 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3048 }
3049 case ISD::EXTRACT_SUBVECTOR: {
3050 // Offset the demanded elts by the subvector index.
3051 SDValue Src = V.getOperand(i: 0);
3052 // We don't support scalable vectors at the moment.
3053 if (Src.getValueType().isScalableVector())
3054 return false;
3055 uint64_t Idx = V.getConstantOperandVal(i: 1);
3056 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3057 APInt UndefSrcElts;
3058 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3059 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3060 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3061 return true;
3062 }
3063 break;
3064 }
3065 case ISD::ANY_EXTEND_VECTOR_INREG:
3066 case ISD::SIGN_EXTEND_VECTOR_INREG:
3067 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3068 // Widen the demanded elts by the src element count.
3069 SDValue Src = V.getOperand(i: 0);
3070 // We don't support scalable vectors at the moment.
3071 if (Src.getValueType().isScalableVector())
3072 return false;
3073 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3074 APInt UndefSrcElts;
3075 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3076 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3077 UndefElts = UndefSrcElts.trunc(width: NumElts);
3078 return true;
3079 }
3080 break;
3081 }
3082 case ISD::BITCAST: {
3083 SDValue Src = V.getOperand(i: 0);
3084 EVT SrcVT = Src.getValueType();
3085 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3086 unsigned BitWidth = VT.getScalarSizeInBits();
3087
3088 // Ignore bitcasts from unsupported types.
3089 // TODO: Add fp support?
3090 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3091 break;
3092
3093 // Bitcast 'small element' vector to 'large element' vector.
3094 if ((BitWidth % SrcBitWidth) == 0) {
3095 // See if each sub element is a splat.
3096 unsigned Scale = BitWidth / SrcBitWidth;
3097 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3098 APInt ScaledDemandedElts =
3099 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3100 for (unsigned I = 0; I != Scale; ++I) {
3101 APInt SubUndefElts;
3102 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3103 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3104 SubDemandedElts &= ScaledDemandedElts;
3105 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3106 return false;
3107 // TODO: Add support for merging sub undef elements.
3108 if (!SubUndefElts.isZero())
3109 return false;
3110 }
3111 return true;
3112 }
3113 break;
3114 }
3115 }
3116
3117 return false;
3118}
3119
3120/// Helper wrapper to main isSplatValue function.
3121bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3122 EVT VT = V.getValueType();
3123 assert(VT.isVector() && "Vector type expected");
3124
3125 APInt UndefElts;
3126 // Since the number of lanes in a scalable vector is unknown at compile time,
3127 // we track one bit which is implicitly broadcast to all lanes. This means
3128 // that all lanes in a scalable vector are considered demanded.
3129 APInt DemandedElts
3130 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3131 return isSplatValue(V, DemandedElts, UndefElts) &&
3132 (AllowUndefs || !UndefElts);
3133}
3134
3135SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3136 V = peekThroughExtractSubvectors(V);
3137
3138 EVT VT = V.getValueType();
3139 unsigned Opcode = V.getOpcode();
3140 switch (Opcode) {
3141 default: {
3142 APInt UndefElts;
3143 // Since the number of lanes in a scalable vector is unknown at compile time,
3144 // we track one bit which is implicitly broadcast to all lanes. This means
3145 // that all lanes in a scalable vector are considered demanded.
3146 APInt DemandedElts
3147 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3148
3149 if (isSplatValue(V, DemandedElts, UndefElts)) {
3150 if (VT.isScalableVector()) {
3151 // DemandedElts and UndefElts are ignored for scalable vectors, since
3152 // the only supported cases are SPLAT_VECTOR nodes.
3153 SplatIdx = 0;
3154 } else {
3155 // Handle case where all demanded elements are UNDEF.
3156 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3157 SplatIdx = 0;
3158 return getUNDEF(VT);
3159 }
3160 SplatIdx = (UndefElts & DemandedElts).countr_one();
3161 }
3162 return V;
3163 }
3164 break;
3165 }
3166 case ISD::SPLAT_VECTOR:
3167 SplatIdx = 0;
3168 return V;
3169 case ISD::VECTOR_SHUFFLE: {
3170 assert(!VT.isScalableVector());
3171 // Check if this is a shuffle node doing a splat.
3172 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3173 // getTargetVShiftNode currently struggles without the splat source.
3174 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3175 if (!SVN->isSplat())
3176 break;
3177 int Idx = SVN->getSplatIndex();
3178 int NumElts = V.getValueType().getVectorNumElements();
3179 SplatIdx = Idx % NumElts;
3180 return V.getOperand(i: Idx / NumElts);
3181 }
3182 }
3183
3184 return SDValue();
3185}
3186
3187SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3188 int SplatIdx;
3189 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3190 EVT SVT = SrcVector.getValueType().getScalarType();
3191 EVT LegalSVT = SVT;
3192 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3193 if (!SVT.isInteger())
3194 return SDValue();
3195 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3196 if (LegalSVT.bitsLT(VT: SVT))
3197 return SDValue();
3198 }
3199 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3200 }
3201 return SDValue();
3202}
3203
3204std::optional<ConstantRange>
3205SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3206 unsigned Depth) const {
3207 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3208 V.getOpcode() == ISD::SRA) &&
3209 "Unknown shift node");
3210 // Shifting more than the bitwidth is not valid.
3211 unsigned BitWidth = V.getScalarValueSizeInBits();
3212
3213 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3214 const APInt &ShAmt = Cst->getAPIntValue();
3215 if (ShAmt.uge(RHS: BitWidth))
3216 return std::nullopt;
3217 return ConstantRange(ShAmt);
3218 }
3219
3220 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3221 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3222 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3223 if (!DemandedElts[i])
3224 continue;
3225 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3226 if (!SA) {
3227 MinAmt = MaxAmt = nullptr;
3228 break;
3229 }
3230 const APInt &ShAmt = SA->getAPIntValue();
3231 if (ShAmt.uge(RHS: BitWidth))
3232 return std::nullopt;
3233 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3234 MinAmt = &ShAmt;
3235 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3236 MaxAmt = &ShAmt;
3237 }
3238 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3239 "Failed to find matching min/max shift amounts");
3240 if (MinAmt && MaxAmt)
3241 return ConstantRange(*MinAmt, *MaxAmt + 1);
3242 }
3243
3244 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3245 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3246 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3247 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3248 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3249
3250 return std::nullopt;
3251}
3252
3253std::optional<unsigned>
3254SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3255 unsigned Depth) const {
3256 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3257 V.getOpcode() == ISD::SRA) &&
3258 "Unknown shift node");
3259 if (std::optional<ConstantRange> AmtRange =
3260 getValidShiftAmountRange(V, DemandedElts, Depth))
3261 if (const APInt *ShAmt = AmtRange->getSingleElement())
3262 return ShAmt->getZExtValue();
3263 return std::nullopt;
3264}
3265
3266std::optional<unsigned>
3267SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3268 EVT VT = V.getValueType();
3269 APInt DemandedElts = VT.isFixedLengthVector()
3270 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3271 : APInt(1, 1);
3272 return getValidShiftAmount(V, DemandedElts, Depth);
3273}
3274
3275std::optional<unsigned>
3276SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3277 unsigned Depth) const {
3278 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3279 V.getOpcode() == ISD::SRA) &&
3280 "Unknown shift node");
3281 if (std::optional<ConstantRange> AmtRange =
3282 getValidShiftAmountRange(V, DemandedElts, Depth))
3283 return AmtRange->getUnsignedMin().getZExtValue();
3284 return std::nullopt;
3285}
3286
3287std::optional<unsigned>
3288SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3289 EVT VT = V.getValueType();
3290 APInt DemandedElts = VT.isFixedLengthVector()
3291 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3292 : APInt(1, 1);
3293 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3294}
3295
3296std::optional<unsigned>
3297SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3298 unsigned Depth) const {
3299 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3300 V.getOpcode() == ISD::SRA) &&
3301 "Unknown shift node");
3302 if (std::optional<ConstantRange> AmtRange =
3303 getValidShiftAmountRange(V, DemandedElts, Depth))
3304 return AmtRange->getUnsignedMax().getZExtValue();
3305 return std::nullopt;
3306}
3307
3308std::optional<unsigned>
3309SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3310 EVT VT = V.getValueType();
3311 APInt DemandedElts = VT.isFixedLengthVector()
3312 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3313 : APInt(1, 1);
3314 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3315}
3316
3317/// Determine which bits of Op are known to be either zero or one and return
3318/// them in Known. For vectors, the known bits are those that are shared by
3319/// every vector element.
3320KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3321 EVT VT = Op.getValueType();
3322
3323 // Since the number of lanes in a scalable vector is unknown at compile time,
3324 // we track one bit which is implicitly broadcast to all lanes. This means
3325 // that all lanes in a scalable vector are considered demanded.
3326 APInt DemandedElts = VT.isFixedLengthVector()
3327 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3328 : APInt(1, 1);
3329 return computeKnownBits(Op, DemandedElts, Depth);
3330}
3331
3332/// Determine which bits of Op are known to be either zero or one and return
3333/// them in Known. The DemandedElts argument allows us to only collect the known
3334/// bits that are shared by the requested vector elements.
3335KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3336 unsigned Depth) const {
3337 unsigned BitWidth = Op.getScalarValueSizeInBits();
3338
3339 KnownBits Known(BitWidth); // Don't know anything.
3340
3341 if (auto OptAPInt = Op->bitcastToAPInt()) {
3342 // We know all of the bits for a constant!
3343 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3344 }
3345
3346 if (Depth >= MaxRecursionDepth)
3347 return Known; // Limit search depth.
3348
3349 KnownBits Known2;
3350 unsigned NumElts = DemandedElts.getBitWidth();
3351 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3352 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3353 assert((!Op.getValueType().isFixedLengthVector() ||
3354 NumElts == Op.getValueType().getVectorNumElements()) &&
3355 "Unexpected vector size");
3356
3357 if (!DemandedElts)
3358 return Known; // No demanded elts, better to assume we don't know anything.
3359
3360 unsigned Opcode = Op.getOpcode();
3361 switch (Opcode) {
3362 case ISD::MERGE_VALUES:
3363 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3364 Depth: Depth + 1);
3365 case ISD::SPLAT_VECTOR: {
3366 SDValue SrcOp = Op.getOperand(i: 0);
3367 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3368 "Expected SPLAT_VECTOR implicit truncation");
3369 // Implicitly truncate the bits to match the official semantics of
3370 // SPLAT_VECTOR.
3371 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3372 break;
3373 }
3374 case ISD::SPLAT_VECTOR_PARTS: {
3375 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3376 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3377 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3378 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3379 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3380 }
3381 break;
3382 }
3383 case ISD::STEP_VECTOR: {
3384 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3385
3386 if (Step.isPowerOf2())
3387 Known.Zero.setLowBits(Step.logBase2());
3388
3389 const Function &F = getMachineFunction().getFunction();
3390
3391 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3392 break;
3393 const APInt MinNumElts =
3394 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3395
3396 bool Overflow;
3397 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3398 .getUnsignedMax()
3399 .umul_ov(RHS: MinNumElts, Overflow);
3400 if (Overflow)
3401 break;
3402
3403 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3404 if (Overflow)
3405 break;
3406
3407 Known.Zero.setHighBits(MaxValue.countl_zero());
3408 break;
3409 }
3410 case ISD::BUILD_VECTOR:
3411 assert(!Op.getValueType().isScalableVector());
3412 // Collect the known bits that are shared by every demanded vector element.
3413 Known.setAllConflict();
3414 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3415 if (!DemandedElts[i])
3416 continue;
3417
3418 SDValue SrcOp = Op.getOperand(i);
3419 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3420
3421 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3422 if (SrcOp.getValueSizeInBits() != BitWidth) {
3423 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3424 "Expected BUILD_VECTOR implicit truncation");
3425 Known2 = Known2.trunc(BitWidth);
3426 }
3427
3428 // Known bits are the values that are shared by every demanded element.
3429 Known = Known.intersectWith(RHS: Known2);
3430
3431 // If we don't know any bits, early out.
3432 if (Known.isUnknown())
3433 break;
3434 }
3435 break;
3436 case ISD::VECTOR_COMPRESS: {
3437 SDValue Vec = Op.getOperand(i: 0);
3438 SDValue PassThru = Op.getOperand(i: 2);
3439 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3440 // If we don't know any bits, early out.
3441 if (Known.isUnknown())
3442 break;
3443 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3444 Known = Known.intersectWith(RHS: Known2);
3445 break;
3446 }
3447 case ISD::VECTOR_SHUFFLE: {
3448 assert(!Op.getValueType().isScalableVector());
3449 // Collect the known bits that are shared by every vector element referenced
3450 // by the shuffle.
3451 APInt DemandedLHS, DemandedRHS;
3452 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3453 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3454 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3455 DemandedLHS, DemandedRHS))
3456 break;
3457
3458 // Known bits are the values that are shared by every demanded element.
3459 Known.setAllConflict();
3460 if (!!DemandedLHS) {
3461 SDValue LHS = Op.getOperand(i: 0);
3462 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3463 Known = Known.intersectWith(RHS: Known2);
3464 }
3465 // If we don't know any bits, early out.
3466 if (Known.isUnknown())
3467 break;
3468 if (!!DemandedRHS) {
3469 SDValue RHS = Op.getOperand(i: 1);
3470 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3471 Known = Known.intersectWith(RHS: Known2);
3472 }
3473 break;
3474 }
3475 case ISD::VSCALE: {
3476 const Function &F = getMachineFunction().getFunction();
3477 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3478 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3479 break;
3480 }
3481 case ISD::CONCAT_VECTORS: {
3482 if (Op.getValueType().isScalableVector())
3483 break;
3484 // Split DemandedElts and test each of the demanded subvectors.
3485 Known.setAllConflict();
3486 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3487 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3488 unsigned NumSubVectors = Op.getNumOperands();
3489 for (unsigned i = 0; i != NumSubVectors; ++i) {
3490 APInt DemandedSub =
3491 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3492 if (!!DemandedSub) {
3493 SDValue Sub = Op.getOperand(i);
3494 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3495 Known = Known.intersectWith(RHS: Known2);
3496 }
3497 // If we don't know any bits, early out.
3498 if (Known.isUnknown())
3499 break;
3500 }
3501 break;
3502 }
3503 case ISD::INSERT_SUBVECTOR: {
3504 if (Op.getValueType().isScalableVector())
3505 break;
3506 // Demand any elements from the subvector and the remainder from the src its
3507 // inserted into.
3508 SDValue Src = Op.getOperand(i: 0);
3509 SDValue Sub = Op.getOperand(i: 1);
3510 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3511 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3512 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3513 APInt DemandedSrcElts = DemandedElts;
3514 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3515
3516 Known.setAllConflict();
3517 if (!!DemandedSubElts) {
3518 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3519 if (Known.isUnknown())
3520 break; // early-out.
3521 }
3522 if (!!DemandedSrcElts) {
3523 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3524 Known = Known.intersectWith(RHS: Known2);
3525 }
3526 break;
3527 }
3528 case ISD::EXTRACT_SUBVECTOR: {
3529 // Offset the demanded elts by the subvector index.
3530 SDValue Src = Op.getOperand(i: 0);
3531
3532 APInt DemandedSrcElts;
3533 if (Src.getValueType().isScalableVector())
3534 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3535 else {
3536 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3537 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3538 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3539 }
3540 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3541 break;
3542 }
3543 case ISD::SCALAR_TO_VECTOR: {
3544 if (Op.getValueType().isScalableVector())
3545 break;
3546 // We know about scalar_to_vector as much as we know about it source,
3547 // which becomes the first element of otherwise unknown vector.
3548 if (DemandedElts != 1)
3549 break;
3550
3551 SDValue N0 = Op.getOperand(i: 0);
3552 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3553 if (N0.getValueSizeInBits() != BitWidth)
3554 Known = Known.trunc(BitWidth);
3555
3556 break;
3557 }
3558 case ISD::BITCAST: {
3559 if (Op.getValueType().isScalableVector())
3560 break;
3561
3562 SDValue N0 = Op.getOperand(i: 0);
3563 EVT SubVT = N0.getValueType();
3564 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3565
3566 // Ignore bitcasts from unsupported types.
3567 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3568 break;
3569
3570 // Fast handling of 'identity' bitcasts.
3571 if (BitWidth == SubBitWidth) {
3572 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3573 break;
3574 }
3575
3576 bool IsLE = getDataLayout().isLittleEndian();
3577
3578 // Bitcast 'small element' vector to 'large element' scalar/vector.
3579 if ((BitWidth % SubBitWidth) == 0) {
3580 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3581
3582 // Collect known bits for the (larger) output by collecting the known
3583 // bits from each set of sub elements and shift these into place.
3584 // We need to separately call computeKnownBits for each set of
3585 // sub elements as the knownbits for each is likely to be different.
3586 unsigned SubScale = BitWidth / SubBitWidth;
3587 APInt SubDemandedElts(NumElts * SubScale, 0);
3588 for (unsigned i = 0; i != NumElts; ++i)
3589 if (DemandedElts[i])
3590 SubDemandedElts.setBit(i * SubScale);
3591
3592 for (unsigned i = 0; i != SubScale; ++i) {
3593 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3594 Depth: Depth + 1);
3595 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3596 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3597 }
3598 }
3599
3600 // Bitcast 'large element' scalar/vector to 'small element' vector.
3601 if ((SubBitWidth % BitWidth) == 0) {
3602 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3603
3604 // Collect known bits for the (smaller) output by collecting the known
3605 // bits from the overlapping larger input elements and extracting the
3606 // sub sections we actually care about.
3607 unsigned SubScale = SubBitWidth / BitWidth;
3608 APInt SubDemandedElts =
3609 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3610 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3611
3612 Known.setAllConflict();
3613 for (unsigned i = 0; i != NumElts; ++i)
3614 if (DemandedElts[i]) {
3615 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3616 unsigned Offset = (Shifts % SubScale) * BitWidth;
3617 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3618 // If we don't know any bits, early out.
3619 if (Known.isUnknown())
3620 break;
3621 }
3622 }
3623 break;
3624 }
3625 case ISD::AND:
3626 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3627 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3628
3629 Known &= Known2;
3630 break;
3631 case ISD::OR:
3632 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3633 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3634
3635 Known |= Known2;
3636 break;
3637 case ISD::XOR:
3638 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3639 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3640
3641 Known ^= Known2;
3642 break;
3643 case ISD::MUL: {
3644 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3645 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3646 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3647 // TODO: SelfMultiply can be poison, but not undef.
3648 if (SelfMultiply)
3649 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3650 Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
3651 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3652
3653 // If the multiplication is known not to overflow, the product of a number
3654 // with itself is non-negative. Only do this if we didn't already computed
3655 // the opposite value for the sign bit.
3656 if (Op->getFlags().hasNoSignedWrap() &&
3657 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3658 !Known.isNegative())
3659 Known.makeNonNegative();
3660 break;
3661 }
3662 case ISD::MULHU: {
3663 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3664 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3665 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3666 break;
3667 }
3668 case ISD::MULHS: {
3669 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3670 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3671 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3672 break;
3673 }
3674 case ISD::ABDU: {
3675 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3676 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3677 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3678 break;
3679 }
3680 case ISD::ABDS: {
3681 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3682 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3683 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3684 unsigned SignBits1 =
3685 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3686 if (SignBits1 == 1)
3687 break;
3688 unsigned SignBits0 =
3689 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3690 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3691 break;
3692 }
3693 case ISD::UMUL_LOHI: {
3694 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3695 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3696 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3697 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3698 if (Op.getResNo() == 0)
3699 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3700 else
3701 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3702 break;
3703 }
3704 case ISD::SMUL_LOHI: {
3705 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3706 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3707 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3708 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3709 if (Op.getResNo() == 0)
3710 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3711 else
3712 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3713 break;
3714 }
3715 case ISD::AVGFLOORU: {
3716 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3717 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3718 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3719 break;
3720 }
3721 case ISD::AVGCEILU: {
3722 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3723 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3724 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3725 break;
3726 }
3727 case ISD::AVGFLOORS: {
3728 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3729 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3730 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3731 break;
3732 }
3733 case ISD::AVGCEILS: {
3734 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3735 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3736 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3737 break;
3738 }
3739 case ISD::SELECT:
3740 case ISD::VSELECT:
3741 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3742 // If we don't know any bits, early out.
3743 if (Known.isUnknown())
3744 break;
3745 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3746
3747 // Only known if known in both the LHS and RHS.
3748 Known = Known.intersectWith(RHS: Known2);
3749 break;
3750 case ISD::SELECT_CC:
3751 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3752 // If we don't know any bits, early out.
3753 if (Known.isUnknown())
3754 break;
3755 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3756
3757 // Only known if known in both the LHS and RHS.
3758 Known = Known.intersectWith(RHS: Known2);
3759 break;
3760 case ISD::SMULO:
3761 case ISD::UMULO:
3762 if (Op.getResNo() != 1)
3763 break;
3764 // The boolean result conforms to getBooleanContents.
3765 // If we know the result of a setcc has the top bits zero, use this info.
3766 // We know that we have an integer-based boolean since these operations
3767 // are only available for integer.
3768 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3769 TargetLowering::ZeroOrOneBooleanContent &&
3770 BitWidth > 1)
3771 Known.Zero.setBitsFrom(1);
3772 break;
3773 case ISD::SETCC:
3774 case ISD::SETCCCARRY:
3775 case ISD::STRICT_FSETCC:
3776 case ISD::STRICT_FSETCCS: {
3777 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3778 // If we know the result of a setcc has the top bits zero, use this info.
3779 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3780 TargetLowering::ZeroOrOneBooleanContent &&
3781 BitWidth > 1)
3782 Known.Zero.setBitsFrom(1);
3783 break;
3784 }
3785 case ISD::SHL: {
3786 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3787 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3788
3789 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3790 bool NSW = Op->getFlags().hasNoSignedWrap();
3791
3792 bool ShAmtNonZero = Known2.isNonZero();
3793
3794 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3795
3796 // Minimum shift low bits are known zero.
3797 if (std::optional<unsigned> ShMinAmt =
3798 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3799 Known.Zero.setLowBits(*ShMinAmt);
3800 break;
3801 }
3802 case ISD::SRL:
3803 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3804 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3805 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3806 Exact: Op->getFlags().hasExact());
3807
3808 // Minimum shift high bits are known zero.
3809 if (std::optional<unsigned> ShMinAmt =
3810 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3811 Known.Zero.setHighBits(*ShMinAmt);
3812 break;
3813 case ISD::SRA:
3814 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3815 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3816 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3817 Exact: Op->getFlags().hasExact());
3818 break;
3819 case ISD::ROTL:
3820 case ISD::ROTR:
3821 if (ConstantSDNode *C =
3822 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3823 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3824
3825 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3826
3827 // Canonicalize to ROTR.
3828 if (Opcode == ISD::ROTL && Amt != 0)
3829 Amt = BitWidth - Amt;
3830
3831 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3832 Known.One = Known.One.rotr(rotateAmt: Amt);
3833 }
3834 break;
3835 case ISD::FSHL:
3836 case ISD::FSHR:
3837 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3838 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3839
3840 // For fshl, 0-shift returns the 1st arg.
3841 // For fshr, 0-shift returns the 2nd arg.
3842 if (Amt == 0) {
3843 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3844 DemandedElts, Depth: Depth + 1);
3845 break;
3846 }
3847
3848 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3849 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3850 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3851 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3852 if (Opcode == ISD::FSHL) {
3853 Known <<= Amt;
3854 Known2 >>= BitWidth - Amt;
3855 } else {
3856 Known <<= BitWidth - Amt;
3857 Known2 >>= Amt;
3858 }
3859 Known = Known.unionWith(RHS: Known2);
3860 }
3861 break;
3862 case ISD::SHL_PARTS:
3863 case ISD::SRA_PARTS:
3864 case ISD::SRL_PARTS: {
3865 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3866
3867 // Collect lo/hi source values and concatenate.
3868 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3869 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3870 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3871 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3872 Known = Known2.concat(Lo: Known);
3873
3874 // Collect shift amount.
3875 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3876
3877 if (Opcode == ISD::SHL_PARTS)
3878 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3879 else if (Opcode == ISD::SRA_PARTS)
3880 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3881 else // if (Opcode == ISD::SRL_PARTS)
3882 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3883
3884 // TODO: Minimum shift low/high bits are known zero.
3885
3886 if (Op.getResNo() == 0)
3887 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3888 else
3889 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3890 break;
3891 }
3892 case ISD::SIGN_EXTEND_INREG: {
3893 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3894 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3895 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3896 break;
3897 }
3898 case ISD::CTTZ:
3899 case ISD::CTTZ_ZERO_UNDEF: {
3900 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3901 // If we have a known 1, its position is our upper bound.
3902 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3903 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3904 Known.Zero.setBitsFrom(LowBits);
3905 break;
3906 }
3907 case ISD::CTLZ:
3908 case ISD::CTLZ_ZERO_UNDEF: {
3909 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3910 // If we have a known 1, its position is our upper bound.
3911 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3912 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3913 Known.Zero.setBitsFrom(LowBits);
3914 break;
3915 }
3916 case ISD::CTLS: {
3917 unsigned MinRedundantSignBits =
3918 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3919 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3920 APInt(BitWidth, BitWidth));
3921 Known = Range.toKnownBits();
3922 break;
3923 }
3924 case ISD::CTPOP: {
3925 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3926 // If we know some of the bits are zero, they can't be one.
3927 unsigned PossibleOnes = Known2.countMaxPopulation();
3928 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3929 break;
3930 }
3931 case ISD::PARITY: {
3932 // Parity returns 0 everywhere but the LSB.
3933 Known.Zero.setBitsFrom(1);
3934 break;
3935 }
3936 case ISD::CLMUL: {
3937 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3938 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3939 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3940 break;
3941 }
3942 case ISD::MGATHER:
3943 case ISD::MLOAD: {
3944 ISD::LoadExtType ETy =
3945 (Opcode == ISD::MGATHER)
3946 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3947 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3948 if (ETy == ISD::ZEXTLOAD) {
3949 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3950 KnownBits Known0(MemVT.getScalarSizeInBits());
3951 return Known0.zext(BitWidth);
3952 }
3953 break;
3954 }
3955 case ISD::LOAD: {
3956 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3957 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3958 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3959 // Determine any common known bits from the loaded constant pool value.
3960 Type *CstTy = Cst->getType();
3961 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3962 !Op.getValueType().isScalableVector()) {
3963 // If its a vector splat, then we can (quickly) reuse the scalar path.
3964 // NOTE: We assume all elements match and none are UNDEF.
3965 if (CstTy->isVectorTy()) {
3966 if (const Constant *Splat = Cst->getSplatValue()) {
3967 Cst = Splat;
3968 CstTy = Cst->getType();
3969 }
3970 }
3971 // TODO - do we need to handle different bitwidths?
3972 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3973 // Iterate across all vector elements finding common known bits.
3974 Known.setAllConflict();
3975 for (unsigned i = 0; i != NumElts; ++i) {
3976 if (!DemandedElts[i])
3977 continue;
3978 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
3979 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
3980 const APInt &Value = CInt->getValue();
3981 Known.One &= Value;
3982 Known.Zero &= ~Value;
3983 continue;
3984 }
3985 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
3986 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3987 Known.One &= Value;
3988 Known.Zero &= ~Value;
3989 continue;
3990 }
3991 }
3992 Known.One.clearAllBits();
3993 Known.Zero.clearAllBits();
3994 break;
3995 }
3996 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3997 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
3998 Known = KnownBits::makeConstant(C: CInt->getValue());
3999 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4000 Known =
4001 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4002 }
4003 }
4004 }
4005 } else if (Op.getResNo() == 0) {
4006 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4007 KnownBits KnownScalarMemory(ScalarMemorySize);
4008 if (const MDNode *MD = LD->getRanges())
4009 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4010
4011 // Extend the Known bits from memory to the size of the scalar result.
4012 if (ISD::isZEXTLoad(N: Op.getNode()))
4013 Known = KnownScalarMemory.zext(BitWidth);
4014 else if (ISD::isSEXTLoad(N: Op.getNode()))
4015 Known = KnownScalarMemory.sext(BitWidth);
4016 else if (ISD::isEXTLoad(N: Op.getNode()))
4017 Known = KnownScalarMemory.anyext(BitWidth);
4018 else
4019 Known = KnownScalarMemory;
4020 assert(Known.getBitWidth() == BitWidth);
4021 return Known;
4022 }
4023 break;
4024 }
4025 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4026 if (Op.getValueType().isScalableVector())
4027 break;
4028 EVT InVT = Op.getOperand(i: 0).getValueType();
4029 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4030 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4031 Known = Known.zext(BitWidth);
4032 break;
4033 }
4034 case ISD::ZERO_EXTEND: {
4035 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4036 Known = Known.zext(BitWidth);
4037 break;
4038 }
4039 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4040 if (Op.getValueType().isScalableVector())
4041 break;
4042 EVT InVT = Op.getOperand(i: 0).getValueType();
4043 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4044 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4045 // If the sign bit is known to be zero or one, then sext will extend
4046 // it to the top bits, else it will just zext.
4047 Known = Known.sext(BitWidth);
4048 break;
4049 }
4050 case ISD::SIGN_EXTEND: {
4051 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4052 // If the sign bit is known to be zero or one, then sext will extend
4053 // it to the top bits, else it will just zext.
4054 Known = Known.sext(BitWidth);
4055 break;
4056 }
4057 case ISD::ANY_EXTEND_VECTOR_INREG: {
4058 if (Op.getValueType().isScalableVector())
4059 break;
4060 EVT InVT = Op.getOperand(i: 0).getValueType();
4061 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4062 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4063 Known = Known.anyext(BitWidth);
4064 break;
4065 }
4066 case ISD::ANY_EXTEND: {
4067 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4068 Known = Known.anyext(BitWidth);
4069 break;
4070 }
4071 case ISD::TRUNCATE: {
4072 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4073 Known = Known.trunc(BitWidth);
4074 break;
4075 }
4076 case ISD::TRUNCATE_SSAT_S: {
4077 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4078 Known = Known.truncSSat(BitWidth);
4079 break;
4080 }
4081 case ISD::TRUNCATE_SSAT_U: {
4082 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4083 Known = Known.truncSSatU(BitWidth);
4084 break;
4085 }
4086 case ISD::TRUNCATE_USAT_U: {
4087 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4088 Known = Known.truncUSat(BitWidth);
4089 break;
4090 }
4091 case ISD::AssertZext: {
4092 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4093 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4094 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4095 Known.Zero |= (~InMask);
4096 Known.One &= (~Known.Zero);
4097 break;
4098 }
4099 case ISD::AssertAlign: {
4100 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4101 assert(LogOfAlign != 0);
4102
4103 // TODO: Should use maximum with source
4104 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4105 // well as clearing one bits.
4106 Known.Zero.setLowBits(LogOfAlign);
4107 Known.One.clearLowBits(loBits: LogOfAlign);
4108 break;
4109 }
4110 case ISD::AssertNoFPClass: {
4111 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4112
4113 FPClassTest NoFPClass =
4114 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4115 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4116 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4117 // Cannot be negative.
4118 Known.makeNonNegative();
4119 }
4120
4121 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4122 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4123 // Cannot be positive.
4124 Known.makeNegative();
4125 }
4126
4127 break;
4128 }
4129 case ISD::FGETSIGN:
4130 // All bits are zero except the low bit.
4131 Known.Zero.setBitsFrom(1);
4132 break;
4133 case ISD::ADD:
4134 case ISD::SUB: {
4135 SDNodeFlags Flags = Op.getNode()->getFlags();
4136 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4137 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4138 Known = KnownBits::computeForAddSub(
4139 Add: Op.getOpcode() == ISD::ADD, NSW: Flags.hasNoSignedWrap(),
4140 NUW: Flags.hasNoUnsignedWrap(), LHS: Known, RHS: Known2);
4141 break;
4142 }
4143 case ISD::USUBO:
4144 case ISD::SSUBO:
4145 case ISD::USUBO_CARRY:
4146 case ISD::SSUBO_CARRY:
4147 if (Op.getResNo() == 1) {
4148 // If we know the result of a setcc has the top bits zero, use this info.
4149 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4150 TargetLowering::ZeroOrOneBooleanContent &&
4151 BitWidth > 1)
4152 Known.Zero.setBitsFrom(1);
4153 break;
4154 }
4155 [[fallthrough]];
4156 case ISD::SUBC: {
4157 assert(Op.getResNo() == 0 &&
4158 "We only compute knownbits for the difference here.");
4159
4160 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4161 KnownBits Borrow(1);
4162 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4163 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4164 // Borrow has bit width 1
4165 Borrow = Borrow.trunc(BitWidth: 1);
4166 } else {
4167 Borrow.setAllZero();
4168 }
4169
4170 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4171 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4172 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4173 break;
4174 }
4175 case ISD::UADDO:
4176 case ISD::SADDO:
4177 case ISD::UADDO_CARRY:
4178 case ISD::SADDO_CARRY:
4179 if (Op.getResNo() == 1) {
4180 // If we know the result of a setcc has the top bits zero, use this info.
4181 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4182 TargetLowering::ZeroOrOneBooleanContent &&
4183 BitWidth > 1)
4184 Known.Zero.setBitsFrom(1);
4185 break;
4186 }
4187 [[fallthrough]];
4188 case ISD::ADDC:
4189 case ISD::ADDE: {
4190 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4191
4192 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4193 KnownBits Carry(1);
4194 if (Opcode == ISD::ADDE)
4195 // Can't track carry from glue, set carry to unknown.
4196 Carry.resetAll();
4197 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4198 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4199 // Carry has bit width 1
4200 Carry = Carry.trunc(BitWidth: 1);
4201 } else {
4202 Carry.setAllZero();
4203 }
4204
4205 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4206 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4207 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4208 break;
4209 }
4210 case ISD::UDIV: {
4211 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4212 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4213 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4214 break;
4215 }
4216 case ISD::SDIV: {
4217 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4218 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4219 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4220 break;
4221 }
4222 case ISD::SREM: {
4223 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4224 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4225 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4226 break;
4227 }
4228 case ISD::UREM: {
4229 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4230 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4231 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4232 break;
4233 }
4234 case ISD::EXTRACT_ELEMENT: {
4235 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4236 const unsigned Index = Op.getConstantOperandVal(i: 1);
4237 const unsigned EltBitWidth = Op.getValueSizeInBits();
4238
4239 // Remove low part of known bits mask
4240 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4241 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4242
4243 // Remove high part of known bit mask
4244 Known = Known.trunc(BitWidth: EltBitWidth);
4245 break;
4246 }
4247 case ISD::EXTRACT_VECTOR_ELT: {
4248 SDValue InVec = Op.getOperand(i: 0);
4249 SDValue EltNo = Op.getOperand(i: 1);
4250 EVT VecVT = InVec.getValueType();
4251 // computeKnownBits not yet implemented for scalable vectors.
4252 if (VecVT.isScalableVector())
4253 break;
4254 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4255 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4256
4257 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4258 // anything about the extended bits.
4259 if (BitWidth > EltBitWidth)
4260 Known = Known.trunc(BitWidth: EltBitWidth);
4261
4262 // If we know the element index, just demand that vector element, else for
4263 // an unknown element index, ignore DemandedElts and demand them all.
4264 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4265 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4266 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4267 DemandedSrcElts =
4268 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4269
4270 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4271 if (BitWidth > EltBitWidth)
4272 Known = Known.anyext(BitWidth);
4273 break;
4274 }
4275 case ISD::INSERT_VECTOR_ELT: {
4276 if (Op.getValueType().isScalableVector())
4277 break;
4278
4279 // If we know the element index, split the demand between the
4280 // source vector and the inserted element, otherwise assume we need
4281 // the original demanded vector elements and the value.
4282 SDValue InVec = Op.getOperand(i: 0);
4283 SDValue InVal = Op.getOperand(i: 1);
4284 SDValue EltNo = Op.getOperand(i: 2);
4285 bool DemandedVal = true;
4286 APInt DemandedVecElts = DemandedElts;
4287 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4288 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4289 unsigned EltIdx = CEltNo->getZExtValue();
4290 DemandedVal = !!DemandedElts[EltIdx];
4291 DemandedVecElts.clearBit(BitPosition: EltIdx);
4292 }
4293 Known.setAllConflict();
4294 if (DemandedVal) {
4295 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4296 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4297 }
4298 if (!!DemandedVecElts) {
4299 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4300 Known = Known.intersectWith(RHS: Known2);
4301 }
4302 break;
4303 }
4304 case ISD::BITREVERSE: {
4305 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4306 Known = Known2.reverseBits();
4307 break;
4308 }
4309 case ISD::BSWAP: {
4310 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4311 Known = Known2.byteSwap();
4312 break;
4313 }
4314 case ISD::ABS: {
4315 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4316 Known = Known2.abs();
4317 Known.Zero.setHighBits(
4318 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4319 break;
4320 }
4321 case ISD::USUBSAT: {
4322 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4323 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4324 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4325 break;
4326 }
4327 case ISD::UMIN: {
4328 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4329 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4330 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4331 break;
4332 }
4333 case ISD::UMAX: {
4334 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4335 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4336 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4337 break;
4338 }
4339 case ISD::SMIN:
4340 case ISD::SMAX: {
4341 // If we have a clamp pattern, we know that the number of sign bits will be
4342 // the minimum of the clamp min/max range.
4343 bool IsMax = (Opcode == ISD::SMAX);
4344 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4345 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4346 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4347 CstHigh =
4348 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4349 if (CstLow && CstHigh) {
4350 if (!IsMax)
4351 std::swap(a&: CstLow, b&: CstHigh);
4352
4353 const APInt &ValueLow = CstLow->getAPIntValue();
4354 const APInt &ValueHigh = CstHigh->getAPIntValue();
4355 if (ValueLow.sle(RHS: ValueHigh)) {
4356 unsigned LowSignBits = ValueLow.getNumSignBits();
4357 unsigned HighSignBits = ValueHigh.getNumSignBits();
4358 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4359 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4360 Known.One.setHighBits(MinSignBits);
4361 break;
4362 }
4363 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4364 Known.Zero.setHighBits(MinSignBits);
4365 break;
4366 }
4367 }
4368 }
4369
4370 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4371 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4372 if (IsMax)
4373 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4374 else
4375 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4376
4377 // For SMAX, if CstLow is non-negative we know the result will be
4378 // non-negative and thus all sign bits are 0.
4379 // TODO: There's an equivalent of this for smin with negative constant for
4380 // known ones.
4381 if (IsMax && CstLow) {
4382 const APInt &ValueLow = CstLow->getAPIntValue();
4383 if (ValueLow.isNonNegative()) {
4384 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4385 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4386 }
4387 }
4388
4389 break;
4390 }
4391 case ISD::UINT_TO_FP: {
4392 Known.makeNonNegative();
4393 break;
4394 }
4395 case ISD::SINT_TO_FP: {
4396 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4397 if (Known2.isNonNegative())
4398 Known.makeNonNegative();
4399 else if (Known2.isNegative())
4400 Known.makeNegative();
4401 break;
4402 }
4403 case ISD::FP_TO_UINT_SAT: {
4404 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4405 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4406 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4407 break;
4408 }
4409 case ISD::ATOMIC_LOAD: {
4410 // If we are looking at the loaded value.
4411 if (Op.getResNo() == 0) {
4412 auto *AT = cast<AtomicSDNode>(Val&: Op);
4413 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4414 KnownBits KnownScalarMemory(ScalarMemorySize);
4415 if (const MDNode *MD = AT->getRanges())
4416 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4417
4418 switch (AT->getExtensionType()) {
4419 case ISD::ZEXTLOAD:
4420 Known = KnownScalarMemory.zext(BitWidth);
4421 break;
4422 case ISD::SEXTLOAD:
4423 Known = KnownScalarMemory.sext(BitWidth);
4424 break;
4425 case ISD::EXTLOAD:
4426 switch (TLI->getExtendForAtomicOps()) {
4427 case ISD::ZERO_EXTEND:
4428 Known = KnownScalarMemory.zext(BitWidth);
4429 break;
4430 case ISD::SIGN_EXTEND:
4431 Known = KnownScalarMemory.sext(BitWidth);
4432 break;
4433 default:
4434 Known = KnownScalarMemory.anyext(BitWidth);
4435 break;
4436 }
4437 break;
4438 case ISD::NON_EXTLOAD:
4439 Known = KnownScalarMemory;
4440 break;
4441 }
4442 assert(Known.getBitWidth() == BitWidth);
4443 }
4444 break;
4445 }
4446 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4447 if (Op.getResNo() == 1) {
4448 // The boolean result conforms to getBooleanContents.
4449 // If we know the result of a setcc has the top bits zero, use this info.
4450 // We know that we have an integer-based boolean since these operations
4451 // are only available for integer.
4452 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4453 TargetLowering::ZeroOrOneBooleanContent &&
4454 BitWidth > 1)
4455 Known.Zero.setBitsFrom(1);
4456 break;
4457 }
4458 [[fallthrough]];
4459 case ISD::ATOMIC_CMP_SWAP:
4460 case ISD::ATOMIC_SWAP:
4461 case ISD::ATOMIC_LOAD_ADD:
4462 case ISD::ATOMIC_LOAD_SUB:
4463 case ISD::ATOMIC_LOAD_AND:
4464 case ISD::ATOMIC_LOAD_CLR:
4465 case ISD::ATOMIC_LOAD_OR:
4466 case ISD::ATOMIC_LOAD_XOR:
4467 case ISD::ATOMIC_LOAD_NAND:
4468 case ISD::ATOMIC_LOAD_MIN:
4469 case ISD::ATOMIC_LOAD_MAX:
4470 case ISD::ATOMIC_LOAD_UMIN:
4471 case ISD::ATOMIC_LOAD_UMAX: {
4472 // If we are looking at the loaded value.
4473 if (Op.getResNo() == 0) {
4474 auto *AT = cast<AtomicSDNode>(Val&: Op);
4475 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4476
4477 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4478 Known.Zero.setBitsFrom(MemBits);
4479 }
4480 break;
4481 }
4482 case ISD::FrameIndex:
4483 case ISD::TargetFrameIndex:
4484 TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4485 Known, MF: getMachineFunction());
4486 break;
4487
4488 default:
4489 if (Opcode < ISD::BUILTIN_OP_END)
4490 break;
4491 [[fallthrough]];
4492 case ISD::INTRINSIC_WO_CHAIN:
4493 case ISD::INTRINSIC_W_CHAIN:
4494 case ISD::INTRINSIC_VOID:
4495 // TODO: Probably okay to remove after audit; here to reduce change size
4496 // in initial enablement patch for scalable vectors
4497 if (Op.getValueType().isScalableVector())
4498 break;
4499
4500 // Allow the target to implement this method for its nodes.
4501 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4502 break;
4503 }
4504
4505 return Known;
4506}
4507
4508/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4509static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4510 switch (OR) {
4511 case ConstantRange::OverflowResult::MayOverflow:
4512 return SelectionDAG::OFK_Sometime;
4513 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4514 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4515 return SelectionDAG::OFK_Always;
4516 case ConstantRange::OverflowResult::NeverOverflows:
4517 return SelectionDAG::OFK_Never;
4518 }
4519 llvm_unreachable("Unknown OverflowResult");
4520}
4521
4522SelectionDAG::OverflowKind
4523SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4524 // X + 0 never overflow
4525 if (isNullConstant(V: N1))
4526 return OFK_Never;
4527
4528 // If both operands each have at least two sign bits, the addition
4529 // cannot overflow.
4530 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4531 return OFK_Never;
4532
4533 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4534 return OFK_Sometime;
4535}
4536
4537SelectionDAG::OverflowKind
4538SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4539 // X + 0 never overflow
4540 if (isNullConstant(V: N1))
4541 return OFK_Never;
4542
4543 // mulhi + 1 never overflow
4544 KnownBits N1Known = computeKnownBits(Op: N1);
4545 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4546 N1Known.getMaxValue().ult(RHS: 2))
4547 return OFK_Never;
4548
4549 KnownBits N0Known = computeKnownBits(Op: N0);
4550 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4551 N0Known.getMaxValue().ult(RHS: 2))
4552 return OFK_Never;
4553
4554 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4555 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4556 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4557 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4558}
4559
4560SelectionDAG::OverflowKind
4561SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4562 // X - 0 never overflow
4563 if (isNullConstant(V: N1))
4564 return OFK_Never;
4565
4566 // If both operands each have at least two sign bits, the subtraction
4567 // cannot overflow.
4568 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4569 return OFK_Never;
4570
4571 KnownBits N0Known = computeKnownBits(Op: N0);
4572 KnownBits N1Known = computeKnownBits(Op: N1);
4573 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4574 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4575 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4576}
4577
4578SelectionDAG::OverflowKind
4579SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4580 // X - 0 never overflow
4581 if (isNullConstant(V: N1))
4582 return OFK_Never;
4583
4584 KnownBits N0Known = computeKnownBits(Op: N0);
4585 KnownBits N1Known = computeKnownBits(Op: N1);
4586 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4587 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4588 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4589}
4590
4591SelectionDAG::OverflowKind
4592SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4593 // X * 0 and X * 1 never overflow.
4594 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4595 return OFK_Never;
4596
4597 KnownBits N0Known = computeKnownBits(Op: N0);
4598 KnownBits N1Known = computeKnownBits(Op: N1);
4599 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4600 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4601 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4602}
4603
4604SelectionDAG::OverflowKind
4605SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4606 // X * 0 and X * 1 never overflow.
4607 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4608 return OFK_Never;
4609
4610 // Get the size of the result.
4611 unsigned BitWidth = N0.getScalarValueSizeInBits();
4612
4613 // Sum of the sign bits.
4614 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4615
4616 // If we have enough sign bits, then there's no overflow.
4617 if (SignBits > BitWidth + 1)
4618 return OFK_Never;
4619
4620 if (SignBits == BitWidth + 1) {
4621 // The overflow occurs when the true multiplication of the
4622 // the operands is the minimum negative number.
4623 KnownBits N0Known = computeKnownBits(Op: N0);
4624 KnownBits N1Known = computeKnownBits(Op: N1);
4625 // If one of the operands is non-negative, then there's no
4626 // overflow.
4627 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4628 return OFK_Never;
4629 }
4630
4631 return OFK_Sometime;
4632}
4633
4634bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
4635 if (Depth >= MaxRecursionDepth)
4636 return false; // Limit search depth.
4637
4638 EVT OpVT = Val.getValueType();
4639 unsigned BitWidth = OpVT.getScalarSizeInBits();
4640
4641 // Is the constant a known power of 2?
4642 if (ISD::matchUnaryPredicate(Op: Val, Match: [BitWidth](ConstantSDNode *C) {
4643 return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4644 }))
4645 return true;
4646
4647 // A left-shift of a constant one will have exactly one bit set because
4648 // shifting the bit off the end is undefined.
4649 if (Val.getOpcode() == ISD::SHL) {
4650 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4651 if (C && C->getAPIntValue() == 1)
4652 return true;
4653 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4654 isKnownNeverZero(Op: Val, Depth);
4655 }
4656
4657 // Similarly, a logical right-shift of a constant sign-bit will have exactly
4658 // one bit set.
4659 if (Val.getOpcode() == ISD::SRL) {
4660 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4661 if (C && C->getAPIntValue().isSignMask())
4662 return true;
4663 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4664 isKnownNeverZero(Op: Val, Depth);
4665 }
4666
4667 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4668 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4669
4670 // Are all operands of a build vector constant powers of two?
4671 if (Val.getOpcode() == ISD::BUILD_VECTOR)
4672 if (llvm::all_of(Range: Val->ops(), P: [BitWidth](SDValue E) {
4673 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: E))
4674 return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4675 return false;
4676 }))
4677 return true;
4678
4679 // Is the operand of a splat vector a constant power of two?
4680 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4681 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4682 if (C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2())
4683 return true;
4684
4685 // vscale(power-of-two) is a power-of-two for some targets
4686 if (Val.getOpcode() == ISD::VSCALE &&
4687 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4688 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1))
4689 return true;
4690
4691 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4692 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4693 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1) &&
4694 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4695
4696 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4697 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), Depth: Depth + 1) &&
4698 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1);
4699
4700 // Looking for `x & -x` pattern:
4701 // If x == 0:
4702 // x & -x -> 0
4703 // If x != 0:
4704 // x & -x -> non-zero pow2
4705 // so if we find the pattern return whether we know `x` is non-zero.
4706 SDValue X;
4707 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))))
4708 return isKnownNeverZero(Op: X, Depth);
4709
4710 if (Val.getOpcode() == ISD::ZERO_EXTEND)
4711 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4712
4713 // More could be done here, though the above checks are enough
4714 // to handle some common cases.
4715 return false;
4716}
4717
4718bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4719 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4720 return C1->getValueAPF().getExactLog2Abs() >= 0;
4721
4722 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4723 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4724
4725 return false;
4726}
4727
4728unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4729 EVT VT = Op.getValueType();
4730
4731 // Since the number of lanes in a scalable vector is unknown at compile time,
4732 // we track one bit which is implicitly broadcast to all lanes. This means
4733 // that all lanes in a scalable vector are considered demanded.
4734 APInt DemandedElts = VT.isFixedLengthVector()
4735 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4736 : APInt(1, 1);
4737 return ComputeNumSignBits(Op, DemandedElts, Depth);
4738}
4739
4740unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4741 unsigned Depth) const {
4742 EVT VT = Op.getValueType();
4743 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4744 unsigned VTBits = VT.getScalarSizeInBits();
4745 unsigned NumElts = DemandedElts.getBitWidth();
4746 unsigned Tmp, Tmp2;
4747 unsigned FirstAnswer = 1;
4748
4749 assert((!VT.isScalableVector() || NumElts == 1) &&
4750 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4751
4752 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4753 const APInt &Val = C->getAPIntValue();
4754 return Val.getNumSignBits();
4755 }
4756
4757 if (Depth >= MaxRecursionDepth)
4758 return 1; // Limit search depth.
4759
4760 if (!DemandedElts)
4761 return 1; // No demanded elts, better to assume we don't know anything.
4762
4763 unsigned Opcode = Op.getOpcode();
4764 switch (Opcode) {
4765 default: break;
4766 case ISD::AssertSext:
4767 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4768 return VTBits-Tmp+1;
4769 case ISD::AssertZext:
4770 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4771 return VTBits-Tmp;
4772 case ISD::FREEZE:
4773 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4774 /*PoisonOnly=*/false))
4775 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4776 break;
4777 case ISD::MERGE_VALUES:
4778 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4779 Depth: Depth + 1);
4780 case ISD::SPLAT_VECTOR: {
4781 // Check if the sign bits of source go down as far as the truncated value.
4782 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4783 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4784 if (NumSrcSignBits > (NumSrcBits - VTBits))
4785 return NumSrcSignBits - (NumSrcBits - VTBits);
4786 break;
4787 }
4788 case ISD::BUILD_VECTOR:
4789 assert(!VT.isScalableVector());
4790 Tmp = VTBits;
4791 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4792 if (!DemandedElts[i])
4793 continue;
4794
4795 SDValue SrcOp = Op.getOperand(i);
4796 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4797 // for constant nodes to ensure we only look at the sign bits.
4798 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4799 APInt T = C->getAPIntValue().trunc(width: VTBits);
4800 Tmp2 = T.getNumSignBits();
4801 } else {
4802 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4803
4804 if (SrcOp.getValueSizeInBits() != VTBits) {
4805 assert(SrcOp.getValueSizeInBits() > VTBits &&
4806 "Expected BUILD_VECTOR implicit truncation");
4807 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4808 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4809 }
4810 }
4811 Tmp = std::min(a: Tmp, b: Tmp2);
4812 }
4813 return Tmp;
4814
4815 case ISD::VECTOR_COMPRESS: {
4816 SDValue Vec = Op.getOperand(i: 0);
4817 SDValue PassThru = Op.getOperand(i: 2);
4818 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
4819 if (Tmp == 1)
4820 return 1;
4821 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
4822 Tmp = std::min(a: Tmp, b: Tmp2);
4823 return Tmp;
4824 }
4825
4826 case ISD::VECTOR_SHUFFLE: {
4827 // Collect the minimum number of sign bits that are shared by every vector
4828 // element referenced by the shuffle.
4829 APInt DemandedLHS, DemandedRHS;
4830 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4831 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4832 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4833 DemandedLHS, DemandedRHS))
4834 return 1;
4835
4836 Tmp = std::numeric_limits<unsigned>::max();
4837 if (!!DemandedLHS)
4838 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
4839 if (!!DemandedRHS) {
4840 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
4841 Tmp = std::min(a: Tmp, b: Tmp2);
4842 }
4843 // If we don't know anything, early out and try computeKnownBits fall-back.
4844 if (Tmp == 1)
4845 break;
4846 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4847 return Tmp;
4848 }
4849
4850 case ISD::BITCAST: {
4851 if (VT.isScalableVector())
4852 break;
4853 SDValue N0 = Op.getOperand(i: 0);
4854 EVT SrcVT = N0.getValueType();
4855 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4856
4857 // Ignore bitcasts from unsupported types..
4858 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4859 break;
4860
4861 // Fast handling of 'identity' bitcasts.
4862 if (VTBits == SrcBits)
4863 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
4864
4865 bool IsLE = getDataLayout().isLittleEndian();
4866
4867 // Bitcast 'large element' scalar/vector to 'small element' vector.
4868 if ((SrcBits % VTBits) == 0) {
4869 assert(VT.isVector() && "Expected bitcast to vector");
4870
4871 unsigned Scale = SrcBits / VTBits;
4872 APInt SrcDemandedElts =
4873 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
4874
4875 // Fast case - sign splat can be simply split across the small elements.
4876 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
4877 if (Tmp == SrcBits)
4878 return VTBits;
4879
4880 // Slow case - determine how far the sign extends into each sub-element.
4881 Tmp2 = VTBits;
4882 for (unsigned i = 0; i != NumElts; ++i)
4883 if (DemandedElts[i]) {
4884 unsigned SubOffset = i % Scale;
4885 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4886 SubOffset = SubOffset * VTBits;
4887 if (Tmp <= SubOffset)
4888 return 1;
4889 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
4890 }
4891 return Tmp2;
4892 }
4893 break;
4894 }
4895
4896 case ISD::FP_TO_SINT_SAT:
4897 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4898 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4899 return VTBits - Tmp + 1;
4900 case ISD::SIGN_EXTEND:
4901 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
4902 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
4903 case ISD::SIGN_EXTEND_INREG:
4904 // Max of the input and what this extends.
4905 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4906 Tmp = VTBits-Tmp+1;
4907 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4908 return std::max(a: Tmp, b: Tmp2);
4909 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4910 if (VT.isScalableVector())
4911 break;
4912 SDValue Src = Op.getOperand(i: 0);
4913 EVT SrcVT = Src.getValueType();
4914 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
4915 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4916 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
4917 }
4918 case ISD::SRA:
4919 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4920 // SRA X, C -> adds C sign bits.
4921 if (std::optional<unsigned> ShAmt =
4922 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
4923 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
4924 return Tmp;
4925 case ISD::SHL:
4926 if (std::optional<ConstantRange> ShAmtRange =
4927 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
4928 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
4929 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
4930 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
4931 // shifted out, then we can compute the number of sign bits for the
4932 // operand being extended. A future improvement could be to pass along the
4933 // "shifted left by" information in the recursive calls to
4934 // ComputeKnownSignBits. Allowing us to handle this more generically.
4935 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
4936 SDValue Ext = Op.getOperand(i: 0);
4937 EVT ExtVT = Ext.getValueType();
4938 SDValue Extendee = Ext.getOperand(i: 0);
4939 EVT ExtendeeVT = Extendee.getValueType();
4940 unsigned SizeDifference =
4941 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
4942 if (SizeDifference <= MinShAmt) {
4943 Tmp = SizeDifference +
4944 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
4945 if (MaxShAmt < Tmp)
4946 return Tmp - MaxShAmt;
4947 }
4948 }
4949 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4950 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4951 if (MaxShAmt < Tmp)
4952 return Tmp - MaxShAmt;
4953 }
4954 break;
4955 case ISD::AND:
4956 case ISD::OR:
4957 case ISD::XOR: // NOT is handled here.
4958 // Logical binary ops preserve the number of sign bits at the worst.
4959 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4960 if (Tmp != 1) {
4961 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4962 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4963 // We computed what we know about the sign bits as our first
4964 // answer. Now proceed to the generic code that uses
4965 // computeKnownBits, and pick whichever answer is better.
4966 }
4967 break;
4968
4969 case ISD::SELECT:
4970 case ISD::VSELECT:
4971 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4972 if (Tmp == 1) return 1; // Early out.
4973 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4974 return std::min(a: Tmp, b: Tmp2);
4975 case ISD::SELECT_CC:
4976 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4977 if (Tmp == 1) return 1; // Early out.
4978 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
4979 return std::min(a: Tmp, b: Tmp2);
4980
4981 case ISD::SMIN:
4982 case ISD::SMAX: {
4983 // If we have a clamp pattern, we know that the number of sign bits will be
4984 // the minimum of the clamp min/max range.
4985 bool IsMax = (Opcode == ISD::SMAX);
4986 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4987 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4988 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4989 CstHigh =
4990 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4991 if (CstLow && CstHigh) {
4992 if (!IsMax)
4993 std::swap(a&: CstLow, b&: CstHigh);
4994 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
4995 Tmp = CstLow->getAPIntValue().getNumSignBits();
4996 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4997 return std::min(a: Tmp, b: Tmp2);
4998 }
4999 }
5000
5001 // Fallback - just get the minimum number of sign bits of the operands.
5002 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5003 if (Tmp == 1)
5004 return 1; // Early out.
5005 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5006 return std::min(a: Tmp, b: Tmp2);
5007 }
5008 case ISD::UMIN:
5009 case ISD::UMAX:
5010 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5011 if (Tmp == 1)
5012 return 1; // Early out.
5013 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5014 return std::min(a: Tmp, b: Tmp2);
5015 case ISD::SSUBO_CARRY:
5016 case ISD::USUBO_CARRY:
5017 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5018 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5019 return VTBits;
5020 [[fallthrough]];
5021 case ISD::SADDO:
5022 case ISD::UADDO:
5023 case ISD::SADDO_CARRY:
5024 case ISD::UADDO_CARRY:
5025 case ISD::SSUBO:
5026 case ISD::USUBO:
5027 case ISD::SMULO:
5028 case ISD::UMULO:
5029 if (Op.getResNo() != 1)
5030 break;
5031 // The boolean result conforms to getBooleanContents. Fall through.
5032 // If setcc returns 0/-1, all bits are sign bits.
5033 // We know that we have an integer-based boolean since these operations
5034 // are only available for integer.
5035 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5036 TargetLowering::ZeroOrNegativeOneBooleanContent)
5037 return VTBits;
5038 break;
5039 case ISD::SETCC:
5040 case ISD::SETCCCARRY:
5041 case ISD::STRICT_FSETCC:
5042 case ISD::STRICT_FSETCCS: {
5043 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5044 // If setcc returns 0/-1, all bits are sign bits.
5045 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5046 TargetLowering::ZeroOrNegativeOneBooleanContent)
5047 return VTBits;
5048 break;
5049 }
5050 case ISD::ROTL:
5051 case ISD::ROTR:
5052 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5053
5054 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
5055 if (Tmp == VTBits)
5056 return VTBits;
5057
5058 if (ConstantSDNode *C =
5059 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
5060 unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
5061
5062 // Handle rotate right by N like a rotate left by 32-N.
5063 if (Opcode == ISD::ROTR)
5064 RotAmt = (VTBits - RotAmt) % VTBits;
5065
5066 // If we aren't rotating out all of the known-in sign bits, return the
5067 // number that are left. This handles rotl(sext(x), 1) for example.
5068 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
5069 }
5070 break;
5071 case ISD::ADD:
5072 case ISD::ADDC:
5073 // TODO: Move Operand 1 check before Operand 0 check
5074 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5075 if (Tmp == 1) return 1; // Early out.
5076
5077 // Special case decrementing a value (ADD X, -1):
5078 if (ConstantSDNode *CRHS =
5079 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5080 if (CRHS->isAllOnes()) {
5081 KnownBits Known =
5082 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5083
5084 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5085 // sign bits set.
5086 if ((Known.Zero | 1).isAllOnes())
5087 return VTBits;
5088
5089 // If we are subtracting one from a positive number, there is no carry
5090 // out of the result.
5091 if (Known.isNonNegative())
5092 return Tmp;
5093 }
5094
5095 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5096 if (Tmp2 == 1) return 1; // Early out.
5097
5098 // Add can have at most one carry bit. Thus we know that the output
5099 // is, at worst, one more bit than the inputs.
5100 return std::min(a: Tmp, b: Tmp2) - 1;
5101 case ISD::SUB:
5102 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5103 if (Tmp2 == 1) return 1; // Early out.
5104
5105 // Handle NEG.
5106 if (ConstantSDNode *CLHS =
5107 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5108 if (CLHS->isZero()) {
5109 KnownBits Known =
5110 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5111 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5112 // sign bits set.
5113 if ((Known.Zero | 1).isAllOnes())
5114 return VTBits;
5115
5116 // If the input is known to be positive (the sign bit is known clear),
5117 // the output of the NEG has the same number of sign bits as the input.
5118 if (Known.isNonNegative())
5119 return Tmp2;
5120
5121 // Otherwise, we treat this like a SUB.
5122 }
5123
5124 // Sub can have at most one carry bit. Thus we know that the output
5125 // is, at worst, one more bit than the inputs.
5126 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5127 if (Tmp == 1) return 1; // Early out.
5128 return std::min(a: Tmp, b: Tmp2) - 1;
5129 case ISD::MUL: {
5130 // The output of the Mul can be at most twice the valid bits in the inputs.
5131 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5132 if (SignBitsOp0 == 1)
5133 break;
5134 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5135 if (SignBitsOp1 == 1)
5136 break;
5137 unsigned OutValidBits =
5138 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5139 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5140 }
5141 case ISD::AVGCEILS:
5142 case ISD::AVGFLOORS:
5143 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5144 if (Tmp == 1)
5145 return 1; // Early out.
5146 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5147 return std::min(a: Tmp, b: Tmp2);
5148 case ISD::SREM:
5149 // The sign bit is the LHS's sign bit, except when the result of the
5150 // remainder is zero. The magnitude of the result should be less than or
5151 // equal to the magnitude of the LHS. Therefore, the result should have
5152 // at least as many sign bits as the left hand side.
5153 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5154 case ISD::TRUNCATE: {
5155 // Check if the sign bits of source go down as far as the truncated value.
5156 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5157 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5158 if (NumSrcSignBits > (NumSrcBits - VTBits))
5159 return NumSrcSignBits - (NumSrcBits - VTBits);
5160 break;
5161 }
5162 case ISD::EXTRACT_ELEMENT: {
5163 if (VT.isScalableVector())
5164 break;
5165 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5166 const int BitWidth = Op.getValueSizeInBits();
5167 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5168
5169 // Get reverse index (starting from 1), Op1 value indexes elements from
5170 // little end. Sign starts at big end.
5171 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5172
5173 // If the sign portion ends in our element the subtraction gives correct
5174 // result. Otherwise it gives either negative or > bitwidth result
5175 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5176 }
5177 case ISD::INSERT_VECTOR_ELT: {
5178 if (VT.isScalableVector())
5179 break;
5180 // If we know the element index, split the demand between the
5181 // source vector and the inserted element, otherwise assume we need
5182 // the original demanded vector elements and the value.
5183 SDValue InVec = Op.getOperand(i: 0);
5184 SDValue InVal = Op.getOperand(i: 1);
5185 SDValue EltNo = Op.getOperand(i: 2);
5186 bool DemandedVal = true;
5187 APInt DemandedVecElts = DemandedElts;
5188 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5189 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5190 unsigned EltIdx = CEltNo->getZExtValue();
5191 DemandedVal = !!DemandedElts[EltIdx];
5192 DemandedVecElts.clearBit(BitPosition: EltIdx);
5193 }
5194 Tmp = std::numeric_limits<unsigned>::max();
5195 if (DemandedVal) {
5196 // TODO - handle implicit truncation of inserted elements.
5197 if (InVal.getScalarValueSizeInBits() != VTBits)
5198 break;
5199 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5200 Tmp = std::min(a: Tmp, b: Tmp2);
5201 }
5202 if (!!DemandedVecElts) {
5203 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5204 Tmp = std::min(a: Tmp, b: Tmp2);
5205 }
5206 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5207 return Tmp;
5208 }
5209 case ISD::EXTRACT_VECTOR_ELT: {
5210 SDValue InVec = Op.getOperand(i: 0);
5211 SDValue EltNo = Op.getOperand(i: 1);
5212 EVT VecVT = InVec.getValueType();
5213 // ComputeNumSignBits not yet implemented for scalable vectors.
5214 if (VecVT.isScalableVector())
5215 break;
5216 const unsigned BitWidth = Op.getValueSizeInBits();
5217 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5218 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5219
5220 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5221 // anything about sign bits. But if the sizes match we can derive knowledge
5222 // about sign bits from the vector operand.
5223 if (BitWidth != EltBitWidth)
5224 break;
5225
5226 // If we know the element index, just demand that vector element, else for
5227 // an unknown element index, ignore DemandedElts and demand them all.
5228 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5229 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5230 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5231 DemandedSrcElts =
5232 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5233
5234 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5235 }
5236 case ISD::EXTRACT_SUBVECTOR: {
5237 // Offset the demanded elts by the subvector index.
5238 SDValue Src = Op.getOperand(i: 0);
5239
5240 APInt DemandedSrcElts;
5241 if (Src.getValueType().isScalableVector())
5242 DemandedSrcElts = APInt(1, 1);
5243 else {
5244 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5245 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5246 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5247 }
5248 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5249 }
5250 case ISD::CONCAT_VECTORS: {
5251 if (VT.isScalableVector())
5252 break;
5253 // Determine the minimum number of sign bits across all demanded
5254 // elts of the input vectors. Early out if the result is already 1.
5255 Tmp = std::numeric_limits<unsigned>::max();
5256 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5257 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5258 unsigned NumSubVectors = Op.getNumOperands();
5259 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5260 APInt DemandedSub =
5261 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5262 if (!DemandedSub)
5263 continue;
5264 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5265 Tmp = std::min(a: Tmp, b: Tmp2);
5266 }
5267 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5268 return Tmp;
5269 }
5270 case ISD::INSERT_SUBVECTOR: {
5271 if (VT.isScalableVector())
5272 break;
5273 // Demand any elements from the subvector and the remainder from the src its
5274 // inserted into.
5275 SDValue Src = Op.getOperand(i: 0);
5276 SDValue Sub = Op.getOperand(i: 1);
5277 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5278 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5279 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5280 APInt DemandedSrcElts = DemandedElts;
5281 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5282
5283 Tmp = std::numeric_limits<unsigned>::max();
5284 if (!!DemandedSubElts) {
5285 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5286 if (Tmp == 1)
5287 return 1; // early-out
5288 }
5289 if (!!DemandedSrcElts) {
5290 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5291 Tmp = std::min(a: Tmp, b: Tmp2);
5292 }
5293 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5294 return Tmp;
5295 }
5296 case ISD::LOAD: {
5297 // If we are looking at the loaded value of the SDNode.
5298 if (Op.getResNo() != 0)
5299 break;
5300
5301 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5302 if (const MDNode *Ranges = LD->getRanges()) {
5303 if (DemandedElts != 1)
5304 break;
5305
5306 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5307 if (VTBits > CR.getBitWidth()) {
5308 switch (LD->getExtensionType()) {
5309 case ISD::SEXTLOAD:
5310 CR = CR.signExtend(BitWidth: VTBits);
5311 break;
5312 case ISD::ZEXTLOAD:
5313 CR = CR.zeroExtend(BitWidth: VTBits);
5314 break;
5315 default:
5316 break;
5317 }
5318 }
5319
5320 if (VTBits != CR.getBitWidth())
5321 break;
5322 return std::min(a: CR.getSignedMin().getNumSignBits(),
5323 b: CR.getSignedMax().getNumSignBits());
5324 }
5325
5326 unsigned ExtType = LD->getExtensionType();
5327 switch (ExtType) {
5328 default:
5329 break;
5330 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5331 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5332 return VTBits - Tmp + 1;
5333 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5334 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5335 return VTBits - Tmp;
5336 case ISD::NON_EXTLOAD:
5337 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5338 // We only need to handle vectors - computeKnownBits should handle
5339 // scalar cases.
5340 Type *CstTy = Cst->getType();
5341 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5342 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5343 VTBits == CstTy->getScalarSizeInBits()) {
5344 Tmp = VTBits;
5345 for (unsigned i = 0; i != NumElts; ++i) {
5346 if (!DemandedElts[i])
5347 continue;
5348 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5349 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5350 const APInt &Value = CInt->getValue();
5351 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5352 continue;
5353 }
5354 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5355 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5356 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5357 continue;
5358 }
5359 }
5360 // Unknown type. Conservatively assume no bits match sign bit.
5361 return 1;
5362 }
5363 return Tmp;
5364 }
5365 }
5366 break;
5367 }
5368
5369 break;
5370 }
5371 case ISD::ATOMIC_CMP_SWAP:
5372 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5373 case ISD::ATOMIC_SWAP:
5374 case ISD::ATOMIC_LOAD_ADD:
5375 case ISD::ATOMIC_LOAD_SUB:
5376 case ISD::ATOMIC_LOAD_AND:
5377 case ISD::ATOMIC_LOAD_CLR:
5378 case ISD::ATOMIC_LOAD_OR:
5379 case ISD::ATOMIC_LOAD_XOR:
5380 case ISD::ATOMIC_LOAD_NAND:
5381 case ISD::ATOMIC_LOAD_MIN:
5382 case ISD::ATOMIC_LOAD_MAX:
5383 case ISD::ATOMIC_LOAD_UMIN:
5384 case ISD::ATOMIC_LOAD_UMAX:
5385 case ISD::ATOMIC_LOAD: {
5386 auto *AT = cast<AtomicSDNode>(Val&: Op);
5387 // If we are looking at the loaded value.
5388 if (Op.getResNo() == 0) {
5389 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5390 if (Tmp == VTBits)
5391 return 1; // early-out
5392
5393 // For atomic_load, prefer to use the extension type.
5394 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5395 switch (AT->getExtensionType()) {
5396 default:
5397 break;
5398 case ISD::SEXTLOAD:
5399 return VTBits - Tmp + 1;
5400 case ISD::ZEXTLOAD:
5401 return VTBits - Tmp;
5402 }
5403 }
5404
5405 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5406 return VTBits - Tmp + 1;
5407 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5408 return VTBits - Tmp;
5409 }
5410 break;
5411 }
5412 }
5413
5414 // Allow the target to implement this method for its nodes.
5415 if (Opcode >= ISD::BUILTIN_OP_END ||
5416 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5417 Opcode == ISD::INTRINSIC_W_CHAIN ||
5418 Opcode == ISD::INTRINSIC_VOID) {
5419 // TODO: This can probably be removed once target code is audited. This
5420 // is here purely to reduce patch size and review complexity.
5421 if (!VT.isScalableVector()) {
5422 unsigned NumBits =
5423 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5424 if (NumBits > 1)
5425 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5426 }
5427 }
5428
5429 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5430 // use this information.
5431 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5432 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5433}
5434
5435unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5436 unsigned Depth) const {
5437 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5438 return Op.getScalarValueSizeInBits() - SignBits + 1;
5439}
5440
5441unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5442 const APInt &DemandedElts,
5443 unsigned Depth) const {
5444 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5445 return Op.getScalarValueSizeInBits() - SignBits + 1;
5446}
5447
5448bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
5449 unsigned Depth) const {
5450 // Early out for FREEZE.
5451 if (Op.getOpcode() == ISD::FREEZE)
5452 return true;
5453
5454 EVT VT = Op.getValueType();
5455 APInt DemandedElts = VT.isFixedLengthVector()
5456 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5457 : APInt(1, 1);
5458 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
5459}
5460
5461bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5462 const APInt &DemandedElts,
5463 bool PoisonOnly,
5464 unsigned Depth) const {
5465 unsigned Opcode = Op.getOpcode();
5466
5467 // Early out for FREEZE.
5468 if (Opcode == ISD::FREEZE)
5469 return true;
5470
5471 if (Depth >= MaxRecursionDepth)
5472 return false; // Limit search depth.
5473
5474 if (isIntOrFPConstant(V: Op))
5475 return true;
5476
5477 switch (Opcode) {
5478 case ISD::CONDCODE:
5479 case ISD::VALUETYPE:
5480 case ISD::FrameIndex:
5481 case ISD::TargetFrameIndex:
5482 case ISD::CopyFromReg:
5483 return true;
5484
5485 case ISD::POISON:
5486 return false;
5487
5488 case ISD::UNDEF:
5489 return PoisonOnly;
5490
5491 case ISD::BUILD_VECTOR:
5492 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5493 // this shouldn't affect the result.
5494 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5495 if (!DemandedElts[i])
5496 continue;
5497 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly,
5498 Depth: Depth + 1))
5499 return false;
5500 }
5501 return true;
5502
5503 case ISD::EXTRACT_SUBVECTOR: {
5504 SDValue Src = Op.getOperand(i: 0);
5505 if (Src.getValueType().isScalableVector())
5506 break;
5507 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5508 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5509 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5510 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5511 Depth: Depth + 1);
5512 }
5513
5514 case ISD::INSERT_SUBVECTOR: {
5515 if (Op.getValueType().isScalableVector())
5516 break;
5517 SDValue Src = Op.getOperand(i: 0);
5518 SDValue Sub = Op.getOperand(i: 1);
5519 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5520 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5521 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5522 APInt DemandedSrcElts = DemandedElts;
5523 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5524
5525 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5526 Op: Sub, DemandedElts: DemandedSubElts, PoisonOnly, Depth: Depth + 1))
5527 return false;
5528 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5529 Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly, Depth: Depth + 1))
5530 return false;
5531 return true;
5532 }
5533
5534 case ISD::EXTRACT_VECTOR_ELT: {
5535 SDValue Src = Op.getOperand(i: 0);
5536 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5537 EVT SrcVT = Src.getValueType();
5538 if (SrcVT.isFixedLengthVector() && IndexC &&
5539 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5540 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5541 BitNo: IndexC->getZExtValue());
5542 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5543 Depth: Depth + 1);
5544 }
5545 break;
5546 }
5547
5548 case ISD::INSERT_VECTOR_ELT: {
5549 SDValue InVec = Op.getOperand(i: 0);
5550 SDValue InVal = Op.getOperand(i: 1);
5551 SDValue EltNo = Op.getOperand(i: 2);
5552 EVT VT = InVec.getValueType();
5553 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5554 if (IndexC && VT.isFixedLengthVector() &&
5555 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5556 if (DemandedElts[IndexC->getZExtValue()] &&
5557 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, PoisonOnly, Depth: Depth + 1))
5558 return false;
5559 APInt InVecDemandedElts = DemandedElts;
5560 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5561 if (!!InVecDemandedElts &&
5562 !isGuaranteedNotToBeUndefOrPoison(
5563 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5564 DemandedElts: InVecDemandedElts, PoisonOnly, Depth: Depth + 1))
5565 return false;
5566 return true;
5567 }
5568 break;
5569 }
5570
5571 case ISD::SCALAR_TO_VECTOR:
5572 // Check upper (known undef) elements.
5573 if (DemandedElts.ugt(RHS: 1) && !PoisonOnly)
5574 return false;
5575 // Check element zero.
5576 if (DemandedElts[0] && !isGuaranteedNotToBeUndefOrPoison(
5577 Op: Op.getOperand(i: 0), PoisonOnly, Depth: Depth + 1))
5578 return false;
5579 return true;
5580
5581 case ISD::SPLAT_VECTOR:
5582 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), PoisonOnly,
5583 Depth: Depth + 1);
5584
5585 case ISD::VECTOR_SHUFFLE: {
5586 APInt DemandedLHS, DemandedRHS;
5587 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5588 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5589 DemandedElts, DemandedLHS, DemandedRHS,
5590 /*AllowUndefElts=*/false))
5591 return false;
5592 if (!DemandedLHS.isZero() &&
5593 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS,
5594 PoisonOnly, Depth: Depth + 1))
5595 return false;
5596 if (!DemandedRHS.isZero() &&
5597 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS,
5598 PoisonOnly, Depth: Depth + 1))
5599 return false;
5600 return true;
5601 }
5602
5603 case ISD::SHL:
5604 case ISD::SRL:
5605 case ISD::SRA:
5606 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5607 // enough to check operand 0 if Op can't create undef/poison.
5608 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5609 /*ConsiderFlags*/ true, Depth) &&
5610 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5611 PoisonOnly, Depth: Depth + 1);
5612
5613 case ISD::BSWAP:
5614 case ISD::CTPOP:
5615 case ISD::BITREVERSE:
5616 case ISD::AND:
5617 case ISD::OR:
5618 case ISD::XOR:
5619 case ISD::ADD:
5620 case ISD::SUB:
5621 case ISD::MUL:
5622 case ISD::SADDSAT:
5623 case ISD::UADDSAT:
5624 case ISD::SSUBSAT:
5625 case ISD::USUBSAT:
5626 case ISD::SSHLSAT:
5627 case ISD::USHLSAT:
5628 case ISD::SMIN:
5629 case ISD::SMAX:
5630 case ISD::UMIN:
5631 case ISD::UMAX:
5632 case ISD::ZERO_EXTEND:
5633 case ISD::SIGN_EXTEND:
5634 case ISD::ANY_EXTEND:
5635 case ISD::TRUNCATE:
5636 case ISD::VSELECT: {
5637 // If Op can't create undef/poison and none of its operands are undef/poison
5638 // then Op is never undef/poison. A difference from the more common check
5639 // below, outside the switch, is that we handle elementwise operations for
5640 // which the DemandedElts mask is valid for all operands here.
5641 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5642 /*ConsiderFlags*/ true, Depth) &&
5643 all_of(Range: Op->ops(), P: [&](SDValue V) {
5644 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts,
5645 PoisonOnly, Depth: Depth + 1);
5646 });
5647 }
5648
5649 // TODO: Search for noundef attributes from library functions.
5650
5651 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5652
5653 default:
5654 // Allow the target to implement this method for its nodes.
5655 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5656 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5657 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5658 Op, DemandedElts, DAG: *this, PoisonOnly, Depth);
5659 break;
5660 }
5661
5662 // If Op can't create undef/poison and none of its operands are undef/poison
5663 // then Op is never undef/poison.
5664 // NOTE: TargetNodes can handle this in themselves in
5665 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5666 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5667 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5668 Depth) &&
5669 all_of(Range: Op->ops(), P: [&](SDValue V) {
5670 return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1);
5671 });
5672}
5673
5674bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
5675 bool ConsiderFlags,
5676 unsigned Depth) const {
5677 EVT VT = Op.getValueType();
5678 APInt DemandedElts = VT.isFixedLengthVector()
5679 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5680 : APInt(1, 1);
5681 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5682 Depth);
5683}
5684
5685bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5686 bool PoisonOnly, bool ConsiderFlags,
5687 unsigned Depth) const {
5688 if (ConsiderFlags && Op->hasPoisonGeneratingFlags())
5689 return true;
5690
5691 unsigned Opcode = Op.getOpcode();
5692 switch (Opcode) {
5693 case ISD::AssertSext:
5694 case ISD::AssertZext:
5695 case ISD::AssertAlign:
5696 case ISD::AssertNoFPClass:
5697 // Assertion nodes can create poison if the assertion fails.
5698 return true;
5699
5700 case ISD::FREEZE:
5701 case ISD::CONCAT_VECTORS:
5702 case ISD::INSERT_SUBVECTOR:
5703 case ISD::EXTRACT_SUBVECTOR:
5704 case ISD::SADDSAT:
5705 case ISD::UADDSAT:
5706 case ISD::SSUBSAT:
5707 case ISD::USUBSAT:
5708 case ISD::MULHU:
5709 case ISD::MULHS:
5710 case ISD::AVGFLOORS:
5711 case ISD::AVGFLOORU:
5712 case ISD::AVGCEILS:
5713 case ISD::AVGCEILU:
5714 case ISD::ABDU:
5715 case ISD::ABDS:
5716 case ISD::SMIN:
5717 case ISD::SMAX:
5718 case ISD::SCMP:
5719 case ISD::UMIN:
5720 case ISD::UMAX:
5721 case ISD::UCMP:
5722 case ISD::AND:
5723 case ISD::XOR:
5724 case ISD::ROTL:
5725 case ISD::ROTR:
5726 case ISD::FSHL:
5727 case ISD::FSHR:
5728 case ISD::BSWAP:
5729 case ISD::CTTZ:
5730 case ISD::CTLZ:
5731 case ISD::CTLS:
5732 case ISD::CTPOP:
5733 case ISD::BITREVERSE:
5734 case ISD::PARITY:
5735 case ISD::SIGN_EXTEND:
5736 case ISD::TRUNCATE:
5737 case ISD::SIGN_EXTEND_INREG:
5738 case ISD::SIGN_EXTEND_VECTOR_INREG:
5739 case ISD::ZERO_EXTEND_VECTOR_INREG:
5740 case ISD::BITCAST:
5741 case ISD::BUILD_VECTOR:
5742 case ISD::BUILD_PAIR:
5743 case ISD::SPLAT_VECTOR:
5744 case ISD::FABS:
5745 return false;
5746
5747 case ISD::ABS:
5748 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
5749 // Different to Intrinsic::abs.
5750 return false;
5751
5752 case ISD::ADDC:
5753 case ISD::SUBC:
5754 case ISD::ADDE:
5755 case ISD::SUBE:
5756 case ISD::SADDO:
5757 case ISD::SSUBO:
5758 case ISD::SMULO:
5759 case ISD::SADDO_CARRY:
5760 case ISD::SSUBO_CARRY:
5761 case ISD::UADDO:
5762 case ISD::USUBO:
5763 case ISD::UMULO:
5764 case ISD::UADDO_CARRY:
5765 case ISD::USUBO_CARRY:
5766 // No poison on result or overflow flags.
5767 return false;
5768
5769 case ISD::SELECT_CC:
5770 case ISD::SETCC: {
5771 // Integer setcc cannot create undef or poison.
5772 if (Op.getOperand(i: 0).getValueType().isInteger())
5773 return false;
5774
5775 // FP compares are more complicated. They can create poison for nan/infinity
5776 // based on options and flags. The options and flags also cause special
5777 // nonan condition codes to be used. Those condition codes may be preserved
5778 // even if the nonan flag is dropped somewhere.
5779 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
5780 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
5781 return (unsigned)CCCode & 0x10U;
5782 }
5783
5784 case ISD::OR:
5785 case ISD::ZERO_EXTEND:
5786 case ISD::SELECT:
5787 case ISD::VSELECT:
5788 case ISD::ADD:
5789 case ISD::SUB:
5790 case ISD::MUL:
5791 case ISD::FNEG:
5792 case ISD::FADD:
5793 case ISD::FSUB:
5794 case ISD::FMUL:
5795 case ISD::FDIV:
5796 case ISD::FREM:
5797 case ISD::FCOPYSIGN:
5798 case ISD::FMA:
5799 case ISD::FMAD:
5800 case ISD::FMULADD:
5801 case ISD::FP_EXTEND:
5802 case ISD::FP_TO_SINT_SAT:
5803 case ISD::FP_TO_UINT_SAT:
5804 case ISD::TRUNCATE_SSAT_S:
5805 case ISD::TRUNCATE_SSAT_U:
5806 case ISD::TRUNCATE_USAT_U:
5807 // No poison except from flags (which is handled above)
5808 return false;
5809
5810 case ISD::SHL:
5811 case ISD::SRL:
5812 case ISD::SRA:
5813 // If the max shift amount isn't in range, then the shift can
5814 // create poison.
5815 return !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
5816
5817 case ISD::CTTZ_ZERO_UNDEF:
5818 case ISD::CTLZ_ZERO_UNDEF:
5819 // If the amount is zero then the result will be poison.
5820 // TODO: Add isKnownNeverZero DemandedElts handling.
5821 return !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5822
5823 case ISD::SCALAR_TO_VECTOR:
5824 // Check if we demand any upper (undef) elements.
5825 return !PoisonOnly && DemandedElts.ugt(RHS: 1);
5826
5827 case ISD::INSERT_VECTOR_ELT:
5828 case ISD::EXTRACT_VECTOR_ELT: {
5829 // Ensure that the element index is in bounds.
5830 EVT VecVT = Op.getOperand(i: 0).getValueType();
5831 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
5832 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
5833 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5834 }
5835
5836 case ISD::VECTOR_SHUFFLE: {
5837 // Check for any demanded shuffle element that is undef.
5838 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5839 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
5840 if (Elt < 0 && DemandedElts[Idx])
5841 return true;
5842 return false;
5843 }
5844
5845 case ISD::VECTOR_COMPRESS:
5846 return false;
5847
5848 default:
5849 // Allow the target to implement this method for its nodes.
5850 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5851 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5852 return TLI->canCreateUndefOrPoisonForTargetNode(
5853 Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth);
5854 break;
5855 }
5856
5857 // Be conservative and return true.
5858 return true;
5859}
5860
5861bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
5862 unsigned Opcode = Op.getOpcode();
5863 if (Opcode == ISD::OR)
5864 return Op->getFlags().hasDisjoint() ||
5865 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
5866 if (Opcode == ISD::XOR)
5867 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
5868 return false;
5869}
5870
5871bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5872 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
5873 (Op.isAnyAdd() || isADDLike(Op));
5874}
5875
5876bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
5877 unsigned Depth) const {
5878 EVT VT = Op.getValueType();
5879
5880 // Since the number of lanes in a scalable vector is unknown at compile time,
5881 // we track one bit which is implicitly broadcast to all lanes. This means
5882 // that all lanes in a scalable vector are considered demanded.
5883 APInt DemandedElts = VT.isFixedLengthVector()
5884 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5885 : APInt(1, 1);
5886
5887 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
5888}
5889
5890bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
5891 bool SNaN, unsigned Depth) const {
5892 assert(!DemandedElts.isZero() && "No demanded elements");
5893
5894 // If we're told that NaNs won't happen, assume they won't.
5895 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5896 return true;
5897
5898 if (Depth >= MaxRecursionDepth)
5899 return false; // Limit search depth.
5900
5901 // If the value is a constant, we can obviously see if it is a NaN or not.
5902 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
5903 return !C->getValueAPF().isNaN() ||
5904 (SNaN && !C->getValueAPF().isSignaling());
5905 }
5906
5907 unsigned Opcode = Op.getOpcode();
5908 switch (Opcode) {
5909 case ISD::FADD:
5910 case ISD::FSUB:
5911 case ISD::FMUL:
5912 case ISD::FDIV:
5913 case ISD::FREM:
5914 case ISD::FSIN:
5915 case ISD::FCOS:
5916 case ISD::FTAN:
5917 case ISD::FASIN:
5918 case ISD::FACOS:
5919 case ISD::FATAN:
5920 case ISD::FATAN2:
5921 case ISD::FSINH:
5922 case ISD::FCOSH:
5923 case ISD::FTANH:
5924 case ISD::FMA:
5925 case ISD::FMULADD:
5926 case ISD::FMAD: {
5927 if (SNaN)
5928 return true;
5929 // TODO: Need isKnownNeverInfinity
5930 return false;
5931 }
5932 case ISD::FCANONICALIZE:
5933 case ISD::FEXP:
5934 case ISD::FEXP2:
5935 case ISD::FEXP10:
5936 case ISD::FTRUNC:
5937 case ISD::FFLOOR:
5938 case ISD::FCEIL:
5939 case ISD::FROUND:
5940 case ISD::FROUNDEVEN:
5941 case ISD::LROUND:
5942 case ISD::LLROUND:
5943 case ISD::FRINT:
5944 case ISD::LRINT:
5945 case ISD::LLRINT:
5946 case ISD::FNEARBYINT:
5947 case ISD::FLDEXP: {
5948 if (SNaN)
5949 return true;
5950 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
5951 }
5952 case ISD::FABS:
5953 case ISD::FNEG:
5954 case ISD::FCOPYSIGN: {
5955 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
5956 }
5957 case ISD::SELECT:
5958 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
5959 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
5960 case ISD::FP_EXTEND:
5961 case ISD::FP_ROUND: {
5962 if (SNaN)
5963 return true;
5964 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
5965 }
5966 case ISD::SINT_TO_FP:
5967 case ISD::UINT_TO_FP:
5968 return true;
5969 case ISD::FSQRT: // Need is known positive
5970 case ISD::FLOG:
5971 case ISD::FLOG2:
5972 case ISD::FLOG10:
5973 case ISD::FPOWI:
5974 case ISD::FPOW: {
5975 if (SNaN)
5976 return true;
5977 // TODO: Refine on operand
5978 return false;
5979 }
5980 case ISD::FMINNUM:
5981 case ISD::FMAXNUM:
5982 case ISD::FMINIMUMNUM:
5983 case ISD::FMAXIMUMNUM: {
5984 // Only one needs to be known not-nan, since it will be returned if the
5985 // other ends up being one.
5986 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
5987 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
5988 }
5989 case ISD::FMINNUM_IEEE:
5990 case ISD::FMAXNUM_IEEE: {
5991 if (SNaN)
5992 return true;
5993 // This can return a NaN if either operand is an sNaN, or if both operands
5994 // are NaN.
5995 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
5996 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
5997 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
5998 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
5999 }
6000 case ISD::FMINIMUM:
6001 case ISD::FMAXIMUM: {
6002 // TODO: Does this quiet or return the origina NaN as-is?
6003 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6004 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6005 }
6006 case ISD::EXTRACT_VECTOR_ELT: {
6007 SDValue Src = Op.getOperand(i: 0);
6008 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6009 EVT SrcVT = Src.getValueType();
6010 if (SrcVT.isFixedLengthVector() && Idx &&
6011 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6012 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6013 BitNo: Idx->getZExtValue());
6014 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6015 }
6016 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6017 }
6018 case ISD::EXTRACT_SUBVECTOR: {
6019 SDValue Src = Op.getOperand(i: 0);
6020 if (Src.getValueType().isFixedLengthVector()) {
6021 unsigned Idx = Op.getConstantOperandVal(i: 1);
6022 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6023 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6024 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6025 }
6026 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6027 }
6028 case ISD::INSERT_SUBVECTOR: {
6029 SDValue BaseVector = Op.getOperand(i: 0);
6030 SDValue SubVector = Op.getOperand(i: 1);
6031 EVT BaseVectorVT = BaseVector.getValueType();
6032 if (BaseVectorVT.isFixedLengthVector()) {
6033 unsigned Idx = Op.getConstantOperandVal(i: 2);
6034 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6035 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6036
6037 // Clear/Extract the bits at the position where the subvector will be
6038 // inserted.
6039 APInt DemandedMask =
6040 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6041 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6042 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6043
6044 bool NeverNaN = true;
6045 if (!DemandedSrcElts.isZero())
6046 NeverNaN &=
6047 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6048 if (NeverNaN && !DemandedSubElts.isZero())
6049 NeverNaN &=
6050 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6051 return NeverNaN;
6052 }
6053 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6054 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6055 }
6056 case ISD::BUILD_VECTOR: {
6057 unsigned NumElts = Op.getNumOperands();
6058 for (unsigned I = 0; I != NumElts; ++I)
6059 if (DemandedElts[I] &&
6060 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6061 return false;
6062 return true;
6063 }
6064 case ISD::AssertNoFPClass: {
6065 FPClassTest NoFPClass =
6066 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6067 if ((NoFPClass & fcNan) == fcNan)
6068 return true;
6069 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6070 return true;
6071 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6072 }
6073 default:
6074 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6075 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6076 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6077 Depth);
6078 }
6079
6080 return false;
6081 }
6082}
6083
6084bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
6085 assert(Op.getValueType().isFloatingPoint() &&
6086 "Floating point type expected");
6087
6088 // If the value is a constant, we can obviously see if it is a zero or not.
6089 return ISD::matchUnaryFpPredicate(
6090 Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); });
6091}
6092
6093bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6094 if (Depth >= MaxRecursionDepth)
6095 return false; // Limit search depth.
6096
6097 assert(!Op.getValueType().isFloatingPoint() &&
6098 "Floating point types unsupported - use isKnownNeverZeroFloat");
6099
6100 // If the value is a constant, we can obviously see if it is a zero or not.
6101 if (ISD::matchUnaryPredicate(Op,
6102 Match: [](ConstantSDNode *C) { return !C->isZero(); }))
6103 return true;
6104
6105 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6106 // some degree.
6107 switch (Op.getOpcode()) {
6108 default:
6109 break;
6110
6111 case ISD::OR:
6112 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
6113 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6114
6115 case ISD::VSELECT:
6116 case ISD::SELECT:
6117 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6118 isKnownNeverZero(Op: Op.getOperand(i: 2), Depth: Depth + 1);
6119
6120 case ISD::SHL: {
6121 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6122 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6123 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6124 // 1 << X is never zero.
6125 if (ValKnown.One[0])
6126 return true;
6127 // If max shift cnt of known ones is non-zero, result is non-zero.
6128 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
6129 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6130 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6131 return true;
6132 break;
6133 }
6134 case ISD::UADDSAT:
6135 case ISD::UMAX:
6136 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
6137 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6138
6139 // For smin/smax: If either operand is known negative/positive
6140 // respectively we don't need the other to be known at all.
6141 case ISD::SMAX: {
6142 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6143 if (Op1.isStrictlyPositive())
6144 return true;
6145
6146 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6147 if (Op0.isStrictlyPositive())
6148 return true;
6149
6150 if (Op1.isNonZero() && Op0.isNonZero())
6151 return true;
6152
6153 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6154 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6155 }
6156 case ISD::SMIN: {
6157 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6158 if (Op1.isNegative())
6159 return true;
6160
6161 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6162 if (Op0.isNegative())
6163 return true;
6164
6165 if (Op1.isNonZero() && Op0.isNonZero())
6166 return true;
6167
6168 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6169 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6170 }
6171 case ISD::UMIN:
6172 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6173 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6174
6175 case ISD::ROTL:
6176 case ISD::ROTR:
6177 case ISD::BITREVERSE:
6178 case ISD::BSWAP:
6179 case ISD::CTPOP:
6180 case ISD::ABS:
6181 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6182
6183 case ISD::SRA:
6184 case ISD::SRL: {
6185 if (Op->getFlags().hasExact())
6186 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6187 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6188 if (ValKnown.isNegative())
6189 return true;
6190 // If max shift cnt of known ones is non-zero, result is non-zero.
6191 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
6192 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6193 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6194 return true;
6195 break;
6196 }
6197 case ISD::UDIV:
6198 case ISD::SDIV:
6199 // div exact can only produce a zero if the dividend is zero.
6200 // TODO: For udiv this is also true if Op1 u<= Op0
6201 if (Op->getFlags().hasExact())
6202 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6203 break;
6204
6205 case ISD::ADD:
6206 if (Op->getFlags().hasNoUnsignedWrap())
6207 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
6208 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6209 return true;
6210 // TODO: There are a lot more cases we can prove for add.
6211 break;
6212
6213 case ISD::SUB: {
6214 if (isNullConstant(V: Op.getOperand(i: 0)))
6215 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6216
6217 std::optional<bool> ne =
6218 KnownBits::ne(LHS: computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1),
6219 RHS: computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1));
6220 return ne && *ne;
6221 }
6222
6223 case ISD::MUL:
6224 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6225 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6226 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6227 return true;
6228 break;
6229
6230 case ISD::ZERO_EXTEND:
6231 case ISD::SIGN_EXTEND:
6232 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6233 case ISD::VSCALE: {
6234 const Function &F = getMachineFunction().getFunction();
6235 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6236 ConstantRange CR =
6237 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6238 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6239 return true;
6240 break;
6241 }
6242 }
6243
6244 return computeKnownBits(Op, Depth).isNonZero();
6245}
6246
6247bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6248 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6249 return !C1->isNegative();
6250
6251 switch (Op.getOpcode()) {
6252 case ISD::FABS:
6253 case ISD::FEXP:
6254 case ISD::FEXP2:
6255 case ISD::FEXP10:
6256 return true;
6257 default:
6258 return false;
6259 }
6260
6261 llvm_unreachable("covered opcode switch");
6262}
6263
6264bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6265 assert(Use.getValueType().isFloatingPoint());
6266 const SDNode *User = Use.getUser();
6267 if (User->getFlags().hasNoSignedZeros())
6268 return true;
6269
6270 unsigned OperandNo = Use.getOperandNo();
6271 // Check if this use is insensitive to the sign of zero
6272 switch (User->getOpcode()) {
6273 case ISD::SETCC:
6274 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6275 case ISD::FABS:
6276 // fabs always produces +0.0.
6277 return true;
6278 case ISD::FCOPYSIGN:
6279 // copysign overwrites the sign bit of the first operand.
6280 return OperandNo == 0;
6281 case ISD::FADD:
6282 case ISD::FSUB: {
6283 // Arithmetic with non-zero constants fixes the uncertainty around the
6284 // sign bit.
6285 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6286 return isKnownNeverZeroFloat(Op: Other);
6287 }
6288 case ISD::FP_TO_SINT:
6289 case ISD::FP_TO_UINT:
6290 // fp-to-int conversions normalize signed zeros.
6291 return true;
6292 default:
6293 return false;
6294 }
6295}
6296
6297bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6298 if (Op->getFlags().hasNoSignedZeros())
6299 return true;
6300 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6301 // regression. Ideally, this should be implemented as a demanded-bits
6302 // optimization that stems from the users.
6303 if (Op->use_size() > 2)
6304 return false;
6305 return all_of(Range: Op->uses(),
6306 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6307}
6308
6309bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6310 // Check the obvious case.
6311 if (A == B) return true;
6312
6313 // For negative and positive zero.
6314 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6315 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6316 if (CA->isZero() && CB->isZero()) return true;
6317
6318 // Otherwise they may not be equal.
6319 return false;
6320}
6321
6322// Only bits set in Mask must be negated, other bits may be arbitrary.
6323SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6324 if (isBitwiseNot(V, AllowUndefs))
6325 return V.getOperand(i: 0);
6326
6327 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6328 // bits in the non-extended part.
6329 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6330 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6331 return SDValue();
6332 SDValue ExtArg = V.getOperand(i: 0);
6333 if (ExtArg.getScalarValueSizeInBits() >=
6334 MaskC->getAPIntValue().getActiveBits() &&
6335 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6336 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6337 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6338 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6339 return SDValue();
6340}
6341
6342static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6343 // Match masked merge pattern (X & ~M) op (Y & M)
6344 // Including degenerate case (X & ~M) op M
6345 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6346 SDValue Other) {
6347 if (SDValue NotOperand =
6348 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6349 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6350 NotOperand->getOpcode() == ISD::TRUNCATE)
6351 NotOperand = NotOperand->getOperand(Num: 0);
6352
6353 if (Other == NotOperand)
6354 return true;
6355 if (Other->getOpcode() == ISD::AND)
6356 return NotOperand == Other->getOperand(Num: 0) ||
6357 NotOperand == Other->getOperand(Num: 1);
6358 }
6359 return false;
6360 };
6361
6362 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6363 A = A->getOperand(Num: 0);
6364
6365 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6366 B = B->getOperand(Num: 0);
6367
6368 if (A->getOpcode() == ISD::AND)
6369 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6370 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6371 return false;
6372}
6373
6374// FIXME: unify with llvm::haveNoCommonBitsSet.
6375bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6376 assert(A.getValueType() == B.getValueType() &&
6377 "Values must have the same type");
6378 if (haveNoCommonBitsSetCommutative(A, B) ||
6379 haveNoCommonBitsSetCommutative(A: B, B: A))
6380 return true;
6381 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6382 RHS: computeKnownBits(Op: B));
6383}
6384
6385static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6386 SelectionDAG &DAG) {
6387 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6388 return DAG.getConstant(Val: 0, DL, VT);
6389
6390 return SDValue();
6391}
6392
6393static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6394 ArrayRef<SDValue> Ops,
6395 SelectionDAG &DAG) {
6396 int NumOps = Ops.size();
6397 assert(NumOps != 0 && "Can't build an empty vector!");
6398 assert(!VT.isScalableVector() &&
6399 "BUILD_VECTOR cannot be used with scalable types");
6400 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6401 "Incorrect element count in BUILD_VECTOR!");
6402
6403 // BUILD_VECTOR of UNDEFs is UNDEF.
6404 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6405 return DAG.getUNDEF(VT);
6406
6407 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6408 SDValue IdentitySrc;
6409 bool IsIdentity = true;
6410 for (int i = 0; i != NumOps; ++i) {
6411 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6412 Ops[i].getOperand(i: 0).getValueType() != VT ||
6413 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6414 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6415 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6416 IsIdentity = false;
6417 break;
6418 }
6419 IdentitySrc = Ops[i].getOperand(i: 0);
6420 }
6421 if (IsIdentity)
6422 return IdentitySrc;
6423
6424 return SDValue();
6425}
6426
6427/// Try to simplify vector concatenation to an input value, undef, or build
6428/// vector.
6429static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6430 ArrayRef<SDValue> Ops,
6431 SelectionDAG &DAG) {
6432 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6433 assert(llvm::all_of(Ops,
6434 [Ops](SDValue Op) {
6435 return Ops[0].getValueType() == Op.getValueType();
6436 }) &&
6437 "Concatenation of vectors with inconsistent value types!");
6438 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6439 VT.getVectorElementCount() &&
6440 "Incorrect element count in vector concatenation!");
6441
6442 if (Ops.size() == 1)
6443 return Ops[0];
6444
6445 // Concat of UNDEFs is UNDEF.
6446 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6447 return DAG.getUNDEF(VT);
6448
6449 // Scan the operands and look for extract operations from a single source
6450 // that correspond to insertion at the same location via this concatenation:
6451 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6452 SDValue IdentitySrc;
6453 bool IsIdentity = true;
6454 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6455 SDValue Op = Ops[i];
6456 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6457 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6458 Op.getOperand(i: 0).getValueType() != VT ||
6459 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
6460 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
6461 IsIdentity = false;
6462 break;
6463 }
6464 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6465 "Unexpected identity source vector for concat of extracts");
6466 IdentitySrc = Op.getOperand(i: 0);
6467 }
6468 if (IsIdentity) {
6469 assert(IdentitySrc && "Failed to set source vector of extracts");
6470 return IdentitySrc;
6471 }
6472
6473 // The code below this point is only designed to work for fixed width
6474 // vectors, so we bail out for now.
6475 if (VT.isScalableVector())
6476 return SDValue();
6477
6478 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
6479 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
6480 // BUILD_VECTOR.
6481 // FIXME: Add support for SCALAR_TO_VECTOR as well.
6482 EVT SVT = VT.getScalarType();
6483 SmallVector<SDValue, 16> Elts;
6484 for (SDValue Op : Ops) {
6485 EVT OpVT = Op.getValueType();
6486 if (Op.isUndef())
6487 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
6488 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
6489 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
6490 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
6491 OpVT.getVectorNumElements() == 1 &&
6492 isNullConstant(V: Op.getOperand(i: 2)))
6493 Elts.push_back(Elt: Op.getOperand(i: 1));
6494 else
6495 return SDValue();
6496 }
6497
6498 // BUILD_VECTOR requires all inputs to be of the same type, find the
6499 // maximum type and extend them all.
6500 for (SDValue Op : Elts)
6501 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
6502
6503 if (SVT.bitsGT(VT: VT.getScalarType())) {
6504 for (SDValue &Op : Elts) {
6505 if (Op.isUndef())
6506 Op = DAG.getUNDEF(VT: SVT);
6507 else
6508 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
6509 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
6510 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
6511 }
6512 }
6513
6514 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
6515 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
6516 return V;
6517}
6518
6519/// Gets or creates the specified node.
6520SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
6521 SDVTList VTs = getVTList(VT);
6522 FoldingSetNodeID ID;
6523 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: {});
6524 void *IP = nullptr;
6525 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
6526 return SDValue(E, 0);
6527
6528 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6529 CSEMap.InsertNode(N, InsertPos: IP);
6530
6531 InsertNode(N);
6532 SDValue V = SDValue(N, 0);
6533 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6534 return V;
6535}
6536
6537SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6538 SDValue N1) {
6539 SDNodeFlags Flags;
6540 if (Inserter)
6541 Flags = Inserter->getFlags();
6542 return getNode(Opcode, DL, VT, Operand: N1, Flags);
6543}
6544
6545SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6546 SDValue N1, const SDNodeFlags Flags) {
6547 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
6548
6549 // Constant fold unary operations with a vector integer or float operand.
6550 switch (Opcode) {
6551 default:
6552 // FIXME: Entirely reasonable to perform folding of other unary
6553 // operations here as the need arises.
6554 break;
6555 case ISD::FNEG:
6556 case ISD::FABS:
6557 case ISD::FCEIL:
6558 case ISD::FTRUNC:
6559 case ISD::FFLOOR:
6560 case ISD::FP_EXTEND:
6561 case ISD::FP_TO_SINT:
6562 case ISD::FP_TO_UINT:
6563 case ISD::FP_TO_FP16:
6564 case ISD::FP_TO_BF16:
6565 case ISD::TRUNCATE:
6566 case ISD::ANY_EXTEND:
6567 case ISD::ZERO_EXTEND:
6568 case ISD::SIGN_EXTEND:
6569 case ISD::UINT_TO_FP:
6570 case ISD::SINT_TO_FP:
6571 case ISD::FP16_TO_FP:
6572 case ISD::BF16_TO_FP:
6573 case ISD::BITCAST:
6574 case ISD::ABS:
6575 case ISD::BITREVERSE:
6576 case ISD::BSWAP:
6577 case ISD::CTLZ:
6578 case ISD::CTLZ_ZERO_UNDEF:
6579 case ISD::CTTZ:
6580 case ISD::CTTZ_ZERO_UNDEF:
6581 case ISD::CTPOP:
6582 case ISD::CTLS:
6583 case ISD::STEP_VECTOR: {
6584 SDValue Ops = {N1};
6585 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
6586 return Fold;
6587 }
6588 }
6589
6590 unsigned OpOpcode = N1.getNode()->getOpcode();
6591 switch (Opcode) {
6592 case ISD::STEP_VECTOR:
6593 assert(VT.isScalableVector() &&
6594 "STEP_VECTOR can only be used with scalable types");
6595 assert(OpOpcode == ISD::TargetConstant &&
6596 VT.getVectorElementType() == N1.getValueType() &&
6597 "Unexpected step operand");
6598 break;
6599 case ISD::FREEZE:
6600 assert(VT == N1.getValueType() && "Unexpected VT!");
6601 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly=*/false))
6602 return N1;
6603 break;
6604 case ISD::TokenFactor:
6605 case ISD::MERGE_VALUES:
6606 case ISD::CONCAT_VECTORS:
6607 return N1; // Factor, merge or concat of one node? No need.
6608 case ISD::BUILD_VECTOR: {
6609 // Attempt to simplify BUILD_VECTOR.
6610 SDValue Ops[] = {N1};
6611 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
6612 return V;
6613 break;
6614 }
6615 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
6616 case ISD::FP_EXTEND:
6617 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
6618 "Invalid FP cast!");
6619 if (N1.getValueType() == VT) return N1; // noop conversion.
6620 assert((!VT.isVector() || VT.getVectorElementCount() ==
6621 N1.getValueType().getVectorElementCount()) &&
6622 "Vector element count mismatch!");
6623 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
6624 if (N1.isUndef())
6625 return getUNDEF(VT);
6626 break;
6627 case ISD::FP_TO_SINT:
6628 case ISD::FP_TO_UINT:
6629 if (N1.isUndef())
6630 return getUNDEF(VT);
6631 break;
6632 case ISD::SINT_TO_FP:
6633 case ISD::UINT_TO_FP:
6634 // [us]itofp(undef) = 0, because the result value is bounded.
6635 if (N1.isUndef())
6636 return getConstantFP(Val: 0.0, DL, VT);
6637 break;
6638 case ISD::SIGN_EXTEND:
6639 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6640 "Invalid SIGN_EXTEND!");
6641 assert(VT.isVector() == N1.getValueType().isVector() &&
6642 "SIGN_EXTEND result type type should be vector iff the operand "
6643 "type is vector!");
6644 if (N1.getValueType() == VT) return N1; // noop extension
6645 assert((!VT.isVector() || VT.getVectorElementCount() ==
6646 N1.getValueType().getVectorElementCount()) &&
6647 "Vector element count mismatch!");
6648 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
6649 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
6650 SDNodeFlags Flags;
6651 if (OpOpcode == ISD::ZERO_EXTEND)
6652 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6653 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6654 transferDbgValues(From: N1, To: NewVal);
6655 return NewVal;
6656 }
6657
6658 if (OpOpcode == ISD::POISON)
6659 return getPOISON(VT);
6660
6661 if (N1.isUndef())
6662 // sext(undef) = 0, because the top bits will all be the same.
6663 return getConstant(Val: 0, DL, VT);
6664
6665 // Skip unnecessary sext_inreg pattern:
6666 // (sext (trunc x)) -> x iff the upper bits are all signbits.
6667 if (OpOpcode == ISD::TRUNCATE) {
6668 SDValue OpOp = N1.getOperand(i: 0);
6669 if (OpOp.getValueType() == VT) {
6670 unsigned NumSignExtBits =
6671 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
6672 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
6673 transferDbgValues(From: N1, To: OpOp);
6674 return OpOp;
6675 }
6676 }
6677 }
6678 break;
6679 case ISD::ZERO_EXTEND:
6680 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6681 "Invalid ZERO_EXTEND!");
6682 assert(VT.isVector() == N1.getValueType().isVector() &&
6683 "ZERO_EXTEND result type type should be vector iff the operand "
6684 "type is vector!");
6685 if (N1.getValueType() == VT) return N1; // noop extension
6686 assert((!VT.isVector() || VT.getVectorElementCount() ==
6687 N1.getValueType().getVectorElementCount()) &&
6688 "Vector element count mismatch!");
6689 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
6690 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
6691 SDNodeFlags Flags;
6692 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6693 SDValue NewVal =
6694 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
6695 transferDbgValues(From: N1, To: NewVal);
6696 return NewVal;
6697 }
6698
6699 if (OpOpcode == ISD::POISON)
6700 return getPOISON(VT);
6701
6702 if (N1.isUndef())
6703 // zext(undef) = 0, because the top bits will be zero.
6704 return getConstant(Val: 0, DL, VT);
6705
6706 // Skip unnecessary zext_inreg pattern:
6707 // (zext (trunc x)) -> x iff the upper bits are known zero.
6708 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
6709 // use to recognise zext_inreg patterns.
6710 if (OpOpcode == ISD::TRUNCATE) {
6711 SDValue OpOp = N1.getOperand(i: 0);
6712 if (OpOp.getValueType() == VT) {
6713 if (OpOp.getOpcode() != ISD::AND) {
6714 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
6715 loBit: N1.getScalarValueSizeInBits());
6716 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
6717 transferDbgValues(From: N1, To: OpOp);
6718 return OpOp;
6719 }
6720 }
6721 }
6722 }
6723 break;
6724 case ISD::ANY_EXTEND:
6725 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6726 "Invalid ANY_EXTEND!");
6727 assert(VT.isVector() == N1.getValueType().isVector() &&
6728 "ANY_EXTEND result type type should be vector iff the operand "
6729 "type is vector!");
6730 if (N1.getValueType() == VT) return N1; // noop extension
6731 assert((!VT.isVector() || VT.getVectorElementCount() ==
6732 N1.getValueType().getVectorElementCount()) &&
6733 "Vector element count mismatch!");
6734 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
6735
6736 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6737 OpOpcode == ISD::ANY_EXTEND) {
6738 SDNodeFlags Flags;
6739 if (OpOpcode == ISD::ZERO_EXTEND)
6740 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6741 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
6742 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6743 }
6744 if (N1.isUndef())
6745 return getUNDEF(VT);
6746
6747 // (ext (trunc x)) -> x
6748 if (OpOpcode == ISD::TRUNCATE) {
6749 SDValue OpOp = N1.getOperand(i: 0);
6750 if (OpOp.getValueType() == VT) {
6751 transferDbgValues(From: N1, To: OpOp);
6752 return OpOp;
6753 }
6754 }
6755 break;
6756 case ISD::TRUNCATE:
6757 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6758 "Invalid TRUNCATE!");
6759 assert(VT.isVector() == N1.getValueType().isVector() &&
6760 "TRUNCATE result type type should be vector iff the operand "
6761 "type is vector!");
6762 if (N1.getValueType() == VT) return N1; // noop truncate
6763 assert((!VT.isVector() || VT.getVectorElementCount() ==
6764 N1.getValueType().getVectorElementCount()) &&
6765 "Vector element count mismatch!");
6766 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
6767 if (OpOpcode == ISD::TRUNCATE)
6768 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6769 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6770 OpOpcode == ISD::ANY_EXTEND) {
6771 // If the source is smaller than the dest, we still need an extend.
6772 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
6773 VT: VT.getScalarType())) {
6774 SDNodeFlags Flags;
6775 if (OpOpcode == ISD::ZERO_EXTEND)
6776 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6777 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6778 }
6779 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
6780 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6781 return N1.getOperand(i: 0);
6782 }
6783 if (N1.isUndef())
6784 return getUNDEF(VT);
6785 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
6786 return getVScale(DL, VT,
6787 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
6788 break;
6789 case ISD::ANY_EXTEND_VECTOR_INREG:
6790 case ISD::ZERO_EXTEND_VECTOR_INREG:
6791 case ISD::SIGN_EXTEND_VECTOR_INREG:
6792 assert(VT.isVector() && "This DAG node is restricted to vector types.");
6793 assert(N1.getValueType().bitsLE(VT) &&
6794 "The input must be the same size or smaller than the result.");
6795 assert(VT.getVectorMinNumElements() <
6796 N1.getValueType().getVectorMinNumElements() &&
6797 "The destination vector type must have fewer lanes than the input.");
6798 break;
6799 case ISD::ABS:
6800 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
6801 if (N1.isUndef())
6802 return getConstant(Val: 0, DL, VT);
6803 break;
6804 case ISD::BSWAP:
6805 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
6806 assert((VT.getScalarSizeInBits() % 16 == 0) &&
6807 "BSWAP types must be a multiple of 16 bits!");
6808 if (N1.isUndef())
6809 return getUNDEF(VT);
6810 // bswap(bswap(X)) -> X.
6811 if (OpOpcode == ISD::BSWAP)
6812 return N1.getOperand(i: 0);
6813 break;
6814 case ISD::BITREVERSE:
6815 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
6816 if (N1.isUndef())
6817 return getUNDEF(VT);
6818 break;
6819 case ISD::BITCAST:
6820 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
6821 "Cannot BITCAST between types of different sizes!");
6822 if (VT == N1.getValueType()) return N1; // noop conversion.
6823 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
6824 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
6825 if (N1.isUndef())
6826 return getUNDEF(VT);
6827 break;
6828 case ISD::SCALAR_TO_VECTOR:
6829 assert(VT.isVector() && !N1.getValueType().isVector() &&
6830 (VT.getVectorElementType() == N1.getValueType() ||
6831 (VT.getVectorElementType().isInteger() &&
6832 N1.getValueType().isInteger() &&
6833 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
6834 "Illegal SCALAR_TO_VECTOR node!");
6835 if (N1.isUndef())
6836 return getUNDEF(VT);
6837 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
6838 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
6839 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
6840 N1.getConstantOperandVal(i: 1) == 0 &&
6841 N1.getOperand(i: 0).getValueType() == VT)
6842 return N1.getOperand(i: 0);
6843 break;
6844 case ISD::FNEG:
6845 // Negation of an unknown bag of bits is still completely undefined.
6846 if (N1.isUndef())
6847 return getUNDEF(VT);
6848
6849 if (OpOpcode == ISD::FNEG) // --X -> X
6850 return N1.getOperand(i: 0);
6851 break;
6852 case ISD::FABS:
6853 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
6854 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
6855 break;
6856 case ISD::VSCALE:
6857 assert(VT == N1.getValueType() && "Unexpected VT!");
6858 break;
6859 case ISD::CTPOP:
6860 if (N1.getValueType().getScalarType() == MVT::i1)
6861 return N1;
6862 break;
6863 case ISD::CTLZ:
6864 case ISD::CTTZ:
6865 if (N1.getValueType().getScalarType() == MVT::i1)
6866 return getNOT(DL, Val: N1, VT: N1.getValueType());
6867 break;
6868 case ISD::CTLS:
6869 if (N1.getValueType().getScalarType() == MVT::i1)
6870 return getConstant(Val: 0, DL, VT);
6871 break;
6872 case ISD::VECREDUCE_ADD:
6873 if (N1.getValueType().getScalarType() == MVT::i1)
6874 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
6875 break;
6876 case ISD::VECREDUCE_SMIN:
6877 case ISD::VECREDUCE_UMAX:
6878 if (N1.getValueType().getScalarType() == MVT::i1)
6879 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
6880 break;
6881 case ISD::VECREDUCE_SMAX:
6882 case ISD::VECREDUCE_UMIN:
6883 if (N1.getValueType().getScalarType() == MVT::i1)
6884 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
6885 break;
6886 case ISD::SPLAT_VECTOR:
6887 assert(VT.isVector() && "Wrong return type!");
6888 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
6889 // that for now.
6890 assert((VT.getVectorElementType() == N1.getValueType() ||
6891 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
6892 (VT.getVectorElementType().isInteger() &&
6893 N1.getValueType().isInteger() &&
6894 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
6895 "Wrong operand type!");
6896 break;
6897 }
6898
6899 SDNode *N;
6900 SDVTList VTs = getVTList(VT);
6901 SDValue Ops[] = {N1};
6902 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
6903 FoldingSetNodeID ID;
6904 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
6905 void *IP = nullptr;
6906 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
6907 E->intersectFlagsWith(Flags);
6908 return SDValue(E, 0);
6909 }
6910
6911 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6912 N->setFlags(Flags);
6913 createOperands(Node: N, Vals: Ops);
6914 CSEMap.InsertNode(N, InsertPos: IP);
6915 } else {
6916 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6917 createOperands(Node: N, Vals: Ops);
6918 }
6919
6920 InsertNode(N);
6921 SDValue V = SDValue(N, 0);
6922 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6923 return V;
6924}
6925
6926static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
6927 const APInt &C2) {
6928 switch (Opcode) {
6929 case ISD::ADD: return C1 + C2;
6930 case ISD::SUB: return C1 - C2;
6931 case ISD::MUL: return C1 * C2;
6932 case ISD::AND: return C1 & C2;
6933 case ISD::OR: return C1 | C2;
6934 case ISD::XOR: return C1 ^ C2;
6935 case ISD::SHL: return C1 << C2;
6936 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
6937 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
6938 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
6939 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
6940 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
6941 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
6942 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
6943 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
6944 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
6945 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
6946 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
6947 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
6948 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
6949 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
6950 case ISD::UDIV:
6951 if (!C2.getBoolValue())
6952 break;
6953 return C1.udiv(RHS: C2);
6954 case ISD::UREM:
6955 if (!C2.getBoolValue())
6956 break;
6957 return C1.urem(RHS: C2);
6958 case ISD::SDIV:
6959 if (!C2.getBoolValue())
6960 break;
6961 return C1.sdiv(RHS: C2);
6962 case ISD::SREM:
6963 if (!C2.getBoolValue())
6964 break;
6965 return C1.srem(RHS: C2);
6966 case ISD::AVGFLOORS:
6967 return APIntOps::avgFloorS(C1, C2);
6968 case ISD::AVGFLOORU:
6969 return APIntOps::avgFloorU(C1, C2);
6970 case ISD::AVGCEILS:
6971 return APIntOps::avgCeilS(C1, C2);
6972 case ISD::AVGCEILU:
6973 return APIntOps::avgCeilU(C1, C2);
6974 case ISD::ABDS:
6975 return APIntOps::abds(A: C1, B: C2);
6976 case ISD::ABDU:
6977 return APIntOps::abdu(A: C1, B: C2);
6978 case ISD::MULHS:
6979 return APIntOps::mulhs(C1, C2);
6980 case ISD::MULHU:
6981 return APIntOps::mulhu(C1, C2);
6982 case ISD::CLMUL:
6983 return APIntOps::clmul(LHS: C1, RHS: C2);
6984 case ISD::CLMULR:
6985 return APIntOps::clmulr(LHS: C1, RHS: C2);
6986 case ISD::CLMULH:
6987 return APIntOps::clmulh(LHS: C1, RHS: C2);
6988 }
6989 return std::nullopt;
6990}
6991// Handle constant folding with UNDEF.
6992// TODO: Handle more cases.
6993static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
6994 bool IsUndef1, const APInt &C2,
6995 bool IsUndef2) {
6996 if (!(IsUndef1 || IsUndef2))
6997 return FoldValue(Opcode, C1, C2);
6998
6999 // Fold and(x, undef) -> 0
7000 // Fold mul(x, undef) -> 0
7001 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7002 return APInt::getZero(numBits: C1.getBitWidth());
7003
7004 return std::nullopt;
7005}
7006
7007SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7008 const GlobalAddressSDNode *GA,
7009 const SDNode *N2) {
7010 if (GA->getOpcode() != ISD::GlobalAddress)
7011 return SDValue();
7012 if (!TLI->isOffsetFoldingLegal(GA))
7013 return SDValue();
7014 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7015 if (!C2)
7016 return SDValue();
7017 int64_t Offset = C2->getSExtValue();
7018 switch (Opcode) {
7019 case ISD::ADD:
7020 case ISD::PTRADD:
7021 break;
7022 case ISD::SUB: Offset = -uint64_t(Offset); break;
7023 default: return SDValue();
7024 }
7025 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7026 Offset: GA->getOffset() + uint64_t(Offset));
7027}
7028
7029bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7030 switch (Opcode) {
7031 case ISD::SDIV:
7032 case ISD::UDIV:
7033 case ISD::SREM:
7034 case ISD::UREM: {
7035 // If a divisor is zero/undef or any element of a divisor vector is
7036 // zero/undef, the whole op is undef.
7037 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7038 SDValue Divisor = Ops[1];
7039 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7040 return true;
7041
7042 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7043 llvm::any_of(Range: Divisor->op_values(),
7044 P: [](SDValue V) { return V.isUndef() ||
7045 isNullConstant(V); });
7046 // TODO: Handle signed overflow.
7047 }
7048 // TODO: Handle oversized shifts.
7049 default:
7050 return false;
7051 }
7052}
7053
7054SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7055 EVT VT, ArrayRef<SDValue> Ops,
7056 SDNodeFlags Flags) {
7057 // If the opcode is a target-specific ISD node, there's nothing we can
7058 // do here and the operand rules may not line up with the below, so
7059 // bail early.
7060 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7061 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7062 // foldCONCAT_VECTORS in getNode before this is called.
7063 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7064 return SDValue();
7065
7066 unsigned NumOps = Ops.size();
7067 if (NumOps == 0)
7068 return SDValue();
7069
7070 if (isUndef(Opcode, Ops))
7071 return getUNDEF(VT);
7072
7073 // Handle unary special cases.
7074 if (NumOps == 1) {
7075 SDValue N1 = Ops[0];
7076
7077 // Constant fold unary operations with an integer constant operand. Even
7078 // opaque constant will be folded, because the folding of unary operations
7079 // doesn't create new constants with different values. Nevertheless, the
7080 // opaque flag is preserved during folding to prevent future folding with
7081 // other constants.
7082 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7083 const APInt &Val = C->getAPIntValue();
7084 switch (Opcode) {
7085 case ISD::SIGN_EXTEND:
7086 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7087 isT: C->isTargetOpcode(), isO: C->isOpaque());
7088 case ISD::TRUNCATE:
7089 if (C->isOpaque())
7090 break;
7091 [[fallthrough]];
7092 case ISD::ZERO_EXTEND:
7093 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7094 isT: C->isTargetOpcode(), isO: C->isOpaque());
7095 case ISD::ANY_EXTEND:
7096 // Some targets like RISCV prefer to sign extend some types.
7097 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7098 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7099 isT: C->isTargetOpcode(), isO: C->isOpaque());
7100 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7101 isT: C->isTargetOpcode(), isO: C->isOpaque());
7102 case ISD::ABS:
7103 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7104 isO: C->isOpaque());
7105 case ISD::BITREVERSE:
7106 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7107 isO: C->isOpaque());
7108 case ISD::BSWAP:
7109 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7110 isO: C->isOpaque());
7111 case ISD::CTPOP:
7112 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7113 isO: C->isOpaque());
7114 case ISD::CTLZ:
7115 case ISD::CTLZ_ZERO_UNDEF:
7116 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7117 isO: C->isOpaque());
7118 case ISD::CTTZ:
7119 case ISD::CTTZ_ZERO_UNDEF:
7120 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7121 isO: C->isOpaque());
7122 case ISD::CTLS:
7123 // CTLS returns the number of extra sign bits so subtract one.
7124 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7125 isT: C->isTargetOpcode(), isO: C->isOpaque());
7126 case ISD::UINT_TO_FP:
7127 case ISD::SINT_TO_FP: {
7128 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7129 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7130 RM: APFloat::rmNearestTiesToEven);
7131 return getConstantFP(V: FPV, DL, VT);
7132 }
7133 case ISD::FP16_TO_FP:
7134 case ISD::BF16_TO_FP: {
7135 bool Ignored;
7136 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7137 : APFloat::BFloat(),
7138 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7139
7140 // This can return overflow, underflow, or inexact; we don't care.
7141 // FIXME need to be more flexible about rounding mode.
7142 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7143 losesInfo: &Ignored);
7144 return getConstantFP(V: FPV, DL, VT);
7145 }
7146 case ISD::STEP_VECTOR:
7147 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7148 return V;
7149 break;
7150 case ISD::BITCAST:
7151 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7152 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7153 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7154 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7155 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7156 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7157 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7158 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7159 break;
7160 }
7161 }
7162
7163 // Constant fold unary operations with a floating point constant operand.
7164 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7165 APFloat V = C->getValueAPF(); // make copy
7166 switch (Opcode) {
7167 case ISD::FNEG:
7168 V.changeSign();
7169 return getConstantFP(V, DL, VT);
7170 case ISD::FABS:
7171 V.clearSign();
7172 return getConstantFP(V, DL, VT);
7173 case ISD::FCEIL: {
7174 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7175 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7176 return getConstantFP(V, DL, VT);
7177 return SDValue();
7178 }
7179 case ISD::FTRUNC: {
7180 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7181 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7182 return getConstantFP(V, DL, VT);
7183 return SDValue();
7184 }
7185 case ISD::FFLOOR: {
7186 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7187 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7188 return getConstantFP(V, DL, VT);
7189 return SDValue();
7190 }
7191 case ISD::FP_EXTEND: {
7192 bool ignored;
7193 // This can return overflow, underflow, or inexact; we don't care.
7194 // FIXME need to be more flexible about rounding mode.
7195 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7196 losesInfo: &ignored);
7197 return getConstantFP(V, DL, VT);
7198 }
7199 case ISD::FP_TO_SINT:
7200 case ISD::FP_TO_UINT: {
7201 bool ignored;
7202 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7203 // FIXME need to be more flexible about rounding mode.
7204 APFloat::opStatus s =
7205 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7206 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7207 break;
7208 return getConstant(Val: IntVal, DL, VT);
7209 }
7210 case ISD::FP_TO_FP16:
7211 case ISD::FP_TO_BF16: {
7212 bool Ignored;
7213 // This can return overflow, underflow, or inexact; we don't care.
7214 // FIXME need to be more flexible about rounding mode.
7215 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7216 : APFloat::BFloat(),
7217 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7218 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7219 }
7220 case ISD::BITCAST:
7221 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7222 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7223 VT);
7224 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7225 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7226 VT);
7227 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7228 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7229 VT);
7230 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7231 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7232 break;
7233 }
7234 }
7235
7236 // Early-out if we failed to constant fold a bitcast.
7237 if (Opcode == ISD::BITCAST)
7238 return SDValue();
7239 }
7240
7241 // Handle binops special cases.
7242 if (NumOps == 2) {
7243 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7244 return CFP;
7245
7246 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7247 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7248 if (C1->isOpaque() || C2->isOpaque())
7249 return SDValue();
7250
7251 std::optional<APInt> FoldAttempt =
7252 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7253 if (!FoldAttempt)
7254 return SDValue();
7255
7256 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7257 assert((!Folded || !VT.isVector()) &&
7258 "Can't fold vectors ops with scalar operands");
7259 return Folded;
7260 }
7261 }
7262
7263 // fold (add Sym, c) -> Sym+c
7264 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7265 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7266 if (TLI->isCommutativeBinOp(Opcode))
7267 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7268 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7269
7270 // fold (sext_in_reg c1) -> c2
7271 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7272 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7273
7274 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7275 unsigned FromBits = EVT.getScalarSizeInBits();
7276 Val <<= Val.getBitWidth() - FromBits;
7277 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7278 return getConstant(Val, DL, VT: ConstantVT);
7279 };
7280
7281 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7282 const APInt &Val = C1->getAPIntValue();
7283 return SignExtendInReg(Val, VT);
7284 }
7285
7286 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7287 SmallVector<SDValue, 8> ScalarOps;
7288 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7289 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7290 SDValue Op = Ops[0].getOperand(i: I);
7291 if (Op.isUndef()) {
7292 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7293 continue;
7294 }
7295 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7296 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7297 }
7298 return getBuildVector(VT, DL, Ops: ScalarOps);
7299 }
7300
7301 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7302 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7303 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7304 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7305 Ops[0].getOperand(i: 0).getValueType()));
7306 }
7307 }
7308
7309 // Handle fshl/fshr special cases.
7310 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7311 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7312 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7313 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7314
7315 if (C1 && C2 && C3) {
7316 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7317 return SDValue();
7318 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7319 &V3 = C3->getAPIntValue();
7320
7321 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7322 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7323 return getConstant(Val: FoldedVal, DL, VT);
7324 }
7325 }
7326
7327 // Handle fma/fmad special cases.
7328 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7329 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7330 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7331 Ops[2].getValueType() == VT && "FMA types must match!");
7332 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
7333 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
7334 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
7335 if (C1 && C2 && C3) {
7336 APFloat V1 = C1->getValueAPF();
7337 const APFloat &V2 = C2->getValueAPF();
7338 const APFloat &V3 = C3->getValueAPF();
7339 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7340 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7341 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7342 } else
7343 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7344 return getConstantFP(V: V1, DL, VT);
7345 }
7346 }
7347
7348 // This is for vector folding only from here on.
7349 if (!VT.isVector())
7350 return SDValue();
7351
7352 ElementCount NumElts = VT.getVectorElementCount();
7353
7354 // See if we can fold through any bitcasted integer ops.
7355 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
7356 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7357 (Ops[0].getOpcode() == ISD::BITCAST ||
7358 Ops[1].getOpcode() == ISD::BITCAST)) {
7359 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
7360 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
7361 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
7362 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
7363 if (BV1 && BV2 && N1.getValueType().isInteger() &&
7364 N2.getValueType().isInteger()) {
7365 bool IsLE = getDataLayout().isLittleEndian();
7366 unsigned EltBits = VT.getScalarSizeInBits();
7367 SmallVector<APInt> RawBits1, RawBits2;
7368 BitVector UndefElts1, UndefElts2;
7369 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
7370 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
7371 SmallVector<APInt> RawBits;
7372 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
7373 std::optional<APInt> Fold = FoldValueWithUndef(
7374 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
7375 if (!Fold)
7376 break;
7377 RawBits.push_back(Elt: *Fold);
7378 }
7379 if (RawBits.size() == NumElts.getFixedValue()) {
7380 // We have constant folded, but we might need to cast this again back
7381 // to the original (possibly legalized) type.
7382 EVT BVVT, BVEltVT;
7383 if (N1.getValueType() == VT) {
7384 BVVT = N1.getValueType();
7385 BVEltVT = BV1->getOperand(Num: 0).getValueType();
7386 } else {
7387 BVVT = N2.getValueType();
7388 BVEltVT = BV2->getOperand(Num: 0).getValueType();
7389 }
7390 unsigned BVEltBits = BVEltVT.getSizeInBits();
7391 SmallVector<APInt> DstBits;
7392 BitVector DstUndefs;
7393 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
7394 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
7395 SrcUndefElements: BitVector(RawBits.size(), false));
7396 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
7397 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
7398 if (DstUndefs[I])
7399 continue;
7400 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
7401 }
7402 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
7403 }
7404 }
7405 }
7406 }
7407
7408 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
7409 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
7410 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
7411 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
7412 APInt RHSVal;
7413 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
7414 APInt NewStep = Opcode == ISD::MUL
7415 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
7416 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
7417 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
7418 }
7419 }
7420
7421 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
7422 return !Op.getValueType().isVector() ||
7423 Op.getValueType().getVectorElementCount() == NumElts;
7424 };
7425
7426 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
7427 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
7428 Op.getOpcode() == ISD::BUILD_VECTOR ||
7429 Op.getOpcode() == ISD::SPLAT_VECTOR;
7430 };
7431
7432 // All operands must be vector types with the same number of elements as
7433 // the result type and must be either UNDEF or a build/splat vector
7434 // or UNDEF scalars.
7435 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
7436 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
7437 return SDValue();
7438
7439 // If we are comparing vectors, then the result needs to be a i1 boolean that
7440 // is then extended back to the legal result type depending on how booleans
7441 // are represented.
7442 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
7443 ISD::NodeType ExtendCode =
7444 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
7445 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
7446 : ISD::SIGN_EXTEND;
7447
7448 // Find legal integer scalar type for constant promotion and
7449 // ensure that its scalar size is at least as large as source.
7450 EVT LegalSVT = VT.getScalarType();
7451 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
7452 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
7453 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
7454 return SDValue();
7455 }
7456
7457 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
7458 // only have one operand to check. For fixed-length vector types we may have
7459 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
7460 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
7461
7462 // Constant fold each scalar lane separately.
7463 SmallVector<SDValue, 4> ScalarResults;
7464 for (unsigned I = 0; I != NumVectorElts; I++) {
7465 SmallVector<SDValue, 4> ScalarOps;
7466 for (SDValue Op : Ops) {
7467 EVT InSVT = Op.getValueType().getScalarType();
7468 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
7469 Op.getOpcode() != ISD::SPLAT_VECTOR) {
7470 if (Op.isUndef())
7471 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
7472 else
7473 ScalarOps.push_back(Elt: Op);
7474 continue;
7475 }
7476
7477 SDValue ScalarOp =
7478 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
7479 EVT ScalarVT = ScalarOp.getValueType();
7480
7481 // Build vector (integer) scalar operands may need implicit
7482 // truncation - do this before constant folding.
7483 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
7484 // Don't create illegally-typed nodes unless they're constants or undef
7485 // - if we fail to constant fold we can't guarantee the (dead) nodes
7486 // we're creating will be cleaned up before being visited for
7487 // legalization.
7488 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
7489 !isa<ConstantSDNode>(Val: ScalarOp) &&
7490 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
7491 TargetLowering::TypeLegal)
7492 return SDValue();
7493 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
7494 }
7495
7496 ScalarOps.push_back(Elt: ScalarOp);
7497 }
7498
7499 // Constant fold the scalar operands.
7500 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
7501
7502 // Scalar folding only succeeded if the result is a constant or UNDEF.
7503 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
7504 ScalarResult.getOpcode() != ISD::ConstantFP)
7505 return SDValue();
7506
7507 // Legalize the (integer) scalar constant if necessary. We only do
7508 // this once we know the folding succeeded, since otherwise we would
7509 // get a node with illegal type which has a user.
7510 if (LegalSVT != SVT)
7511 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
7512
7513 ScalarResults.push_back(Elt: ScalarResult);
7514 }
7515
7516 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
7517 : getBuildVector(VT, DL, Ops: ScalarResults);
7518 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
7519 return V;
7520}
7521
7522SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
7523 EVT VT, ArrayRef<SDValue> Ops) {
7524 // TODO: Add support for unary/ternary fp opcodes.
7525 if (Ops.size() != 2)
7526 return SDValue();
7527
7528 // TODO: We don't do any constant folding for strict FP opcodes here, but we
7529 // should. That will require dealing with a potentially non-default
7530 // rounding mode, checking the "opStatus" return value from the APFloat
7531 // math calculations, and possibly other variations.
7532 SDValue N1 = Ops[0];
7533 SDValue N2 = Ops[1];
7534 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
7535 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
7536 if (N1CFP && N2CFP) {
7537 APFloat C1 = N1CFP->getValueAPF(); // make copy
7538 const APFloat &C2 = N2CFP->getValueAPF();
7539 switch (Opcode) {
7540 case ISD::FADD:
7541 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7542 return getConstantFP(V: C1, DL, VT);
7543 case ISD::FSUB:
7544 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7545 return getConstantFP(V: C1, DL, VT);
7546 case ISD::FMUL:
7547 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7548 return getConstantFP(V: C1, DL, VT);
7549 case ISD::FDIV:
7550 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7551 return getConstantFP(V: C1, DL, VT);
7552 case ISD::FREM:
7553 C1.mod(RHS: C2);
7554 return getConstantFP(V: C1, DL, VT);
7555 case ISD::FCOPYSIGN:
7556 C1.copySign(RHS: C2);
7557 return getConstantFP(V: C1, DL, VT);
7558 case ISD::FMINNUM:
7559 if (C1.isSignaling() || C2.isSignaling())
7560 return SDValue();
7561 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
7562 case ISD::FMAXNUM:
7563 if (C1.isSignaling() || C2.isSignaling())
7564 return SDValue();
7565 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
7566 case ISD::FMINIMUM:
7567 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
7568 case ISD::FMAXIMUM:
7569 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
7570 case ISD::FMINIMUMNUM:
7571 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
7572 case ISD::FMAXIMUMNUM:
7573 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
7574 default: break;
7575 }
7576 }
7577 if (N1CFP && Opcode == ISD::FP_ROUND) {
7578 APFloat C1 = N1CFP->getValueAPF(); // make copy
7579 bool Unused;
7580 // This can return overflow, underflow, or inexact; we don't care.
7581 // FIXME need to be more flexible about rounding mode.
7582 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7583 losesInfo: &Unused);
7584 return getConstantFP(V: C1, DL, VT);
7585 }
7586
7587 switch (Opcode) {
7588 case ISD::FSUB:
7589 // -0.0 - undef --> undef (consistent with "fneg undef")
7590 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
7591 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
7592 return getUNDEF(VT);
7593 [[fallthrough]];
7594
7595 case ISD::FADD:
7596 case ISD::FMUL:
7597 case ISD::FDIV:
7598 case ISD::FREM:
7599 // If both operands are undef, the result is undef. If 1 operand is undef,
7600 // the result is NaN. This should match the behavior of the IR optimizer.
7601 if (N1.isUndef() && N2.isUndef())
7602 return getUNDEF(VT);
7603 if (N1.isUndef() || N2.isUndef())
7604 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
7605 }
7606 return SDValue();
7607}
7608
7609SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
7610 const SDLoc &DL, EVT DstEltVT) {
7611 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
7612
7613 // If this is already the right type, we're done.
7614 if (SrcEltVT == DstEltVT)
7615 return SDValue(BV, 0);
7616
7617 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
7618 unsigned DstBitSize = DstEltVT.getSizeInBits();
7619
7620 // If this is a conversion of N elements of one type to N elements of another
7621 // type, convert each element. This handles FP<->INT cases.
7622 if (SrcBitSize == DstBitSize) {
7623 SmallVector<SDValue, 8> Ops;
7624 for (SDValue Op : BV->op_values()) {
7625 // If the vector element type is not legal, the BUILD_VECTOR operands
7626 // are promoted and implicitly truncated. Make that explicit here.
7627 if (Op.getValueType() != SrcEltVT)
7628 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
7629 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
7630 }
7631 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
7632 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
7633 return getBuildVector(VT, DL, Ops);
7634 }
7635
7636 // Otherwise, we're growing or shrinking the elements. To avoid having to
7637 // handle annoying details of growing/shrinking FP values, we convert them to
7638 // int first.
7639 if (SrcEltVT.isFloatingPoint()) {
7640 // Convert the input float vector to a int vector where the elements are the
7641 // same sizes.
7642 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
7643 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7644 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7645 DstEltVT);
7646 return SDValue();
7647 }
7648
7649 // Now we know the input is an integer vector. If the output is a FP type,
7650 // convert to integer first, then to FP of the right size.
7651 if (DstEltVT.isFloatingPoint()) {
7652 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
7653 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7654 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7655 DstEltVT);
7656 return SDValue();
7657 }
7658
7659 // Okay, we know the src/dst types are both integers of differing types.
7660 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
7661
7662 // Extract the constant raw bit data.
7663 BitVector UndefElements;
7664 SmallVector<APInt> RawBits;
7665 bool IsLE = getDataLayout().isLittleEndian();
7666 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
7667 return SDValue();
7668
7669 SmallVector<SDValue, 8> Ops;
7670 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
7671 if (UndefElements[I])
7672 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
7673 else
7674 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
7675 }
7676
7677 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
7678 return getBuildVector(VT, DL, Ops);
7679}
7680
7681SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
7682 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
7683
7684 // There's no need to assert on a byte-aligned pointer. All pointers are at
7685 // least byte aligned.
7686 if (A == Align(1))
7687 return Val;
7688
7689 SDVTList VTs = getVTList(VT: Val.getValueType());
7690 FoldingSetNodeID ID;
7691 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: VTs, OpList: {Val});
7692 ID.AddInteger(I: A.value());
7693
7694 void *IP = nullptr;
7695 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
7696 return SDValue(E, 0);
7697
7698 auto *N =
7699 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
7700 createOperands(Node: N, Vals: {Val});
7701
7702 CSEMap.InsertNode(N, InsertPos: IP);
7703 InsertNode(N);
7704
7705 SDValue V(N, 0);
7706 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7707 return V;
7708}
7709
7710SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7711 SDValue N1, SDValue N2) {
7712 SDNodeFlags Flags;
7713 if (Inserter)
7714 Flags = Inserter->getFlags();
7715 return getNode(Opcode, DL, VT, N1, N2, Flags);
7716}
7717
7718void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
7719 SDValue &N2) const {
7720 if (!TLI->isCommutativeBinOp(Opcode))
7721 return;
7722
7723 // Canonicalize:
7724 // binop(const, nonconst) -> binop(nonconst, const)
7725 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
7726 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
7727 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
7728 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
7729 if ((N1C && !N2C) || (N1CFP && !N2CFP))
7730 std::swap(a&: N1, b&: N2);
7731
7732 // Canonicalize:
7733 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
7734 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
7735 N2.getOpcode() == ISD::STEP_VECTOR)
7736 std::swap(a&: N1, b&: N2);
7737}
7738
7739SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7740 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
7741 assert(N1.getOpcode() != ISD::DELETED_NODE &&
7742 N2.getOpcode() != ISD::DELETED_NODE &&
7743 "Operand is DELETED_NODE!");
7744
7745 canonicalizeCommutativeBinop(Opcode, N1, N2);
7746
7747 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
7748 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
7749
7750 // Don't allow undefs in vector splats - we might be returning N2 when folding
7751 // to zero etc.
7752 ConstantSDNode *N2CV =
7753 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
7754
7755 switch (Opcode) {
7756 default: break;
7757 case ISD::TokenFactor:
7758 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
7759 N2.getValueType() == MVT::Other && "Invalid token factor!");
7760 // Fold trivial token factors.
7761 if (N1.getOpcode() == ISD::EntryToken) return N2;
7762 if (N2.getOpcode() == ISD::EntryToken) return N1;
7763 if (N1 == N2) return N1;
7764 break;
7765 case ISD::BUILD_VECTOR: {
7766 // Attempt to simplify BUILD_VECTOR.
7767 SDValue Ops[] = {N1, N2};
7768 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7769 return V;
7770 break;
7771 }
7772 case ISD::CONCAT_VECTORS: {
7773 SDValue Ops[] = {N1, N2};
7774 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
7775 return V;
7776 break;
7777 }
7778 case ISD::AND:
7779 assert(VT.isInteger() && "This operator does not apply to FP types!");
7780 assert(N1.getValueType() == N2.getValueType() &&
7781 N1.getValueType() == VT && "Binary operator types must match!");
7782 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
7783 // worth handling here.
7784 if (N2CV && N2CV->isZero())
7785 return N2;
7786 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
7787 return N1;
7788 break;
7789 case ISD::OR:
7790 case ISD::XOR:
7791 case ISD::ADD:
7792 case ISD::PTRADD:
7793 case ISD::SUB:
7794 assert(VT.isInteger() && "This operator does not apply to FP types!");
7795 assert(N1.getValueType() == N2.getValueType() &&
7796 N1.getValueType() == VT && "Binary operator types must match!");
7797 // The equal operand types requirement is unnecessarily strong for PTRADD.
7798 // However, the SelectionDAGBuilder does not generate PTRADDs with different
7799 // operand types, and we'd need to re-implement GEP's non-standard wrapping
7800 // logic everywhere where PTRADDs may be folded or combined to properly
7801 // support them. If/when we introduce pointer types to the SDAG, we will
7802 // need to relax this constraint.
7803
7804 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
7805 // it's worth handling here.
7806 if (N2CV && N2CV->isZero())
7807 return N1;
7808 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
7809 VT.getScalarType() == MVT::i1)
7810 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
7811 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
7812 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
7813 N2.getOpcode() == ISD::VSCALE) {
7814 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
7815 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
7816 return getVScale(DL, VT, MulImm: C1 + C2);
7817 }
7818 break;
7819 case ISD::MUL:
7820 assert(VT.isInteger() && "This operator does not apply to FP types!");
7821 assert(N1.getValueType() == N2.getValueType() &&
7822 N1.getValueType() == VT && "Binary operator types must match!");
7823 if (VT.getScalarType() == MVT::i1)
7824 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
7825 if (N2CV && N2CV->isZero())
7826 return N2;
7827 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7828 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
7829 const APInt &N2CImm = N2C->getAPIntValue();
7830 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
7831 }
7832 break;
7833 case ISD::UDIV:
7834 case ISD::UREM:
7835 case ISD::MULHU:
7836 case ISD::MULHS:
7837 case ISD::SDIV:
7838 case ISD::SREM:
7839 case ISD::SADDSAT:
7840 case ISD::SSUBSAT:
7841 case ISD::UADDSAT:
7842 case ISD::USUBSAT:
7843 assert(VT.isInteger() && "This operator does not apply to FP types!");
7844 assert(N1.getValueType() == N2.getValueType() &&
7845 N1.getValueType() == VT && "Binary operator types must match!");
7846 if (VT.getScalarType() == MVT::i1) {
7847 // fold (add_sat x, y) -> (or x, y) for bool types.
7848 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
7849 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
7850 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
7851 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
7852 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
7853 }
7854 break;
7855 case ISD::SCMP:
7856 case ISD::UCMP:
7857 assert(N1.getValueType() == N2.getValueType() &&
7858 "Types of operands of UCMP/SCMP must match");
7859 assert(N1.getValueType().isVector() == VT.isVector() &&
7860 "Operands and return type of must both be scalars or vectors");
7861 if (VT.isVector())
7862 assert(VT.getVectorElementCount() ==
7863 N1.getValueType().getVectorElementCount() &&
7864 "Result and operands must have the same number of elements");
7865 break;
7866 case ISD::AVGFLOORS:
7867 case ISD::AVGFLOORU:
7868 case ISD::AVGCEILS:
7869 case ISD::AVGCEILU:
7870 assert(VT.isInteger() && "This operator does not apply to FP types!");
7871 assert(N1.getValueType() == N2.getValueType() &&
7872 N1.getValueType() == VT && "Binary operator types must match!");
7873 break;
7874 case ISD::ABDS:
7875 case ISD::ABDU:
7876 assert(VT.isInteger() && "This operator does not apply to FP types!");
7877 assert(N1.getValueType() == N2.getValueType() &&
7878 N1.getValueType() == VT && "Binary operator types must match!");
7879 if (VT.getScalarType() == MVT::i1)
7880 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
7881 break;
7882 case ISD::SMIN:
7883 case ISD::UMAX:
7884 assert(VT.isInteger() && "This operator does not apply to FP types!");
7885 assert(N1.getValueType() == N2.getValueType() &&
7886 N1.getValueType() == VT && "Binary operator types must match!");
7887 if (VT.getScalarType() == MVT::i1)
7888 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
7889 break;
7890 case ISD::SMAX:
7891 case ISD::UMIN:
7892 assert(VT.isInteger() && "This operator does not apply to FP types!");
7893 assert(N1.getValueType() == N2.getValueType() &&
7894 N1.getValueType() == VT && "Binary operator types must match!");
7895 if (VT.getScalarType() == MVT::i1)
7896 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
7897 break;
7898 case ISD::FADD:
7899 case ISD::FSUB:
7900 case ISD::FMUL:
7901 case ISD::FDIV:
7902 case ISD::FREM:
7903 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7904 assert(N1.getValueType() == N2.getValueType() &&
7905 N1.getValueType() == VT && "Binary operator types must match!");
7906 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
7907 return V;
7908 break;
7909 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
7910 assert(N1.getValueType() == VT &&
7911 N1.getValueType().isFloatingPoint() &&
7912 N2.getValueType().isFloatingPoint() &&
7913 "Invalid FCOPYSIGN!");
7914 break;
7915 case ISD::SHL:
7916 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7917 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
7918 const APInt &ShiftImm = N2C->getAPIntValue();
7919 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
7920 }
7921 [[fallthrough]];
7922 case ISD::SRA:
7923 case ISD::SRL:
7924 if (SDValue V = simplifyShift(X: N1, Y: N2))
7925 return V;
7926 [[fallthrough]];
7927 case ISD::ROTL:
7928 case ISD::ROTR:
7929 case ISD::SSHLSAT:
7930 case ISD::USHLSAT:
7931 assert(VT == N1.getValueType() &&
7932 "Shift operators return type must be the same as their first arg");
7933 assert(VT.isInteger() && N2.getValueType().isInteger() &&
7934 "Shifts only work on integers");
7935 assert((!VT.isVector() || VT == N2.getValueType()) &&
7936 "Vector shift amounts must be in the same as their first arg");
7937 // Verify that the shift amount VT is big enough to hold valid shift
7938 // amounts. This catches things like trying to shift an i1024 value by an
7939 // i8, which is easy to fall into in generic code that uses
7940 // TLI.getShiftAmount().
7941 assert(N2.getValueType().getScalarSizeInBits() >=
7942 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
7943 "Invalid use of small shift amount with oversized value!");
7944
7945 // Always fold shifts of i1 values so the code generator doesn't need to
7946 // handle them. Since we know the size of the shift has to be less than the
7947 // size of the value, the shift/rotate count is guaranteed to be zero.
7948 if (VT == MVT::i1)
7949 return N1;
7950 if (N2CV && N2CV->isZero())
7951 return N1;
7952 break;
7953 case ISD::FP_ROUND:
7954 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7955 VT.bitsLE(N1.getValueType()) && N2C &&
7956 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
7957 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
7958 if (N1.getValueType() == VT) return N1; // noop conversion.
7959 break;
7960 case ISD::AssertNoFPClass: {
7961 assert(N1.getValueType().isFloatingPoint() &&
7962 "AssertNoFPClass is used for a non-floating type");
7963 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
7964 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
7965 assert(llvm::to_underlying(NoFPClass) <=
7966 BitmaskEnumDetail::Mask<FPClassTest>() &&
7967 "FPClassTest value too large");
7968 (void)NoFPClass;
7969 break;
7970 }
7971 case ISD::AssertSext:
7972 case ISD::AssertZext: {
7973 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
7974 assert(VT == N1.getValueType() && "Not an inreg extend!");
7975 assert(VT.isInteger() && EVT.isInteger() &&
7976 "Cannot *_EXTEND_INREG FP types");
7977 assert(!EVT.isVector() &&
7978 "AssertSExt/AssertZExt type should be the vector element type "
7979 "rather than the vector type!");
7980 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
7981 if (VT.getScalarType() == EVT) return N1; // noop assertion.
7982 break;
7983 }
7984 case ISD::SIGN_EXTEND_INREG: {
7985 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
7986 assert(VT == N1.getValueType() && "Not an inreg extend!");
7987 assert(VT.isInteger() && EVT.isInteger() &&
7988 "Cannot *_EXTEND_INREG FP types");
7989 assert(EVT.isVector() == VT.isVector() &&
7990 "SIGN_EXTEND_INREG type should be vector iff the operand "
7991 "type is vector!");
7992 assert((!EVT.isVector() ||
7993 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
7994 "Vector element counts must match in SIGN_EXTEND_INREG");
7995 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
7996 if (EVT == VT) return N1; // Not actually extending
7997 break;
7998 }
7999 case ISD::FP_TO_SINT_SAT:
8000 case ISD::FP_TO_UINT_SAT: {
8001 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8002 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8003 assert(N1.getValueType().isVector() == VT.isVector() &&
8004 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8005 "vector!");
8006 assert((!VT.isVector() || VT.getVectorElementCount() ==
8007 N1.getValueType().getVectorElementCount()) &&
8008 "Vector element counts must match in FP_TO_*INT_SAT");
8009 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8010 "Type to saturate to must be a scalar.");
8011 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8012 "Not extending!");
8013 break;
8014 }
8015 case ISD::EXTRACT_VECTOR_ELT:
8016 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8017 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8018 element type of the vector.");
8019
8020 // Extract from an undefined value or using an undefined index is undefined.
8021 if (N1.isUndef() || N2.isUndef())
8022 return getUNDEF(VT);
8023
8024 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
8025 // vectors. For scalable vectors we will provide appropriate support for
8026 // dealing with arbitrary indices.
8027 if (N2C && N1.getValueType().isFixedLengthVector() &&
8028 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8029 return getUNDEF(VT);
8030
8031 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8032 // expanding copies of large vectors from registers. This only works for
8033 // fixed length vectors, since we need to know the exact number of
8034 // elements.
8035 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8036 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8037 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8038 return getExtractVectorElt(DL, VT,
8039 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8040 Idx: N2C->getZExtValue() % Factor);
8041 }
8042
8043 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8044 // lowering is expanding large vector constants.
8045 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8046 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8047 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8048 N1.getValueType().isFixedLengthVector()) &&
8049 "BUILD_VECTOR used for scalable vectors");
8050 unsigned Index =
8051 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8052 SDValue Elt = N1.getOperand(i: Index);
8053
8054 if (VT != Elt.getValueType())
8055 // If the vector element type is not legal, the BUILD_VECTOR operands
8056 // are promoted and implicitly truncated, and the result implicitly
8057 // extended. Make that explicit here.
8058 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8059
8060 return Elt;
8061 }
8062
8063 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8064 // operations are lowered to scalars.
8065 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8066 // If the indices are the same, return the inserted element else
8067 // if the indices are known different, extract the element from
8068 // the original vector.
8069 SDValue N1Op2 = N1.getOperand(i: 2);
8070 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8071
8072 if (N1Op2C && N2C) {
8073 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8074 if (VT == N1.getOperand(i: 1).getValueType())
8075 return N1.getOperand(i: 1);
8076 if (VT.isFloatingPoint()) {
8077 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8078 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8079 }
8080 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8081 }
8082 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8083 }
8084 }
8085
8086 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8087 // when vector types are scalarized and v1iX is legal.
8088 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8089 // Here we are completely ignoring the extract element index (N2),
8090 // which is fine for fixed width vectors, since any index other than 0
8091 // is undefined anyway. However, this cannot be ignored for scalable
8092 // vectors - in theory we could support this, but we don't want to do this
8093 // without a profitability check.
8094 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8095 N1.getValueType().isFixedLengthVector() &&
8096 N1.getValueType().getVectorNumElements() == 1) {
8097 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8098 N2: N1.getOperand(i: 1));
8099 }
8100 break;
8101 case ISD::EXTRACT_ELEMENT:
8102 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8103 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8104 (N1.getValueType().isInteger() == VT.isInteger()) &&
8105 N1.getValueType() != VT &&
8106 "Wrong types for EXTRACT_ELEMENT!");
8107
8108 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8109 // 64-bit integers into 32-bit parts. Instead of building the extract of
8110 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8111 if (N1.getOpcode() == ISD::BUILD_PAIR)
8112 return N1.getOperand(i: N2C->getZExtValue());
8113
8114 // EXTRACT_ELEMENT of a constant int is also very common.
8115 if (N1C) {
8116 unsigned ElementSize = VT.getSizeInBits();
8117 unsigned Shift = ElementSize * N2C->getZExtValue();
8118 const APInt &Val = N1C->getAPIntValue();
8119 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8120 }
8121 break;
8122 case ISD::EXTRACT_SUBVECTOR: {
8123 EVT N1VT = N1.getValueType();
8124 assert(VT.isVector() && N1VT.isVector() &&
8125 "Extract subvector VTs must be vectors!");
8126 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8127 "Extract subvector VTs must have the same element type!");
8128 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8129 "Cannot extract a scalable vector from a fixed length vector!");
8130 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8131 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8132 "Extract subvector must be from larger vector to smaller vector!");
8133 assert(N2C && "Extract subvector index must be a constant");
8134 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8135 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8136 N1VT.getVectorMinNumElements()) &&
8137 "Extract subvector overflow!");
8138 assert(N2C->getAPIntValue().getBitWidth() ==
8139 TLI->getVectorIdxWidth(getDataLayout()) &&
8140 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8141 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8142 "Extract index is not a multiple of the output vector length");
8143
8144 // Trivial extraction.
8145 if (VT == N1VT)
8146 return N1;
8147
8148 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8149 if (N1.isUndef())
8150 return getUNDEF(VT);
8151
8152 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8153 // the concat have the same type as the extract.
8154 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8155 VT == N1.getOperand(i: 0).getValueType()) {
8156 unsigned Factor = VT.getVectorMinNumElements();
8157 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8158 }
8159
8160 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8161 // during shuffle legalization.
8162 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8163 VT == N1.getOperand(i: 1).getValueType())
8164 return N1.getOperand(i: 1);
8165 break;
8166 }
8167 }
8168
8169 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8170 switch (Opcode) {
8171 case ISD::XOR:
8172 case ISD::ADD:
8173 case ISD::PTRADD:
8174 case ISD::SUB:
8175 case ISD::SIGN_EXTEND_INREG:
8176 case ISD::UDIV:
8177 case ISD::SDIV:
8178 case ISD::UREM:
8179 case ISD::SREM:
8180 case ISD::MUL:
8181 case ISD::AND:
8182 case ISD::SSUBSAT:
8183 case ISD::USUBSAT:
8184 case ISD::UMIN:
8185 case ISD::OR:
8186 case ISD::SADDSAT:
8187 case ISD::UADDSAT:
8188 case ISD::UMAX:
8189 case ISD::SMAX:
8190 case ISD::SMIN:
8191 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8192 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8193 }
8194 }
8195
8196 // Canonicalize an UNDEF to the RHS, even over a constant.
8197 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8198 if (TLI->isCommutativeBinOp(Opcode)) {
8199 std::swap(a&: N1, b&: N2);
8200 } else {
8201 switch (Opcode) {
8202 case ISD::PTRADD:
8203 case ISD::SUB:
8204 // fold op(undef, non_undef_arg2) -> undef.
8205 return N1;
8206 case ISD::SIGN_EXTEND_INREG:
8207 case ISD::UDIV:
8208 case ISD::SDIV:
8209 case ISD::UREM:
8210 case ISD::SREM:
8211 case ISD::SSUBSAT:
8212 case ISD::USUBSAT:
8213 // fold op(undef, non_undef_arg2) -> 0.
8214 return getConstant(Val: 0, DL, VT);
8215 }
8216 }
8217 }
8218
8219 // Fold a bunch of operators when the RHS is undef.
8220 if (N2.getOpcode() == ISD::UNDEF) {
8221 switch (Opcode) {
8222 case ISD::XOR:
8223 if (N1.getOpcode() == ISD::UNDEF)
8224 // Handle undef ^ undef -> 0 special case. This is a common
8225 // idiom (misuse).
8226 return getConstant(Val: 0, DL, VT);
8227 [[fallthrough]];
8228 case ISD::ADD:
8229 case ISD::PTRADD:
8230 case ISD::SUB:
8231 // fold op(arg1, undef) -> undef.
8232 return N2;
8233 case ISD::UDIV:
8234 case ISD::SDIV:
8235 case ISD::UREM:
8236 case ISD::SREM:
8237 // fold op(arg1, undef) -> poison.
8238 return getPOISON(VT);
8239 case ISD::MUL:
8240 case ISD::AND:
8241 case ISD::SSUBSAT:
8242 case ISD::USUBSAT:
8243 case ISD::UMIN:
8244 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
8245 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
8246 case ISD::OR:
8247 case ISD::SADDSAT:
8248 case ISD::UADDSAT:
8249 case ISD::UMAX:
8250 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
8251 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
8252 case ISD::SMAX:
8253 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
8254 return N1.getOpcode() == ISD::UNDEF
8255 ? N2
8256 : getConstant(
8257 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
8258 VT);
8259 case ISD::SMIN:
8260 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
8261 return N1.getOpcode() == ISD::UNDEF
8262 ? N2
8263 : getConstant(
8264 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
8265 VT);
8266 }
8267 }
8268
8269 // Perform trivial constant folding.
8270 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
8271 return SV;
8272
8273 // Memoize this node if possible.
8274 SDNode *N;
8275 SDVTList VTs = getVTList(VT);
8276 SDValue Ops[] = {N1, N2};
8277 if (VT != MVT::Glue) {
8278 FoldingSetNodeID ID;
8279 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8280 void *IP = nullptr;
8281 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8282 E->intersectFlagsWith(Flags);
8283 return SDValue(E, 0);
8284 }
8285
8286 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8287 N->setFlags(Flags);
8288 createOperands(Node: N, Vals: Ops);
8289 CSEMap.InsertNode(N, InsertPos: IP);
8290 } else {
8291 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8292 createOperands(Node: N, Vals: Ops);
8293 }
8294
8295 InsertNode(N);
8296 SDValue V = SDValue(N, 0);
8297 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8298 return V;
8299}
8300
8301SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8302 SDValue N1, SDValue N2, SDValue N3) {
8303 SDNodeFlags Flags;
8304 if (Inserter)
8305 Flags = Inserter->getFlags();
8306 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
8307}
8308
8309SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8310 SDValue N1, SDValue N2, SDValue N3,
8311 const SDNodeFlags Flags) {
8312 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8313 N2.getOpcode() != ISD::DELETED_NODE &&
8314 N3.getOpcode() != ISD::DELETED_NODE &&
8315 "Operand is DELETED_NODE!");
8316 // Perform various simplifications.
8317 switch (Opcode) {
8318 case ISD::BUILD_VECTOR: {
8319 // Attempt to simplify BUILD_VECTOR.
8320 SDValue Ops[] = {N1, N2, N3};
8321 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8322 return V;
8323 break;
8324 }
8325 case ISD::CONCAT_VECTORS: {
8326 SDValue Ops[] = {N1, N2, N3};
8327 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8328 return V;
8329 break;
8330 }
8331 case ISD::SETCC: {
8332 assert(VT.isInteger() && "SETCC result type must be an integer!");
8333 assert(N1.getValueType() == N2.getValueType() &&
8334 "SETCC operands must have the same type!");
8335 assert(VT.isVector() == N1.getValueType().isVector() &&
8336 "SETCC type should be vector iff the operand type is vector!");
8337 assert((!VT.isVector() || VT.getVectorElementCount() ==
8338 N1.getValueType().getVectorElementCount()) &&
8339 "SETCC vector element counts must match!");
8340 // Use FoldSetCC to simplify SETCC's.
8341 if (SDValue V = FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL))
8342 return V;
8343 break;
8344 }
8345 case ISD::SELECT:
8346 case ISD::VSELECT:
8347 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
8348 return V;
8349 break;
8350 case ISD::VECTOR_SHUFFLE:
8351 llvm_unreachable("should use getVectorShuffle constructor!");
8352 case ISD::VECTOR_SPLICE_LEFT:
8353 if (isNullConstant(V: N3))
8354 return N1;
8355 break;
8356 case ISD::VECTOR_SPLICE_RIGHT:
8357 if (isNullConstant(V: N3))
8358 return N2;
8359 break;
8360 case ISD::INSERT_VECTOR_ELT: {
8361 assert(VT.isVector() && VT == N1.getValueType() &&
8362 "INSERT_VECTOR_ELT vector type mismatch");
8363 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
8364 "INSERT_VECTOR_ELT scalar fp/int mismatch");
8365 assert((!VT.isFloatingPoint() ||
8366 VT.getVectorElementType() == N2.getValueType()) &&
8367 "INSERT_VECTOR_ELT fp scalar type mismatch");
8368 assert((!VT.isInteger() ||
8369 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
8370 "INSERT_VECTOR_ELT int scalar size mismatch");
8371
8372 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
8373 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
8374 // for scalable vectors where we will generate appropriate code to
8375 // deal with out-of-bounds cases correctly.
8376 if (N3C && VT.isFixedLengthVector() &&
8377 N3C->getZExtValue() >= VT.getVectorNumElements())
8378 return getUNDEF(VT);
8379
8380 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
8381 if (N3.isUndef())
8382 return getUNDEF(VT);
8383
8384 // If inserting poison, just use the input vector.
8385 if (N2.getOpcode() == ISD::POISON)
8386 return N1;
8387
8388 // Inserting undef into undef/poison is still undef.
8389 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8390 return getUNDEF(VT);
8391
8392 // If the inserted element is an UNDEF, just use the input vector.
8393 // But not if skipping the insert could make the result more poisonous.
8394 if (N2.isUndef()) {
8395 if (N3C && VT.isFixedLengthVector()) {
8396 APInt EltMask =
8397 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
8398 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8399 return N1;
8400 } else if (isGuaranteedNotToBePoison(Op: N1))
8401 return N1;
8402 }
8403 break;
8404 }
8405 case ISD::INSERT_SUBVECTOR: {
8406 // If inserting poison, just use the input vector,
8407 if (N2.getOpcode() == ISD::POISON)
8408 return N1;
8409
8410 // Inserting undef into undef/poison is still undef.
8411 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8412 return getUNDEF(VT);
8413
8414 EVT N2VT = N2.getValueType();
8415 assert(VT == N1.getValueType() &&
8416 "Dest and insert subvector source types must match!");
8417 assert(VT.isVector() && N2VT.isVector() &&
8418 "Insert subvector VTs must be vectors!");
8419 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
8420 "Insert subvector VTs must have the same element type!");
8421 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
8422 "Cannot insert a scalable vector into a fixed length vector!");
8423 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8424 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
8425 "Insert subvector must be from smaller vector to larger vector!");
8426 assert(isa<ConstantSDNode>(N3) &&
8427 "Insert subvector index must be constant");
8428 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8429 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
8430 VT.getVectorMinNumElements()) &&
8431 "Insert subvector overflow!");
8432 assert(N3->getAsAPIntVal().getBitWidth() ==
8433 TLI->getVectorIdxWidth(getDataLayout()) &&
8434 "Constant index for INSERT_SUBVECTOR has an invalid size");
8435
8436 // Trivial insertion.
8437 if (VT == N2VT)
8438 return N2;
8439
8440 // If this is an insert of an extracted vector into an undef/poison vector,
8441 // we can just use the input to the extract. But not if skipping the
8442 // extract+insert could make the result more poisonous.
8443 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8444 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
8445 if (N1.getOpcode() == ISD::POISON)
8446 return N2.getOperand(i: 0);
8447 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
8448 unsigned LoBit = N3->getAsZExtVal();
8449 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8450 APInt EltMask =
8451 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8452 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
8453 return N2.getOperand(i: 0);
8454 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
8455 return N2.getOperand(i: 0);
8456 }
8457
8458 // If the inserted subvector is UNDEF, just use the input vector.
8459 // But not if skipping the insert could make the result more poisonous.
8460 if (N2.isUndef()) {
8461 if (VT.isFixedLengthVector()) {
8462 unsigned LoBit = N3->getAsZExtVal();
8463 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8464 APInt EltMask =
8465 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8466 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8467 return N1;
8468 } else if (isGuaranteedNotToBePoison(Op: N1))
8469 return N1;
8470 }
8471 break;
8472 }
8473 case ISD::BITCAST:
8474 // Fold bit_convert nodes from a type to themselves.
8475 if (N1.getValueType() == VT)
8476 return N1;
8477 break;
8478 case ISD::VP_TRUNCATE:
8479 case ISD::VP_SIGN_EXTEND:
8480 case ISD::VP_ZERO_EXTEND:
8481 // Don't create noop casts.
8482 if (N1.getValueType() == VT)
8483 return N1;
8484 break;
8485 case ISD::VECTOR_COMPRESS: {
8486 [[maybe_unused]] EVT VecVT = N1.getValueType();
8487 [[maybe_unused]] EVT MaskVT = N2.getValueType();
8488 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
8489 assert(VT == VecVT && "Vector and result type don't match.");
8490 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
8491 "All inputs must be vectors.");
8492 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
8493 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
8494 "Vector and mask must have same number of elements.");
8495
8496 if (N1.isUndef() || N2.isUndef())
8497 return N3;
8498
8499 break;
8500 }
8501 case ISD::PARTIAL_REDUCE_UMLA:
8502 case ISD::PARTIAL_REDUCE_SMLA:
8503 case ISD::PARTIAL_REDUCE_SUMLA:
8504 case ISD::PARTIAL_REDUCE_FMLA: {
8505 [[maybe_unused]] EVT AccVT = N1.getValueType();
8506 [[maybe_unused]] EVT Input1VT = N2.getValueType();
8507 [[maybe_unused]] EVT Input2VT = N3.getValueType();
8508 assert(Input1VT.isVector() && Input1VT == Input2VT &&
8509 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8510 "node to have the same type!");
8511 assert(VT.isVector() && VT == AccVT &&
8512 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
8513 "the same type as its result!");
8514 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
8515 AccVT.getVectorElementCount()) &&
8516 "Expected the element count of the second and third operands of the "
8517 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
8518 "element count of the first operand and the result!");
8519 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
8520 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8521 "node to have an element type which is the same as or smaller than "
8522 "the element type of the first operand and result!");
8523 break;
8524 }
8525 }
8526
8527 // Perform trivial constant folding for arithmetic operators.
8528 switch (Opcode) {
8529 case ISD::FMA:
8530 case ISD::FMAD:
8531 case ISD::SETCC:
8532 case ISD::FSHL:
8533 case ISD::FSHR:
8534 if (SDValue SV =
8535 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
8536 return SV;
8537 break;
8538 }
8539
8540 // Memoize node if it doesn't produce a glue result.
8541 SDNode *N;
8542 SDVTList VTs = getVTList(VT);
8543 SDValue Ops[] = {N1, N2, N3};
8544 if (VT != MVT::Glue) {
8545 FoldingSetNodeID ID;
8546 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8547 void *IP = nullptr;
8548 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8549 E->intersectFlagsWith(Flags);
8550 return SDValue(E, 0);
8551 }
8552
8553 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8554 N->setFlags(Flags);
8555 createOperands(Node: N, Vals: Ops);
8556 CSEMap.InsertNode(N, InsertPos: IP);
8557 } else {
8558 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8559 createOperands(Node: N, Vals: Ops);
8560 }
8561
8562 InsertNode(N);
8563 SDValue V = SDValue(N, 0);
8564 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8565 return V;
8566}
8567
8568SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8569 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8570 const SDNodeFlags Flags) {
8571 SDValue Ops[] = { N1, N2, N3, N4 };
8572 return getNode(Opcode, DL, VT, Ops, Flags);
8573}
8574
8575SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8576 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
8577 SDNodeFlags Flags;
8578 if (Inserter)
8579 Flags = Inserter->getFlags();
8580 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
8581}
8582
8583SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8584 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8585 SDValue N5, const SDNodeFlags Flags) {
8586 SDValue Ops[] = { N1, N2, N3, N4, N5 };
8587 return getNode(Opcode, DL, VT, Ops, Flags);
8588}
8589
8590SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8591 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8592 SDValue N5) {
8593 SDNodeFlags Flags;
8594 if (Inserter)
8595 Flags = Inserter->getFlags();
8596 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
8597}
8598
8599/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
8600/// the incoming stack arguments to be loaded from the stack.
8601SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
8602 SmallVector<SDValue, 8> ArgChains;
8603
8604 // Include the original chain at the beginning of the list. When this is
8605 // used by target LowerCall hooks, this helps legalize find the
8606 // CALLSEQ_BEGIN node.
8607 ArgChains.push_back(Elt: Chain);
8608
8609 // Add a chain value for each stack argument.
8610 for (SDNode *U : getEntryNode().getNode()->users())
8611 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
8612 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
8613 if (FI->getIndex() < 0)
8614 ArgChains.push_back(Elt: SDValue(L, 1));
8615
8616 // Build a tokenfactor for all the chains.
8617 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
8618}
8619
8620/// getMemsetValue - Vectorized representation of the memset value
8621/// operand.
8622static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
8623 const SDLoc &dl) {
8624 assert(!Value.isUndef());
8625
8626 unsigned NumBits = VT.getScalarSizeInBits();
8627 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
8628 assert(C->getAPIntValue().getBitWidth() == 8);
8629 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
8630 if (VT.isInteger()) {
8631 bool IsOpaque = VT.getSizeInBits() > 64 ||
8632 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
8633 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
8634 }
8635 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
8636 }
8637
8638 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
8639 EVT IntVT = VT.getScalarType();
8640 if (!IntVT.isInteger())
8641 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
8642
8643 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
8644 if (NumBits > 8) {
8645 // Use a multiplication with 0x010101... to extend the input to the
8646 // required length.
8647 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
8648 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
8649 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
8650 }
8651
8652 if (VT != Value.getValueType() && !VT.isInteger())
8653 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
8654 if (VT != Value.getValueType())
8655 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
8656
8657 return Value;
8658}
8659
8660/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
8661/// used when a memcpy is turned into a memset when the source is a constant
8662/// string ptr.
8663static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
8664 const TargetLowering &TLI,
8665 const ConstantDataArraySlice &Slice) {
8666 // Handle vector with all elements zero.
8667 if (Slice.Array == nullptr) {
8668 if (VT.isInteger())
8669 return DAG.getConstant(Val: 0, DL: dl, VT);
8670 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
8671 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
8672 }
8673
8674 assert(!VT.isVector() && "Can't handle vector type here!");
8675 unsigned NumVTBits = VT.getSizeInBits();
8676 unsigned NumVTBytes = NumVTBits / 8;
8677 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
8678
8679 APInt Val(NumVTBits, 0);
8680 if (DAG.getDataLayout().isLittleEndian()) {
8681 for (unsigned i = 0; i != NumBytes; ++i)
8682 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
8683 } else {
8684 for (unsigned i = 0; i != NumBytes; ++i)
8685 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
8686 }
8687
8688 // If the "cost" of materializing the integer immediate is less than the cost
8689 // of a load, then it is cost effective to turn the load into the immediate.
8690 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
8691 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
8692 return DAG.getConstant(Val, DL: dl, VT);
8693 return SDValue();
8694}
8695
8696SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
8697 const SDLoc &DL,
8698 const SDNodeFlags Flags) {
8699 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
8700 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
8701}
8702
8703SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
8704 const SDLoc &DL,
8705 const SDNodeFlags Flags) {
8706 assert(Offset.getValueType().isInteger());
8707 EVT BasePtrVT = Ptr.getValueType();
8708 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
8709 PtrVT: BasePtrVT))
8710 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
8711 // InBounds only applies to PTRADD, don't set it if we generate ADD.
8712 SDNodeFlags AddFlags = Flags;
8713 AddFlags.setInBounds(false);
8714 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
8715}
8716
8717/// Returns true if memcpy source is constant data.
8718static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
8719 uint64_t SrcDelta = 0;
8720 GlobalAddressSDNode *G = nullptr;
8721 if (Src.getOpcode() == ISD::GlobalAddress)
8722 G = cast<GlobalAddressSDNode>(Val&: Src);
8723 else if (Src->isAnyAdd() &&
8724 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
8725 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
8726 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
8727 SrcDelta = Src.getConstantOperandVal(i: 1);
8728 }
8729 if (!G)
8730 return false;
8731
8732 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
8733 Offset: SrcDelta + G->getOffset());
8734}
8735
8736static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
8737 SelectionDAG &DAG) {
8738 // On Darwin, -Os means optimize for size without hurting performance, so
8739 // only really optimize for size when -Oz (MinSize) is used.
8740 if (MF.getTarget().getTargetTriple().isOSDarwin())
8741 return MF.getFunction().hasMinSize();
8742 return DAG.shouldOptForSize();
8743}
8744
8745static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
8746 SmallVector<SDValue, 32> &OutChains, unsigned From,
8747 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
8748 SmallVector<SDValue, 16> &OutStoreChains) {
8749 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
8750 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
8751 SmallVector<SDValue, 16> GluedLoadChains;
8752 for (unsigned i = From; i < To; ++i) {
8753 OutChains.push_back(Elt: OutLoadChains[i]);
8754 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
8755 }
8756
8757 // Chain for all loads.
8758 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
8759 Ops: GluedLoadChains);
8760
8761 for (unsigned i = From; i < To; ++i) {
8762 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
8763 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
8764 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
8765 MMO: ST->getMemOperand());
8766 OutChains.push_back(Elt: NewStore);
8767 }
8768}
8769
8770static SDValue getMemcpyLoadsAndStores(
8771 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
8772 uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline,
8773 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
8774 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
8775 // Turn a memcpy of undef to nop.
8776 // FIXME: We need to honor volatile even is Src is undef.
8777 if (Src.isUndef())
8778 return Chain;
8779
8780 // Expand memcpy to a series of load and store ops if the size operand falls
8781 // below a certain threshold.
8782 // TODO: In the AlwaysInline case, if the size is big then generate a loop
8783 // rather than maybe a humongous number of loads and stores.
8784 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8785 const DataLayout &DL = DAG.getDataLayout();
8786 LLVMContext &C = *DAG.getContext();
8787 std::vector<EVT> MemOps;
8788 bool DstAlignCanChange = false;
8789 MachineFunction &MF = DAG.getMachineFunction();
8790 MachineFrameInfo &MFI = MF.getFrameInfo();
8791 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8792 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
8793 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
8794 DstAlignCanChange = true;
8795 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
8796 if (!SrcAlign || Alignment > *SrcAlign)
8797 SrcAlign = Alignment;
8798 assert(SrcAlign && "SrcAlign must be set");
8799 ConstantDataArraySlice Slice;
8800 // If marked as volatile, perform a copy even when marked as constant.
8801 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
8802 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
8803 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
8804 const MemOp Op = isZeroConstant
8805 ? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment,
8806 /*IsZeroMemset*/ true, IsVolatile: isVol)
8807 : MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment,
8808 SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
8809 if (!TLI.findOptimalMemOpLowering(
8810 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
8811 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
8812 return SDValue();
8813
8814 if (DstAlignCanChange) {
8815 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
8816 Align NewAlign = DL.getABITypeAlign(Ty);
8817
8818 // Don't promote to an alignment that would require dynamic stack
8819 // realignment which may conflict with optimizations such as tail call
8820 // optimization.
8821 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
8822 if (!TRI->hasStackRealignment(MF))
8823 if (MaybeAlign StackAlign = DL.getStackAlignment())
8824 NewAlign = std::min(a: NewAlign, b: *StackAlign);
8825
8826 if (NewAlign > Alignment) {
8827 // Give the stack frame object a larger alignment if needed.
8828 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
8829 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
8830 Alignment = NewAlign;
8831 }
8832 }
8833
8834 // Prepare AAInfo for loads/stores after lowering this memcpy.
8835 AAMDNodes NewAAInfo = AAInfo;
8836 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8837
8838 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
8839 bool isConstant =
8840 BatchAA && SrcVal &&
8841 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
8842
8843 MachineMemOperand::Flags MMOFlags =
8844 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
8845 SmallVector<SDValue, 16> OutLoadChains;
8846 SmallVector<SDValue, 16> OutStoreChains;
8847 SmallVector<SDValue, 32> OutChains;
8848 unsigned NumMemOps = MemOps.size();
8849 uint64_t SrcOff = 0, DstOff = 0;
8850 for (unsigned i = 0; i != NumMemOps; ++i) {
8851 EVT VT = MemOps[i];
8852 unsigned VTSize = VT.getSizeInBits() / 8;
8853 SDValue Value, Store;
8854
8855 if (VTSize > Size) {
8856 // Issuing an unaligned load / store pair that overlaps with the previous
8857 // pair. Adjust the offset accordingly.
8858 assert(i == NumMemOps-1 && i != 0);
8859 SrcOff -= VTSize - Size;
8860 DstOff -= VTSize - Size;
8861 }
8862
8863 if (CopyFromConstant &&
8864 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
8865 // It's unlikely a store of a vector immediate can be done in a single
8866 // instruction. It would require a load from a constantpool first.
8867 // We only handle zero vectors here.
8868 // FIXME: Handle other cases where store of vector immediate is done in
8869 // a single instruction.
8870 ConstantDataArraySlice SubSlice;
8871 if (SrcOff < Slice.Length) {
8872 SubSlice = Slice;
8873 SubSlice.move(Delta: SrcOff);
8874 } else {
8875 // This is an out-of-bounds access and hence UB. Pretend we read zero.
8876 SubSlice.Array = nullptr;
8877 SubSlice.Offset = 0;
8878 SubSlice.Length = VTSize;
8879 }
8880 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
8881 if (Value.getNode()) {
8882 Store = DAG.getStore(
8883 Chain, dl, Val: Value,
8884 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
8885 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
8886 OutChains.push_back(Elt: Store);
8887 }
8888 }
8889
8890 if (!Store.getNode()) {
8891 // The type might not be legal for the target. This should only happen
8892 // if the type is smaller than a legal type, as on PPC, so the right
8893 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
8894 // to Load/Store if NVT==VT.
8895 // FIXME does the case above also need this?
8896 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
8897 assert(NVT.bitsGE(VT));
8898
8899 bool isDereferenceable =
8900 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
8901 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
8902 if (isDereferenceable)
8903 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
8904 if (isConstant)
8905 SrcMMOFlags |= MachineMemOperand::MOInvariant;
8906
8907 Value = DAG.getExtLoad(
8908 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
8909 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
8910 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
8911 Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
8912 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
8913
8914 Store = DAG.getTruncStore(
8915 Chain, dl, Val: Value,
8916 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
8917 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo);
8918 OutStoreChains.push_back(Elt: Store);
8919 }
8920 SrcOff += VTSize;
8921 DstOff += VTSize;
8922 Size -= VTSize;
8923 }
8924
8925 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
8926 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
8927 unsigned NumLdStInMemcpy = OutStoreChains.size();
8928
8929 if (NumLdStInMemcpy) {
8930 // It may be that memcpy might be converted to memset if it's memcpy
8931 // of constants. In such a case, we won't have loads and stores, but
8932 // just stores. In the absence of loads, there is nothing to gang up.
8933 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
8934 // If target does not care, just leave as it.
8935 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
8936 OutChains.push_back(Elt: OutLoadChains[i]);
8937 OutChains.push_back(Elt: OutStoreChains[i]);
8938 }
8939 } else {
8940 // Ld/St less than/equal limit set by target.
8941 if (NumLdStInMemcpy <= GluedLdStLimit) {
8942 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
8943 To: NumLdStInMemcpy, OutLoadChains,
8944 OutStoreChains);
8945 } else {
8946 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
8947 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
8948 unsigned GlueIter = 0;
8949
8950 // Residual ld/st.
8951 if (RemainingLdStInMemcpy) {
8952 chainLoadsAndStoresForMemcpy(
8953 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
8954 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
8955 }
8956
8957 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
8958 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
8959 GlueIter - GluedLdStLimit;
8960 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
8961 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
8962 OutLoadChains, OutStoreChains);
8963 GlueIter += GluedLdStLimit;
8964 }
8965 }
8966 }
8967 }
8968 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
8969}
8970
8971static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
8972 SDValue Chain, SDValue Dst, SDValue Src,
8973 uint64_t Size, Align Alignment,
8974 bool isVol, bool AlwaysInline,
8975 MachinePointerInfo DstPtrInfo,
8976 MachinePointerInfo SrcPtrInfo,
8977 const AAMDNodes &AAInfo) {
8978 // Turn a memmove of undef to nop.
8979 // FIXME: We need to honor volatile even is Src is undef.
8980 if (Src.isUndef())
8981 return Chain;
8982
8983 // Expand memmove to a series of load and store ops if the size operand falls
8984 // below a certain threshold.
8985 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8986 const DataLayout &DL = DAG.getDataLayout();
8987 LLVMContext &C = *DAG.getContext();
8988 std::vector<EVT> MemOps;
8989 bool DstAlignCanChange = false;
8990 MachineFunction &MF = DAG.getMachineFunction();
8991 MachineFrameInfo &MFI = MF.getFrameInfo();
8992 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8993 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
8994 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
8995 DstAlignCanChange = true;
8996 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
8997 if (!SrcAlign || Alignment > *SrcAlign)
8998 SrcAlign = Alignment;
8999 assert(SrcAlign && "SrcAlign must be set");
9000 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9001 if (!TLI.findOptimalMemOpLowering(
9002 Context&: C, MemOps, Limit,
9003 Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign,
9004 /*IsVolatile*/ true),
9005 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9006 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9007 return SDValue();
9008
9009 if (DstAlignCanChange) {
9010 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9011 Align NewAlign = DL.getABITypeAlign(Ty);
9012
9013 // Don't promote to an alignment that would require dynamic stack
9014 // realignment which may conflict with optimizations such as tail call
9015 // optimization.
9016 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9017 if (!TRI->hasStackRealignment(MF))
9018 if (MaybeAlign StackAlign = DL.getStackAlignment())
9019 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9020
9021 if (NewAlign > Alignment) {
9022 // Give the stack frame object a larger alignment if needed.
9023 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9024 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9025 Alignment = NewAlign;
9026 }
9027 }
9028
9029 // Prepare AAInfo for loads/stores after lowering this memmove.
9030 AAMDNodes NewAAInfo = AAInfo;
9031 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9032
9033 MachineMemOperand::Flags MMOFlags =
9034 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9035 uint64_t SrcOff = 0, DstOff = 0;
9036 SmallVector<SDValue, 8> LoadValues;
9037 SmallVector<SDValue, 8> LoadChains;
9038 SmallVector<SDValue, 8> OutChains;
9039 unsigned NumMemOps = MemOps.size();
9040 for (unsigned i = 0; i < NumMemOps; i++) {
9041 EVT VT = MemOps[i];
9042 unsigned VTSize = VT.getSizeInBits() / 8;
9043 SDValue Value;
9044
9045 bool isDereferenceable =
9046 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9047 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9048 if (isDereferenceable)
9049 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9050
9051 Value = DAG.getLoad(
9052 VT, dl, Chain,
9053 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9054 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: *SrcAlign, MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9055 LoadValues.push_back(Elt: Value);
9056 LoadChains.push_back(Elt: Value.getValue(R: 1));
9057 SrcOff += VTSize;
9058 }
9059 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
9060 OutChains.clear();
9061 for (unsigned i = 0; i < NumMemOps; i++) {
9062 EVT VT = MemOps[i];
9063 unsigned VTSize = VT.getSizeInBits() / 8;
9064 SDValue Store;
9065
9066 Store = DAG.getStore(
9067 Chain, dl, Val: LoadValues[i],
9068 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9069 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
9070 OutChains.push_back(Elt: Store);
9071 DstOff += VTSize;
9072 }
9073
9074 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9075}
9076
9077/// Lower the call to 'memset' intrinsic function into a series of store
9078/// operations.
9079///
9080/// \param DAG Selection DAG where lowered code is placed.
9081/// \param dl Link to corresponding IR location.
9082/// \param Chain Control flow dependency.
9083/// \param Dst Pointer to destination memory location.
9084/// \param Src Value of byte to write into the memory.
9085/// \param Size Number of bytes to write.
9086/// \param Alignment Alignment of the destination in bytes.
9087/// \param isVol True if destination is volatile.
9088/// \param AlwaysInline Makes sure no function call is generated.
9089/// \param DstPtrInfo IR information on the memory pointer.
9090/// \returns New head in the control flow, if lowering was successful, empty
9091/// SDValue otherwise.
9092///
9093/// The function tries to replace 'llvm.memset' intrinsic with several store
9094/// operations and value calculation code. This is usually profitable for small
9095/// memory size or when the semantic requires inlining.
9096static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9097 SDValue Chain, SDValue Dst, SDValue Src,
9098 uint64_t Size, Align Alignment, bool isVol,
9099 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9100 const AAMDNodes &AAInfo) {
9101 // Turn a memset of undef to nop.
9102 // FIXME: We need to honor volatile even is Src is undef.
9103 if (Src.isUndef())
9104 return Chain;
9105
9106 // Expand memset to a series of load/store ops if the size operand
9107 // falls below a certain threshold.
9108 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9109 std::vector<EVT> MemOps;
9110 bool DstAlignCanChange = false;
9111 LLVMContext &C = *DAG.getContext();
9112 MachineFunction &MF = DAG.getMachineFunction();
9113 MachineFrameInfo &MFI = MF.getFrameInfo();
9114 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9115 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9116 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9117 DstAlignCanChange = true;
9118 bool IsZeroVal = isNullConstant(V: Src);
9119 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9120
9121 EVT LargestVT;
9122 if (!TLI.findOptimalMemOpLowering(
9123 Context&: C, MemOps, Limit,
9124 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9125 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9126 LargestVT: &LargestVT))
9127 return SDValue();
9128
9129 if (DstAlignCanChange) {
9130 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9131 const DataLayout &DL = DAG.getDataLayout();
9132 Align NewAlign = DL.getABITypeAlign(Ty);
9133
9134 // Don't promote to an alignment that would require dynamic stack
9135 // realignment which may conflict with optimizations such as tail call
9136 // optimization.
9137 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9138 if (!TRI->hasStackRealignment(MF))
9139 if (MaybeAlign StackAlign = DL.getStackAlignment())
9140 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9141
9142 if (NewAlign > Alignment) {
9143 // Give the stack frame object a larger alignment if needed.
9144 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9145 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9146 Alignment = NewAlign;
9147 }
9148 }
9149
9150 SmallVector<SDValue, 8> OutChains;
9151 uint64_t DstOff = 0;
9152 unsigned NumMemOps = MemOps.size();
9153
9154 // Find the largest store and generate the bit pattern for it.
9155 // If target didn't set LargestVT, compute it from MemOps.
9156 if (!LargestVT.isSimple()) {
9157 LargestVT = MemOps[0];
9158 for (unsigned i = 1; i < NumMemOps; i++)
9159 if (MemOps[i].bitsGT(VT: LargestVT))
9160 LargestVT = MemOps[i];
9161 }
9162 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9163
9164 // Prepare AAInfo for loads/stores after lowering this memset.
9165 AAMDNodes NewAAInfo = AAInfo;
9166 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9167
9168 for (unsigned i = 0; i < NumMemOps; i++) {
9169 EVT VT = MemOps[i];
9170 unsigned VTSize = VT.getSizeInBits() / 8;
9171 // The target should specify store types that exactly cover the memset size
9172 // (with the last store potentially being oversized for overlapping stores).
9173 assert(Size > 0 && "Target specified more stores than needed in "
9174 "findOptimalMemOpLowering");
9175 if (VTSize > Size) {
9176 // Issuing an unaligned load / store pair that overlaps with the previous
9177 // pair. Adjust the offset accordingly.
9178 assert(i == NumMemOps-1 && i != 0);
9179 DstOff -= VTSize - Size;
9180 }
9181
9182 // If this store is smaller than the largest store see whether we can get
9183 // the smaller value for free with a truncate or extract vector element and
9184 // then store.
9185 SDValue Value = MemSetValue;
9186 if (VT.bitsLT(VT: LargestVT)) {
9187 unsigned Index;
9188 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9189 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
9190 if (!LargestVT.isVector() && !VT.isVector() &&
9191 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
9192 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
9193 else if (LargestVT.isVector() && !VT.isVector() &&
9194 TLI.shallExtractConstSplatVectorElementToStore(
9195 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
9196 ElemSizeInBits: VT.getSizeInBits(), Index) &&
9197 TLI.isTypeLegal(VT: SVT) &&
9198 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
9199 // Target which can combine store(extractelement VectorTy, Idx) can get
9200 // the smaller value for free.
9201 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
9202 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
9203 } else
9204 Value = getMemsetValue(Value: Src, VT, DAG, dl);
9205 }
9206 assert(Value.getValueType() == VT && "Value with wrong type.");
9207 SDValue Store = DAG.getStore(
9208 Chain, dl, Val: Value,
9209 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9210 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
9211 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
9212 AAInfo: NewAAInfo);
9213 OutChains.push_back(Elt: Store);
9214 DstOff += VT.getSizeInBits() / 8;
9215 // For oversized overlapping stores, only subtract the remaining bytes.
9216 // For normal stores, subtract the full store size.
9217 if (VTSize > Size) {
9218 Size = 0;
9219 } else {
9220 Size -= VTSize;
9221 }
9222 }
9223
9224 // After processing all stores, Size should be exactly 0. Any remaining bytes
9225 // indicate a bug in the target's findOptimalMemOpLowering implementation.
9226 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
9227 "stores that exactly cover the memset size");
9228
9229 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9230}
9231
9232static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
9233 unsigned AS) {
9234 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
9235 // pointer operands can be losslessly bitcasted to pointers of address space 0
9236 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
9237 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
9238 Twine(AS));
9239 }
9240}
9241
9242static bool isInTailCallPositionWrapper(const CallInst *CI,
9243 const SelectionDAG *SelDAG,
9244 bool AllowReturnsFirstArg) {
9245 if (!CI || !CI->isTailCall())
9246 return false;
9247 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
9248 // helper symbol we lower to.
9249 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
9250 ReturnsFirstArg: AllowReturnsFirstArg &&
9251 funcReturnsFirstArgOfCall(CI: *CI));
9252}
9253
9254static std::pair<SDValue, SDValue>
9255getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
9256 TargetLowering::ArgListTy &&Args,
9257 const CallInst *CI, RTLIB::Libcall Call,
9258 SelectionDAG *DAG, const TargetLowering *TLI) {
9259 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
9260
9261 if (LCImpl == RTLIB::Unsupported)
9262 return {};
9263
9264 TargetLowering::CallLoweringInfo CLI(*DAG);
9265 bool IsTailCall =
9266 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true);
9267 SDValue Callee =
9268 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
9269
9270 CLI.setDebugLoc(dl)
9271 .setChain(Chain)
9272 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9273 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
9274 .setTailCall(IsTailCall);
9275
9276 return TLI->LowerCallTo(CLI);
9277}
9278
9279std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
9280 const SDLoc &dl, SDValue S1,
9281 SDValue S2,
9282 const CallInst *CI) {
9283 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9284 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9285 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9286 Call: RTLIB::STRCMP, DAG: this, TLI);
9287}
9288
9289std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
9290 const SDLoc &dl, SDValue S1,
9291 SDValue S2,
9292 const CallInst *CI) {
9293 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9294 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9295 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9296 Call: RTLIB::STRSTR, DAG: this, TLI);
9297}
9298
9299std::pair<SDValue, SDValue>
9300SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
9301 SDValue Mem1, SDValue Size, const CallInst *CI) {
9302 RTLIB::LibcallImpl MemcmpImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMCMP);
9303 if (MemcmpImpl == RTLIB::Unsupported)
9304 return {};
9305
9306 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9307 TargetLowering::ArgListTy Args = {
9308 {Mem0, PT},
9309 {Mem1, PT},
9310 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9311
9312 TargetLowering::CallLoweringInfo CLI(*this);
9313 bool IsTailCall =
9314 isInTailCallPositionWrapper(CI, SelDAG: this, /*AllowReturnsFirstArg*/ true);
9315
9316 CLI.setDebugLoc(dl)
9317 .setChain(Chain)
9318 .setLibCallee(
9319 CC: Libcalls->getLibcallImplCallingConv(Call: MemcmpImpl),
9320 ResultType: Type::getInt32Ty(C&: *getContext()),
9321 Target: getExternalSymbol(Libcall: MemcmpImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9322 ArgsList: std::move(Args))
9323 .setTailCall(IsTailCall);
9324
9325 return TLI->LowerCallTo(CLI);
9326}
9327
9328std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
9329 const SDLoc &dl,
9330 SDValue Dst, SDValue Src,
9331 const CallInst *CI) {
9332 RTLIB::LibcallImpl LCImpl = Libcalls->getLibcallImpl(Call: RTLIB::STRCPY);
9333 if (LCImpl == RTLIB::Unsupported)
9334 return {};
9335
9336 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9337 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
9338
9339 TargetLowering::CallLoweringInfo CLI(*this);
9340 bool IsTailCall =
9341 isInTailCallPositionWrapper(CI, SelDAG: this, /*AllowReturnsFirstArg=*/true);
9342
9343 CLI.setDebugLoc(dl)
9344 .setChain(Chain)
9345 .setLibCallee(
9346 CC: Libcalls->getLibcallImplCallingConv(Call: LCImpl), ResultType: CI->getType(),
9347 Target: getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9348 ArgsList: std::move(Args))
9349 .setTailCall(IsTailCall);
9350
9351 return TLI->LowerCallTo(CLI);
9352}
9353
9354std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
9355 const SDLoc &dl,
9356 SDValue Src,
9357 const CallInst *CI) {
9358 RTLIB::LibcallImpl StrlenImpl = Libcalls->getLibcallImpl(Call: RTLIB::STRLEN);
9359 if (StrlenImpl == RTLIB::Unsupported)
9360 return {};
9361
9362 // Emit a library call.
9363 TargetLowering::ArgListTy Args = {
9364 {Src, PointerType::getUnqual(C&: *getContext())}};
9365
9366 TargetLowering::CallLoweringInfo CLI(*this);
9367 bool IsTailCall =
9368 isInTailCallPositionWrapper(CI, SelDAG: this, /*AllowReturnsFirstArg*/ true);
9369
9370 CLI.setDebugLoc(dl)
9371 .setChain(Chain)
9372 .setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: StrlenImpl),
9373 ResultType: CI->getType(),
9374 Target: getExternalSymbol(
9375 Libcall: StrlenImpl, VT: TLI->getProgramPointerTy(DL: getDataLayout())),
9376 ArgsList: std::move(Args))
9377 .setTailCall(IsTailCall);
9378
9379 return TLI->LowerCallTo(CLI);
9380}
9381
9382SDValue SelectionDAG::getMemcpy(
9383 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
9384 Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
9385 std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
9386 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9387 BatchAAResults *BatchAA) {
9388 // Check to see if we should lower the memcpy to loads and stores first.
9389 // For cases within the target-specified limits, this is the best choice.
9390 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9391 if (ConstantSize) {
9392 // Memcpy with size zero? Just return the original chain.
9393 if (ConstantSize->isZero())
9394 return Chain;
9395
9396 SDValue Result = getMemcpyLoadsAndStores(
9397 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9398 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9399 if (Result.getNode())
9400 return Result;
9401 }
9402
9403 // Then check to see if we should lower the memcpy with target-specific
9404 // code. If the target chooses to do this, this is the next best.
9405 if (TSI) {
9406 SDValue Result = TSI->EmitTargetCodeForMemcpy(
9407 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
9408 DstPtrInfo, SrcPtrInfo);
9409 if (Result.getNode())
9410 return Result;
9411 }
9412
9413 // If we really need inline code and the target declined to provide it,
9414 // use a (potentially long) sequence of loads and stores.
9415 if (AlwaysInline) {
9416 assert(ConstantSize && "AlwaysInline requires a constant size!");
9417 return getMemcpyLoadsAndStores(
9418 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9419 isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9420 }
9421
9422 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9423 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9424
9425 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
9426 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
9427 // respect volatile, so they may do things like read or write memory
9428 // beyond the given memory regions. But fixing this isn't easy, and most
9429 // people don't care.
9430
9431 // Emit a library call.
9432 TargetLowering::ArgListTy Args;
9433 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9434 Args.emplace_back(args&: Dst, args&: PtrTy);
9435 Args.emplace_back(args&: Src, args&: PtrTy);
9436 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9437 // FIXME: pass in SDLoc
9438 TargetLowering::CallLoweringInfo CLI(*this);
9439 bool IsTailCall = false;
9440 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
9441
9442 if (OverrideTailCall.has_value()) {
9443 IsTailCall = *OverrideTailCall;
9444 } else {
9445 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
9446 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
9447 }
9448
9449 CLI.setDebugLoc(dl)
9450 .setChain(Chain)
9451 .setLibCallee(
9452 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
9453 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9454 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9455 ArgsList: std::move(Args))
9456 .setDiscardResult()
9457 .setTailCall(IsTailCall);
9458
9459 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9460 return CallResult.second;
9461}
9462
9463SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
9464 SDValue Dst, SDValue Src, SDValue Size,
9465 Type *SizeTy, unsigned ElemSz,
9466 bool isTailCall,
9467 MachinePointerInfo DstPtrInfo,
9468 MachinePointerInfo SrcPtrInfo) {
9469 // Emit a library call.
9470 TargetLowering::ArgListTy Args;
9471 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
9472 Args.emplace_back(args&: Dst, args&: ArgTy);
9473 Args.emplace_back(args&: Src, args&: ArgTy);
9474 Args.emplace_back(args&: Size, args&: SizeTy);
9475
9476 RTLIB::Libcall LibraryCall =
9477 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9478 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9479 if (LibcallImpl == RTLIB::Unsupported)
9480 report_fatal_error(reason: "Unsupported element size");
9481
9482 TargetLowering::CallLoweringInfo CLI(*this);
9483 CLI.setDebugLoc(dl)
9484 .setChain(Chain)
9485 .setLibCallee(
9486 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9487 ResultType: Type::getVoidTy(C&: *getContext()),
9488 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9489 ArgsList: std::move(Args))
9490 .setDiscardResult()
9491 .setTailCall(isTailCall);
9492
9493 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9494 return CallResult.second;
9495}
9496
9497SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
9498 SDValue Src, SDValue Size, Align Alignment,
9499 bool isVol, const CallInst *CI,
9500 std::optional<bool> OverrideTailCall,
9501 MachinePointerInfo DstPtrInfo,
9502 MachinePointerInfo SrcPtrInfo,
9503 const AAMDNodes &AAInfo,
9504 BatchAAResults *BatchAA) {
9505 // Check to see if we should lower the memmove to loads and stores first.
9506 // For cases within the target-specified limits, this is the best choice.
9507 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9508 if (ConstantSize) {
9509 // Memmove with size zero? Just return the original chain.
9510 if (ConstantSize->isZero())
9511 return Chain;
9512
9513 SDValue Result = getMemmoveLoadsAndStores(
9514 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9515 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
9516 if (Result.getNode())
9517 return Result;
9518 }
9519
9520 // Then check to see if we should lower the memmove with target-specific
9521 // code. If the target chooses to do this, this is the next best.
9522 if (TSI) {
9523 SDValue Result =
9524 TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size,
9525 Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo);
9526 if (Result.getNode())
9527 return Result;
9528 }
9529
9530 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9531 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9532
9533 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
9534 // not be safe. See memcpy above for more details.
9535
9536 // Emit a library call.
9537 TargetLowering::ArgListTy Args;
9538 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9539 Args.emplace_back(args&: Dst, args&: PtrTy);
9540 Args.emplace_back(args&: Src, args&: PtrTy);
9541 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9542 // FIXME: pass in SDLoc
9543 TargetLowering::CallLoweringInfo CLI(*this);
9544
9545 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
9546
9547 bool IsTailCall = false;
9548 if (OverrideTailCall.has_value()) {
9549 IsTailCall = *OverrideTailCall;
9550 } else {
9551 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
9552 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
9553 }
9554
9555 CLI.setDebugLoc(dl)
9556 .setChain(Chain)
9557 .setLibCallee(
9558 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
9559 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9560 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9561 ArgsList: std::move(Args))
9562 .setDiscardResult()
9563 .setTailCall(IsTailCall);
9564
9565 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9566 return CallResult.second;
9567}
9568
9569SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
9570 SDValue Dst, SDValue Src, SDValue Size,
9571 Type *SizeTy, unsigned ElemSz,
9572 bool isTailCall,
9573 MachinePointerInfo DstPtrInfo,
9574 MachinePointerInfo SrcPtrInfo) {
9575 // Emit a library call.
9576 TargetLowering::ArgListTy Args;
9577 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
9578 Args.emplace_back(args&: Dst, args&: IntPtrTy);
9579 Args.emplace_back(args&: Src, args&: IntPtrTy);
9580 Args.emplace_back(args&: Size, args&: SizeTy);
9581
9582 RTLIB::Libcall LibraryCall =
9583 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9584 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9585 if (LibcallImpl == RTLIB::Unsupported)
9586 report_fatal_error(reason: "Unsupported element size");
9587
9588 TargetLowering::CallLoweringInfo CLI(*this);
9589 CLI.setDebugLoc(dl)
9590 .setChain(Chain)
9591 .setLibCallee(
9592 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9593 ResultType: Type::getVoidTy(C&: *getContext()),
9594 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9595 ArgsList: std::move(Args))
9596 .setDiscardResult()
9597 .setTailCall(isTailCall);
9598
9599 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9600 return CallResult.second;
9601}
9602
9603SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
9604 SDValue Src, SDValue Size, Align Alignment,
9605 bool isVol, bool AlwaysInline,
9606 const CallInst *CI,
9607 MachinePointerInfo DstPtrInfo,
9608 const AAMDNodes &AAInfo) {
9609 // Check to see if we should lower the memset to stores first.
9610 // For cases within the target-specified limits, this is the best choice.
9611 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9612 if (ConstantSize) {
9613 // Memset with size zero? Just return the original chain.
9614 if (ConstantSize->isZero())
9615 return Chain;
9616
9617 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9618 Size: ConstantSize->getZExtValue(), Alignment,
9619 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
9620
9621 if (Result.getNode())
9622 return Result;
9623 }
9624
9625 // Then check to see if we should lower the memset with target-specific
9626 // code. If the target chooses to do this, this is the next best.
9627 if (TSI) {
9628 SDValue Result = TSI->EmitTargetCodeForMemset(
9629 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
9630 if (Result.getNode())
9631 return Result;
9632 }
9633
9634 // If we really need inline code and the target declined to provide it,
9635 // use a (potentially long) sequence of loads and stores.
9636 if (AlwaysInline) {
9637 assert(ConstantSize && "AlwaysInline requires a constant size!");
9638 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9639 Size: ConstantSize->getZExtValue(), Alignment,
9640 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
9641 assert(Result &&
9642 "getMemsetStores must return a valid sequence when AlwaysInline");
9643 return Result;
9644 }
9645
9646 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9647
9648 // Emit a library call.
9649 auto &Ctx = *getContext();
9650 const auto& DL = getDataLayout();
9651
9652 TargetLowering::CallLoweringInfo CLI(*this);
9653 // FIXME: pass in SDLoc
9654 CLI.setDebugLoc(dl).setChain(Chain);
9655
9656 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
9657 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
9658
9659 // If zeroing out and bzero is present, use it.
9660 if (UseBZero) {
9661 TargetLowering::ArgListTy Args;
9662 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
9663 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
9664 CLI.setLibCallee(
9665 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
9666 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
9667 } else {
9668 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
9669
9670 TargetLowering::ArgListTy Args;
9671 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
9672 Args.emplace_back(args&: Src, args: Src.getValueType().getTypeForEVT(Context&: Ctx));
9673 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
9674 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
9675 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
9676 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
9677 ArgsList: std::move(Args));
9678 }
9679
9680 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
9681 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
9682
9683 // If we're going to use bzero, make sure not to tail call unless the
9684 // subsequent return doesn't need a value, as bzero doesn't return the first
9685 // arg unlike memset.
9686 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
9687 bool IsTailCall =
9688 CI && CI->isTailCall() &&
9689 isInTailCallPosition(Call: *CI, TM: getTarget(), ReturnsFirstArg: ReturnsFirstArg && LowersToMemset);
9690 CLI.setDiscardResult().setTailCall(IsTailCall);
9691
9692 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9693 return CallResult.second;
9694}
9695
9696SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
9697 SDValue Dst, SDValue Value, SDValue Size,
9698 Type *SizeTy, unsigned ElemSz,
9699 bool isTailCall,
9700 MachinePointerInfo DstPtrInfo) {
9701 // Emit a library call.
9702 TargetLowering::ArgListTy Args;
9703 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
9704 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
9705 Args.emplace_back(args&: Size, args&: SizeTy);
9706
9707 RTLIB::Libcall LibraryCall =
9708 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9709 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9710 if (LibcallImpl == RTLIB::Unsupported)
9711 report_fatal_error(reason: "Unsupported element size");
9712
9713 TargetLowering::CallLoweringInfo CLI(*this);
9714 CLI.setDebugLoc(dl)
9715 .setChain(Chain)
9716 .setLibCallee(
9717 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9718 ResultType: Type::getVoidTy(C&: *getContext()),
9719 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9720 ArgsList: std::move(Args))
9721 .setDiscardResult()
9722 .setTailCall(isTailCall);
9723
9724 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9725 return CallResult.second;
9726}
9727
9728SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9729 SDVTList VTList, ArrayRef<SDValue> Ops,
9730 MachineMemOperand *MMO,
9731 ISD::LoadExtType ExtType) {
9732 FoldingSetNodeID ID;
9733 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
9734 ID.AddInteger(I: MemVT.getRawBits());
9735 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
9736 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
9737 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9738 ID.AddInteger(I: MMO->getFlags());
9739 void* IP = nullptr;
9740 if (auto *E = cast_or_null<AtomicSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
9741 E->refineAlignment(NewMMO: MMO);
9742 E->refineRanges(NewMMO: MMO);
9743 return SDValue(E, 0);
9744 }
9745
9746 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
9747 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
9748 createOperands(Node: N, Vals: Ops);
9749
9750 CSEMap.InsertNode(N, InsertPos: IP);
9751 InsertNode(N);
9752 SDValue V(N, 0);
9753 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9754 return V;
9755}
9756
9757SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
9758 EVT MemVT, SDVTList VTs, SDValue Chain,
9759 SDValue Ptr, SDValue Cmp, SDValue Swp,
9760 MachineMemOperand *MMO) {
9761 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
9762 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
9763 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
9764
9765 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
9766 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9767}
9768
9769SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9770 SDValue Chain, SDValue Ptr, SDValue Val,
9771 MachineMemOperand *MMO) {
9772 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
9773 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
9774 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
9775 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
9776 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
9777 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
9778 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
9779 Opcode == ISD::ATOMIC_LOAD_FMIN ||
9780 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
9781 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
9782 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
9783 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
9784 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
9785 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
9786 Opcode == ISD::ATOMIC_STORE) &&
9787 "Invalid Atomic Op");
9788
9789 EVT VT = Val.getValueType();
9790
9791 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
9792 getVTList(VT1: VT, VT2: MVT::Other);
9793 SDValue Ops[] = {Chain, Ptr, Val};
9794 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9795}
9796
9797SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
9798 EVT MemVT, EVT VT, SDValue Chain,
9799 SDValue Ptr, MachineMemOperand *MMO) {
9800 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
9801 SDValue Ops[] = {Chain, Ptr};
9802 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
9803}
9804
9805/// getMergeValues - Create a MERGE_VALUES node from the given operands.
9806SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
9807 if (Ops.size() == 1)
9808 return Ops[0];
9809
9810 SmallVector<EVT, 4> VTs;
9811 VTs.reserve(N: Ops.size());
9812 for (const SDValue &Op : Ops)
9813 VTs.push_back(Elt: Op.getValueType());
9814 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
9815}
9816
9817SDValue SelectionDAG::getMemIntrinsicNode(
9818 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
9819 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
9820 MachineMemOperand::Flags Flags, LocationSize Size,
9821 const AAMDNodes &AAInfo) {
9822 if (Size.hasValue() && !Size.getValue())
9823 Size = LocationSize::precise(Value: MemVT.getStoreSize());
9824
9825 MachineFunction &MF = getMachineFunction();
9826 MachineMemOperand *MMO =
9827 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, AAInfo);
9828
9829 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
9830}
9831
9832SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
9833 SDVTList VTList,
9834 ArrayRef<SDValue> Ops, EVT MemVT,
9835 MachineMemOperand *MMO) {
9836 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
9837}
9838
9839SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
9840 SDVTList VTList,
9841 ArrayRef<SDValue> Ops, EVT MemVT,
9842 ArrayRef<MachineMemOperand *> MMOs) {
9843 assert(!MMOs.empty() && "Must have at least one MMO");
9844 assert(
9845 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
9846 Opcode == ISD::PREFETCH ||
9847 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
9848 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
9849 "Opcode is not a memory-accessing opcode!");
9850
9851 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
9852 if (MMOs.size() == 1) {
9853 MemRefs = MMOs[0];
9854 } else {
9855 // Allocate: [size_t count][MMO*][MMO*]...
9856 size_t AllocSize =
9857 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
9858 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
9859 size_t *CountPtr = static_cast<size_t *>(Buffer);
9860 *CountPtr = MMOs.size();
9861 MachineMemOperand **Array =
9862 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
9863 llvm::copy(Range&: MMOs, Out: Array);
9864 MemRefs = Array;
9865 }
9866
9867 // Memoize the node unless it returns a glue result.
9868 MemIntrinsicSDNode *N;
9869 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9870 FoldingSetNodeID ID;
9871 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
9872 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
9873 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
9874 ID.AddInteger(I: MemVT.getRawBits());
9875 for (const MachineMemOperand *MMO : MMOs) {
9876 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9877 ID.AddInteger(I: MMO->getFlags());
9878 }
9879 void *IP = nullptr;
9880 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9881 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
9882 return SDValue(E, 0);
9883 }
9884
9885 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9886 Args&: VTList, Args&: MemVT, Args&: MemRefs);
9887 createOperands(Node: N, Vals: Ops);
9888 CSEMap.InsertNode(N, InsertPos: IP);
9889 } else {
9890 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9891 Args&: VTList, Args&: MemVT, Args&: MemRefs);
9892 createOperands(Node: N, Vals: Ops);
9893 }
9894 InsertNode(N);
9895 SDValue V(N, 0);
9896 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9897 return V;
9898}
9899
9900SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
9901 SDValue Chain, int FrameIndex) {
9902 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
9903 const auto VTs = getVTList(VT: MVT::Other);
9904 SDValue Ops[2] = {
9905 Chain,
9906 getFrameIndex(FI: FrameIndex,
9907 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
9908 isTarget: true)};
9909
9910 FoldingSetNodeID ID;
9911 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9912 ID.AddInteger(I: FrameIndex);
9913 void *IP = nullptr;
9914 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9915 return SDValue(E, 0);
9916
9917 LifetimeSDNode *N =
9918 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
9919 createOperands(Node: N, Vals: Ops);
9920 CSEMap.InsertNode(N, InsertPos: IP);
9921 InsertNode(N);
9922 SDValue V(N, 0);
9923 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9924 return V;
9925}
9926
9927SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
9928 uint64_t Guid, uint64_t Index,
9929 uint32_t Attr) {
9930 const unsigned Opcode = ISD::PSEUDO_PROBE;
9931 const auto VTs = getVTList(VT: MVT::Other);
9932 SDValue Ops[] = {Chain};
9933 FoldingSetNodeID ID;
9934 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9935 ID.AddInteger(I: Guid);
9936 ID.AddInteger(I: Index);
9937 void *IP = nullptr;
9938 if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
9939 return SDValue(E, 0);
9940
9941 auto *N = newSDNode<PseudoProbeSDNode>(
9942 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
9943 createOperands(Node: N, Vals: Ops);
9944 CSEMap.InsertNode(N, InsertPos: IP);
9945 InsertNode(N);
9946 SDValue V(N, 0);
9947 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9948 return V;
9949}
9950
9951/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
9952/// MachinePointerInfo record from it. This is particularly useful because the
9953/// code generator has many cases where it doesn't bother passing in a
9954/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
9955static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
9956 SelectionDAG &DAG, SDValue Ptr,
9957 int64_t Offset = 0) {
9958 // If this is FI+Offset, we can model it.
9959 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
9960 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
9961 FI: FI->getIndex(), Offset);
9962
9963 // If this is (FI+Offset1)+Offset2, we can model it.
9964 if (Ptr.getOpcode() != ISD::ADD ||
9965 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
9966 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
9967 return Info;
9968
9969 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
9970 return MachinePointerInfo::getFixedStack(
9971 MF&: DAG.getMachineFunction(), FI,
9972 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
9973}
9974
9975/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
9976/// MachinePointerInfo record from it. This is particularly useful because the
9977/// code generator has many cases where it doesn't bother passing in a
9978/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
9979static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
9980 SelectionDAG &DAG, SDValue Ptr,
9981 SDValue OffsetOp) {
9982 // If the 'Offset' value isn't a constant, we can't handle this.
9983 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
9984 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
9985 if (OffsetOp.isUndef())
9986 return InferPointerInfo(Info, DAG, Ptr);
9987 return Info;
9988}
9989
9990SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
9991 EVT VT, const SDLoc &dl, SDValue Chain,
9992 SDValue Ptr, SDValue Offset,
9993 MachinePointerInfo PtrInfo, EVT MemVT,
9994 Align Alignment,
9995 MachineMemOperand::Flags MMOFlags,
9996 const AAMDNodes &AAInfo, const MDNode *Ranges) {
9997 assert(Chain.getValueType() == MVT::Other &&
9998 "Invalid chain type");
9999
10000 MMOFlags |= MachineMemOperand::MOLoad;
10001 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10002 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10003 // clients.
10004 if (PtrInfo.V.isNull())
10005 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10006
10007 TypeSize Size = MemVT.getStoreSize();
10008 MachineFunction &MF = getMachineFunction();
10009 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10010 BaseAlignment: Alignment, AAInfo, Ranges);
10011 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10012}
10013
10014SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10015 EVT VT, const SDLoc &dl, SDValue Chain,
10016 SDValue Ptr, SDValue Offset, EVT MemVT,
10017 MachineMemOperand *MMO) {
10018 if (VT == MemVT) {
10019 ExtType = ISD::NON_EXTLOAD;
10020 } else if (ExtType == ISD::NON_EXTLOAD) {
10021 assert(VT == MemVT && "Non-extending load from different memory type!");
10022 } else {
10023 // Extending load.
10024 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10025 "Should only be an extending load, not truncating!");
10026 assert(VT.isInteger() == MemVT.isInteger() &&
10027 "Cannot convert from FP to Int or Int -> FP!");
10028 assert(VT.isVector() == MemVT.isVector() &&
10029 "Cannot use an ext load to convert to or from a vector!");
10030 assert((!VT.isVector() ||
10031 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10032 "Cannot use an ext load to change the number of vector elements!");
10033 }
10034
10035 assert((!MMO->getRanges() ||
10036 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10037 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10038 MemVT.isInteger())) &&
10039 "Range metadata and load type must match!");
10040
10041 bool Indexed = AM != ISD::UNINDEXED;
10042 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10043
10044 SDVTList VTs = Indexed ?
10045 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10046 SDValue Ops[] = { Chain, Ptr, Offset };
10047 FoldingSetNodeID ID;
10048 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
10049 ID.AddInteger(I: MemVT.getRawBits());
10050 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10051 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10052 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10053 ID.AddInteger(I: MMO->getFlags());
10054 void *IP = nullptr;
10055 if (auto *E = cast_or_null<LoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10056 E->refineAlignment(NewMMO: MMO);
10057 E->refineRanges(NewMMO: MMO);
10058 return SDValue(E, 0);
10059 }
10060 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10061 Args&: ExtType, Args&: MemVT, Args&: MMO);
10062 createOperands(Node: N, Vals: Ops);
10063
10064 CSEMap.InsertNode(N, InsertPos: IP);
10065 InsertNode(N);
10066 SDValue V(N, 0);
10067 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10068 return V;
10069}
10070
10071SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10072 SDValue Ptr, MachinePointerInfo PtrInfo,
10073 MaybeAlign Alignment,
10074 MachineMemOperand::Flags MMOFlags,
10075 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10076 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10077 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10078 PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
10079}
10080
10081SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10082 SDValue Ptr, MachineMemOperand *MMO) {
10083 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10084 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10085 MemVT: VT, MMO);
10086}
10087
10088SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10089 EVT VT, SDValue Chain, SDValue Ptr,
10090 MachinePointerInfo PtrInfo, EVT MemVT,
10091 MaybeAlign Alignment,
10092 MachineMemOperand::Flags MMOFlags,
10093 const AAMDNodes &AAInfo) {
10094 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10095 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10096 MemVT, Alignment, MMOFlags, AAInfo);
10097}
10098
10099SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10100 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10101 MachineMemOperand *MMO) {
10102 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10103 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10104 MemVT, MMO);
10105}
10106
10107SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10108 SDValue Base, SDValue Offset,
10109 ISD::MemIndexedMode AM) {
10110 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10111 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10112 // Don't propagate the invariant or dereferenceable flags.
10113 auto MMOFlags =
10114 LD->getMemOperand()->getFlags() &
10115 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10116 return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10117 Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
10118 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
10119}
10120
10121SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10122 SDValue Ptr, MachinePointerInfo PtrInfo,
10123 Align Alignment,
10124 MachineMemOperand::Flags MMOFlags,
10125 const AAMDNodes &AAInfo) {
10126 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10127
10128 MMOFlags |= MachineMemOperand::MOStore;
10129 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10130
10131 if (PtrInfo.V.isNull())
10132 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10133
10134 MachineFunction &MF = getMachineFunction();
10135 TypeSize Size = Val.getValueType().getStoreSize();
10136 MachineMemOperand *MMO =
10137 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, AAInfo);
10138 return getStore(Chain, dl, Val, Ptr, MMO);
10139}
10140
10141SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10142 SDValue Ptr, MachineMemOperand *MMO) {
10143 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10144 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10145 AM: ISD::UNINDEXED);
10146}
10147
10148SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10149 SDValue Ptr, SDValue Offset, EVT SVT,
10150 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
10151 bool IsTruncating) {
10152 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10153 EVT VT = Val.getValueType();
10154 if (VT == SVT) {
10155 IsTruncating = false;
10156 } else if (!IsTruncating) {
10157 assert(VT == SVT && "No-truncating store from different memory type!");
10158 } else {
10159 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10160 "Should only be a truncating store, not extending!");
10161 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10162 assert(VT.isVector() == SVT.isVector() &&
10163 "Cannot use trunc store to convert to or from a vector!");
10164 assert((!VT.isVector() ||
10165 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10166 "Cannot use trunc store to change the number of vector elements!");
10167 }
10168
10169 bool Indexed = AM != ISD::UNINDEXED;
10170 assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");
10171 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10172 : getVTList(VT: MVT::Other);
10173 SDValue Ops[] = {Chain, Val, Ptr, Offset};
10174 FoldingSetNodeID ID;
10175 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
10176 ID.AddInteger(I: SVT.getRawBits());
10177 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
10178 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
10179 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10180 ID.AddInteger(I: MMO->getFlags());
10181 void *IP = nullptr;
10182 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10183 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10184 return SDValue(E, 0);
10185 }
10186 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10187 Args&: IsTruncating, Args&: SVT, Args&: MMO);
10188 createOperands(Node: N, Vals: Ops);
10189
10190 CSEMap.InsertNode(N, InsertPos: IP);
10191 InsertNode(N);
10192 SDValue V(N, 0);
10193 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10194 return V;
10195}
10196
10197SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10198 SDValue Ptr, MachinePointerInfo PtrInfo,
10199 EVT SVT, Align Alignment,
10200 MachineMemOperand::Flags MMOFlags,
10201 const AAMDNodes &AAInfo) {
10202 assert(Chain.getValueType() == MVT::Other &&
10203 "Invalid chain type");
10204
10205 MMOFlags |= MachineMemOperand::MOStore;
10206 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10207
10208 if (PtrInfo.V.isNull())
10209 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10210
10211 MachineFunction &MF = getMachineFunction();
10212 MachineMemOperand *MMO = MF.getMachineMemOperand(
10213 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10214 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
10215}
10216
10217SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10218 SDValue Ptr, EVT SVT,
10219 MachineMemOperand *MMO) {
10220 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10221 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
10222}
10223
10224SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
10225 SDValue Base, SDValue Offset,
10226 ISD::MemIndexedMode AM) {
10227 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
10228 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
10229 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
10230 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
10231 IsTruncating: ST->isTruncatingStore());
10232}
10233
10234SDValue SelectionDAG::getLoadVP(
10235 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
10236 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
10237 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
10238 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
10239 const MDNode *Ranges, bool IsExpanding) {
10240 MMOFlags |= MachineMemOperand::MOLoad;
10241 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10242 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10243 // clients.
10244 if (PtrInfo.V.isNull())
10245 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10246
10247 TypeSize Size = MemVT.getStoreSize();
10248 MachineFunction &MF = getMachineFunction();
10249 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10250 BaseAlignment: Alignment, AAInfo, Ranges);
10251 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
10252 MMO, IsExpanding);
10253}
10254
10255SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
10256 ISD::LoadExtType ExtType, EVT VT,
10257 const SDLoc &dl, SDValue Chain, SDValue Ptr,
10258 SDValue Offset, SDValue Mask, SDValue EVL,
10259 EVT MemVT, MachineMemOperand *MMO,
10260 bool IsExpanding) {
10261 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10262 assert(Mask.getValueType().getVectorElementCount() ==
10263 VT.getVectorElementCount() &&
10264 "Vector width mismatch between mask and data");
10265
10266 bool Indexed = AM != ISD::UNINDEXED;
10267 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10268
10269 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10270 : getVTList(VT1: VT, VT2: MVT::Other);
10271 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
10272 FoldingSetNodeID ID;
10273 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
10274 ID.AddInteger(I: MemVT.getRawBits());
10275 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
10276 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10277 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10278 ID.AddInteger(I: MMO->getFlags());
10279 void *IP = nullptr;
10280 if (auto *E = cast_or_null<VPLoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10281 E->refineAlignment(NewMMO: MMO);
10282 E->refineRanges(NewMMO: MMO);
10283 return SDValue(E, 0);
10284 }
10285 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10286 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10287 createOperands(Node: N, Vals: Ops);
10288
10289 CSEMap.InsertNode(N, InsertPos: IP);
10290 InsertNode(N);
10291 SDValue V(N, 0);
10292 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10293 return V;
10294}
10295
10296SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10297 SDValue Ptr, SDValue Mask, SDValue EVL,
10298 MachinePointerInfo PtrInfo,
10299 MaybeAlign Alignment,
10300 MachineMemOperand::Flags MMOFlags,
10301 const AAMDNodes &AAInfo, const MDNode *Ranges,
10302 bool IsExpanding) {
10303 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10304 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10305 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
10306 IsExpanding);
10307}
10308
10309SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10310 SDValue Ptr, SDValue Mask, SDValue EVL,
10311 MachineMemOperand *MMO, bool IsExpanding) {
10312 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10313 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10314 Mask, EVL, MemVT: VT, MMO, IsExpanding);
10315}
10316
10317SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10318 EVT VT, SDValue Chain, SDValue Ptr,
10319 SDValue Mask, SDValue EVL,
10320 MachinePointerInfo PtrInfo, EVT MemVT,
10321 MaybeAlign Alignment,
10322 MachineMemOperand::Flags MMOFlags,
10323 const AAMDNodes &AAInfo, bool IsExpanding) {
10324 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10325 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10326 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
10327 IsExpanding);
10328}
10329
10330SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10331 EVT VT, SDValue Chain, SDValue Ptr,
10332 SDValue Mask, SDValue EVL, EVT MemVT,
10333 MachineMemOperand *MMO, bool IsExpanding) {
10334 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10335 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10336 EVL, MemVT, MMO, IsExpanding);
10337}
10338
10339SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
10340 SDValue Base, SDValue Offset,
10341 ISD::MemIndexedMode AM) {
10342 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
10343 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10344 // Don't propagate the invariant or dereferenceable flags.
10345 auto MMOFlags =
10346 LD->getMemOperand()->getFlags() &
10347 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10348 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10349 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
10350 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
10351 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
10352 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
10353}
10354
10355SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
10356 SDValue Ptr, SDValue Offset, SDValue Mask,
10357 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
10358 ISD::MemIndexedMode AM, bool IsTruncating,
10359 bool IsCompressing) {
10360 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10361 assert(Mask.getValueType().getVectorElementCount() ==
10362 Val.getValueType().getVectorElementCount() &&
10363 "Vector width mismatch between mask and data");
10364
10365 bool Indexed = AM != ISD::UNINDEXED;
10366 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10367 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10368 : getVTList(VT: MVT::Other);
10369 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
10370 FoldingSetNodeID ID;
10371 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10372 ID.AddInteger(I: MemVT.getRawBits());
10373 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10374 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10375 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10376 ID.AddInteger(I: MMO->getFlags());
10377 void *IP = nullptr;
10378 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10379 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10380 return SDValue(E, 0);
10381 }
10382 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10383 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10384 createOperands(Node: N, Vals: Ops);
10385
10386 CSEMap.InsertNode(N, InsertPos: IP);
10387 InsertNode(N);
10388 SDValue V(N, 0);
10389 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10390 return V;
10391}
10392
10393SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10394 SDValue Val, SDValue Ptr, SDValue Mask,
10395 SDValue EVL, MachinePointerInfo PtrInfo,
10396 EVT SVT, Align Alignment,
10397 MachineMemOperand::Flags MMOFlags,
10398 const AAMDNodes &AAInfo,
10399 bool IsCompressing) {
10400 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10401
10402 MMOFlags |= MachineMemOperand::MOStore;
10403 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10404
10405 if (PtrInfo.V.isNull())
10406 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10407
10408 MachineFunction &MF = getMachineFunction();
10409 MachineMemOperand *MMO = MF.getMachineMemOperand(
10410 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10411 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
10412 IsCompressing);
10413}
10414
10415SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10416 SDValue Val, SDValue Ptr, SDValue Mask,
10417 SDValue EVL, EVT SVT,
10418 MachineMemOperand *MMO,
10419 bool IsCompressing) {
10420 EVT VT = Val.getValueType();
10421
10422 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10423 if (VT == SVT)
10424 return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
10425 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10426 /*IsTruncating*/ false, IsCompressing);
10427
10428 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10429 "Should only be a truncating store, not extending!");
10430 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10431 assert(VT.isVector() == SVT.isVector() &&
10432 "Cannot use trunc store to convert to or from a vector!");
10433 assert((!VT.isVector() ||
10434 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10435 "Cannot use trunc store to change the number of vector elements!");
10436
10437 SDVTList VTs = getVTList(VT: MVT::Other);
10438 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10439 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
10440 FoldingSetNodeID ID;
10441 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10442 ID.AddInteger(I: SVT.getRawBits());
10443 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10444 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10445 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10446 ID.AddInteger(I: MMO->getFlags());
10447 void *IP = nullptr;
10448 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10449 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10450 return SDValue(E, 0);
10451 }
10452 auto *N =
10453 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10454 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
10455 createOperands(Node: N, Vals: Ops);
10456
10457 CSEMap.InsertNode(N, InsertPos: IP);
10458 InsertNode(N);
10459 SDValue V(N, 0);
10460 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10461 return V;
10462}
10463
10464SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
10465 SDValue Base, SDValue Offset,
10466 ISD::MemIndexedMode AM) {
10467 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
10468 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
10469 SDVTList VTs = getVTList(VT1: Base.getValueType(), VT2: MVT::Other);
10470 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
10471 Offset, ST->getMask(), ST->getVectorLength()};
10472 FoldingSetNodeID ID;
10473 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10474 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
10475 ID.AddInteger(I: ST->getRawSubclassData());
10476 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
10477 ID.AddInteger(I: ST->getMemOperand()->getFlags());
10478 void *IP = nullptr;
10479 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10480 return SDValue(E, 0);
10481
10482 auto *N = newSDNode<VPStoreSDNode>(
10483 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
10484 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
10485 createOperands(Node: N, Vals: Ops);
10486
10487 CSEMap.InsertNode(N, InsertPos: IP);
10488 InsertNode(N);
10489 SDValue V(N, 0);
10490 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10491 return V;
10492}
10493
10494SDValue SelectionDAG::getStridedLoadVP(
10495 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
10496 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
10497 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
10498 bool Indexed = AM != ISD::UNINDEXED;
10499 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10500
10501 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
10502 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10503 : getVTList(VT1: VT, VT2: MVT::Other);
10504 FoldingSetNodeID ID;
10505 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
10506 ID.AddInteger(I: VT.getRawBits());
10507 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
10508 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10509 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10510
10511 void *IP = nullptr;
10512 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10513 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10514 return SDValue(E, 0);
10515 }
10516
10517 auto *N =
10518 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
10519 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10520 createOperands(Node: N, Vals: Ops);
10521 CSEMap.InsertNode(N, InsertPos: IP);
10522 InsertNode(N);
10523 SDValue V(N, 0);
10524 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10525 return V;
10526}
10527
10528SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
10529 SDValue Ptr, SDValue Stride,
10530 SDValue Mask, SDValue EVL,
10531 MachineMemOperand *MMO,
10532 bool IsExpanding) {
10533 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10534 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
10535 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
10536}
10537
10538SDValue SelectionDAG::getExtStridedLoadVP(
10539 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
10540 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
10541 MachineMemOperand *MMO, bool IsExpanding) {
10542 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10543 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
10544 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
10545}
10546
10547SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
10548 SDValue Val, SDValue Ptr,
10549 SDValue Offset, SDValue Stride,
10550 SDValue Mask, SDValue EVL, EVT MemVT,
10551 MachineMemOperand *MMO,
10552 ISD::MemIndexedMode AM,
10553 bool IsTruncating, bool IsCompressing) {
10554 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10555 bool Indexed = AM != ISD::UNINDEXED;
10556 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10557 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10558 : getVTList(VT: MVT::Other);
10559 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
10560 FoldingSetNodeID ID;
10561 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10562 ID.AddInteger(I: MemVT.getRawBits());
10563 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10564 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10565 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10566 void *IP = nullptr;
10567 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10568 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10569 return SDValue(E, 0);
10570 }
10571 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10572 Args&: VTs, Args&: AM, Args&: IsTruncating,
10573 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10574 createOperands(Node: N, Vals: Ops);
10575
10576 CSEMap.InsertNode(N, InsertPos: IP);
10577 InsertNode(N);
10578 SDValue V(N, 0);
10579 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10580 return V;
10581}
10582
10583SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
10584 SDValue Val, SDValue Ptr,
10585 SDValue Stride, SDValue Mask,
10586 SDValue EVL, EVT SVT,
10587 MachineMemOperand *MMO,
10588 bool IsCompressing) {
10589 EVT VT = Val.getValueType();
10590
10591 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10592 if (VT == SVT)
10593 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
10594 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10595 /*IsTruncating*/ false, IsCompressing);
10596
10597 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10598 "Should only be a truncating store, not extending!");
10599 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10600 assert(VT.isVector() == SVT.isVector() &&
10601 "Cannot use trunc store to convert to or from a vector!");
10602 assert((!VT.isVector() ||
10603 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10604 "Cannot use trunc store to change the number of vector elements!");
10605
10606 SDVTList VTs = getVTList(VT: MVT::Other);
10607 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10608 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
10609 FoldingSetNodeID ID;
10610 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10611 ID.AddInteger(I: SVT.getRawBits());
10612 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10613 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10614 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10615 void *IP = nullptr;
10616 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10617 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10618 return SDValue(E, 0);
10619 }
10620 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10621 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
10622 Args&: IsCompressing, Args&: SVT, Args&: MMO);
10623 createOperands(Node: N, Vals: Ops);
10624
10625 CSEMap.InsertNode(N, InsertPos: IP);
10626 InsertNode(N);
10627 SDValue V(N, 0);
10628 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10629 return V;
10630}
10631
10632SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10633 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
10634 ISD::MemIndexType IndexType) {
10635 assert(Ops.size() == 6 && "Incompatible number of operands");
10636
10637 FoldingSetNodeID ID;
10638 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
10639 ID.AddInteger(I: VT.getRawBits());
10640 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
10641 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10642 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10643 ID.AddInteger(I: MMO->getFlags());
10644 void *IP = nullptr;
10645 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10646 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10647 return SDValue(E, 0);
10648 }
10649
10650 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10651 Args&: VT, Args&: MMO, Args&: IndexType);
10652 createOperands(Node: N, Vals: Ops);
10653
10654 assert(N->getMask().getValueType().getVectorElementCount() ==
10655 N->getValueType(0).getVectorElementCount() &&
10656 "Vector width mismatch between mask and data");
10657 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10658 N->getValueType(0).getVectorElementCount().isScalable() &&
10659 "Scalable flags of index and data do not match");
10660 assert(ElementCount::isKnownGE(
10661 N->getIndex().getValueType().getVectorElementCount(),
10662 N->getValueType(0).getVectorElementCount()) &&
10663 "Vector width mismatch between index and data");
10664 assert(isa<ConstantSDNode>(N->getScale()) &&
10665 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10666 "Scale should be a constant power of 2");
10667
10668 CSEMap.InsertNode(N, InsertPos: IP);
10669 InsertNode(N);
10670 SDValue V(N, 0);
10671 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10672 return V;
10673}
10674
10675SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10676 ArrayRef<SDValue> Ops,
10677 MachineMemOperand *MMO,
10678 ISD::MemIndexType IndexType) {
10679 assert(Ops.size() == 7 && "Incompatible number of operands");
10680
10681 FoldingSetNodeID ID;
10682 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
10683 ID.AddInteger(I: VT.getRawBits());
10684 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
10685 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10686 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10687 ID.AddInteger(I: MMO->getFlags());
10688 void *IP = nullptr;
10689 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10690 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10691 return SDValue(E, 0);
10692 }
10693 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10694 Args&: VT, Args&: MMO, Args&: IndexType);
10695 createOperands(Node: N, Vals: Ops);
10696
10697 assert(N->getMask().getValueType().getVectorElementCount() ==
10698 N->getValue().getValueType().getVectorElementCount() &&
10699 "Vector width mismatch between mask and data");
10700 assert(
10701 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10702 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10703 "Scalable flags of index and data do not match");
10704 assert(ElementCount::isKnownGE(
10705 N->getIndex().getValueType().getVectorElementCount(),
10706 N->getValue().getValueType().getVectorElementCount()) &&
10707 "Vector width mismatch between index and data");
10708 assert(isa<ConstantSDNode>(N->getScale()) &&
10709 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10710 "Scale should be a constant power of 2");
10711
10712 CSEMap.InsertNode(N, InsertPos: IP);
10713 InsertNode(N);
10714 SDValue V(N, 0);
10715 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10716 return V;
10717}
10718
10719SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10720 SDValue Base, SDValue Offset, SDValue Mask,
10721 SDValue PassThru, EVT MemVT,
10722 MachineMemOperand *MMO,
10723 ISD::MemIndexedMode AM,
10724 ISD::LoadExtType ExtTy, bool isExpanding) {
10725 bool Indexed = AM != ISD::UNINDEXED;
10726 assert((Indexed || Offset.isUndef()) &&
10727 "Unindexed masked load with an offset!");
10728 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
10729 : getVTList(VT1: VT, VT2: MVT::Other);
10730 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
10731 FoldingSetNodeID ID;
10732 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
10733 ID.AddInteger(I: MemVT.getRawBits());
10734 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
10735 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
10736 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10737 ID.AddInteger(I: MMO->getFlags());
10738 void *IP = nullptr;
10739 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10740 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10741 return SDValue(E, 0);
10742 }
10743 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10744 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
10745 createOperands(Node: N, Vals: Ops);
10746
10747 CSEMap.InsertNode(N, InsertPos: IP);
10748 InsertNode(N);
10749 SDValue V(N, 0);
10750 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10751 return V;
10752}
10753
10754SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
10755 SDValue Base, SDValue Offset,
10756 ISD::MemIndexedMode AM) {
10757 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
10758 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
10759 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
10760 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
10761 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
10762 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
10763}
10764
10765SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
10766 SDValue Val, SDValue Base, SDValue Offset,
10767 SDValue Mask, EVT MemVT,
10768 MachineMemOperand *MMO,
10769 ISD::MemIndexedMode AM, bool IsTruncating,
10770 bool IsCompressing) {
10771 assert(Chain.getValueType() == MVT::Other &&
10772 "Invalid chain type");
10773 bool Indexed = AM != ISD::UNINDEXED;
10774 assert((Indexed || Offset.isUndef()) &&
10775 "Unindexed masked store with an offset!");
10776 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
10777 : getVTList(VT: MVT::Other);
10778 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
10779 FoldingSetNodeID ID;
10780 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
10781 ID.AddInteger(I: MemVT.getRawBits());
10782 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
10783 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10784 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10785 ID.AddInteger(I: MMO->getFlags());
10786 void *IP = nullptr;
10787 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10788 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10789 return SDValue(E, 0);
10790 }
10791 auto *N =
10792 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10793 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10794 createOperands(Node: N, Vals: Ops);
10795
10796 CSEMap.InsertNode(N, InsertPos: IP);
10797 InsertNode(N);
10798 SDValue V(N, 0);
10799 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10800 return V;
10801}
10802
10803SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
10804 SDValue Base, SDValue Offset,
10805 ISD::MemIndexedMode AM) {
10806 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
10807 assert(ST->getOffset().isUndef() &&
10808 "Masked store is already a indexed store!");
10809 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
10810 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
10811 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
10812}
10813
10814SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10815 ArrayRef<SDValue> Ops,
10816 MachineMemOperand *MMO,
10817 ISD::MemIndexType IndexType,
10818 ISD::LoadExtType ExtTy) {
10819 assert(Ops.size() == 6 && "Incompatible number of operands");
10820
10821 FoldingSetNodeID ID;
10822 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
10823 ID.AddInteger(I: MemVT.getRawBits());
10824 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
10825 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
10826 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10827 ID.AddInteger(I: MMO->getFlags());
10828 void *IP = nullptr;
10829 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10830 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10831 return SDValue(E, 0);
10832 }
10833
10834 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10835 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
10836 createOperands(Node: N, Vals: Ops);
10837
10838 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
10839 "Incompatible type of the PassThru value in MaskedGatherSDNode");
10840 assert(N->getMask().getValueType().getVectorElementCount() ==
10841 N->getValueType(0).getVectorElementCount() &&
10842 "Vector width mismatch between mask and data");
10843 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10844 N->getValueType(0).getVectorElementCount().isScalable() &&
10845 "Scalable flags of index and data do not match");
10846 assert(ElementCount::isKnownGE(
10847 N->getIndex().getValueType().getVectorElementCount(),
10848 N->getValueType(0).getVectorElementCount()) &&
10849 "Vector width mismatch between index and data");
10850 assert(isa<ConstantSDNode>(N->getScale()) &&
10851 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10852 "Scale should be a constant power of 2");
10853
10854 CSEMap.InsertNode(N, InsertPos: IP);
10855 InsertNode(N);
10856 SDValue V(N, 0);
10857 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10858 return V;
10859}
10860
10861SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10862 ArrayRef<SDValue> Ops,
10863 MachineMemOperand *MMO,
10864 ISD::MemIndexType IndexType,
10865 bool IsTrunc) {
10866 assert(Ops.size() == 6 && "Incompatible number of operands");
10867
10868 FoldingSetNodeID ID;
10869 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
10870 ID.AddInteger(I: MemVT.getRawBits());
10871 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
10872 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
10873 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10874 ID.AddInteger(I: MMO->getFlags());
10875 void *IP = nullptr;
10876 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10877 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10878 return SDValue(E, 0);
10879 }
10880
10881 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10882 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
10883 createOperands(Node: N, Vals: Ops);
10884
10885 assert(N->getMask().getValueType().getVectorElementCount() ==
10886 N->getValue().getValueType().getVectorElementCount() &&
10887 "Vector width mismatch between mask and data");
10888 assert(
10889 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10890 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10891 "Scalable flags of index and data do not match");
10892 assert(ElementCount::isKnownGE(
10893 N->getIndex().getValueType().getVectorElementCount(),
10894 N->getValue().getValueType().getVectorElementCount()) &&
10895 "Vector width mismatch between index and data");
10896 assert(isa<ConstantSDNode>(N->getScale()) &&
10897 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10898 "Scale should be a constant power of 2");
10899
10900 CSEMap.InsertNode(N, InsertPos: IP);
10901 InsertNode(N);
10902 SDValue V(N, 0);
10903 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10904 return V;
10905}
10906
10907SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
10908 const SDLoc &dl, ArrayRef<SDValue> Ops,
10909 MachineMemOperand *MMO,
10910 ISD::MemIndexType IndexType) {
10911 assert(Ops.size() == 7 && "Incompatible number of operands");
10912
10913 FoldingSetNodeID ID;
10914 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTList: VTs, OpList: Ops);
10915 ID.AddInteger(I: MemVT.getRawBits());
10916 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
10917 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
10918 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10919 ID.AddInteger(I: MMO->getFlags());
10920 void *IP = nullptr;
10921 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10922 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10923 return SDValue(E, 0);
10924 }
10925
10926 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10927 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
10928 createOperands(Node: N, Vals: Ops);
10929
10930 assert(N->getMask().getValueType().getVectorElementCount() ==
10931 N->getIndex().getValueType().getVectorElementCount() &&
10932 "Vector width mismatch between mask and data");
10933 assert(isa<ConstantSDNode>(N->getScale()) &&
10934 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10935 "Scale should be a constant power of 2");
10936 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
10937
10938 CSEMap.InsertNode(N, InsertPos: IP);
10939 InsertNode(N);
10940 SDValue V(N, 0);
10941 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10942 return V;
10943}
10944
10945SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
10946 SDValue Ptr, SDValue Mask, SDValue EVL,
10947 MachineMemOperand *MMO) {
10948 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
10949 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
10950 FoldingSetNodeID ID;
10951 AddNodeIDNode(ID, OpC: ISD::VP_LOAD_FF, VTList: VTs, OpList: Ops);
10952 ID.AddInteger(I: VT.getRawBits());
10953 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
10954 Args&: VTs, Args&: VT, Args&: MMO));
10955 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10956 ID.AddInteger(I: MMO->getFlags());
10957 void *IP = nullptr;
10958 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10959 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10960 return SDValue(E, 0);
10961 }
10962 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
10963 Args&: VT, Args&: MMO);
10964 createOperands(Node: N, Vals: Ops);
10965
10966 CSEMap.InsertNode(N, InsertPos: IP);
10967 InsertNode(N);
10968 SDValue V(N, 0);
10969 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10970 return V;
10971}
10972
10973SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
10974 EVT MemVT, MachineMemOperand *MMO) {
10975 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10976 SDVTList VTs = getVTList(VT: MVT::Other);
10977 SDValue Ops[] = {Chain, Ptr};
10978 FoldingSetNodeID ID;
10979 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
10980 ID.AddInteger(I: MemVT.getRawBits());
10981 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
10982 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
10983 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10984 ID.AddInteger(I: MMO->getFlags());
10985 void *IP = nullptr;
10986 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10987 return SDValue(E, 0);
10988
10989 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
10990 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
10991 createOperands(Node: N, Vals: Ops);
10992
10993 CSEMap.InsertNode(N, InsertPos: IP);
10994 InsertNode(N);
10995 SDValue V(N, 0);
10996 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10997 return V;
10998}
10999
11000SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11001 EVT MemVT, MachineMemOperand *MMO) {
11002 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11003 SDVTList VTs = getVTList(VT: MVT::Other);
11004 SDValue Ops[] = {Chain, Ptr};
11005 FoldingSetNodeID ID;
11006 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
11007 ID.AddInteger(I: MemVT.getRawBits());
11008 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11009 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11010 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11011 ID.AddInteger(I: MMO->getFlags());
11012 void *IP = nullptr;
11013 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11014 return SDValue(E, 0);
11015
11016 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11017 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11018 createOperands(Node: N, Vals: Ops);
11019
11020 CSEMap.InsertNode(N, InsertPos: IP);
11021 InsertNode(N);
11022 SDValue V(N, 0);
11023 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11024 return V;
11025}
11026
11027SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11028 // select undef, T, F --> T (if T is a constant), otherwise F
11029 // select, ?, undef, F --> F
11030 // select, ?, T, undef --> T
11031 if (Cond.isUndef())
11032 return isConstantValueOfAnyType(N: T) ? T : F;
11033 if (T.isUndef())
11034 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11035 if (F.isUndef())
11036 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11037
11038 // select true, T, F --> T
11039 // select false, T, F --> F
11040 if (auto C = isBoolConstant(N: Cond))
11041 return *C ? T : F;
11042
11043 // select ?, T, T --> T
11044 if (T == F)
11045 return T;
11046
11047 return SDValue();
11048}
11049
11050SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11051 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11052 if (X.isUndef())
11053 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11054 // shift X, undef --> undef (because it may shift by the bitwidth)
11055 if (Y.isUndef())
11056 return getUNDEF(VT: X.getValueType());
11057
11058 // shift 0, Y --> 0
11059 // shift X, 0 --> X
11060 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11061 return X;
11062
11063 // shift X, C >= bitwidth(X) --> undef
11064 // All vector elements must be too big (or undef) to avoid partial undefs.
11065 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11066 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11067 };
11068 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11069 return getUNDEF(VT: X.getValueType());
11070
11071 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11072 if (X.getValueType().getScalarType() == MVT::i1)
11073 return X;
11074
11075 return SDValue();
11076}
11077
11078SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11079 SDNodeFlags Flags) {
11080 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11081 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11082 // operation is poison. That result can be relaxed to undef.
11083 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11084 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11085 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11086 (YC && YC->getValueAPF().isNaN());
11087 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11088 (YC && YC->getValueAPF().isInfinity());
11089
11090 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11091 return getUNDEF(VT: X.getValueType());
11092
11093 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11094 return getUNDEF(VT: X.getValueType());
11095
11096 if (!YC)
11097 return SDValue();
11098
11099 // X + -0.0 --> X
11100 if (Opcode == ISD::FADD)
11101 if (YC->getValueAPF().isNegZero())
11102 return X;
11103
11104 // X - +0.0 --> X
11105 if (Opcode == ISD::FSUB)
11106 if (YC->getValueAPF().isPosZero())
11107 return X;
11108
11109 // X * 1.0 --> X
11110 // X / 1.0 --> X
11111 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11112 if (YC->getValueAPF().isExactlyValue(V: 1.0))
11113 return X;
11114
11115 // X * 0.0 --> 0.0
11116 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11117 if (YC->getValueAPF().isZero())
11118 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11119
11120 return SDValue();
11121}
11122
11123SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11124 SDValue Ptr, SDValue SV, unsigned Align) {
11125 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11126 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11127}
11128
11129SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11130 ArrayRef<SDUse> Ops) {
11131 switch (Ops.size()) {
11132 case 0: return getNode(Opcode, DL, VT);
11133 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11134 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11135 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11136 default: break;
11137 }
11138
11139 // Copy from an SDUse array into an SDValue array for use with
11140 // the regular getNode logic.
11141 SmallVector<SDValue, 8> NewOps(Ops);
11142 return getNode(Opcode, DL, VT, Ops: NewOps);
11143}
11144
11145SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11146 ArrayRef<SDValue> Ops) {
11147 SDNodeFlags Flags;
11148 if (Inserter)
11149 Flags = Inserter->getFlags();
11150 return getNode(Opcode, DL, VT, Ops, Flags);
11151}
11152
11153SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11154 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11155 unsigned NumOps = Ops.size();
11156 switch (NumOps) {
11157 case 0: return getNode(Opcode, DL, VT);
11158 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11159 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11160 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11161 default: break;
11162 }
11163
11164#ifndef NDEBUG
11165 for (const auto &Op : Ops)
11166 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11167 "Operand is DELETED_NODE!");
11168#endif
11169
11170 switch (Opcode) {
11171 default: break;
11172 case ISD::BUILD_VECTOR:
11173 // Attempt to simplify BUILD_VECTOR.
11174 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11175 return V;
11176 break;
11177 case ISD::CONCAT_VECTORS:
11178 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11179 return V;
11180 break;
11181 case ISD::SELECT_CC:
11182 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11183 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11184 "LHS and RHS of condition must have same type!");
11185 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11186 "True and False arms of SelectCC must have same type!");
11187 assert(Ops[2].getValueType() == VT &&
11188 "select_cc node must be of same type as true and false value!");
11189 assert((!Ops[0].getValueType().isVector() ||
11190 Ops[0].getValueType().getVectorElementCount() ==
11191 VT.getVectorElementCount()) &&
11192 "Expected select_cc with vector result to have the same sized "
11193 "comparison type!");
11194 break;
11195 case ISD::BR_CC:
11196 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11197 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11198 "LHS/RHS of comparison should match types!");
11199 break;
11200 case ISD::VP_ADD:
11201 case ISD::VP_SUB:
11202 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
11203 if (VT.getScalarType() == MVT::i1)
11204 Opcode = ISD::VP_XOR;
11205 break;
11206 case ISD::VP_MUL:
11207 // If it is VP_MUL mask operation then turn it to VP_AND
11208 if (VT.getScalarType() == MVT::i1)
11209 Opcode = ISD::VP_AND;
11210 break;
11211 case ISD::VP_REDUCE_MUL:
11212 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11213 if (VT == MVT::i1)
11214 Opcode = ISD::VP_REDUCE_AND;
11215 break;
11216 case ISD::VP_REDUCE_ADD:
11217 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11218 if (VT == MVT::i1)
11219 Opcode = ISD::VP_REDUCE_XOR;
11220 break;
11221 case ISD::VP_REDUCE_SMAX:
11222 case ISD::VP_REDUCE_UMIN:
11223 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11224 // VP_REDUCE_AND.
11225 if (VT == MVT::i1)
11226 Opcode = ISD::VP_REDUCE_AND;
11227 break;
11228 case ISD::VP_REDUCE_SMIN:
11229 case ISD::VP_REDUCE_UMAX:
11230 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11231 // VP_REDUCE_OR.
11232 if (VT == MVT::i1)
11233 Opcode = ISD::VP_REDUCE_OR;
11234 break;
11235 }
11236
11237 // Memoize nodes.
11238 SDNode *N;
11239 SDVTList VTs = getVTList(VT);
11240
11241 if (VT != MVT::Glue) {
11242 FoldingSetNodeID ID;
11243 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
11244 void *IP = nullptr;
11245
11246 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11247 E->intersectFlagsWith(Flags);
11248 return SDValue(E, 0);
11249 }
11250
11251 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11252 createOperands(Node: N, Vals: Ops);
11253
11254 CSEMap.InsertNode(N, InsertPos: IP);
11255 } else {
11256 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11257 createOperands(Node: N, Vals: Ops);
11258 }
11259
11260 N->setFlags(Flags);
11261 InsertNode(N);
11262 SDValue V(N, 0);
11263 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11264 return V;
11265}
11266
11267SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11268 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
11269 SDNodeFlags Flags;
11270 if (Inserter)
11271 Flags = Inserter->getFlags();
11272 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11273}
11274
11275SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11276 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
11277 const SDNodeFlags Flags) {
11278 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11279}
11280
11281SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11282 ArrayRef<SDValue> Ops) {
11283 SDNodeFlags Flags;
11284 if (Inserter)
11285 Flags = Inserter->getFlags();
11286 return getNode(Opcode, DL, VTList, Ops, Flags);
11287}
11288
11289SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11290 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11291 if (VTList.NumVTs == 1)
11292 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
11293
11294#ifndef NDEBUG
11295 for (const auto &Op : Ops)
11296 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11297 "Operand is DELETED_NODE!");
11298#endif
11299
11300 switch (Opcode) {
11301 case ISD::SADDO:
11302 case ISD::UADDO:
11303 case ISD::SSUBO:
11304 case ISD::USUBO: {
11305 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11306 "Invalid add/sub overflow op!");
11307 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11308 Ops[0].getValueType() == Ops[1].getValueType() &&
11309 Ops[0].getValueType() == VTList.VTs[0] &&
11310 "Binary operator types must match!");
11311 SDValue N1 = Ops[0], N2 = Ops[1];
11312 canonicalizeCommutativeBinop(Opcode, N1, N2);
11313
11314 // (X +- 0) -> X with zero-overflow.
11315 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
11316 /*AllowTruncation*/ true);
11317 if (N2CV && N2CV->isZero()) {
11318 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
11319 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
11320 }
11321
11322 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
11323 VTList.VTs[1].getScalarType() == MVT::i1) {
11324 SDValue F1 = getFreeze(V: N1);
11325 SDValue F2 = getFreeze(V: N2);
11326 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
11327 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
11328 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11329 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11330 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
11331 Flags);
11332 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
11333 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
11334 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
11335 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11336 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11337 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
11338 Flags);
11339 }
11340 }
11341 break;
11342 }
11343 case ISD::SADDO_CARRY:
11344 case ISD::UADDO_CARRY:
11345 case ISD::SSUBO_CARRY:
11346 case ISD::USUBO_CARRY:
11347 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
11348 "Invalid add/sub overflow op!");
11349 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11350 Ops[0].getValueType() == Ops[1].getValueType() &&
11351 Ops[0].getValueType() == VTList.VTs[0] &&
11352 Ops[2].getValueType() == VTList.VTs[1] &&
11353 "Binary operator types must match!");
11354 break;
11355 case ISD::SMUL_LOHI:
11356 case ISD::UMUL_LOHI: {
11357 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
11358 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
11359 VTList.VTs[0] == Ops[0].getValueType() &&
11360 VTList.VTs[0] == Ops[1].getValueType() &&
11361 "Binary operator types must match!");
11362 // Constant fold.
11363 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
11364 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
11365 if (LHS && RHS) {
11366 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
11367 unsigned OutWidth = Width * 2;
11368 APInt Val = LHS->getAPIntValue();
11369 APInt Mul = RHS->getAPIntValue();
11370 if (Opcode == ISD::SMUL_LOHI) {
11371 Val = Val.sext(width: OutWidth);
11372 Mul = Mul.sext(width: OutWidth);
11373 } else {
11374 Val = Val.zext(width: OutWidth);
11375 Mul = Mul.zext(width: OutWidth);
11376 }
11377 Val *= Mul;
11378
11379 SDValue Hi =
11380 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
11381 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
11382 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
11383 }
11384 break;
11385 }
11386 case ISD::FFREXP: {
11387 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
11388 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
11389 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
11390
11391 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
11392 int FrexpExp;
11393 APFloat FrexpMant =
11394 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
11395 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
11396 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
11397 DL, VT: VTList.VTs[1]);
11398 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
11399 }
11400
11401 break;
11402 }
11403 case ISD::STRICT_FP_EXTEND:
11404 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11405 "Invalid STRICT_FP_EXTEND!");
11406 assert(VTList.VTs[0].isFloatingPoint() &&
11407 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
11408 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11409 "STRICT_FP_EXTEND result type should be vector iff the operand "
11410 "type is vector!");
11411 assert((!VTList.VTs[0].isVector() ||
11412 VTList.VTs[0].getVectorElementCount() ==
11413 Ops[1].getValueType().getVectorElementCount()) &&
11414 "Vector element count mismatch!");
11415 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
11416 "Invalid fpext node, dst <= src!");
11417 break;
11418 case ISD::STRICT_FP_ROUND:
11419 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
11420 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11421 "STRICT_FP_ROUND result type should be vector iff the operand "
11422 "type is vector!");
11423 assert((!VTList.VTs[0].isVector() ||
11424 VTList.VTs[0].getVectorElementCount() ==
11425 Ops[1].getValueType().getVectorElementCount()) &&
11426 "Vector element count mismatch!");
11427 assert(VTList.VTs[0].isFloatingPoint() &&
11428 Ops[1].getValueType().isFloatingPoint() &&
11429 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
11430 Ops[2].getOpcode() == ISD::TargetConstant &&
11431 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
11432 "Invalid STRICT_FP_ROUND!");
11433 break;
11434 }
11435
11436 // Memoize the node unless it returns a glue result.
11437 SDNode *N;
11438 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
11439 FoldingSetNodeID ID;
11440 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
11441 void *IP = nullptr;
11442 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11443 E->intersectFlagsWith(Flags);
11444 return SDValue(E, 0);
11445 }
11446
11447 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11448 createOperands(Node: N, Vals: Ops);
11449 CSEMap.InsertNode(N, InsertPos: IP);
11450 } else {
11451 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11452 createOperands(Node: N, Vals: Ops);
11453 }
11454
11455 N->setFlags(Flags);
11456 InsertNode(N);
11457 SDValue V(N, 0);
11458 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11459 return V;
11460}
11461
11462SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11463 SDVTList VTList) {
11464 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
11465}
11466
11467SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11468 SDValue N1) {
11469 SDValue Ops[] = { N1 };
11470 return getNode(Opcode, DL, VTList, Ops);
11471}
11472
11473SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11474 SDValue N1, SDValue N2) {
11475 SDValue Ops[] = { N1, N2 };
11476 return getNode(Opcode, DL, VTList, Ops);
11477}
11478
11479SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11480 SDValue N1, SDValue N2, SDValue N3) {
11481 SDValue Ops[] = { N1, N2, N3 };
11482 return getNode(Opcode, DL, VTList, Ops);
11483}
11484
11485SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11486 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
11487 SDValue Ops[] = { N1, N2, N3, N4 };
11488 return getNode(Opcode, DL, VTList, Ops);
11489}
11490
11491SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11492 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
11493 SDValue N5) {
11494 SDValue Ops[] = { N1, N2, N3, N4, N5 };
11495 return getNode(Opcode, DL, VTList, Ops);
11496}
11497
11498SDVTList SelectionDAG::getVTList(EVT VT) {
11499 if (!VT.isExtended())
11500 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
11501
11502 return makeVTList(VTs: &(*EVTs.insert(x: VT).first), NumVTs: 1);
11503}
11504
11505SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
11506 FoldingSetNodeID ID;
11507 ID.AddInteger(I: 2U);
11508 ID.AddInteger(I: VT1.getRawBits());
11509 ID.AddInteger(I: VT2.getRawBits());
11510
11511 void *IP = nullptr;
11512 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11513 if (!Result) {
11514 EVT *Array = Allocator.Allocate<EVT>(Num: 2);
11515 Array[0] = VT1;
11516 Array[1] = VT2;
11517 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
11518 VTListMap.InsertNode(N: Result, InsertPos: IP);
11519 }
11520 return Result->getSDVTList();
11521}
11522
11523SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
11524 FoldingSetNodeID ID;
11525 ID.AddInteger(I: 3U);
11526 ID.AddInteger(I: VT1.getRawBits());
11527 ID.AddInteger(I: VT2.getRawBits());
11528 ID.AddInteger(I: VT3.getRawBits());
11529
11530 void *IP = nullptr;
11531 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11532 if (!Result) {
11533 EVT *Array = Allocator.Allocate<EVT>(Num: 3);
11534 Array[0] = VT1;
11535 Array[1] = VT2;
11536 Array[2] = VT3;
11537 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
11538 VTListMap.InsertNode(N: Result, InsertPos: IP);
11539 }
11540 return Result->getSDVTList();
11541}
11542
11543SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
11544 FoldingSetNodeID ID;
11545 ID.AddInteger(I: 4U);
11546 ID.AddInteger(I: VT1.getRawBits());
11547 ID.AddInteger(I: VT2.getRawBits());
11548 ID.AddInteger(I: VT3.getRawBits());
11549 ID.AddInteger(I: VT4.getRawBits());
11550
11551 void *IP = nullptr;
11552 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11553 if (!Result) {
11554 EVT *Array = Allocator.Allocate<EVT>(Num: 4);
11555 Array[0] = VT1;
11556 Array[1] = VT2;
11557 Array[2] = VT3;
11558 Array[3] = VT4;
11559 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
11560 VTListMap.InsertNode(N: Result, InsertPos: IP);
11561 }
11562 return Result->getSDVTList();
11563}
11564
11565SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
11566 unsigned NumVTs = VTs.size();
11567 FoldingSetNodeID ID;
11568 ID.AddInteger(I: NumVTs);
11569 for (unsigned index = 0; index < NumVTs; index++) {
11570 ID.AddInteger(I: VTs[index].getRawBits());
11571 }
11572
11573 void *IP = nullptr;
11574 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11575 if (!Result) {
11576 EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
11577 llvm::copy(Range&: VTs, Out: Array);
11578 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
11579 VTListMap.InsertNode(N: Result, InsertPos: IP);
11580 }
11581 return Result->getSDVTList();
11582}
11583
11584
11585/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
11586/// specified operands. If the resultant node already exists in the DAG,
11587/// this does not modify the specified node, instead it returns the node that
11588/// already exists. If the resultant node does not exist in the DAG, the
11589/// input node is returned. As a degenerate case, if you specify the same
11590/// input operands as the node already has, the input node is returned.
11591SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
11592 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
11593
11594 // Check to see if there is no change.
11595 if (Op == N->getOperand(Num: 0)) return N;
11596
11597 // See if the modified node already exists.
11598 void *InsertPos = nullptr;
11599 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
11600 return Existing;
11601
11602 // Nope it doesn't. Remove the node from its current place in the maps.
11603 if (InsertPos)
11604 if (!RemoveNodeFromCSEMaps(N))
11605 InsertPos = nullptr;
11606
11607 // Now we update the operands.
11608 N->OperandList[0].set(Op);
11609
11610 updateDivergence(N);
11611 // If this gets put into a CSE map, add it.
11612 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11613 return N;
11614}
11615
11616SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
11617 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
11618
11619 // Check to see if there is no change.
11620 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
11621 return N; // No operands changed, just return the input node.
11622
11623 // See if the modified node already exists.
11624 void *InsertPos = nullptr;
11625 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
11626 return Existing;
11627
11628 // Nope it doesn't. Remove the node from its current place in the maps.
11629 if (InsertPos)
11630 if (!RemoveNodeFromCSEMaps(N))
11631 InsertPos = nullptr;
11632
11633 // Now we update the operands.
11634 if (N->OperandList[0] != Op1)
11635 N->OperandList[0].set(Op1);
11636 if (N->OperandList[1] != Op2)
11637 N->OperandList[1].set(Op2);
11638
11639 updateDivergence(N);
11640 // If this gets put into a CSE map, add it.
11641 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11642 return N;
11643}
11644
11645SDNode *SelectionDAG::
11646UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
11647 SDValue Ops[] = { Op1, Op2, Op3 };
11648 return UpdateNodeOperands(N, Ops);
11649}
11650
11651SDNode *SelectionDAG::
11652UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11653 SDValue Op3, SDValue Op4) {
11654 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
11655 return UpdateNodeOperands(N, Ops);
11656}
11657
11658SDNode *SelectionDAG::
11659UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11660 SDValue Op3, SDValue Op4, SDValue Op5) {
11661 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
11662 return UpdateNodeOperands(N, Ops);
11663}
11664
11665SDNode *SelectionDAG::
11666UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
11667 unsigned NumOps = Ops.size();
11668 assert(N->getNumOperands() == NumOps &&
11669 "Update with wrong number of operands");
11670
11671 // If no operands changed just return the input node.
11672 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
11673 return N;
11674
11675 // See if the modified node already exists.
11676 void *InsertPos = nullptr;
11677 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
11678 return Existing;
11679
11680 // Nope it doesn't. Remove the node from its current place in the maps.
11681 if (InsertPos)
11682 if (!RemoveNodeFromCSEMaps(N))
11683 InsertPos = nullptr;
11684
11685 // Now we update the operands.
11686 for (unsigned i = 0; i != NumOps; ++i)
11687 if (N->OperandList[i] != Ops[i])
11688 N->OperandList[i].set(Ops[i]);
11689
11690 updateDivergence(N);
11691 // If this gets put into a CSE map, add it.
11692 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11693 return N;
11694}
11695
11696/// DropOperands - Release the operands and set this node to have
11697/// zero operands.
11698void SDNode::DropOperands() {
11699 // Unlike the code in MorphNodeTo that does this, we don't need to
11700 // watch for dead nodes here.
11701 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
11702 SDUse &Use = *I++;
11703 Use.set(SDValue());
11704 }
11705}
11706
11707void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
11708 ArrayRef<MachineMemOperand *> NewMemRefs) {
11709 if (NewMemRefs.empty()) {
11710 N->clearMemRefs();
11711 return;
11712 }
11713
11714 // Check if we can avoid allocating by storing a single reference directly.
11715 if (NewMemRefs.size() == 1) {
11716 N->MemRefs = NewMemRefs[0];
11717 N->NumMemRefs = 1;
11718 return;
11719 }
11720
11721 MachineMemOperand **MemRefsBuffer =
11722 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
11723 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
11724 N->MemRefs = MemRefsBuffer;
11725 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
11726}
11727
11728/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
11729/// machine opcode.
11730///
11731SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11732 EVT VT) {
11733 SDVTList VTs = getVTList(VT);
11734 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11735}
11736
11737SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11738 EVT VT, SDValue Op1) {
11739 SDVTList VTs = getVTList(VT);
11740 SDValue Ops[] = { Op1 };
11741 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11742}
11743
11744SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11745 EVT VT, SDValue Op1,
11746 SDValue Op2) {
11747 SDVTList VTs = getVTList(VT);
11748 SDValue Ops[] = { Op1, Op2 };
11749 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11750}
11751
11752SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11753 EVT VT, SDValue Op1,
11754 SDValue Op2, SDValue Op3) {
11755 SDVTList VTs = getVTList(VT);
11756 SDValue Ops[] = { Op1, Op2, Op3 };
11757 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11758}
11759
11760SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11761 EVT VT, ArrayRef<SDValue> Ops) {
11762 SDVTList VTs = getVTList(VT);
11763 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11764}
11765
11766SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11767 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
11768 SDVTList VTs = getVTList(VT1, VT2);
11769 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11770}
11771
11772SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11773 EVT VT1, EVT VT2) {
11774 SDVTList VTs = getVTList(VT1, VT2);
11775 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11776}
11777
11778SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11779 EVT VT1, EVT VT2, EVT VT3,
11780 ArrayRef<SDValue> Ops) {
11781 SDVTList VTs = getVTList(VT1, VT2, VT3);
11782 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11783}
11784
11785SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11786 EVT VT1, EVT VT2,
11787 SDValue Op1, SDValue Op2) {
11788 SDVTList VTs = getVTList(VT1, VT2);
11789 SDValue Ops[] = { Op1, Op2 };
11790 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11791}
11792
11793SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11794 SDVTList VTs,ArrayRef<SDValue> Ops) {
11795 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
11796 // Reset the NodeID to -1.
11797 New->setNodeId(-1);
11798 if (New != N) {
11799 ReplaceAllUsesWith(From: N, To: New);
11800 RemoveDeadNode(N);
11801 }
11802 return New;
11803}
11804
11805/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
11806/// the line number information on the merged node since it is not possible to
11807/// preserve the information that operation is associated with multiple lines.
11808/// This will make the debugger working better at -O0, were there is a higher
11809/// probability having other instructions associated with that line.
11810///
11811/// For IROrder, we keep the smaller of the two
11812SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
11813 DebugLoc NLoc = N->getDebugLoc();
11814 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
11815 N->setDebugLoc(DebugLoc());
11816 }
11817 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
11818 N->setIROrder(Order);
11819 return N;
11820}
11821
11822/// MorphNodeTo - This *mutates* the specified node to have the specified
11823/// return type, opcode, and operands.
11824///
11825/// Note that MorphNodeTo returns the resultant node. If there is already a
11826/// node of the specified opcode and operands, it returns that node instead of
11827/// the current one. Note that the SDLoc need not be the same.
11828///
11829/// Using MorphNodeTo is faster than creating a new node and swapping it in
11830/// with ReplaceAllUsesWith both because it often avoids allocating a new
11831/// node, and because it doesn't require CSE recalculation for any of
11832/// the node's users.
11833///
11834/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
11835/// As a consequence it isn't appropriate to use from within the DAG combiner or
11836/// the legalizer which maintain worklists that would need to be updated when
11837/// deleting things.
11838SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
11839 SDVTList VTs, ArrayRef<SDValue> Ops) {
11840 // If an identical node already exists, use it.
11841 void *IP = nullptr;
11842 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
11843 FoldingSetNodeID ID;
11844 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
11845 if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
11846 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
11847 }
11848
11849 if (!RemoveNodeFromCSEMaps(N))
11850 IP = nullptr;
11851
11852 // Start the morphing.
11853 N->NodeType = Opc;
11854 N->ValueList = VTs.VTs;
11855 N->NumValues = VTs.NumVTs;
11856
11857 // Clear the operands list, updating used nodes to remove this from their
11858 // use list. Keep track of any operands that become dead as a result.
11859 SmallPtrSet<SDNode*, 16> DeadNodeSet;
11860 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
11861 SDUse &Use = *I++;
11862 SDNode *Used = Use.getNode();
11863 Use.set(SDValue());
11864 if (Used->use_empty())
11865 DeadNodeSet.insert(Ptr: Used);
11866 }
11867
11868 // For MachineNode, initialize the memory references information.
11869 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
11870 MN->clearMemRefs();
11871
11872 // Swap for an appropriately sized array from the recycler.
11873 removeOperands(Node: N);
11874 createOperands(Node: N, Vals: Ops);
11875
11876 // Delete any nodes that are still dead after adding the uses for the
11877 // new operands.
11878 if (!DeadNodeSet.empty()) {
11879 SmallVector<SDNode *, 16> DeadNodes;
11880 for (SDNode *N : DeadNodeSet)
11881 if (N->use_empty())
11882 DeadNodes.push_back(Elt: N);
11883 RemoveDeadNodes(DeadNodes);
11884 }
11885
11886 if (IP)
11887 CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
11888 return N;
11889}
11890
11891SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
11892 unsigned OrigOpc = Node->getOpcode();
11893 unsigned NewOpc;
11894 switch (OrigOpc) {
11895 default:
11896 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
11897#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11898 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
11899#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11900 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
11901#include "llvm/IR/ConstrainedOps.def"
11902 }
11903
11904 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
11905
11906 // We're taking this node out of the chain, so we need to re-link things.
11907 SDValue InputChain = Node->getOperand(Num: 0);
11908 SDValue OutputChain = SDValue(Node, 1);
11909 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
11910
11911 SmallVector<SDValue, 3> Ops;
11912 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
11913 Ops.push_back(Elt: Node->getOperand(Num: i));
11914
11915 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
11916 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
11917
11918 // MorphNodeTo can operate in two ways: if an existing node with the
11919 // specified operands exists, it can just return it. Otherwise, it
11920 // updates the node in place to have the requested operands.
11921 if (Res == Node) {
11922 // If we updated the node in place, reset the node ID. To the isel,
11923 // this should be just like a newly allocated machine node.
11924 Res->setNodeId(-1);
11925 } else {
11926 ReplaceAllUsesWith(From: Node, To: Res);
11927 RemoveDeadNode(N: Node);
11928 }
11929
11930 return Res;
11931}
11932
11933/// getMachineNode - These are used for target selectors to create a new node
11934/// with specified return type(s), MachineInstr opcode, and operands.
11935///
11936/// Note that getMachineNode returns the resultant node. If there is already a
11937/// node of the specified opcode and operands, it returns that node instead of
11938/// the current one.
11939MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11940 EVT VT) {
11941 SDVTList VTs = getVTList(VT);
11942 return getMachineNode(Opcode, dl, VTs, Ops: {});
11943}
11944
11945MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11946 EVT VT, SDValue Op1) {
11947 SDVTList VTs = getVTList(VT);
11948 SDValue Ops[] = { Op1 };
11949 return getMachineNode(Opcode, dl, VTs, Ops);
11950}
11951
11952MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11953 EVT VT, SDValue Op1, SDValue Op2) {
11954 SDVTList VTs = getVTList(VT);
11955 SDValue Ops[] = { Op1, Op2 };
11956 return getMachineNode(Opcode, dl, VTs, Ops);
11957}
11958
11959MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11960 EVT VT, SDValue Op1, SDValue Op2,
11961 SDValue Op3) {
11962 SDVTList VTs = getVTList(VT);
11963 SDValue Ops[] = { Op1, Op2, Op3 };
11964 return getMachineNode(Opcode, dl, VTs, Ops);
11965}
11966
11967MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11968 EVT VT, ArrayRef<SDValue> Ops) {
11969 SDVTList VTs = getVTList(VT);
11970 return getMachineNode(Opcode, dl, VTs, Ops);
11971}
11972
11973MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11974 EVT VT1, EVT VT2, SDValue Op1,
11975 SDValue Op2) {
11976 SDVTList VTs = getVTList(VT1, VT2);
11977 SDValue Ops[] = { Op1, Op2 };
11978 return getMachineNode(Opcode, dl, VTs, Ops);
11979}
11980
11981MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11982 EVT VT1, EVT VT2, SDValue Op1,
11983 SDValue Op2, SDValue Op3) {
11984 SDVTList VTs = getVTList(VT1, VT2);
11985 SDValue Ops[] = { Op1, Op2, Op3 };
11986 return getMachineNode(Opcode, dl, VTs, Ops);
11987}
11988
11989MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11990 EVT VT1, EVT VT2,
11991 ArrayRef<SDValue> Ops) {
11992 SDVTList VTs = getVTList(VT1, VT2);
11993 return getMachineNode(Opcode, dl, VTs, Ops);
11994}
11995
11996MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11997 EVT VT1, EVT VT2, EVT VT3,
11998 SDValue Op1, SDValue Op2) {
11999 SDVTList VTs = getVTList(VT1, VT2, VT3);
12000 SDValue Ops[] = { Op1, Op2 };
12001 return getMachineNode(Opcode, dl, VTs, Ops);
12002}
12003
12004MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12005 EVT VT1, EVT VT2, EVT VT3,
12006 SDValue Op1, SDValue Op2,
12007 SDValue Op3) {
12008 SDVTList VTs = getVTList(VT1, VT2, VT3);
12009 SDValue Ops[] = { Op1, Op2, Op3 };
12010 return getMachineNode(Opcode, dl, VTs, Ops);
12011}
12012
12013MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12014 EVT VT1, EVT VT2, EVT VT3,
12015 ArrayRef<SDValue> Ops) {
12016 SDVTList VTs = getVTList(VT1, VT2, VT3);
12017 return getMachineNode(Opcode, dl, VTs, Ops);
12018}
12019
12020MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12021 ArrayRef<EVT> ResultTys,
12022 ArrayRef<SDValue> Ops) {
12023 SDVTList VTs = getVTList(VTs: ResultTys);
12024 return getMachineNode(Opcode, dl, VTs, Ops);
12025}
12026
12027MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12028 SDVTList VTs,
12029 ArrayRef<SDValue> Ops) {
12030 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12031 MachineSDNode *N;
12032 void *IP = nullptr;
12033
12034 if (DoCSE) {
12035 FoldingSetNodeID ID;
12036 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
12037 IP = nullptr;
12038 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
12039 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12040 }
12041 }
12042
12043 // Allocate a new MachineSDNode.
12044 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12045 createOperands(Node: N, Vals: Ops);
12046
12047 if (DoCSE)
12048 CSEMap.InsertNode(N, InsertPos: IP);
12049
12050 InsertNode(N);
12051 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12052 return N;
12053}
12054
12055/// getTargetExtractSubreg - A convenience function for creating
12056/// TargetOpcode::EXTRACT_SUBREG nodes.
12057SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12058 SDValue Operand) {
12059 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12060 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12061 VT, Op1: Operand, Op2: SRIdxVal);
12062 return SDValue(Subreg, 0);
12063}
12064
12065/// getTargetInsertSubreg - A convenience function for creating
12066/// TargetOpcode::INSERT_SUBREG nodes.
12067SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12068 SDValue Operand, SDValue Subreg) {
12069 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12070 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12071 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12072 return SDValue(Result, 0);
12073}
12074
12075/// getNodeIfExists - Get the specified node if it's already available, or
12076/// else return NULL.
12077SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12078 ArrayRef<SDValue> Ops,
12079 bool AllowCommute) {
12080 SDNodeFlags Flags;
12081 if (Inserter)
12082 Flags = Inserter->getFlags();
12083 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12084}
12085
12086SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12087 ArrayRef<SDValue> Ops,
12088 const SDNodeFlags Flags,
12089 bool AllowCommute) {
12090 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12091 return nullptr;
12092
12093 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12094 FoldingSetNodeID ID;
12095 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: LookupOps);
12096 void *IP = nullptr;
12097 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) {
12098 E->intersectFlagsWith(Flags);
12099 return E;
12100 }
12101 return nullptr;
12102 };
12103
12104 if (SDNode *Existing = Lookup(Ops))
12105 return Existing;
12106
12107 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12108 return Lookup({Ops[1], Ops[0]});
12109
12110 return nullptr;
12111}
12112
12113/// doesNodeExist - Check if a node exists without modifying its flags.
12114bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12115 ArrayRef<SDValue> Ops) {
12116 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12117 FoldingSetNodeID ID;
12118 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12119 void *IP = nullptr;
12120 if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
12121 return true;
12122 }
12123 return false;
12124}
12125
12126/// getDbgValue - Creates a SDDbgValue node.
12127///
12128/// SDNode
12129SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12130 SDNode *N, unsigned R, bool IsIndirect,
12131 const DebugLoc &DL, unsigned O) {
12132 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12133 "Expected inlined-at fields to agree");
12134 return new (DbgInfo->getAlloc())
12135 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12136 {}, IsIndirect, DL, O,
12137 /*IsVariadic=*/false);
12138}
12139
12140/// Constant
12141SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12142 DIExpression *Expr,
12143 const Value *C,
12144 const DebugLoc &DL, unsigned O) {
12145 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12146 "Expected inlined-at fields to agree");
12147 return new (DbgInfo->getAlloc())
12148 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12149 /*IsIndirect=*/false, DL, O,
12150 /*IsVariadic=*/false);
12151}
12152
12153/// FrameIndex
12154SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12155 DIExpression *Expr, unsigned FI,
12156 bool IsIndirect,
12157 const DebugLoc &DL,
12158 unsigned O) {
12159 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12160 "Expected inlined-at fields to agree");
12161 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12162}
12163
12164/// FrameIndex with dependencies
12165SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12166 DIExpression *Expr, unsigned FI,
12167 ArrayRef<SDNode *> Dependencies,
12168 bool IsIndirect,
12169 const DebugLoc &DL,
12170 unsigned O) {
12171 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12172 "Expected inlined-at fields to agree");
12173 return new (DbgInfo->getAlloc())
12174 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12175 Dependencies, IsIndirect, DL, O,
12176 /*IsVariadic=*/false);
12177}
12178
12179/// VReg
12180SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12181 Register VReg, bool IsIndirect,
12182 const DebugLoc &DL, unsigned O) {
12183 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12184 "Expected inlined-at fields to agree");
12185 return new (DbgInfo->getAlloc())
12186 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12187 {}, IsIndirect, DL, O,
12188 /*IsVariadic=*/false);
12189}
12190
12191SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12192 ArrayRef<SDDbgOperand> Locs,
12193 ArrayRef<SDNode *> Dependencies,
12194 bool IsIndirect, const DebugLoc &DL,
12195 unsigned O, bool IsVariadic) {
12196 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12197 "Expected inlined-at fields to agree");
12198 return new (DbgInfo->getAlloc())
12199 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12200 DL, O, IsVariadic);
12201}
12202
12203void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12204 unsigned OffsetInBits, unsigned SizeInBits,
12205 bool InvalidateDbg) {
12206 SDNode *FromNode = From.getNode();
12207 SDNode *ToNode = To.getNode();
12208 assert(FromNode && ToNode && "Can't modify dbg values");
12209
12210 // PR35338
12211 // TODO: assert(From != To && "Redundant dbg value transfer");
12212 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12213 if (From == To || FromNode == ToNode)
12214 return;
12215
12216 if (!FromNode->getHasDebugValue())
12217 return;
12218
12219 SDDbgOperand FromLocOp =
12220 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12221 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12222
12223 SmallVector<SDDbgValue *, 2> ClonedDVs;
12224 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12225 if (Dbg->isInvalidated())
12226 continue;
12227
12228 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12229
12230 // Create a new location ops vector that is equal to the old vector, but
12231 // with each instance of FromLocOp replaced with ToLocOp.
12232 bool Changed = false;
12233 auto NewLocOps = Dbg->copyLocationOps();
12234 std::replace_if(
12235 first: NewLocOps.begin(), last: NewLocOps.end(),
12236 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12237 bool Match = Op == FromLocOp;
12238 Changed |= Match;
12239 return Match;
12240 },
12241 new_value: ToLocOp);
12242 // Ignore this SDDbgValue if we didn't find a matching location.
12243 if (!Changed)
12244 continue;
12245
12246 DIVariable *Var = Dbg->getVariable();
12247 auto *Expr = Dbg->getExpression();
12248 // If a fragment is requested, update the expression.
12249 if (SizeInBits) {
12250 // When splitting a larger (e.g., sign-extended) value whose
12251 // lower bits are described with an SDDbgValue, do not attempt
12252 // to transfer the SDDbgValue to the upper bits.
12253 if (auto FI = Expr->getFragmentInfo())
12254 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12255 continue;
12256 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12257 SizeInBits);
12258 if (!Fragment)
12259 continue;
12260 Expr = *Fragment;
12261 }
12262
12263 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12264 // Clone the SDDbgValue and move it to To.
12265 SDDbgValue *Clone = getDbgValueList(
12266 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12267 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12268 IsVariadic: Dbg->isVariadic());
12269 ClonedDVs.push_back(Elt: Clone);
12270
12271 if (InvalidateDbg) {
12272 // Invalidate value and indicate the SDDbgValue should not be emitted.
12273 Dbg->setIsInvalidated();
12274 Dbg->setIsEmitted();
12275 }
12276 }
12277
12278 for (SDDbgValue *Dbg : ClonedDVs) {
12279 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12280 "Transferred DbgValues should depend on the new SDNode");
12281 AddDbgValue(DB: Dbg, isParameter: false);
12282 }
12283}
12284
12285void SelectionDAG::salvageDebugInfo(SDNode &N) {
12286 if (!N.getHasDebugValue())
12287 return;
12288
12289 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
12290 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
12291 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
12292 return SDDbgOperand::fromNode(Node, ResNo);
12293 };
12294
12295 SmallVector<SDDbgValue *, 2> ClonedDVs;
12296 for (auto *DV : GetDbgValues(SD: &N)) {
12297 if (DV->isInvalidated())
12298 continue;
12299 switch (N.getOpcode()) {
12300 default:
12301 break;
12302 case ISD::ADD: {
12303 SDValue N0 = N.getOperand(Num: 0);
12304 SDValue N1 = N.getOperand(Num: 1);
12305 if (!isa<ConstantSDNode>(Val: N0)) {
12306 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
12307 uint64_t Offset;
12308 if (RHSConstant)
12309 Offset = N.getConstantOperandVal(Num: 1);
12310 // We are not allowed to turn indirect debug values variadic, so
12311 // don't salvage those.
12312 if (!RHSConstant && DV->isIndirect())
12313 continue;
12314
12315 // Rewrite an ADD constant node into a DIExpression. Since we are
12316 // performing arithmetic to compute the variable's *value* in the
12317 // DIExpression, we need to mark the expression with a
12318 // DW_OP_stack_value.
12319 auto *DIExpr = DV->getExpression();
12320 auto NewLocOps = DV->copyLocationOps();
12321 bool Changed = false;
12322 size_t OrigLocOpsSize = NewLocOps.size();
12323 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
12324 // We're not given a ResNo to compare against because the whole
12325 // node is going away. We know that any ISD::ADD only has one
12326 // result, so we can assume any node match is using the result.
12327 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12328 NewLocOps[i].getSDNode() != &N)
12329 continue;
12330 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12331 if (RHSConstant) {
12332 SmallVector<uint64_t, 3> ExprOps;
12333 DIExpression::appendOffset(Ops&: ExprOps, Offset);
12334 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12335 } else {
12336 // Convert to a variadic expression (if not already).
12337 // convertToVariadicExpression() returns a const pointer, so we use
12338 // a temporary const variable here.
12339 const auto *TmpDIExpr =
12340 DIExpression::convertToVariadicExpression(Expr: DIExpr);
12341 SmallVector<uint64_t, 3> ExprOps;
12342 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
12343 ExprOps.push_back(Elt: NewLocOps.size());
12344 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
12345 SDDbgOperand RHS =
12346 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
12347 NewLocOps.push_back(Elt: RHS);
12348 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12349 }
12350 Changed = true;
12351 }
12352 (void)Changed;
12353 assert(Changed && "Salvage target doesn't use N");
12354
12355 bool IsVariadic =
12356 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
12357
12358 auto AdditionalDependencies = DV->getAdditionalDependencies();
12359 SDDbgValue *Clone = getDbgValueList(
12360 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
12361 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
12362 ClonedDVs.push_back(Elt: Clone);
12363 DV->setIsInvalidated();
12364 DV->setIsEmitted();
12365 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
12366 N0.getNode()->dumprFull(this);
12367 dbgs() << " into " << *DIExpr << '\n');
12368 }
12369 break;
12370 }
12371 case ISD::TRUNCATE: {
12372 SDValue N0 = N.getOperand(Num: 0);
12373 TypeSize FromSize = N0.getValueSizeInBits();
12374 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
12375
12376 DIExpression *DbgExpression = DV->getExpression();
12377 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
12378 auto NewLocOps = DV->copyLocationOps();
12379 bool Changed = false;
12380 for (size_t i = 0; i < NewLocOps.size(); ++i) {
12381 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12382 NewLocOps[i].getSDNode() != &N)
12383 continue;
12384
12385 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12386 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
12387 Changed = true;
12388 }
12389 assert(Changed && "Salvage target doesn't use N");
12390 (void)Changed;
12391
12392 SDDbgValue *Clone =
12393 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
12394 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
12395 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
12396
12397 ClonedDVs.push_back(Elt: Clone);
12398 DV->setIsInvalidated();
12399 DV->setIsEmitted();
12400 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
12401 dbgs() << " into " << *DbgExpression << '\n');
12402 break;
12403 }
12404 }
12405 }
12406
12407 for (SDDbgValue *Dbg : ClonedDVs) {
12408 assert((!Dbg->getSDNodes().empty() ||
12409 llvm::any_of(Dbg->getLocationOps(),
12410 [&](const SDDbgOperand &Op) {
12411 return Op.getKind() == SDDbgOperand::FRAMEIX;
12412 })) &&
12413 "Salvaged DbgValue should depend on a new SDNode");
12414 AddDbgValue(DB: Dbg, isParameter: false);
12415 }
12416}
12417
12418/// Creates a SDDbgLabel node.
12419SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
12420 const DebugLoc &DL, unsigned O) {
12421 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
12422 "Expected inlined-at fields to agree");
12423 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
12424}
12425
12426namespace {
12427
12428/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
12429/// pointed to by a use iterator is deleted, increment the use iterator
12430/// so that it doesn't dangle.
12431///
12432class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
12433 SDNode::use_iterator &UI;
12434 SDNode::use_iterator &UE;
12435
12436 void NodeDeleted(SDNode *N, SDNode *E) override {
12437 // Increment the iterator as needed.
12438 while (UI != UE && N == UI->getUser())
12439 ++UI;
12440 }
12441
12442public:
12443 RAUWUpdateListener(SelectionDAG &d,
12444 SDNode::use_iterator &ui,
12445 SDNode::use_iterator &ue)
12446 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
12447};
12448
12449} // end anonymous namespace
12450
12451/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12452/// This can cause recursive merging of nodes in the DAG.
12453///
12454/// This version assumes From has a single result value.
12455///
12456void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
12457 SDNode *From = FromN.getNode();
12458 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
12459 "Cannot replace with this method!");
12460 assert(From != To.getNode() && "Cannot replace uses of with self");
12461
12462 // Preserve Debug Values
12463 transferDbgValues(From: FromN, To);
12464 // Preserve extra info.
12465 copyExtraInfo(From, To: To.getNode());
12466
12467 // Iterate over all the existing uses of From. New uses will be added
12468 // to the beginning of the use list, which we avoid visiting.
12469 // This specifically avoids visiting uses of From that arise while the
12470 // replacement is happening, because any such uses would be the result
12471 // of CSE: If an existing node looks like From after one of its operands
12472 // is replaced by To, we don't want to replace of all its users with To
12473 // too. See PR3018 for more info.
12474 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12475 RAUWUpdateListener Listener(*this, UI, UE);
12476 while (UI != UE) {
12477 SDNode *User = UI->getUser();
12478
12479 // This node is about to morph, remove its old self from the CSE maps.
12480 RemoveNodeFromCSEMaps(N: User);
12481
12482 // A user can appear in a use list multiple times, and when this
12483 // happens the uses are usually next to each other in the list.
12484 // To help reduce the number of CSE recomputations, process all
12485 // the uses of this user that we can find this way.
12486 do {
12487 SDUse &Use = *UI;
12488 ++UI;
12489 Use.set(To);
12490 if (To->isDivergent() != From->isDivergent())
12491 updateDivergence(N: User);
12492 } while (UI != UE && UI->getUser() == User);
12493 // Now that we have modified User, add it back to the CSE maps. If it
12494 // already exists there, recursively merge the results together.
12495 AddModifiedNodeToCSEMaps(N: User);
12496 }
12497
12498 // If we just RAUW'd the root, take note.
12499 if (FromN == getRoot())
12500 setRoot(To);
12501}
12502
12503/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12504/// This can cause recursive merging of nodes in the DAG.
12505///
12506/// This version assumes that for each value of From, there is a
12507/// corresponding value in To in the same position with the same type.
12508///
12509void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
12510#ifndef NDEBUG
12511 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12512 assert((!From->hasAnyUseOfValue(i) ||
12513 From->getValueType(i) == To->getValueType(i)) &&
12514 "Cannot use this version of ReplaceAllUsesWith!");
12515#endif
12516
12517 // Handle the trivial case.
12518 if (From == To)
12519 return;
12520
12521 // Preserve Debug Info. Only do this if there's a use.
12522 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12523 if (From->hasAnyUseOfValue(Value: i)) {
12524 assert((i < To->getNumValues()) && "Invalid To location");
12525 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
12526 }
12527 // Preserve extra info.
12528 copyExtraInfo(From, To);
12529
12530 // Iterate over just the existing users of From. See the comments in
12531 // the ReplaceAllUsesWith above.
12532 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12533 RAUWUpdateListener Listener(*this, UI, UE);
12534 while (UI != UE) {
12535 SDNode *User = UI->getUser();
12536
12537 // This node is about to morph, remove its old self from the CSE maps.
12538 RemoveNodeFromCSEMaps(N: User);
12539
12540 // A user can appear in a use list multiple times, and when this
12541 // happens the uses are usually next to each other in the list.
12542 // To help reduce the number of CSE recomputations, process all
12543 // the uses of this user that we can find this way.
12544 do {
12545 SDUse &Use = *UI;
12546 ++UI;
12547 Use.setNode(To);
12548 if (To->isDivergent() != From->isDivergent())
12549 updateDivergence(N: User);
12550 } while (UI != UE && UI->getUser() == User);
12551
12552 // Now that we have modified User, add it back to the CSE maps. If it
12553 // already exists there, recursively merge the results together.
12554 AddModifiedNodeToCSEMaps(N: User);
12555 }
12556
12557 // If we just RAUW'd the root, take note.
12558 if (From == getRoot().getNode())
12559 setRoot(SDValue(To, getRoot().getResNo()));
12560}
12561
12562/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12563/// This can cause recursive merging of nodes in the DAG.
12564///
12565/// This version can replace From with any result values. To must match the
12566/// number and types of values returned by From.
12567void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
12568 if (From->getNumValues() == 1) // Handle the simple case efficiently.
12569 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
12570
12571 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
12572 // Preserve Debug Info.
12573 transferDbgValues(From: SDValue(From, i), To: To[i]);
12574 // Preserve extra info.
12575 copyExtraInfo(From, To: To[i].getNode());
12576 }
12577
12578 // Iterate over just the existing users of From. See the comments in
12579 // the ReplaceAllUsesWith above.
12580 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12581 RAUWUpdateListener Listener(*this, UI, UE);
12582 while (UI != UE) {
12583 SDNode *User = UI->getUser();
12584
12585 // This node is about to morph, remove its old self from the CSE maps.
12586 RemoveNodeFromCSEMaps(N: User);
12587
12588 // A user can appear in a use list multiple times, and when this happens the
12589 // uses are usually next to each other in the list. To help reduce the
12590 // number of CSE and divergence recomputations, process all the uses of this
12591 // user that we can find this way.
12592 bool To_IsDivergent = false;
12593 do {
12594 SDUse &Use = *UI;
12595 const SDValue &ToOp = To[Use.getResNo()];
12596 ++UI;
12597 Use.set(ToOp);
12598 if (ToOp.getValueType() != MVT::Other)
12599 To_IsDivergent |= ToOp->isDivergent();
12600 } while (UI != UE && UI->getUser() == User);
12601
12602 if (To_IsDivergent != From->isDivergent())
12603 updateDivergence(N: User);
12604
12605 // Now that we have modified User, add it back to the CSE maps. If it
12606 // already exists there, recursively merge the results together.
12607 AddModifiedNodeToCSEMaps(N: User);
12608 }
12609
12610 // If we just RAUW'd the root, take note.
12611 if (From == getRoot().getNode())
12612 setRoot(SDValue(To[getRoot().getResNo()]));
12613}
12614
12615/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
12616/// uses of other values produced by From.getNode() alone. The Deleted
12617/// vector is handled the same way as for ReplaceAllUsesWith.
12618void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
12619 // Handle the really simple, really trivial case efficiently.
12620 if (From == To) return;
12621
12622 // Handle the simple, trivial, case efficiently.
12623 if (From.getNode()->getNumValues() == 1) {
12624 ReplaceAllUsesWith(FromN: From, To);
12625 return;
12626 }
12627
12628 // Preserve Debug Info.
12629 transferDbgValues(From, To);
12630 copyExtraInfo(From: From.getNode(), To: To.getNode());
12631
12632 // Iterate over just the existing users of From. See the comments in
12633 // the ReplaceAllUsesWith above.
12634 SDNode::use_iterator UI = From.getNode()->use_begin(),
12635 UE = From.getNode()->use_end();
12636 RAUWUpdateListener Listener(*this, UI, UE);
12637 while (UI != UE) {
12638 SDNode *User = UI->getUser();
12639 bool UserRemovedFromCSEMaps = false;
12640
12641 // A user can appear in a use list multiple times, and when this
12642 // happens the uses are usually next to each other in the list.
12643 // To help reduce the number of CSE recomputations, process all
12644 // the uses of this user that we can find this way.
12645 do {
12646 SDUse &Use = *UI;
12647
12648 // Skip uses of different values from the same node.
12649 if (Use.getResNo() != From.getResNo()) {
12650 ++UI;
12651 continue;
12652 }
12653
12654 // If this node hasn't been modified yet, it's still in the CSE maps,
12655 // so remove its old self from the CSE maps.
12656 if (!UserRemovedFromCSEMaps) {
12657 RemoveNodeFromCSEMaps(N: User);
12658 UserRemovedFromCSEMaps = true;
12659 }
12660
12661 ++UI;
12662 Use.set(To);
12663 if (To->isDivergent() != From->isDivergent())
12664 updateDivergence(N: User);
12665 } while (UI != UE && UI->getUser() == User);
12666 // We are iterating over all uses of the From node, so if a use
12667 // doesn't use the specific value, no changes are made.
12668 if (!UserRemovedFromCSEMaps)
12669 continue;
12670
12671 // Now that we have modified User, add it back to the CSE maps. If it
12672 // already exists there, recursively merge the results together.
12673 AddModifiedNodeToCSEMaps(N: User);
12674 }
12675
12676 // If we just RAUW'd the root, take note.
12677 if (From == getRoot())
12678 setRoot(To);
12679}
12680
12681namespace {
12682
12683/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
12684/// to record information about a use.
12685struct UseMemo {
12686 SDNode *User;
12687 unsigned Index;
12688 SDUse *Use;
12689};
12690
12691/// operator< - Sort Memos by User.
12692bool operator<(const UseMemo &L, const UseMemo &R) {
12693 return (intptr_t)L.User < (intptr_t)R.User;
12694}
12695
12696/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
12697/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
12698/// the node already has been taken care of recursively.
12699class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
12700 SmallVectorImpl<UseMemo> &Uses;
12701
12702 void NodeDeleted(SDNode *N, SDNode *E) override {
12703 for (UseMemo &Memo : Uses)
12704 if (Memo.User == N)
12705 Memo.User = nullptr;
12706 }
12707
12708public:
12709 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
12710 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
12711};
12712
12713} // end anonymous namespace
12714
12715/// Return true if a glue output should propagate divergence information.
12716static bool gluePropagatesDivergence(const SDNode *Node) {
12717 switch (Node->getOpcode()) {
12718 case ISD::CopyFromReg:
12719 case ISD::CopyToReg:
12720 return false;
12721 default:
12722 return true;
12723 }
12724
12725 llvm_unreachable("covered opcode switch");
12726}
12727
12728bool SelectionDAG::calculateDivergence(SDNode *N) {
12729 if (TLI->isSDNodeAlwaysUniform(N)) {
12730 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
12731 "Conflicting divergence information!");
12732 return false;
12733 }
12734 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
12735 return true;
12736 for (const auto &Op : N->ops()) {
12737 EVT VT = Op.getValueType();
12738
12739 // Skip Chain. It does not carry divergence.
12740 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
12741 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
12742 return true;
12743 }
12744 return false;
12745}
12746
12747void SelectionDAG::updateDivergence(SDNode *N) {
12748 SmallVector<SDNode *, 16> Worklist(1, N);
12749 do {
12750 N = Worklist.pop_back_val();
12751 bool IsDivergent = calculateDivergence(N);
12752 if (N->SDNodeBits.IsDivergent != IsDivergent) {
12753 N->SDNodeBits.IsDivergent = IsDivergent;
12754 llvm::append_range(C&: Worklist, R: N->users());
12755 }
12756 } while (!Worklist.empty());
12757}
12758
12759void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
12760 DenseMap<SDNode *, unsigned> Degree;
12761 Order.reserve(n: AllNodes.size());
12762 for (auto &N : allnodes()) {
12763 unsigned NOps = N.getNumOperands();
12764 Degree[&N] = NOps;
12765 if (0 == NOps)
12766 Order.push_back(x: &N);
12767 }
12768 for (size_t I = 0; I != Order.size(); ++I) {
12769 SDNode *N = Order[I];
12770 for (auto *U : N->users()) {
12771 unsigned &UnsortedOps = Degree[U];
12772 if (0 == --UnsortedOps)
12773 Order.push_back(x: U);
12774 }
12775 }
12776}
12777
12778#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
12779void SelectionDAG::VerifyDAGDivergence() {
12780 std::vector<SDNode *> TopoOrder;
12781 CreateTopologicalOrder(TopoOrder);
12782 for (auto *N : TopoOrder) {
12783 assert(calculateDivergence(N) == N->isDivergent() &&
12784 "Divergence bit inconsistency detected");
12785 }
12786}
12787#endif
12788
12789/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
12790/// uses of other values produced by From.getNode() alone. The same value
12791/// may appear in both the From and To list. The Deleted vector is
12792/// handled the same way as for ReplaceAllUsesWith.
12793void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
12794 const SDValue *To,
12795 unsigned Num){
12796 // Handle the simple, trivial case efficiently.
12797 if (Num == 1)
12798 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
12799
12800 transferDbgValues(From: *From, To: *To);
12801 copyExtraInfo(From: From->getNode(), To: To->getNode());
12802
12803 // Read up all the uses and make records of them. This helps
12804 // processing new uses that are introduced during the
12805 // replacement process.
12806 SmallVector<UseMemo, 4> Uses;
12807 for (unsigned i = 0; i != Num; ++i) {
12808 unsigned FromResNo = From[i].getResNo();
12809 SDNode *FromNode = From[i].getNode();
12810 for (SDUse &Use : FromNode->uses()) {
12811 if (Use.getResNo() == FromResNo) {
12812 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
12813 Uses.push_back(Elt: Memo);
12814 }
12815 }
12816 }
12817
12818 // Sort the uses, so that all the uses from a given User are together.
12819 llvm::sort(C&: Uses);
12820 RAUOVWUpdateListener Listener(*this, Uses);
12821
12822 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
12823 UseIndex != UseIndexEnd; ) {
12824 // We know that this user uses some value of From. If it is the right
12825 // value, update it.
12826 SDNode *User = Uses[UseIndex].User;
12827 // If the node has been deleted by recursive CSE updates when updating
12828 // another node, then just skip this entry.
12829 if (User == nullptr) {
12830 ++UseIndex;
12831 continue;
12832 }
12833
12834 // This node is about to morph, remove its old self from the CSE maps.
12835 RemoveNodeFromCSEMaps(N: User);
12836
12837 // The Uses array is sorted, so all the uses for a given User
12838 // are next to each other in the list.
12839 // To help reduce the number of CSE recomputations, process all
12840 // the uses of this user that we can find this way.
12841 do {
12842 unsigned i = Uses[UseIndex].Index;
12843 SDUse &Use = *Uses[UseIndex].Use;
12844 ++UseIndex;
12845
12846 Use.set(To[i]);
12847 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
12848
12849 // Now that we have modified User, add it back to the CSE maps. If it
12850 // already exists there, recursively merge the results together.
12851 AddModifiedNodeToCSEMaps(N: User);
12852 }
12853}
12854
12855/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
12856/// based on their topological order. It returns the maximum id and a vector
12857/// of the SDNodes* in assigned order by reference.
12858unsigned SelectionDAG::AssignTopologicalOrder() {
12859 unsigned DAGSize = 0;
12860
12861 // SortedPos tracks the progress of the algorithm. Nodes before it are
12862 // sorted, nodes after it are unsorted. When the algorithm completes
12863 // it is at the end of the list.
12864 allnodes_iterator SortedPos = allnodes_begin();
12865
12866 // Visit all the nodes. Move nodes with no operands to the front of
12867 // the list immediately. Annotate nodes that do have operands with their
12868 // operand count. Before we do this, the Node Id fields of the nodes
12869 // may contain arbitrary values. After, the Node Id fields for nodes
12870 // before SortedPos will contain the topological sort index, and the
12871 // Node Id fields for nodes At SortedPos and after will contain the
12872 // count of outstanding operands.
12873 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
12874 checkForCycles(N: &N, DAG: this);
12875 unsigned Degree = N.getNumOperands();
12876 if (Degree == 0) {
12877 // A node with no uses, add it to the result array immediately.
12878 N.setNodeId(DAGSize++);
12879 allnodes_iterator Q(&N);
12880 if (Q != SortedPos)
12881 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
12882 assert(SortedPos != AllNodes.end() && "Overran node list");
12883 ++SortedPos;
12884 } else {
12885 // Temporarily use the Node Id as scratch space for the degree count.
12886 N.setNodeId(Degree);
12887 }
12888 }
12889
12890 // Visit all the nodes. As we iterate, move nodes into sorted order,
12891 // such that by the time the end is reached all nodes will be sorted.
12892 for (SDNode &Node : allnodes()) {
12893 SDNode *N = &Node;
12894 checkForCycles(N, DAG: this);
12895 // N is in sorted position, so all its uses have one less operand
12896 // that needs to be sorted.
12897 for (SDNode *P : N->users()) {
12898 unsigned Degree = P->getNodeId();
12899 assert(Degree != 0 && "Invalid node degree");
12900 --Degree;
12901 if (Degree == 0) {
12902 // All of P's operands are sorted, so P may sorted now.
12903 P->setNodeId(DAGSize++);
12904 if (P->getIterator() != SortedPos)
12905 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
12906 assert(SortedPos != AllNodes.end() && "Overran node list");
12907 ++SortedPos;
12908 } else {
12909 // Update P's outstanding operand count.
12910 P->setNodeId(Degree);
12911 }
12912 }
12913 if (Node.getIterator() == SortedPos) {
12914#ifndef NDEBUG
12915 allnodes_iterator I(N);
12916 SDNode *S = &*++I;
12917 dbgs() << "Overran sorted position:\n";
12918 S->dumprFull(this); dbgs() << "\n";
12919 dbgs() << "Checking if this is due to cycles\n";
12920 checkForCycles(this, true);
12921#endif
12922 llvm_unreachable(nullptr);
12923 }
12924 }
12925
12926 assert(SortedPos == AllNodes.end() &&
12927 "Topological sort incomplete!");
12928 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
12929 "First node in topological sort is not the entry token!");
12930 assert(AllNodes.front().getNodeId() == 0 &&
12931 "First node in topological sort has non-zero id!");
12932 assert(AllNodes.front().getNumOperands() == 0 &&
12933 "First node in topological sort has operands!");
12934 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
12935 "Last node in topologic sort has unexpected id!");
12936 assert(AllNodes.back().use_empty() &&
12937 "Last node in topologic sort has users!");
12938 assert(DAGSize == allnodes_size() && "Node count mismatch!");
12939 return DAGSize;
12940}
12941
12942void SelectionDAG::getTopologicallyOrderedNodes(
12943 SmallVectorImpl<const SDNode *> &SortedNodes) const {
12944 SortedNodes.clear();
12945 // Node -> remaining number of outstanding operands.
12946 DenseMap<const SDNode *, unsigned> RemainingOperands;
12947
12948 // Put nodes without any operands into SortedNodes first.
12949 for (const SDNode &N : allnodes()) {
12950 checkForCycles(N: &N, DAG: this);
12951 unsigned NumOperands = N.getNumOperands();
12952 if (NumOperands == 0)
12953 SortedNodes.push_back(Elt: &N);
12954 else
12955 // Record their total number of outstanding operands.
12956 RemainingOperands[&N] = NumOperands;
12957 }
12958
12959 // A node is pushed into SortedNodes when all of its operands (predecessors in
12960 // the graph) are also in SortedNodes.
12961 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
12962 const SDNode *N = SortedNodes[i];
12963 for (const SDNode *U : N->users()) {
12964 // HandleSDNode is never part of a DAG and therefore has no entry in
12965 // RemainingOperands.
12966 if (U->getOpcode() == ISD::HANDLENODE)
12967 continue;
12968 unsigned &NumRemOperands = RemainingOperands[U];
12969 assert(NumRemOperands && "Invalid number of remaining operands");
12970 --NumRemOperands;
12971 if (!NumRemOperands)
12972 SortedNodes.push_back(Elt: U);
12973 }
12974 }
12975
12976 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
12977 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
12978 "First node in topological sort is not the entry token");
12979 assert(SortedNodes.front()->getNumOperands() == 0 &&
12980 "First node in topological sort has operands");
12981}
12982
12983/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
12984/// value is produced by SD.
12985void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
12986 for (SDNode *SD : DB->getSDNodes()) {
12987 if (!SD)
12988 continue;
12989 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
12990 SD->setHasDebugValue(true);
12991 }
12992 DbgInfo->add(V: DB, isParameter);
12993}
12994
12995void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
12996
12997SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
12998 SDValue NewMemOpChain) {
12999 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13000 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13001 // The new memory operation must have the same position as the old load in
13002 // terms of memory dependency. Create a TokenFactor for the old load and new
13003 // memory operation and update uses of the old load's output chain to use that
13004 // TokenFactor.
13005 if (OldChain == NewMemOpChain || OldChain.use_empty())
13006 return NewMemOpChain;
13007
13008 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13009 N1: OldChain, N2: NewMemOpChain);
13010 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13011 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13012 return TokenFactor;
13013}
13014
13015SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13016 SDValue NewMemOp) {
13017 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13018 SDValue OldChain = SDValue(OldLoad, 1);
13019 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13020 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13021}
13022
13023SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13024 Function **OutFunction) {
13025 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13026
13027 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13028 auto *Module = MF->getFunction().getParent();
13029 auto *Function = Module->getFunction(Name: Symbol);
13030
13031 if (OutFunction != nullptr)
13032 *OutFunction = Function;
13033
13034 if (Function != nullptr) {
13035 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13036 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13037 }
13038
13039 std::string ErrorStr;
13040 raw_string_ostream ErrorFormatter(ErrorStr);
13041 ErrorFormatter << "Undefined external symbol ";
13042 ErrorFormatter << '"' << Symbol << '"';
13043 report_fatal_error(reason: Twine(ErrorStr));
13044}
13045
13046//===----------------------------------------------------------------------===//
13047// SDNode Class
13048//===----------------------------------------------------------------------===//
13049
13050bool llvm::isNullConstant(SDValue V) {
13051 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13052 return Const != nullptr && Const->isZero();
13053}
13054
13055bool llvm::isNullConstantOrUndef(SDValue V) {
13056 return V.isUndef() || isNullConstant(V);
13057}
13058
13059bool llvm::isNullFPConstant(SDValue V) {
13060 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13061 return Const != nullptr && Const->isZero() && !Const->isNegative();
13062}
13063
13064bool llvm::isAllOnesConstant(SDValue V) {
13065 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13066 return Const != nullptr && Const->isAllOnes();
13067}
13068
13069bool llvm::isOneConstant(SDValue V) {
13070 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13071 return Const != nullptr && Const->isOne();
13072}
13073
13074bool llvm::isMinSignedConstant(SDValue V) {
13075 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13076 return Const != nullptr && Const->isMinSignedValue();
13077}
13078
13079bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
13080 unsigned OperandNo) {
13081 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13082 // TODO: Target-specific opcodes could be added.
13083 if (auto *ConstV = isConstOrConstSplat(N: V, /*AllowUndefs*/ false,
13084 /*AllowTruncation*/ true)) {
13085 APInt Const = ConstV->getAPIntValue().trunc(width: V.getScalarValueSizeInBits());
13086 switch (Opcode) {
13087 case ISD::ADD:
13088 case ISD::OR:
13089 case ISD::XOR:
13090 case ISD::UMAX:
13091 return Const.isZero();
13092 case ISD::MUL:
13093 return Const.isOne();
13094 case ISD::AND:
13095 case ISD::UMIN:
13096 return Const.isAllOnes();
13097 case ISD::SMAX:
13098 return Const.isMinSignedValue();
13099 case ISD::SMIN:
13100 return Const.isMaxSignedValue();
13101 case ISD::SUB:
13102 case ISD::SHL:
13103 case ISD::SRA:
13104 case ISD::SRL:
13105 return OperandNo == 1 && Const.isZero();
13106 case ISD::UDIV:
13107 case ISD::SDIV:
13108 return OperandNo == 1 && Const.isOne();
13109 }
13110 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) {
13111 switch (Opcode) {
13112 case ISD::FADD:
13113 return ConstFP->isZero() &&
13114 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13115 case ISD::FSUB:
13116 return OperandNo == 1 && ConstFP->isZero() &&
13117 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13118 case ISD::FMUL:
13119 return ConstFP->isExactlyValue(V: 1.0);
13120 case ISD::FDIV:
13121 return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0);
13122 case ISD::FMINNUM:
13123 case ISD::FMAXNUM: {
13124 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
13125 EVT VT = V.getValueType();
13126 const fltSemantics &Semantics = VT.getFltSemantics();
13127 APFloat NeutralAF = !Flags.hasNoNaNs()
13128 ? APFloat::getQNaN(Sem: Semantics)
13129 : !Flags.hasNoInfs()
13130 ? APFloat::getInf(Sem: Semantics)
13131 : APFloat::getLargest(Sem: Semantics);
13132 if (Opcode == ISD::FMAXNUM)
13133 NeutralAF.changeSign();
13134
13135 return ConstFP->isExactlyValue(V: NeutralAF);
13136 }
13137 }
13138 }
13139 return false;
13140}
13141
13142SDValue llvm::peekThroughBitcasts(SDValue V) {
13143 while (V.getOpcode() == ISD::BITCAST)
13144 V = V.getOperand(i: 0);
13145 return V;
13146}
13147
13148SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13149 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13150 V = V.getOperand(i: 0);
13151 return V;
13152}
13153
13154SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13155 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13156 V = V.getOperand(i: 0);
13157 return V;
13158}
13159
13160SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13161 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13162 SDValue InVec = V.getOperand(i: 0);
13163 SDValue EltNo = V.getOperand(i: 2);
13164 EVT VT = InVec.getValueType();
13165 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13166 if (IndexC && VT.isFixedLengthVector() &&
13167 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13168 !DemandedElts[IndexC->getZExtValue()]) {
13169 V = InVec;
13170 continue;
13171 }
13172 break;
13173 }
13174 return V;
13175}
13176
13177SDValue llvm::peekThroughTruncates(SDValue V) {
13178 while (V.getOpcode() == ISD::TRUNCATE)
13179 V = V.getOperand(i: 0);
13180 return V;
13181}
13182
13183bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13184 if (V.getOpcode() != ISD::XOR)
13185 return false;
13186 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13187 unsigned NumBits = V.getScalarValueSizeInBits();
13188 ConstantSDNode *C =
13189 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13190 return C && (C->getAPIntValue().countr_one() >= NumBits);
13191}
13192
13193ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13194 bool AllowTruncation) {
13195 EVT VT = N.getValueType();
13196 APInt DemandedElts = VT.isFixedLengthVector()
13197 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
13198 : APInt(1, 1);
13199 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13200}
13201
13202ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13203 bool AllowUndefs,
13204 bool AllowTruncation) {
13205 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13206 return CN;
13207
13208 // SplatVectors can truncate their operands. Ignore that case here unless
13209 // AllowTruncation is set.
13210 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13211 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13212 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13213 EVT CVT = CN->getValueType(ResNo: 0);
13214 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13215 if (AllowTruncation || CVT == VecEltVT)
13216 return CN;
13217 }
13218 }
13219
13220 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13221 BitVector UndefElements;
13222 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13223
13224 // BuildVectors can truncate their operands. Ignore that case here unless
13225 // AllowTruncation is set.
13226 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13227 if (CN && (UndefElements.none() || AllowUndefs)) {
13228 EVT CVT = CN->getValueType(ResNo: 0);
13229 EVT NSVT = N.getValueType().getScalarType();
13230 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13231 if (AllowTruncation || (CVT == NSVT))
13232 return CN;
13233 }
13234 }
13235
13236 return nullptr;
13237}
13238
13239ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13240 EVT VT = N.getValueType();
13241 APInt DemandedElts = VT.isFixedLengthVector()
13242 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
13243 : APInt(1, 1);
13244 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13245}
13246
13247ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13248 const APInt &DemandedElts,
13249 bool AllowUndefs) {
13250 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13251 return CN;
13252
13253 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13254 BitVector UndefElements;
13255 ConstantFPSDNode *CN =
13256 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13257 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13258 if (CN && (UndefElements.none() || AllowUndefs))
13259 return CN;
13260 }
13261
13262 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13263 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13264 return CN;
13265
13266 return nullptr;
13267}
13268
13269bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13270 // TODO: may want to use peekThroughBitcast() here.
13271 ConstantSDNode *C =
13272 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13273 return C && C->isZero();
13274}
13275
13276bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
13277 ConstantSDNode *C =
13278 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
13279 return C && C->isOne();
13280}
13281
13282bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
13283 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13284 return C && C->isExactlyValue(V: 1.0);
13285}
13286
13287bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
13288 N = peekThroughBitcasts(V: N);
13289 unsigned BitWidth = N.getScalarValueSizeInBits();
13290 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13291 return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth;
13292}
13293
13294bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
13295 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13296 return C && APInt::isSameValue(I1: C->getAPIntValue(),
13297 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
13298}
13299
13300bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
13301 N = peekThroughBitcasts(V: N);
13302 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
13303 return C && C->isZero();
13304}
13305
13306bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
13307 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13308 return C && C->isZero();
13309}
13310
13311HandleSDNode::~HandleSDNode() {
13312 DropOperands();
13313}
13314
13315MemSDNode::MemSDNode(
13316 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
13317 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
13318 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
13319 bool IsVolatile = false;
13320 bool IsNonTemporal = false;
13321 bool IsDereferenceable = true;
13322 bool IsInvariant = true;
13323 for (const MachineMemOperand *MMO : memoperands()) {
13324 IsVolatile |= MMO->isVolatile();
13325 IsNonTemporal |= MMO->isNonTemporal();
13326 IsDereferenceable &= MMO->isDereferenceable();
13327 IsInvariant &= MMO->isInvariant();
13328 }
13329 MemSDNodeBits.IsVolatile = IsVolatile;
13330 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
13331 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
13332 MemSDNodeBits.IsInvariant = IsInvariant;
13333
13334 // For the single-MMO case, we check here that the size of the memory operand
13335 // fits within the size of the MMO. This is because the MMO might indicate
13336 // only a possible address range instead of specifying the affected memory
13337 // addresses precisely.
13338 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
13339 TypeSize::isKnownLE(memvt.getStoreSize(),
13340 getMemOperand()->getSize().getValue())) &&
13341 "Size mismatch!");
13342}
13343
13344/// Profile - Gather unique data for the node.
13345///
13346void SDNode::Profile(FoldingSetNodeID &ID) const {
13347 AddNodeIDNode(ID, N: this);
13348}
13349
13350namespace {
13351
13352 struct EVTArray {
13353 std::vector<EVT> VTs;
13354
13355 EVTArray() {
13356 VTs.reserve(n: MVT::VALUETYPE_SIZE);
13357 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
13358 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
13359 }
13360 };
13361
13362} // end anonymous namespace
13363
13364/// getValueTypeList - Return a pointer to the specified value type.
13365///
13366const EVT *SDNode::getValueTypeList(MVT VT) {
13367 static EVTArray SimpleVTArray;
13368
13369 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
13370 return &SimpleVTArray.VTs[VT.SimpleTy];
13371}
13372
13373/// hasAnyUseOfValue - Return true if there are any use of the indicated
13374/// value. This method ignores uses of other values defined by this operation.
13375bool SDNode::hasAnyUseOfValue(unsigned Value) const {
13376 assert(Value < getNumValues() && "Bad value!");
13377
13378 for (SDUse &U : uses())
13379 if (U.getResNo() == Value)
13380 return true;
13381
13382 return false;
13383}
13384
13385/// isOnlyUserOf - Return true if this node is the only use of N.
13386bool SDNode::isOnlyUserOf(const SDNode *N) const {
13387 bool Seen = false;
13388 for (const SDNode *User : N->users()) {
13389 if (User == this)
13390 Seen = true;
13391 else
13392 return false;
13393 }
13394
13395 return Seen;
13396}
13397
13398/// Return true if the only users of N are contained in Nodes.
13399bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
13400 bool Seen = false;
13401 for (const SDNode *User : N->users()) {
13402 if (llvm::is_contained(Range&: Nodes, Element: User))
13403 Seen = true;
13404 else
13405 return false;
13406 }
13407
13408 return Seen;
13409}
13410
13411/// Return true if the referenced return value is an operand of N.
13412bool SDValue::isOperandOf(const SDNode *N) const {
13413 return is_contained(Range: N->op_values(), Element: *this);
13414}
13415
13416bool SDNode::isOperandOf(const SDNode *N) const {
13417 return any_of(Range: N->op_values(),
13418 P: [this](SDValue Op) { return this == Op.getNode(); });
13419}
13420
13421/// reachesChainWithoutSideEffects - Return true if this operand (which must
13422/// be a chain) reaches the specified operand without crossing any
13423/// side-effecting instructions on any chain path. In practice, this looks
13424/// through token factors and non-volatile loads. In order to remain efficient,
13425/// this only looks a couple of nodes in, it does not do an exhaustive search.
13426///
13427/// Note that we only need to examine chains when we're searching for
13428/// side-effects; SelectionDAG requires that all side-effects are represented
13429/// by chains, even if another operand would force a specific ordering. This
13430/// constraint is necessary to allow transformations like splitting loads.
13431bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
13432 unsigned Depth) const {
13433 if (*this == Dest) return true;
13434
13435 // Don't search too deeply, we just want to be able to see through
13436 // TokenFactor's etc.
13437 if (Depth == 0) return false;
13438
13439 // If this is a token factor, all inputs to the TF happen in parallel.
13440 if (getOpcode() == ISD::TokenFactor) {
13441 // First, try a shallow search.
13442 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
13443 // We found the chain we want as an operand of this TokenFactor.
13444 // Essentially, we reach the chain without side-effects if we could
13445 // serialize the TokenFactor into a simple chain of operations with
13446 // Dest as the last operation. This is automatically true if the
13447 // chain has one use: there are no other ordering constraints.
13448 // If the chain has more than one use, we give up: some other
13449 // use of Dest might force a side-effect between Dest and the current
13450 // node.
13451 if (Dest.hasOneUse())
13452 return true;
13453 }
13454 // Next, try a deep search: check whether every operand of the TokenFactor
13455 // reaches Dest.
13456 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
13457 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
13458 });
13459 }
13460
13461 // Loads don't have side effects, look through them.
13462 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
13463 if (Ld->isUnordered())
13464 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
13465 }
13466 return false;
13467}
13468
13469bool SDNode::hasPredecessor(const SDNode *N) const {
13470 SmallPtrSet<const SDNode *, 32> Visited;
13471 SmallVector<const SDNode *, 16> Worklist;
13472 Worklist.push_back(Elt: this);
13473 return hasPredecessorHelper(N, Visited, Worklist);
13474}
13475
13476void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
13477 this->Flags &= Flags;
13478}
13479
13480SDValue
13481SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
13482 ArrayRef<ISD::NodeType> CandidateBinOps,
13483 bool AllowPartials) {
13484 // The pattern must end in an extract from index 0.
13485 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
13486 !isNullConstant(V: Extract->getOperand(Num: 1)))
13487 return SDValue();
13488
13489 // Match against one of the candidate binary ops.
13490 SDValue Op = Extract->getOperand(Num: 0);
13491 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
13492 return Op.getOpcode() == unsigned(BinOp);
13493 }))
13494 return SDValue();
13495
13496 // Floating-point reductions may require relaxed constraints on the final step
13497 // of the reduction because they may reorder intermediate operations.
13498 unsigned CandidateBinOp = Op.getOpcode();
13499 if (Op.getValueType().isFloatingPoint()) {
13500 SDNodeFlags Flags = Op->getFlags();
13501 switch (CandidateBinOp) {
13502 case ISD::FADD:
13503 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
13504 return SDValue();
13505 break;
13506 default:
13507 llvm_unreachable("Unhandled FP opcode for binop reduction");
13508 }
13509 }
13510
13511 // Matching failed - attempt to see if we did enough stages that a partial
13512 // reduction from a subvector is possible.
13513 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
13514 if (!AllowPartials || !Op)
13515 return SDValue();
13516 EVT OpVT = Op.getValueType();
13517 EVT OpSVT = OpVT.getScalarType();
13518 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
13519 if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
13520 return SDValue();
13521 BinOp = (ISD::NodeType)CandidateBinOp;
13522 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
13523 };
13524
13525 // At each stage, we're looking for something that looks like:
13526 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
13527 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
13528 // i32 undef, i32 undef, i32 undef, i32 undef>
13529 // %a = binop <8 x i32> %op, %s
13530 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
13531 // we expect something like:
13532 // <4,5,6,7,u,u,u,u>
13533 // <2,3,u,u,u,u,u,u>
13534 // <1,u,u,u,u,u,u,u>
13535 // While a partial reduction match would be:
13536 // <2,3,u,u,u,u,u,u>
13537 // <1,u,u,u,u,u,u,u>
13538 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
13539 SDValue PrevOp;
13540 for (unsigned i = 0; i < Stages; ++i) {
13541 unsigned MaskEnd = (1 << i);
13542
13543 if (Op.getOpcode() != CandidateBinOp)
13544 return PartialReduction(PrevOp, MaskEnd);
13545
13546 SDValue Op0 = Op.getOperand(i: 0);
13547 SDValue Op1 = Op.getOperand(i: 1);
13548
13549 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
13550 if (Shuffle) {
13551 Op = Op1;
13552 } else {
13553 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
13554 Op = Op0;
13555 }
13556
13557 // The first operand of the shuffle should be the same as the other operand
13558 // of the binop.
13559 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
13560 return PartialReduction(PrevOp, MaskEnd);
13561
13562 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
13563 for (int Index = 0; Index < (int)MaskEnd; ++Index)
13564 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
13565 return PartialReduction(PrevOp, MaskEnd);
13566
13567 PrevOp = Op;
13568 }
13569
13570 // Handle subvector reductions, which tend to appear after the shuffle
13571 // reduction stages.
13572 while (Op.getOpcode() == CandidateBinOp) {
13573 unsigned NumElts = Op.getValueType().getVectorNumElements();
13574 SDValue Op0 = Op.getOperand(i: 0);
13575 SDValue Op1 = Op.getOperand(i: 1);
13576 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13577 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13578 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
13579 break;
13580 SDValue Src = Op0.getOperand(i: 0);
13581 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
13582 if (NumSrcElts != (2 * NumElts))
13583 break;
13584 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
13585 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
13586 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
13587 Op0.getConstantOperandAPInt(i: 1) == NumElts))
13588 break;
13589 Op = Src;
13590 }
13591
13592 BinOp = (ISD::NodeType)CandidateBinOp;
13593 return Op;
13594}
13595
13596SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
13597 EVT VT = N->getValueType(ResNo: 0);
13598 EVT EltVT = VT.getVectorElementType();
13599 unsigned NE = VT.getVectorNumElements();
13600
13601 SDLoc dl(N);
13602
13603 // If ResNE is 0, fully unroll the vector op.
13604 if (ResNE == 0)
13605 ResNE = NE;
13606 else if (NE > ResNE)
13607 NE = ResNE;
13608
13609 if (N->getNumValues() == 2) {
13610 SmallVector<SDValue, 8> Scalars0, Scalars1;
13611 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13612 EVT VT1 = N->getValueType(ResNo: 1);
13613 EVT EltVT1 = VT1.getVectorElementType();
13614
13615 unsigned i;
13616 for (i = 0; i != NE; ++i) {
13617 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13618 SDValue Operand = N->getOperand(Num: j);
13619 EVT OperandVT = Operand.getValueType();
13620
13621 // A vector operand; extract a single element.
13622 EVT OperandEltVT = OperandVT.getVectorElementType();
13623 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13624 }
13625
13626 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
13627 Scalars0.push_back(Elt: EltOp);
13628 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
13629 }
13630
13631 for (; i < ResNE; ++i) {
13632 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
13633 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
13634 }
13635
13636 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
13637 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, NumElements: ResNE);
13638 SDValue Vec0 = getBuildVector(VT: VecVT, DL: dl, Ops: Scalars0);
13639 SDValue Vec1 = getBuildVector(VT: VecVT1, DL: dl, Ops: Scalars1);
13640 return getMergeValues(Ops: {Vec0, Vec1}, dl);
13641 }
13642
13643 assert(N->getNumValues() == 1 &&
13644 "Can't unroll a vector with multiple results!");
13645
13646 SmallVector<SDValue, 8> Scalars;
13647 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13648
13649 unsigned i;
13650 for (i= 0; i != NE; ++i) {
13651 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13652 SDValue Operand = N->getOperand(Num: j);
13653 EVT OperandVT = Operand.getValueType();
13654 if (OperandVT.isVector()) {
13655 // A vector operand; extract a single element.
13656 EVT OperandEltVT = OperandVT.getVectorElementType();
13657 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13658 } else {
13659 // A scalar operand; just use it as is.
13660 Operands[j] = Operand;
13661 }
13662 }
13663
13664 switch (N->getOpcode()) {
13665 default: {
13666 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
13667 Flags: N->getFlags()));
13668 break;
13669 }
13670 case ISD::VSELECT:
13671 Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
13672 break;
13673 case ISD::SHL:
13674 case ISD::SRA:
13675 case ISD::SRL:
13676 case ISD::ROTL:
13677 case ISD::ROTR:
13678 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
13679 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
13680 Op: Operands[1])));
13681 break;
13682 case ISD::SIGN_EXTEND_INREG: {
13683 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
13684 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
13685 N1: Operands[0],
13686 N2: getValueType(VT: ExtVT)));
13687 break;
13688 }
13689 case ISD::ADDRSPACECAST: {
13690 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
13691 Scalars.push_back(Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0],
13692 SrcAS: ASC->getSrcAddressSpace(),
13693 DestAS: ASC->getDestAddressSpace()));
13694 break;
13695 }
13696 }
13697 }
13698
13699 for (; i < ResNE; ++i)
13700 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
13701
13702 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
13703 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
13704}
13705
13706std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
13707 SDNode *N, unsigned ResNE) {
13708 unsigned Opcode = N->getOpcode();
13709 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
13710 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
13711 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
13712 "Expected an overflow opcode");
13713
13714 EVT ResVT = N->getValueType(ResNo: 0);
13715 EVT OvVT = N->getValueType(ResNo: 1);
13716 EVT ResEltVT = ResVT.getVectorElementType();
13717 EVT OvEltVT = OvVT.getVectorElementType();
13718 SDLoc dl(N);
13719
13720 // If ResNE is 0, fully unroll the vector op.
13721 unsigned NE = ResVT.getVectorNumElements();
13722 if (ResNE == 0)
13723 ResNE = NE;
13724 else if (NE > ResNE)
13725 NE = ResNE;
13726
13727 SmallVector<SDValue, 8> LHSScalars;
13728 SmallVector<SDValue, 8> RHSScalars;
13729 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
13730 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
13731
13732 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
13733 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
13734 SmallVector<SDValue, 8> ResScalars;
13735 SmallVector<SDValue, 8> OvScalars;
13736 for (unsigned i = 0; i < NE; ++i) {
13737 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
13738 SDValue Ov =
13739 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
13740 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
13741 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
13742
13743 ResScalars.push_back(Elt: Res);
13744 OvScalars.push_back(Elt: Ov);
13745 }
13746
13747 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
13748 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
13749
13750 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
13751 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
13752 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
13753 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
13754}
13755
13756bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
13757 LoadSDNode *Base,
13758 unsigned Bytes,
13759 int Dist) const {
13760 if (LD->isVolatile() || Base->isVolatile())
13761 return false;
13762 // TODO: probably too restrictive for atomics, revisit
13763 if (!LD->isSimple())
13764 return false;
13765 if (LD->isIndexed() || Base->isIndexed())
13766 return false;
13767 if (LD->getChain() != Base->getChain())
13768 return false;
13769 EVT VT = LD->getMemoryVT();
13770 if (VT.getSizeInBits() / 8 != Bytes)
13771 return false;
13772
13773 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
13774 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
13775
13776 int64_t Offset = 0;
13777 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
13778 return (Dist * (int64_t)Bytes == Offset);
13779 return false;
13780}
13781
13782/// InferPtrAlignment - Infer alignment of a load / store address. Return
13783/// std::nullopt if it cannot be inferred.
13784MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
13785 // If this is a GlobalAddress + cst, return the alignment.
13786 const GlobalValue *GV = nullptr;
13787 int64_t GVOffset = 0;
13788 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
13789 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
13790 KnownBits Known(PtrWidth);
13791 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
13792 unsigned AlignBits = Known.countMinTrailingZeros();
13793 if (AlignBits)
13794 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
13795 }
13796
13797 // If this is a direct reference to a stack slot, use information about the
13798 // stack slot's alignment.
13799 int FrameIdx = INT_MIN;
13800 int64_t FrameOffset = 0;
13801 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
13802 FrameIdx = FI->getIndex();
13803 } else if (isBaseWithConstantOffset(Op: Ptr) &&
13804 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
13805 // Handle FI+Cst
13806 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
13807 FrameOffset = Ptr.getConstantOperandVal(i: 1);
13808 }
13809
13810 if (FrameIdx != INT_MIN) {
13811 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
13812 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
13813 }
13814
13815 return std::nullopt;
13816}
13817
13818/// Split the scalar node with EXTRACT_ELEMENT using the provided
13819/// VTs and return the low/high part.
13820std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
13821 const SDLoc &DL,
13822 const EVT &LoVT,
13823 const EVT &HiVT) {
13824 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
13825 "Split node must be a scalar type");
13826 SDValue Lo =
13827 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
13828 SDValue Hi =
13829 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
13830 return std::make_pair(x&: Lo, y&: Hi);
13831}
13832
13833/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
13834/// which is split (or expanded) into two not necessarily identical pieces.
13835std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
13836 // Currently all types are split in half.
13837 EVT LoVT, HiVT;
13838 if (!VT.isVector())
13839 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
13840 else
13841 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
13842
13843 return std::make_pair(x&: LoVT, y&: HiVT);
13844}
13845
13846/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
13847/// type, dependent on an enveloping VT that has been split into two identical
13848/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
13849std::pair<EVT, EVT>
13850SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
13851 bool *HiIsEmpty) const {
13852 EVT EltTp = VT.getVectorElementType();
13853 // Examples:
13854 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
13855 // custom VL=9 with enveloping VL=8/8 yields 8/1
13856 // custom VL=10 with enveloping VL=8/8 yields 8/2
13857 // etc.
13858 ElementCount VTNumElts = VT.getVectorElementCount();
13859 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
13860 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
13861 "Mixing fixed width and scalable vectors when enveloping a type");
13862 EVT LoVT, HiVT;
13863 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
13864 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
13865 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
13866 *HiIsEmpty = false;
13867 } else {
13868 // Flag that hi type has zero storage size, but return split envelop type
13869 // (this would be easier if vector types with zero elements were allowed).
13870 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
13871 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
13872 *HiIsEmpty = true;
13873 }
13874 return std::make_pair(x&: LoVT, y&: HiVT);
13875}
13876
13877/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
13878/// low/high part.
13879std::pair<SDValue, SDValue>
13880SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
13881 const EVT &HiVT) {
13882 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
13883 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
13884 "Splitting vector with an invalid mixture of fixed and scalable "
13885 "vector types");
13886 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
13887 N.getValueType().getVectorMinNumElements() &&
13888 "More vector elements requested than available!");
13889 SDValue Lo, Hi;
13890 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
13891 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
13892 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
13893 // IDX with the runtime scaling factor of the result vector type. For
13894 // fixed-width result vectors, that runtime scaling factor is 1.
13895 Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
13896 N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
13897 return std::make_pair(x&: Lo, y&: Hi);
13898}
13899
13900std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
13901 const SDLoc &DL) {
13902 // Split the vector length parameter.
13903 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
13904 EVT VT = N.getValueType();
13905 assert(VecVT.getVectorElementCount().isKnownEven() &&
13906 "Expecting the mask to be an evenly-sized vector");
13907 SDValue HalfNumElts = getElementCount(
13908 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
13909 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
13910 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
13911 return std::make_pair(x&: Lo, y&: Hi);
13912}
13913
13914/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
13915SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
13916 EVT VT = N.getValueType();
13917 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
13918 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
13919 return getInsertSubvector(DL, Vec: getUNDEF(VT: WideVT), SubVec: N, Idx: 0);
13920}
13921
13922void SelectionDAG::ExtractVectorElements(SDValue Op,
13923 SmallVectorImpl<SDValue> &Args,
13924 unsigned Start, unsigned Count,
13925 EVT EltVT) {
13926 EVT VT = Op.getValueType();
13927 if (Count == 0)
13928 Count = VT.getVectorNumElements();
13929 if (EltVT == EVT())
13930 EltVT = VT.getVectorElementType();
13931 SDLoc SL(Op);
13932 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
13933 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
13934 }
13935}
13936
13937// getAddressSpace - Return the address space this GlobalAddress belongs to.
13938unsigned GlobalAddressSDNode::getAddressSpace() const {
13939 return getGlobal()->getType()->getAddressSpace();
13940}
13941
13942Type *ConstantPoolSDNode::getType() const {
13943 if (isMachineConstantPoolEntry())
13944 return Val.MachineCPVal->getType();
13945 return Val.ConstVal->getType();
13946}
13947
13948bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
13949 unsigned &SplatBitSize,
13950 bool &HasAnyUndefs,
13951 unsigned MinSplatBits,
13952 bool IsBigEndian) const {
13953 EVT VT = getValueType(ResNo: 0);
13954 assert(VT.isVector() && "Expected a vector type");
13955 unsigned VecWidth = VT.getSizeInBits();
13956 if (MinSplatBits > VecWidth)
13957 return false;
13958
13959 // FIXME: The widths are based on this node's type, but build vectors can
13960 // truncate their operands.
13961 SplatValue = APInt(VecWidth, 0);
13962 SplatUndef = APInt(VecWidth, 0);
13963
13964 // Get the bits. Bits with undefined values (when the corresponding element
13965 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
13966 // in SplatValue. If any of the values are not constant, give up and return
13967 // false.
13968 unsigned int NumOps = getNumOperands();
13969 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
13970 unsigned EltWidth = VT.getScalarSizeInBits();
13971
13972 for (unsigned j = 0; j < NumOps; ++j) {
13973 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
13974 SDValue OpVal = getOperand(Num: i);
13975 unsigned BitPos = j * EltWidth;
13976
13977 if (OpVal.isUndef())
13978 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
13979 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
13980 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
13981 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
13982 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
13983 else
13984 return false;
13985 }
13986
13987 // The build_vector is all constants or undefs. Find the smallest element
13988 // size that splats the vector.
13989 HasAnyUndefs = (SplatUndef != 0);
13990
13991 // FIXME: This does not work for vectors with elements less than 8 bits.
13992 while (VecWidth > 8) {
13993 // If we can't split in half, stop here.
13994 if (VecWidth & 1)
13995 break;
13996
13997 unsigned HalfSize = VecWidth / 2;
13998 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
13999 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14000 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14001 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14002
14003 // If the two halves do not match (ignoring undef bits), stop here.
14004 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14005 MinSplatBits > HalfSize)
14006 break;
14007
14008 SplatValue = HighValue | LowValue;
14009 SplatUndef = HighUndef & LowUndef;
14010
14011 VecWidth = HalfSize;
14012 }
14013
14014 // FIXME: The loop above only tries to split in halves. But if the input
14015 // vector for example is <3 x i16> it wouldn't be able to detect a
14016 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14017 // optimizations. I guess that back in the days when this helper was created
14018 // vectors normally was power-of-2 sized.
14019
14020 SplatBitSize = VecWidth;
14021 return true;
14022}
14023
14024SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14025 BitVector *UndefElements) const {
14026 unsigned NumOps = getNumOperands();
14027 if (UndefElements) {
14028 UndefElements->clear();
14029 UndefElements->resize(N: NumOps);
14030 }
14031 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14032 if (!DemandedElts)
14033 return SDValue();
14034 SDValue Splatted;
14035 for (unsigned i = 0; i != NumOps; ++i) {
14036 if (!DemandedElts[i])
14037 continue;
14038 SDValue Op = getOperand(Num: i);
14039 if (Op.isUndef()) {
14040 if (UndefElements)
14041 (*UndefElements)[i] = true;
14042 } else if (!Splatted) {
14043 Splatted = Op;
14044 } else if (Splatted != Op) {
14045 return SDValue();
14046 }
14047 }
14048
14049 if (!Splatted) {
14050 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14051 assert(getOperand(FirstDemandedIdx).isUndef() &&
14052 "Can only have a splat without a constant for all undefs.");
14053 return getOperand(Num: FirstDemandedIdx);
14054 }
14055
14056 return Splatted;
14057}
14058
14059SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14060 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14061 return getSplatValue(DemandedElts, UndefElements);
14062}
14063
14064bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14065 SmallVectorImpl<SDValue> &Sequence,
14066 BitVector *UndefElements) const {
14067 unsigned NumOps = getNumOperands();
14068 Sequence.clear();
14069 if (UndefElements) {
14070 UndefElements->clear();
14071 UndefElements->resize(N: NumOps);
14072 }
14073 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14074 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14075 return false;
14076
14077 // Set the undefs even if we don't find a sequence (like getSplatValue).
14078 if (UndefElements)
14079 for (unsigned I = 0; I != NumOps; ++I)
14080 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14081 (*UndefElements)[I] = true;
14082
14083 // Iteratively widen the sequence length looking for repetitions.
14084 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14085 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14086 for (unsigned I = 0; I != NumOps; ++I) {
14087 if (!DemandedElts[I])
14088 continue;
14089 SDValue &SeqOp = Sequence[I % SeqLen];
14090 SDValue Op = getOperand(Num: I);
14091 if (Op.isUndef()) {
14092 if (!SeqOp)
14093 SeqOp = Op;
14094 continue;
14095 }
14096 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14097 Sequence.clear();
14098 break;
14099 }
14100 SeqOp = Op;
14101 }
14102 if (!Sequence.empty())
14103 return true;
14104 }
14105
14106 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14107 return false;
14108}
14109
14110bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14111 BitVector *UndefElements) const {
14112 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14113 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14114}
14115
14116ConstantSDNode *
14117BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14118 BitVector *UndefElements) const {
14119 return dyn_cast_or_null<ConstantSDNode>(
14120 Val: getSplatValue(DemandedElts, UndefElements));
14121}
14122
14123ConstantSDNode *
14124BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14125 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14126}
14127
14128ConstantFPSDNode *
14129BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14130 BitVector *UndefElements) const {
14131 return dyn_cast_or_null<ConstantFPSDNode>(
14132 Val: getSplatValue(DemandedElts, UndefElements));
14133}
14134
14135ConstantFPSDNode *
14136BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14137 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14138}
14139
14140int32_t
14141BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14142 uint32_t BitWidth) const {
14143 if (ConstantFPSDNode *CN =
14144 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14145 bool IsExact;
14146 APSInt IntVal(BitWidth);
14147 const APFloat &APF = CN->getValueAPF();
14148 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14149 APFloat::opOK ||
14150 !IsExact)
14151 return -1;
14152
14153 return IntVal.exactLogBase2();
14154 }
14155 return -1;
14156}
14157
14158bool BuildVectorSDNode::getConstantRawBits(
14159 bool IsLittleEndian, unsigned DstEltSizeInBits,
14160 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14161 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14162 if (!isConstant())
14163 return false;
14164
14165 unsigned NumSrcOps = getNumOperands();
14166 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14167 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14168 "Invalid bitcast scale");
14169
14170 // Extract raw src bits.
14171 SmallVector<APInt> SrcBitElements(NumSrcOps,
14172 APInt::getZero(numBits: SrcEltSizeInBits));
14173 BitVector SrcUndeElements(NumSrcOps, false);
14174
14175 for (unsigned I = 0; I != NumSrcOps; ++I) {
14176 SDValue Op = getOperand(Num: I);
14177 if (Op.isUndef()) {
14178 SrcUndeElements.set(I);
14179 continue;
14180 }
14181 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14182 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14183 assert((CInt || CFP) && "Unknown constant");
14184 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14185 : CFP->getValueAPF().bitcastToAPInt();
14186 }
14187
14188 // Recast to dst width.
14189 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14190 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14191 return true;
14192}
14193
14194void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14195 unsigned DstEltSizeInBits,
14196 SmallVectorImpl<APInt> &DstBitElements,
14197 ArrayRef<APInt> SrcBitElements,
14198 BitVector &DstUndefElements,
14199 const BitVector &SrcUndefElements) {
14200 unsigned NumSrcOps = SrcBitElements.size();
14201 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14202 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14203 "Invalid bitcast scale");
14204 assert(NumSrcOps == SrcUndefElements.size() &&
14205 "Vector size mismatch");
14206
14207 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14208 DstUndefElements.clear();
14209 DstUndefElements.resize(N: NumDstOps, t: false);
14210 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14211
14212 // Concatenate src elements constant bits together into dst element.
14213 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14214 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14215 for (unsigned I = 0; I != NumDstOps; ++I) {
14216 DstUndefElements.set(I);
14217 APInt &DstBits = DstBitElements[I];
14218 for (unsigned J = 0; J != Scale; ++J) {
14219 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14220 if (SrcUndefElements[Idx])
14221 continue;
14222 DstUndefElements.reset(Idx: I);
14223 const APInt &SrcBits = SrcBitElements[Idx];
14224 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14225 "Illegal constant bitwidths");
14226 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14227 }
14228 }
14229 return;
14230 }
14231
14232 // Split src element constant bits into dst elements.
14233 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14234 for (unsigned I = 0; I != NumSrcOps; ++I) {
14235 if (SrcUndefElements[I]) {
14236 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14237 continue;
14238 }
14239 const APInt &SrcBits = SrcBitElements[I];
14240 for (unsigned J = 0; J != Scale; ++J) {
14241 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14242 APInt &DstBits = DstBitElements[Idx];
14243 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14244 }
14245 }
14246}
14247
14248bool BuildVectorSDNode::isConstant() const {
14249 for (const SDValue &Op : op_values()) {
14250 unsigned Opc = Op.getOpcode();
14251 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14252 return false;
14253 }
14254 return true;
14255}
14256
14257std::optional<std::pair<APInt, APInt>>
14258BuildVectorSDNode::isArithmeticSequence() const {
14259 unsigned NumOps = getNumOperands();
14260 if (NumOps < 2)
14261 return std::nullopt;
14262
14263 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14264 APInt Start, Stride;
14265 int FirstIdx = -1, SecondIdx = -1;
14266
14267 // Find the first two non-undef constant elements to determine Start and
14268 // Stride, then verify all remaining elements match the sequence.
14269 for (unsigned I = 0; I < NumOps; ++I) {
14270 SDValue Op = getOperand(Num: I);
14271 if (Op->isUndef())
14272 continue;
14273 if (!isa<ConstantSDNode>(Val: Op))
14274 return std::nullopt;
14275
14276 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
14277 if (FirstIdx < 0) {
14278 FirstIdx = I;
14279 Start = Val;
14280 } else if (SecondIdx < 0) {
14281 SecondIdx = I;
14282 // Compute stride using modular arithmetic. Simple division would handle
14283 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
14284 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
14285 // Note that modular arithmetic is agnostic to signed/unsigned.
14286 unsigned IdxDiff = I - FirstIdx;
14287 APInt ValDiff = Val - Start;
14288
14289 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
14290 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
14291 if (ValDiff.countr_zero() < CommonPow2Bits)
14292 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
14293 IdxDiff >>= CommonPow2Bits;
14294 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
14295
14296 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
14297 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
14298 // one, but we could try all candidates to handle more cases.
14299 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
14300 if (Stride.isZero())
14301 return std::nullopt;
14302
14303 // Step 3: Adjust Start based on the first defined element's index.
14304 Start -= Stride * FirstIdx;
14305 } else {
14306 // Verify this element matches the sequence.
14307 if (Val != Start + Stride * I)
14308 return std::nullopt;
14309 }
14310 }
14311
14312 // Need at least two defined elements.
14313 if (SecondIdx < 0)
14314 return std::nullopt;
14315
14316 return std::make_pair(x&: Start, y&: Stride);
14317}
14318
14319bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
14320 // Find the first non-undef value in the shuffle mask.
14321 unsigned i, e;
14322 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
14323 /* search */;
14324
14325 // If all elements are undefined, this shuffle can be considered a splat
14326 // (although it should eventually get simplified away completely).
14327 if (i == e)
14328 return true;
14329
14330 // Make sure all remaining elements are either undef or the same as the first
14331 // non-undef value.
14332 for (int Idx = Mask[i]; i != e; ++i)
14333 if (Mask[i] >= 0 && Mask[i] != Idx)
14334 return false;
14335 return true;
14336}
14337
14338// Returns true if it is a constant integer BuildVector or constant integer,
14339// possibly hidden by a bitcast.
14340bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
14341 SDValue N, bool AllowOpaques) const {
14342 N = peekThroughBitcasts(V: N);
14343
14344 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
14345 return AllowOpaques || !C->isOpaque();
14346
14347 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
14348 return true;
14349
14350 // Treat a GlobalAddress supporting constant offset folding as a
14351 // constant integer.
14352 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
14353 if (GA->getOpcode() == ISD::GlobalAddress &&
14354 TLI->isOffsetFoldingLegal(GA))
14355 return true;
14356
14357 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14358 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
14359 return true;
14360 return false;
14361}
14362
14363// Returns true if it is a constant float BuildVector or constant float.
14364bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
14365 if (isa<ConstantFPSDNode>(Val: N))
14366 return true;
14367
14368 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
14369 return true;
14370
14371 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14372 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
14373 return true;
14374
14375 return false;
14376}
14377
14378std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
14379 ConstantSDNode *Const =
14380 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
14381 if (!Const)
14382 return std::nullopt;
14383
14384 EVT VT = N->getValueType(ResNo: 0);
14385 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
14386 switch (TLI->getBooleanContents(Type: N.getValueType())) {
14387 case TargetLowering::ZeroOrOneBooleanContent:
14388 if (CVal.isOne())
14389 return true;
14390 if (CVal.isZero())
14391 return false;
14392 return std::nullopt;
14393 case TargetLowering::ZeroOrNegativeOneBooleanContent:
14394 if (CVal.isAllOnes())
14395 return true;
14396 if (CVal.isZero())
14397 return false;
14398 return std::nullopt;
14399 case TargetLowering::UndefinedBooleanContent:
14400 return CVal[0];
14401 }
14402 llvm_unreachable("Unknown BooleanContent enum");
14403}
14404
14405void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
14406 assert(!Node->OperandList && "Node already has operands");
14407 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
14408 "too many operands to fit into SDNode");
14409 SDUse *Ops = OperandRecycler.allocate(
14410 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
14411
14412 bool IsDivergent = false;
14413 for (unsigned I = 0; I != Vals.size(); ++I) {
14414 Ops[I].setUser(Node);
14415 Ops[I].setInitial(Vals[I]);
14416 EVT VT = Ops[I].getValueType();
14417
14418 // Skip Chain. It does not carry divergence.
14419 if (VT != MVT::Other &&
14420 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
14421 Ops[I].getNode()->isDivergent()) {
14422 IsDivergent = true;
14423 }
14424 }
14425 Node->NumOperands = Vals.size();
14426 Node->OperandList = Ops;
14427 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
14428 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
14429 Node->SDNodeBits.IsDivergent = IsDivergent;
14430 }
14431 checkForCycles(N: Node);
14432}
14433
14434SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
14435 SmallVectorImpl<SDValue> &Vals) {
14436 size_t Limit = SDNode::getMaxNumOperands();
14437 while (Vals.size() > Limit) {
14438 unsigned SliceIdx = Vals.size() - Limit;
14439 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
14440 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
14441 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
14442 Vals.emplace_back(Args&: NewTF);
14443 }
14444 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
14445}
14446
14447SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
14448 EVT VT, SDNodeFlags Flags) {
14449 switch (Opcode) {
14450 default:
14451 return SDValue();
14452 case ISD::ADD:
14453 case ISD::OR:
14454 case ISD::XOR:
14455 case ISD::UMAX:
14456 return getConstant(Val: 0, DL, VT);
14457 case ISD::MUL:
14458 return getConstant(Val: 1, DL, VT);
14459 case ISD::AND:
14460 case ISD::UMIN:
14461 return getAllOnesConstant(DL, VT);
14462 case ISD::SMAX:
14463 return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
14464 case ISD::SMIN:
14465 return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
14466 case ISD::FADD:
14467 // If flags allow, prefer positive zero since it's generally cheaper
14468 // to materialize on most targets.
14469 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
14470 case ISD::FMUL:
14471 return getConstantFP(Val: 1.0, DL, VT);
14472 case ISD::FMINNUM:
14473 case ISD::FMAXNUM: {
14474 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
14475 const fltSemantics &Semantics = VT.getFltSemantics();
14476 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
14477 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
14478 APFloat::getLargest(Sem: Semantics);
14479 if (Opcode == ISD::FMAXNUM)
14480 NeutralAF.changeSign();
14481
14482 return getConstantFP(V: NeutralAF, DL, VT);
14483 }
14484 case ISD::FMINIMUM:
14485 case ISD::FMAXIMUM: {
14486 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
14487 const fltSemantics &Semantics = VT.getFltSemantics();
14488 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
14489 : APFloat::getLargest(Sem: Semantics);
14490 if (Opcode == ISD::FMAXIMUM)
14491 NeutralAF.changeSign();
14492
14493 return getConstantFP(V: NeutralAF, DL, VT);
14494 }
14495
14496 }
14497}
14498
14499/// Helper used to make a call to a library function that has one argument of
14500/// pointer type.
14501///
14502/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
14503/// used to get or set floating-point state. They have one argument of pointer
14504/// type, which points to the memory region containing bits of the
14505/// floating-point state. The value returned by such function is ignored in the
14506/// created call.
14507///
14508/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
14509/// \param Ptr Pointer used to save/load state.
14510/// \param InChain Ingoing token chain.
14511/// \returns Outgoing chain token.
14512SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
14513 SDValue InChain,
14514 const SDLoc &DLoc) {
14515 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
14516 TargetLowering::ArgListTy Args;
14517 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
14518 RTLIB::LibcallImpl LibcallImpl =
14519 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
14520 if (LibcallImpl == RTLIB::Unsupported)
14521 reportFatalUsageError(reason: "emitting call to unsupported libcall");
14522
14523 SDValue Callee =
14524 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
14525 TargetLowering::CallLoweringInfo CLI(*this);
14526 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
14527 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
14528 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
14529 return TLI->LowerCallTo(CLI).second;
14530}
14531
14532void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
14533 assert(From && To && "Invalid SDNode; empty source SDValue?");
14534 auto I = SDEI.find(Val: From);
14535 if (I == SDEI.end())
14536 return;
14537
14538 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
14539 // the iterator, hence the need to make a copy to prevent a use-after-free.
14540 NodeExtraInfo NEI = I->second;
14541 if (LLVM_LIKELY(!NEI.PCSections)) {
14542 // No deep copy required for the types of extra info set.
14543 //
14544 // FIXME: Investigate if other types of extra info also need deep copy. This
14545 // depends on the types of nodes they can be attached to: if some extra info
14546 // is only ever attached to nodes where a replacement To node is always the
14547 // node where later use and propagation of the extra info has the intended
14548 // semantics, no deep copy is required.
14549 SDEI[To] = std::move(NEI);
14550 return;
14551 }
14552
14553 const SDNode *EntrySDN = getEntryNode().getNode();
14554
14555 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
14556 // through the replacement of From with To. Otherwise, replacements of a node
14557 // (From) with more complex nodes (To and its operands) may result in lost
14558 // extra info where the root node (To) is insignificant in further propagating
14559 // and using extra info when further lowering to MIR.
14560 //
14561 // In the first step pre-populate the visited set with the nodes reachable
14562 // from the old From node. This avoids copying NodeExtraInfo to parts of the
14563 // DAG that is not new and should be left untouched.
14564 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
14565 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
14566 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
14567 if (MaxDepth == 0) {
14568 // Remember this node in case we need to increase MaxDepth and continue
14569 // populating FromReach from this node.
14570 Leafs.emplace_back(Args&: N);
14571 return;
14572 }
14573 if (!FromReach.insert(V: N).second)
14574 return;
14575 for (const SDValue &Op : N->op_values())
14576 Self(Self, Op.getNode(), MaxDepth - 1);
14577 };
14578
14579 // Copy extra info to To and all its transitive operands (that are new).
14580 SmallPtrSet<const SDNode *, 8> Visited;
14581 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
14582 if (FromReach.contains(V: N))
14583 return true;
14584 if (!Visited.insert(Ptr: N).second)
14585 return true;
14586 if (EntrySDN == N)
14587 return false;
14588 for (const SDValue &Op : N->op_values()) {
14589 if (N == To && Op.getNode() == EntrySDN) {
14590 // Special case: New node's operand is the entry node; just need to
14591 // copy extra info to new node.
14592 break;
14593 }
14594 if (!Self(Self, Op.getNode()))
14595 return false;
14596 }
14597 // Copy only if entry node was not reached.
14598 SDEI[N] = std::move(NEI);
14599 return true;
14600 };
14601
14602 // We first try with a lower MaxDepth, assuming that the path to common
14603 // operands between From and To is relatively short. This significantly
14604 // improves performance in the common case. The initial MaxDepth is big
14605 // enough to avoid retry in the common case; the last MaxDepth is large
14606 // enough to avoid having to use the fallback below (and protects from
14607 // potential stack exhaustion from recursion).
14608 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
14609 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
14610 // StartFrom is the previous (or initial) set of leafs reachable at the
14611 // previous maximum depth.
14612 SmallVector<const SDNode *> StartFrom;
14613 std::swap(LHS&: StartFrom, RHS&: Leafs);
14614 for (const SDNode *N : StartFrom)
14615 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
14616 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
14617 return;
14618 // This should happen very rarely (reached the entry node).
14619 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
14620 assert(!Leafs.empty());
14621 }
14622
14623 // This should not happen - but if it did, that means the subgraph reachable
14624 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
14625 // could not visit all reachable common operands. Consequently, we were able
14626 // to reach the entry node.
14627 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
14628 assert(false && "From subgraph too complex - increase max. MaxDepth?");
14629 // Best-effort fallback if assertions disabled.
14630 SDEI[To] = std::move(NEI);
14631}
14632
14633#ifndef NDEBUG
14634static void checkForCyclesHelper(const SDNode *N,
14635 SmallPtrSetImpl<const SDNode*> &Visited,
14636 SmallPtrSetImpl<const SDNode*> &Checked,
14637 const llvm::SelectionDAG *DAG) {
14638 // If this node has already been checked, don't check it again.
14639 if (Checked.count(N))
14640 return;
14641
14642 // If a node has already been visited on this depth-first walk, reject it as
14643 // a cycle.
14644 if (!Visited.insert(N).second) {
14645 errs() << "Detected cycle in SelectionDAG\n";
14646 dbgs() << "Offending node:\n";
14647 N->dumprFull(DAG); dbgs() << "\n";
14648 abort();
14649 }
14650
14651 for (const SDValue &Op : N->op_values())
14652 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
14653
14654 Checked.insert(N);
14655 Visited.erase(N);
14656}
14657#endif
14658
14659void llvm::checkForCycles(const llvm::SDNode *N,
14660 const llvm::SelectionDAG *DAG,
14661 bool force) {
14662#ifndef NDEBUG
14663 bool check = force;
14664#ifdef EXPENSIVE_CHECKS
14665 check = true;
14666#endif // EXPENSIVE_CHECKS
14667 if (check) {
14668 assert(N && "Checking nonexistent SDNode");
14669 SmallPtrSet<const SDNode*, 32> visited;
14670 SmallPtrSet<const SDNode*, 32> checked;
14671 checkForCyclesHelper(N, visited, checked, DAG);
14672 }
14673#endif // !NDEBUG
14674}
14675
14676void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
14677 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
14678}
14679