1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/FunctionLoweringInfo.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineConstantPool.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/RuntimeLibcallUtil.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetFrameLowering.h"
46#include "llvm/CodeGen/TargetLowering.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGen/ValueTypes.h"
50#include "llvm/CodeGenTypes/MachineValueType.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfoMetadata.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
61#include "llvm/Support/Casting.h"
62#include "llvm/Support/CodeGen.h"
63#include "llvm/Support/Compiler.h"
64#include "llvm/Support/Debug.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/KnownFPClass.h"
68#include "llvm/Support/MathExtras.h"
69#include "llvm/Support/raw_ostream.h"
70#include "llvm/Target/TargetMachine.h"
71#include "llvm/Target/TargetOptions.h"
72#include "llvm/TargetParser/Triple.h"
73#include "llvm/Transforms/Utils/SizeOpts.h"
74#include <algorithm>
75#include <cassert>
76#include <cstdint>
77#include <cstdlib>
78#include <limits>
79#include <optional>
80#include <string>
81#include <utility>
82#include <vector>
83
84using namespace llvm;
85using namespace llvm::SDPatternMatch;
86
87/// makeVTList - Return an instance of the SDVTList struct initialized with the
88/// specified members.
89static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
90 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
91 return Res;
92}
93
94// Default null implementations of the callbacks.
95void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
96void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
97void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
98
99void SelectionDAG::DAGNodeDeletedListener::anchor() {}
100void SelectionDAG::DAGNodeInsertedListener::anchor() {}
101
102#define DEBUG_TYPE "selectiondag"
103
104static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
105 cl::Hidden, cl::init(Val: true),
106 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
107
108static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
109 cl::desc("Number limit for gluing ld/st of memcpy."),
110 cl::Hidden, cl::init(Val: 0));
111
112static cl::opt<unsigned>
113 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
114 cl::desc("DAG combiner limit number of steps when searching DAG "
115 "for predecessor nodes"));
116
117static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
118 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
119}
120
121unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
122
123//===----------------------------------------------------------------------===//
124// ConstantFPSDNode Class
125//===----------------------------------------------------------------------===//
126
127/// isExactlyValue - We don't rely on operator== working on double values, as
128/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
129/// As such, this method can be used to do an exact bit-for-bit comparison of
130/// two floating point values.
131bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
132 return getValueAPF().bitwiseIsEqual(RHS: V);
133}
134
135bool ConstantFPSDNode::isValueValidForType(EVT VT,
136 const APFloat& Val) {
137 assert(VT.isFloatingPoint() && "Can only convert between FP types");
138
139 // convert modifies in place, so make a copy.
140 APFloat Val2 = APFloat(Val);
141 bool losesInfo;
142 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
143 losesInfo: &losesInfo);
144 return !losesInfo;
145}
146
147//===----------------------------------------------------------------------===//
148// ISD Namespace
149//===----------------------------------------------------------------------===//
150
151bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
152 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
153 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
154 unsigned EltSize =
155 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
156 SplatVal = OptAPInt->trunc(width: EltSize);
157 return true;
158 }
159 }
160
161 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
162 if (!BV)
163 return false;
164
165 APInt SplatUndef;
166 unsigned SplatBitSize;
167 bool HasUndefs;
168 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
169 // Endianness does not matter here. We are checking for a splat given the
170 // element size of the vector, and if we find such a splat for little endian
171 // layout, then that should be valid also for big endian (as the full vector
172 // size is known to be a multiple of the element size).
173 const bool IsBigEndian = false;
174 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
175 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
176 EltSize == SplatBitSize;
177}
178
179// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
180// specializations of the more general isConstantSplatVector()?
181
182bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
183 // Look through a bit convert.
184 while (N->getOpcode() == ISD::BITCAST)
185 N = N->getOperand(Num: 0).getNode();
186
187 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
188 APInt SplatVal;
189 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
190 }
191
192 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
193
194 unsigned i = 0, e = N->getNumOperands();
195
196 // Skip over all of the undef values.
197 while (i != e && N->getOperand(Num: i).isUndef())
198 ++i;
199
200 // Do not accept an all-undef vector.
201 if (i == e) return false;
202
203 // Do not accept build_vectors that aren't all constants or which have non-~0
204 // elements. We have to be a bit careful here, as the type of the constant
205 // may not be the same as the type of the vector elements due to type
206 // legalization (the elements are promoted to a legal type for the target and
207 // a vector of a type may be legal when the base element type is not).
208 // We only want to check enough bits to cover the vector elements, because
209 // we care if the resultant vector is all ones, not whether the individual
210 // constants are.
211 SDValue NotZero = N->getOperand(Num: i);
212 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
213 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
214 if (OptAPInt->countr_one() < EltSize)
215 return false;
216 } else
217 return false;
218
219 // Okay, we have at least one ~0 value, check to see if the rest match or are
220 // undefs. Even with the above element type twiddling, this should be OK, as
221 // the same type legalization should have applied to all the elements.
222 for (++i; i != e; ++i)
223 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
224 return false;
225 return true;
226}
227
228bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
229 // Look through a bit convert.
230 while (N->getOpcode() == ISD::BITCAST)
231 N = N->getOperand(Num: 0).getNode();
232
233 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
234 APInt SplatVal;
235 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
236 }
237
238 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
239
240 bool IsAllUndef = true;
241 for (const SDValue &Op : N->op_values()) {
242 if (Op.isUndef())
243 continue;
244 IsAllUndef = false;
245 // Do not accept build_vectors that aren't all constants or which have non-0
246 // elements. We have to be a bit careful here, as the type of the constant
247 // may not be the same as the type of the vector elements due to type
248 // legalization (the elements are promoted to a legal type for the target
249 // and a vector of a type may be legal when the base element type is not).
250 // We only want to check enough bits to cover the vector elements, because
251 // we care if the resultant vector is all zeros, not whether the individual
252 // constants are.
253 if (auto OptAPInt = Op->bitcastToAPInt()) {
254 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
255 if (OptAPInt->countr_zero() < EltSize)
256 return false;
257 } else
258 return false;
259 }
260
261 // Do not accept an all-undef vector.
262 if (IsAllUndef)
263 return false;
264 return true;
265}
266
267bool ISD::isBuildVectorAllOnes(const SDNode *N) {
268 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
269}
270
271bool ISD::isBuildVectorAllZeros(const SDNode *N) {
272 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
273}
274
275bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
276 if (N->getOpcode() != ISD::BUILD_VECTOR)
277 return false;
278
279 for (const SDValue &Op : N->op_values()) {
280 if (Op.isUndef())
281 continue;
282 if (!isa<ConstantSDNode>(Val: Op))
283 return false;
284 }
285 return true;
286}
287
288bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
289 if (N->getOpcode() != ISD::BUILD_VECTOR)
290 return false;
291
292 for (const SDValue &Op : N->op_values()) {
293 if (Op.isUndef())
294 continue;
295 if (!isa<ConstantFPSDNode>(Val: Op))
296 return false;
297 }
298 return true;
299}
300
301bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
302 bool Signed) {
303 assert(N->getValueType(0).isVector() && "Expected a vector!");
304
305 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
306 if (EltSize <= NewEltSize)
307 return false;
308
309 if (N->getOpcode() == ISD::ZERO_EXTEND) {
310 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
311 NewEltSize) &&
312 !Signed;
313 }
314 if (N->getOpcode() == ISD::SIGN_EXTEND) {
315 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
316 NewEltSize) &&
317 Signed;
318 }
319 if (N->getOpcode() != ISD::BUILD_VECTOR)
320 return false;
321
322 for (const SDValue &Op : N->op_values()) {
323 if (Op.isUndef())
324 continue;
325 if (!isa<ConstantSDNode>(Val: Op))
326 return false;
327
328 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
329 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
330 return false;
331 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
332 return false;
333 }
334
335 return true;
336}
337
338bool ISD::allOperandsUndef(const SDNode *N) {
339 // Return false if the node has no operands.
340 // This is "logically inconsistent" with the definition of "all" but
341 // is probably the desired behavior.
342 if (N->getNumOperands() == 0)
343 return false;
344 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
345}
346
347bool ISD::isFreezeUndef(const SDNode *N) {
348 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
349}
350
351template <typename ConstNodeType>
352bool ISD::matchUnaryPredicateImpl(SDValue Op,
353 std::function<bool(ConstNodeType *)> Match,
354 bool AllowUndefs, bool AllowTruncation) {
355 // FIXME: Add support for scalar UNDEF cases?
356 if (auto *C = dyn_cast<ConstNodeType>(Op))
357 return Match(C);
358
359 // FIXME: Add support for vector UNDEF cases?
360 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
361 ISD::SPLAT_VECTOR != Op.getOpcode())
362 return false;
363
364 EVT SVT = Op.getValueType().getScalarType();
365 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
366 if (AllowUndefs && Op.getOperand(i).isUndef()) {
367 if (!Match(nullptr))
368 return false;
369 continue;
370 }
371
372 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
373 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
374 !Match(Cst))
375 return false;
376 }
377 return true;
378}
379// Build used template types.
380template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
381 SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);
382template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
383 SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);
384
385bool ISD::matchBinaryPredicate(
386 SDValue LHS, SDValue RHS,
387 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
388 bool AllowUndefs, bool AllowTypeMismatch) {
389 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
390 return false;
391
392 // TODO: Add support for scalar UNDEF cases?
393 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
394 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
395 return Match(LHSCst, RHSCst);
396
397 // TODO: Add support for vector UNDEF cases?
398 if (LHS.getOpcode() != RHS.getOpcode() ||
399 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
400 LHS.getOpcode() != ISD::SPLAT_VECTOR))
401 return false;
402
403 EVT SVT = LHS.getValueType().getScalarType();
404 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
405 SDValue LHSOp = LHS.getOperand(i);
406 SDValue RHSOp = RHS.getOperand(i);
407 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
408 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
409 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
410 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
411 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
412 return false;
413 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
414 LHSOp.getValueType() != RHSOp.getValueType()))
415 return false;
416 if (!Match(LHSCst, RHSCst))
417 return false;
418 }
419 return true;
420}
421
422ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
423 switch (MinMaxOpc) {
424 default:
425 llvm_unreachable("unrecognized opcode");
426 case ISD::UMIN:
427 return ISD::UMAX;
428 case ISD::UMAX:
429 return ISD::UMIN;
430 case ISD::SMIN:
431 return ISD::SMAX;
432 case ISD::SMAX:
433 return ISD::SMIN;
434 }
435}
436
437ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
438 switch (MinMaxOpc) {
439 default:
440 llvm_unreachable("unrecognized min/max opcode");
441 case ISD::SMIN:
442 return ISD::UMIN;
443 case ISD::SMAX:
444 return ISD::UMAX;
445 case ISD::UMIN:
446 return ISD::SMIN;
447 case ISD::UMAX:
448 return ISD::SMAX;
449 }
450}
451
452ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
453 switch (VecReduceOpcode) {
454 default:
455 llvm_unreachable("Expected VECREDUCE opcode");
456 case ISD::VECREDUCE_FADD:
457 case ISD::VECREDUCE_SEQ_FADD:
458 case ISD::VP_REDUCE_FADD:
459 case ISD::VP_REDUCE_SEQ_FADD:
460 return ISD::FADD;
461 case ISD::VECREDUCE_FMUL:
462 case ISD::VECREDUCE_SEQ_FMUL:
463 case ISD::VP_REDUCE_FMUL:
464 case ISD::VP_REDUCE_SEQ_FMUL:
465 return ISD::FMUL;
466 case ISD::VECREDUCE_ADD:
467 case ISD::VP_REDUCE_ADD:
468 return ISD::ADD;
469 case ISD::VECREDUCE_MUL:
470 case ISD::VP_REDUCE_MUL:
471 return ISD::MUL;
472 case ISD::VECREDUCE_AND:
473 case ISD::VP_REDUCE_AND:
474 return ISD::AND;
475 case ISD::VECREDUCE_OR:
476 case ISD::VP_REDUCE_OR:
477 return ISD::OR;
478 case ISD::VECREDUCE_XOR:
479 case ISD::VP_REDUCE_XOR:
480 return ISD::XOR;
481 case ISD::VECREDUCE_SMAX:
482 case ISD::VP_REDUCE_SMAX:
483 return ISD::SMAX;
484 case ISD::VECREDUCE_SMIN:
485 case ISD::VP_REDUCE_SMIN:
486 return ISD::SMIN;
487 case ISD::VECREDUCE_UMAX:
488 case ISD::VP_REDUCE_UMAX:
489 return ISD::UMAX;
490 case ISD::VECREDUCE_UMIN:
491 case ISD::VP_REDUCE_UMIN:
492 return ISD::UMIN;
493 case ISD::VECREDUCE_FMAX:
494 case ISD::VP_REDUCE_FMAX:
495 return ISD::FMAXNUM;
496 case ISD::VECREDUCE_FMIN:
497 case ISD::VP_REDUCE_FMIN:
498 return ISD::FMINNUM;
499 case ISD::VECREDUCE_FMAXIMUM:
500 case ISD::VP_REDUCE_FMAXIMUM:
501 return ISD::FMAXIMUM;
502 case ISD::VECREDUCE_FMINIMUM:
503 case ISD::VP_REDUCE_FMINIMUM:
504 return ISD::FMINIMUM;
505 }
506}
507
508ISD::NodeType ISD::getUnmaskedBinOpOpcode(unsigned MaskedOpc) {
509 switch (MaskedOpc) {
510 case ISD::MASKED_UDIV:
511 return ISD::UDIV;
512 case ISD::MASKED_SDIV:
513 return ISD::SDIV;
514 case ISD::MASKED_UREM:
515 return ISD::UREM;
516 case ISD::MASKED_SREM:
517 return ISD::SREM;
518 default:
519 llvm_unreachable("Expected masked binop opcode");
520 }
521}
522
523bool ISD::isVPOpcode(unsigned Opcode) {
524 switch (Opcode) {
525 default:
526 return false;
527#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
528 case ISD::VPSD: \
529 return true;
530#include "llvm/IR/VPIntrinsics.def"
531 }
532}
533
534bool ISD::isVPBinaryOp(unsigned Opcode) {
535 switch (Opcode) {
536 default:
537 break;
538#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
539#define VP_PROPERTY_BINARYOP return true;
540#define END_REGISTER_VP_SDNODE(VPSD) break;
541#include "llvm/IR/VPIntrinsics.def"
542 }
543 return false;
544}
545
546bool ISD::isVPReduction(unsigned Opcode) {
547 switch (Opcode) {
548 default:
549 return false;
550 case ISD::VP_REDUCE_ADD:
551 case ISD::VP_REDUCE_MUL:
552 case ISD::VP_REDUCE_AND:
553 case ISD::VP_REDUCE_OR:
554 case ISD::VP_REDUCE_XOR:
555 case ISD::VP_REDUCE_SMAX:
556 case ISD::VP_REDUCE_SMIN:
557 case ISD::VP_REDUCE_UMAX:
558 case ISD::VP_REDUCE_UMIN:
559 case ISD::VP_REDUCE_FMAX:
560 case ISD::VP_REDUCE_FMIN:
561 case ISD::VP_REDUCE_FMAXIMUM:
562 case ISD::VP_REDUCE_FMINIMUM:
563 case ISD::VP_REDUCE_FADD:
564 case ISD::VP_REDUCE_FMUL:
565 case ISD::VP_REDUCE_SEQ_FADD:
566 case ISD::VP_REDUCE_SEQ_FMUL:
567 return true;
568 }
569}
570
571/// The operand position of the vector mask.
572std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
573 switch (Opcode) {
574 default:
575 return std::nullopt;
576#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
577 case ISD::VPSD: \
578 return MASKPOS;
579#include "llvm/IR/VPIntrinsics.def"
580 }
581}
582
583/// The operand position of the explicit vector length parameter.
584std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
585 switch (Opcode) {
586 default:
587 return std::nullopt;
588#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
589 case ISD::VPSD: \
590 return EVLPOS;
591#include "llvm/IR/VPIntrinsics.def"
592 }
593}
594
595std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
596 bool hasFPExcept) {
597 // FIXME: Return strict opcodes in case of fp exceptions.
598 switch (VPOpcode) {
599 default:
600 return std::nullopt;
601#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
602#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
603#define END_REGISTER_VP_SDNODE(VPOPC) break;
604#include "llvm/IR/VPIntrinsics.def"
605 }
606 return std::nullopt;
607}
608
609std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
610 switch (Opcode) {
611 default:
612 return std::nullopt;
613#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
614#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
615#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
616#include "llvm/IR/VPIntrinsics.def"
617 }
618}
619
620ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
621 switch (ExtType) {
622 case ISD::EXTLOAD:
623 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
624 case ISD::SEXTLOAD:
625 return ISD::SIGN_EXTEND;
626 case ISD::ZEXTLOAD:
627 return ISD::ZERO_EXTEND;
628 default:
629 break;
630 }
631
632 llvm_unreachable("Invalid LoadExtType");
633}
634
635ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
636 // To perform this operation, we just need to swap the L and G bits of the
637 // operation.
638 unsigned OldL = (Operation >> 2) & 1;
639 unsigned OldG = (Operation >> 1) & 1;
640 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
641 (OldL << 1) | // New G bit
642 (OldG << 2)); // New L bit.
643}
644
645static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
646 unsigned Operation = Op;
647 if (isIntegerLike)
648 Operation ^= 7; // Flip L, G, E bits, but not U.
649 else
650 Operation ^= 15; // Flip all of the condition bits.
651
652 if (Operation > ISD::SETTRUE2)
653 Operation &= ~8; // Don't let N and U bits get set.
654
655 return ISD::CondCode(Operation);
656}
657
658ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
659 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
660}
661
662ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
663 bool isIntegerLike) {
664 return getSetCCInverseImpl(Op, isIntegerLike);
665}
666
667/// For an integer comparison, return 1 if the comparison is a signed operation
668/// and 2 if the result is an unsigned comparison. Return zero if the operation
669/// does not depend on the sign of the input (setne and seteq).
670static int isSignedOp(ISD::CondCode Opcode) {
671 switch (Opcode) {
672 default: llvm_unreachable("Illegal integer setcc operation!");
673 case ISD::SETEQ:
674 case ISD::SETNE: return 0;
675 case ISD::SETLT:
676 case ISD::SETLE:
677 case ISD::SETGT:
678 case ISD::SETGE: return 1;
679 case ISD::SETULT:
680 case ISD::SETULE:
681 case ISD::SETUGT:
682 case ISD::SETUGE: return 2;
683 }
684}
685
686ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
687 EVT Type) {
688 bool IsInteger = Type.isInteger();
689 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
690 // Cannot fold a signed integer setcc with an unsigned integer setcc.
691 return ISD::SETCC_INVALID;
692
693 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
694
695 // If the N and U bits get set, then the resultant comparison DOES suddenly
696 // care about orderedness, and it is true when ordered.
697 if (Op > ISD::SETTRUE2)
698 Op &= ~16; // Clear the U bit if the N bit is set.
699
700 // Canonicalize illegal integer setcc's.
701 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
702 Op = ISD::SETNE;
703
704 return ISD::CondCode(Op);
705}
706
707ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
708 EVT Type) {
709 bool IsInteger = Type.isInteger();
710 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
711 // Cannot fold a signed setcc with an unsigned setcc.
712 return ISD::SETCC_INVALID;
713
714 // Combine all of the condition bits.
715 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
716
717 // Canonicalize illegal integer setcc's.
718 if (IsInteger) {
719 switch (Result) {
720 default: break;
721 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
722 case ISD::SETOEQ: // SETEQ & SETU[LG]E
723 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
724 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
725 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
726 }
727 }
728
729 return Result;
730}
731
732//===----------------------------------------------------------------------===//
733// SDNode Profile Support
734//===----------------------------------------------------------------------===//
735
736/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
737static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
738 ID.AddInteger(I: OpC);
739}
740
741/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
742/// solely with their pointer.
743static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
744 ID.AddPointer(Ptr: VTList.VTs);
745}
746
747/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
748static void AddNodeIDOperands(FoldingSetNodeID &ID,
749 ArrayRef<SDValue> Ops) {
750 for (const auto &Op : Ops) {
751 ID.AddPointer(Ptr: Op.getNode());
752 ID.AddInteger(I: Op.getResNo());
753 }
754}
755
756/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
757static void AddNodeIDOperands(FoldingSetNodeID &ID,
758 ArrayRef<SDUse> Ops) {
759 for (const auto &Op : Ops) {
760 ID.AddPointer(Ptr: Op.getNode());
761 ID.AddInteger(I: Op.getResNo());
762 }
763}
764
765static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
766 SDVTList VTList, ArrayRef<SDValue> OpList) {
767 AddNodeIDOpcode(ID, OpC);
768 AddNodeIDValueTypes(ID, VTList);
769 AddNodeIDOperands(ID, Ops: OpList);
770}
771
772/// If this is an SDNode with special info, add this info to the NodeID data.
773static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
774 switch (N->getOpcode()) {
775 case ISD::TargetExternalSymbol:
776 case ISD::ExternalSymbol:
777 case ISD::MCSymbol:
778 llvm_unreachable("Should only be used on nodes with operands");
779 default: break; // Normal nodes don't need extra info.
780 case ISD::TargetConstant:
781 case ISD::Constant: {
782 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
783 ID.AddPointer(Ptr: C->getConstantIntValue());
784 ID.AddBoolean(B: C->isOpaque());
785 break;
786 }
787 case ISD::TargetConstantFP:
788 case ISD::ConstantFP:
789 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
790 break;
791 case ISD::TargetGlobalAddress:
792 case ISD::GlobalAddress:
793 case ISD::TargetGlobalTLSAddress:
794 case ISD::GlobalTLSAddress: {
795 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
796 ID.AddPointer(Ptr: GA->getGlobal());
797 ID.AddInteger(I: GA->getOffset());
798 ID.AddInteger(I: GA->getTargetFlags());
799 break;
800 }
801 case ISD::BasicBlock:
802 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
803 break;
804 case ISD::Register:
805 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
806 break;
807 case ISD::RegisterMask:
808 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
809 break;
810 case ISD::SRCVALUE:
811 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
812 break;
813 case ISD::FrameIndex:
814 case ISD::TargetFrameIndex:
815 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
816 break;
817 case ISD::PSEUDO_PROBE:
818 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
819 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
820 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
821 break;
822 case ISD::JumpTable:
823 case ISD::TargetJumpTable:
824 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
825 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
826 break;
827 case ISD::ConstantPool:
828 case ISD::TargetConstantPool: {
829 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
830 ID.AddInteger(I: CP->getAlign().value());
831 ID.AddInteger(I: CP->getOffset());
832 if (CP->isMachineConstantPoolEntry())
833 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
834 else
835 ID.AddPointer(Ptr: CP->getConstVal());
836 ID.AddInteger(I: CP->getTargetFlags());
837 break;
838 }
839 case ISD::TargetIndex: {
840 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
841 ID.AddInteger(I: TI->getIndex());
842 ID.AddInteger(I: TI->getOffset());
843 ID.AddInteger(I: TI->getTargetFlags());
844 break;
845 }
846 case ISD::LOAD: {
847 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
848 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
849 ID.AddInteger(I: LD->getRawSubclassData());
850 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
851 ID.AddInteger(I: LD->getMemOperand()->getFlags());
852 break;
853 }
854 case ISD::STORE: {
855 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
856 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
857 ID.AddInteger(I: ST->getRawSubclassData());
858 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
859 ID.AddInteger(I: ST->getMemOperand()->getFlags());
860 break;
861 }
862 case ISD::VP_LOAD: {
863 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
864 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
865 ID.AddInteger(I: ELD->getRawSubclassData());
866 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
867 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
868 break;
869 }
870 case ISD::VP_LOAD_FF: {
871 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
872 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
873 ID.AddInteger(I: LD->getRawSubclassData());
874 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
875 ID.AddInteger(I: LD->getMemOperand()->getFlags());
876 break;
877 }
878 case ISD::VP_STORE: {
879 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
880 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
881 ID.AddInteger(I: EST->getRawSubclassData());
882 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
883 ID.AddInteger(I: EST->getMemOperand()->getFlags());
884 break;
885 }
886 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
887 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
888 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
889 ID.AddInteger(I: SLD->getRawSubclassData());
890 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
891 break;
892 }
893 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
894 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
895 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
896 ID.AddInteger(I: SST->getRawSubclassData());
897 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
898 break;
899 }
900 case ISD::VP_GATHER: {
901 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
902 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
903 ID.AddInteger(I: EG->getRawSubclassData());
904 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
905 ID.AddInteger(I: EG->getMemOperand()->getFlags());
906 break;
907 }
908 case ISD::VP_SCATTER: {
909 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
910 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
911 ID.AddInteger(I: ES->getRawSubclassData());
912 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
913 ID.AddInteger(I: ES->getMemOperand()->getFlags());
914 break;
915 }
916 case ISD::MLOAD: {
917 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
918 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
919 ID.AddInteger(I: MLD->getRawSubclassData());
920 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
921 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
922 break;
923 }
924 case ISD::MSTORE: {
925 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
926 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
927 ID.AddInteger(I: MST->getRawSubclassData());
928 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
929 ID.AddInteger(I: MST->getMemOperand()->getFlags());
930 break;
931 }
932 case ISD::MGATHER: {
933 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
934 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
935 ID.AddInteger(I: MG->getRawSubclassData());
936 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
937 ID.AddInteger(I: MG->getMemOperand()->getFlags());
938 break;
939 }
940 case ISD::MSCATTER: {
941 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
942 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
943 ID.AddInteger(I: MS->getRawSubclassData());
944 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
945 ID.AddInteger(I: MS->getMemOperand()->getFlags());
946 break;
947 }
948 case ISD::ATOMIC_CMP_SWAP:
949 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
950 case ISD::ATOMIC_SWAP:
951 case ISD::ATOMIC_LOAD_ADD:
952 case ISD::ATOMIC_LOAD_SUB:
953 case ISD::ATOMIC_LOAD_AND:
954 case ISD::ATOMIC_LOAD_CLR:
955 case ISD::ATOMIC_LOAD_OR:
956 case ISD::ATOMIC_LOAD_XOR:
957 case ISD::ATOMIC_LOAD_NAND:
958 case ISD::ATOMIC_LOAD_MIN:
959 case ISD::ATOMIC_LOAD_MAX:
960 case ISD::ATOMIC_LOAD_UMIN:
961 case ISD::ATOMIC_LOAD_UMAX:
962 case ISD::ATOMIC_LOAD:
963 case ISD::ATOMIC_STORE: {
964 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
965 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
966 ID.AddInteger(I: AT->getRawSubclassData());
967 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
968 ID.AddInteger(I: AT->getMemOperand()->getFlags());
969 break;
970 }
971 case ISD::VECTOR_SHUFFLE: {
972 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
973 for (int M : Mask)
974 ID.AddInteger(I: M);
975 break;
976 }
977 case ISD::ADDRSPACECAST: {
978 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
979 ID.AddInteger(I: ASC->getSrcAddressSpace());
980 ID.AddInteger(I: ASC->getDestAddressSpace());
981 break;
982 }
983 case ISD::TargetBlockAddress:
984 case ISD::BlockAddress: {
985 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
986 ID.AddPointer(Ptr: BA->getBlockAddress());
987 ID.AddInteger(I: BA->getOffset());
988 ID.AddInteger(I: BA->getTargetFlags());
989 break;
990 }
991 case ISD::AssertAlign:
992 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
993 break;
994 case ISD::PREFETCH:
995 case ISD::INTRINSIC_VOID:
996 case ISD::INTRINSIC_W_CHAIN:
997 // Handled by MemIntrinsicSDNode check after the switch.
998 break;
999 case ISD::MDNODE_SDNODE:
1000 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
1001 break;
1002 } // end switch (N->getOpcode())
1003
1004 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
1005 // to check.
1006 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
1007 ID.AddInteger(I: MN->getRawSubclassData());
1008 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
1009 for (const MachineMemOperand *MMO : MN->memoperands()) {
1010 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
1011 ID.AddInteger(I: MMO->getFlags());
1012 }
1013 }
1014}
1015
1016/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
1017/// data.
1018static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1019 AddNodeIDOpcode(ID, OpC: N->getOpcode());
1020 // Add the return value info.
1021 AddNodeIDValueTypes(ID, VTList: N->getVTList());
1022 // Add the operand info.
1023 AddNodeIDOperands(ID, Ops: N->ops());
1024
1025 // Handle SDNode leafs with special info.
1026 AddNodeIDCustom(ID, N);
1027}
1028
1029//===----------------------------------------------------------------------===//
1030// SelectionDAG Class
1031//===----------------------------------------------------------------------===//
1032
1033/// doNotCSE - Return true if CSE should not be performed for this node.
1034static bool doNotCSE(SDNode *N) {
1035 if (N->getValueType(ResNo: 0) == MVT::Glue)
1036 return true; // Never CSE anything that produces a glue result.
1037
1038 switch (N->getOpcode()) {
1039 default: break;
1040 case ISD::HANDLENODE:
1041 case ISD::EH_LABEL:
1042 return true; // Never CSE these nodes.
1043 }
1044
1045 // Check that remaining values produced are not flags.
1046 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1047 if (N->getValueType(ResNo: i) == MVT::Glue)
1048 return true; // Never CSE anything that produces a glue result.
1049
1050 return false;
1051}
1052
1053/// Construct a DemandedElts mask which demands all elements of \p V.
1054/// If \p V is not a fixed-length vector, then this will return a single bit.
1055static APInt getDemandAllEltsMask(SDValue V) {
1056 EVT VT = V.getValueType();
1057 // Since the number of lanes in a scalable vector is unknown at compile time,
1058 // we track one bit which is implicitly broadcast to all lanes. This means
1059 // that all lanes in a scalable vector are considered demanded.
1060 return VT.isFixedLengthVector() ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1061 : APInt(1, 1);
1062}
1063
1064/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1065/// SelectionDAG.
1066void SelectionDAG::RemoveDeadNodes() {
1067 // Create a dummy node (which is not added to allnodes), that adds a reference
1068 // to the root node, preventing it from being deleted.
1069 HandleSDNode Dummy(getRoot());
1070
1071 SmallVector<SDNode*, 128> DeadNodes;
1072
1073 // Add all obviously-dead nodes to the DeadNodes worklist.
1074 for (SDNode &Node : allnodes())
1075 if (Node.use_empty())
1076 DeadNodes.push_back(Elt: &Node);
1077
1078 RemoveDeadNodes(DeadNodes);
1079
1080 // If the root changed (e.g. it was a dead load, update the root).
1081 setRoot(Dummy.getValue());
1082}
1083
1084/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1085/// given list, and any nodes that become unreachable as a result.
1086void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1087
1088 // Process the worklist, deleting the nodes and adding their uses to the
1089 // worklist.
1090 while (!DeadNodes.empty()) {
1091 SDNode *N = DeadNodes.pop_back_val();
1092 // Skip to next node if we've already managed to delete the node. This could
1093 // happen if replacing a node causes a node previously added to the node to
1094 // be deleted.
1095 if (N->getOpcode() == ISD::DELETED_NODE)
1096 continue;
1097
1098 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1099 DUL->NodeDeleted(N, nullptr);
1100
1101 // Take the node out of the appropriate CSE map.
1102 RemoveNodeFromCSEMaps(N);
1103
1104 // Next, brutally remove the operand list. This is safe to do, as there are
1105 // no cycles in the graph.
1106 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1107 SDUse &Use = *I++;
1108 SDNode *Operand = Use.getNode();
1109 Use.set(SDValue());
1110
1111 // Now that we removed this operand, see if there are no uses of it left.
1112 if (Operand->use_empty())
1113 DeadNodes.push_back(Elt: Operand);
1114 }
1115
1116 DeallocateNode(N);
1117 }
1118}
1119
1120void SelectionDAG::RemoveDeadNode(SDNode *N){
1121 SmallVector<SDNode*, 16> DeadNodes(1, N);
1122
1123 // Create a dummy node that adds a reference to the root node, preventing
1124 // it from being deleted. (This matters if the root is an operand of the
1125 // dead node.)
1126 HandleSDNode Dummy(getRoot());
1127
1128 RemoveDeadNodes(DeadNodes);
1129}
1130
1131void SelectionDAG::DeleteNode(SDNode *N) {
1132 // First take this out of the appropriate CSE map.
1133 RemoveNodeFromCSEMaps(N);
1134
1135 // Finally, remove uses due to operands of this node, remove from the
1136 // AllNodes list, and delete the node.
1137 DeleteNodeNotInCSEMaps(N);
1138}
1139
1140void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1141 assert(N->getIterator() != AllNodes.begin() &&
1142 "Cannot delete the entry node!");
1143 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1144
1145 // Drop all of the operands and decrement used node's use counts.
1146 N->DropOperands();
1147
1148 DeallocateNode(N);
1149}
1150
1151void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1152 assert(!(V->isVariadic() && isParameter));
1153 if (isParameter)
1154 ByvalParmDbgValues.push_back(Elt: V);
1155 else
1156 DbgValues.push_back(Elt: V);
1157 for (const SDNode *Node : V->getSDNodes())
1158 if (Node)
1159 DbgValMap[Node].push_back(Elt: V);
1160}
1161
1162void SDDbgInfo::erase(const SDNode *Node) {
1163 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1164 if (I == DbgValMap.end())
1165 return;
1166 for (auto &Val: I->second)
1167 Val->setIsInvalidated();
1168 DbgValMap.erase(I);
1169}
1170
1171void SelectionDAG::DeallocateNode(SDNode *N) {
1172 // If we have operands, deallocate them.
1173 removeOperands(Node: N);
1174
1175 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1176
1177 // Set the opcode to DELETED_NODE to help catch bugs when node
1178 // memory is reallocated.
1179 // FIXME: There are places in SDag that have grown a dependency on the opcode
1180 // value in the released node.
1181 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1182 N->NodeType = ISD::DELETED_NODE;
1183
1184 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1185 // them and forget about that node.
1186 DbgInfo->erase(Node: N);
1187
1188 // Invalidate extra info.
1189 SDEI.erase(Val: N);
1190}
1191
1192#ifndef NDEBUG
1193/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1194void SelectionDAG::verifyNode(SDNode *N) const {
1195 switch (N->getOpcode()) {
1196 default:
1197 if (N->isTargetOpcode())
1198 getSelectionDAGInfo().verifyTargetNode(*this, N);
1199 break;
1200 case ISD::BUILD_PAIR: {
1201 EVT VT = N->getValueType(0);
1202 assert(N->getNumValues() == 1 && "Too many results!");
1203 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1204 "Wrong return type!");
1205 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1206 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1207 "Mismatched operand types!");
1208 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1209 "Wrong operand type!");
1210 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1211 "Wrong return type size");
1212 break;
1213 }
1214 case ISD::BUILD_VECTOR: {
1215 assert(N->getNumValues() == 1 && "Too many results!");
1216 assert(N->getValueType(0).isVector() && "Wrong return type!");
1217 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1218 "Wrong number of operands!");
1219 EVT EltVT = N->getValueType(0).getVectorElementType();
1220 for (const SDUse &Op : N->ops()) {
1221 assert((Op.getValueType() == EltVT ||
1222 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1223 EltVT.bitsLE(Op.getValueType()))) &&
1224 "Wrong operand type!");
1225 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1226 "Operands must all have the same type");
1227 }
1228 break;
1229 }
1230 case ISD::SADDO:
1231 case ISD::UADDO:
1232 case ISD::SSUBO:
1233 case ISD::USUBO:
1234 assert(N->getNumValues() == 2 && "Wrong number of results!");
1235 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1236 "Invalid add/sub overflow op!");
1237 assert(N->getVTList().VTs[0].isInteger() &&
1238 N->getVTList().VTs[1].isInteger() &&
1239 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1240 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1241 "Binary operator types must match!");
1242 break;
1243 }
1244}
1245#endif // NDEBUG
1246
1247/// Insert a newly allocated node into the DAG.
1248///
1249/// Handles insertion into the all nodes list and CSE map, as well as
1250/// verification and other common operations when a new node is allocated.
1251void SelectionDAG::InsertNode(SDNode *N) {
1252 AllNodes.push_back(val: N);
1253#ifndef NDEBUG
1254 N->PersistentId = NextPersistentId++;
1255 verifyNode(N);
1256#endif
1257 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1258 DUL->NodeInserted(N);
1259}
1260
1261/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1262/// correspond to it. This is useful when we're about to delete or repurpose
1263/// the node. We don't want future request for structurally identical nodes
1264/// to return N anymore.
1265bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1266 bool Erased = false;
1267 switch (N->getOpcode()) {
1268 case ISD::HANDLENODE: return false; // noop.
1269 case ISD::CONDCODE:
1270 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1271 "Cond code doesn't exist!");
1272 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1273 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1274 break;
1275 case ISD::ExternalSymbol:
1276 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1277 break;
1278 case ISD::TargetExternalSymbol: {
1279 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1280 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1281 ESN->getSymbol(), ESN->getTargetFlags()));
1282 break;
1283 }
1284 case ISD::MCSymbol: {
1285 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1286 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1287 break;
1288 }
1289 case ISD::VALUETYPE: {
1290 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1291 if (VT.isExtended()) {
1292 Erased = ExtendedValueTypeNodes.erase(x: VT);
1293 } else {
1294 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1295 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1296 }
1297 break;
1298 }
1299 default:
1300 // Remove it from the CSE Map.
1301 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1302 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1303 Erased = CSEMap.RemoveNode(N);
1304 break;
1305 }
1306#ifndef NDEBUG
1307 // Verify that the node was actually in one of the CSE maps, unless it has a
1308 // glue result (which cannot be CSE'd) or is one of the special cases that are
1309 // not subject to CSE.
1310 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1311 !N->isMachineOpcode() && !doNotCSE(N)) {
1312 N->dump(this);
1313 dbgs() << "\n";
1314 llvm_unreachable("Node is not in map!");
1315 }
1316#endif
1317 return Erased;
1318}
1319
1320/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1321/// maps and modified in place. Add it back to the CSE maps, unless an identical
1322/// node already exists, in which case transfer all its users to the existing
1323/// node. This transfer can potentially trigger recursive merging.
1324void
1325SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1326 // For node types that aren't CSE'd, just act as if no identical node
1327 // already exists.
1328 if (!doNotCSE(N)) {
1329 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1330 if (Existing != N) {
1331 // If there was already an existing matching node, use ReplaceAllUsesWith
1332 // to replace the dead one with the existing one. This can cause
1333 // recursive merging of other unrelated nodes down the line.
1334 Existing->intersectFlagsWith(Flags: N->getFlags());
1335 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing))
1336 MemNode->refineRanges(NewMMOs: cast<MemSDNode>(Val: N)->memoperands());
1337 ReplaceAllUsesWith(From: N, To: Existing);
1338
1339 // N is now dead. Inform the listeners and delete it.
1340 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1341 DUL->NodeDeleted(N, Existing);
1342 DeleteNodeNotInCSEMaps(N);
1343 return;
1344 }
1345 }
1346
1347 // If the node doesn't already exist, we updated it. Inform listeners.
1348 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1349 DUL->NodeUpdated(N);
1350}
1351
1352/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1353/// were replaced with those specified. If this node is never memoized,
1354/// return null, otherwise return a pointer to the slot it would take. If a
1355/// node already exists with these operands, the slot will be non-null.
1356SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1357 void *&InsertPos) {
1358 if (doNotCSE(N))
1359 return nullptr;
1360
1361 SDValue Ops[] = { Op };
1362 FoldingSetNodeID ID;
1363 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1364 AddNodeIDCustom(ID, N);
1365 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1366 if (Node)
1367 Node->intersectFlagsWith(Flags: N->getFlags());
1368 return Node;
1369}
1370
1371/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1372/// were replaced with those specified. If this node is never memoized,
1373/// return null, otherwise return a pointer to the slot it would take. If a
1374/// node already exists with these operands, the slot will be non-null.
1375SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1376 SDValue Op1, SDValue Op2,
1377 void *&InsertPos) {
1378 if (doNotCSE(N))
1379 return nullptr;
1380
1381 SDValue Ops[] = { Op1, Op2 };
1382 FoldingSetNodeID ID;
1383 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1384 AddNodeIDCustom(ID, N);
1385 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1386 if (Node)
1387 Node->intersectFlagsWith(Flags: N->getFlags());
1388 return Node;
1389}
1390
1391/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1392/// were replaced with those specified. If this node is never memoized,
1393/// return null, otherwise return a pointer to the slot it would take. If a
1394/// node already exists with these operands, the slot will be non-null.
1395SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1396 void *&InsertPos) {
1397 if (doNotCSE(N))
1398 return nullptr;
1399
1400 FoldingSetNodeID ID;
1401 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1402 AddNodeIDCustom(ID, N);
1403 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1404 if (Node)
1405 Node->intersectFlagsWith(Flags: N->getFlags());
1406 return Node;
1407}
1408
1409Align SelectionDAG::getEVTAlign(EVT VT) const {
1410 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1411 : VT.getTypeForEVT(Context&: *getContext());
1412
1413 return getDataLayout().getABITypeAlign(Ty);
1414}
1415
1416// EntryNode could meaningfully have debug info if we can find it...
1417SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1418 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1419 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1420 Root(getEntryNode()) {
1421 InsertNode(N: &EntryNode);
1422 DbgInfo = new SDDbgInfo();
1423}
1424
1425void SelectionDAG::init(MachineFunction &NewMF,
1426 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1427 const TargetLibraryInfo *LibraryInfo,
1428 const LibcallLoweringInfo *LibcallsInfo,
1429 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1430 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
1431 FunctionVarLocs const *VarLocs) {
1432 MF = &NewMF;
1433 SDAGISelPass = PassPtr;
1434 ORE = &NewORE;
1435 TLI = getSubtarget().getTargetLowering();
1436 TSI = getSubtarget().getSelectionDAGInfo();
1437 LibInfo = LibraryInfo;
1438 Libcalls = LibcallsInfo;
1439 Context = &MF->getFunction().getContext();
1440 UA = NewUA;
1441 PSI = PSIin;
1442 BFI = BFIin;
1443 MMI = &MMIin;
1444 FnVarLocs = VarLocs;
1445}
1446
1447SelectionDAG::~SelectionDAG() {
1448 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1449 allnodes_clear();
1450 OperandRecycler.clear(OperandAllocator);
1451 delete DbgInfo;
1452}
1453
1454bool SelectionDAG::shouldOptForSize() const {
1455 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1456}
1457
1458void SelectionDAG::allnodes_clear() {
1459 assert(&*AllNodes.begin() == &EntryNode);
1460 AllNodes.remove(IT: AllNodes.begin());
1461 while (!AllNodes.empty())
1462 DeallocateNode(N: &AllNodes.front());
1463#ifndef NDEBUG
1464 NextPersistentId = 0;
1465#endif
1466}
1467
1468SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1469 void *&InsertPos) {
1470 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1471 if (N) {
1472 switch (N->getOpcode()) {
1473 default: break;
1474 case ISD::Constant:
1475 case ISD::ConstantFP:
1476 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1477 "debug location. Use another overload.");
1478 }
1479 }
1480 return N;
1481}
1482
1483SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1484 const SDLoc &DL, void *&InsertPos) {
1485 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1486 if (N) {
1487 switch (N->getOpcode()) {
1488 case ISD::Constant:
1489 case ISD::ConstantFP:
1490 // Erase debug location from the node if the node is used at several
1491 // different places. Do not propagate one location to all uses as it
1492 // will cause a worse single stepping debugging experience.
1493 if (N->getDebugLoc() != DL.getDebugLoc())
1494 N->setDebugLoc(DebugLoc());
1495 break;
1496 default:
1497 // When the node's point of use is located earlier in the instruction
1498 // sequence than its prior point of use, update its debug info to the
1499 // earlier location.
1500 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1501 N->setDebugLoc(DL.getDebugLoc());
1502 break;
1503 }
1504 }
1505 return N;
1506}
1507
1508void SelectionDAG::clear() {
1509 allnodes_clear();
1510 OperandRecycler.clear(OperandAllocator);
1511 OperandAllocator.Reset();
1512 CSEMap.clear();
1513
1514 ExtendedValueTypeNodes.clear();
1515 ExternalSymbols.clear();
1516 TargetExternalSymbols.clear();
1517 MCSymbols.clear();
1518 SDEI.clear();
1519 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1520 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1521
1522 EntryNode.UseList = nullptr;
1523 InsertNode(N: &EntryNode);
1524 Root = getEntryNode();
1525 DbgInfo->clear();
1526}
1527
1528SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1529 return VT.bitsGT(VT: Op.getValueType())
1530 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1531 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1532 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1533}
1534
1535std::pair<SDValue, SDValue>
1536SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1537 const SDLoc &DL, EVT VT) {
1538 assert(!VT.bitsEq(Op.getValueType()) &&
1539 "Strict no-op FP extend/round not allowed.");
1540 SDValue Res =
1541 VT.bitsGT(VT: Op.getValueType())
1542 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1543 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1544 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1545
1546 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1547}
1548
1549SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1550 return VT.bitsGT(VT: Op.getValueType()) ?
1551 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1552 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1553}
1554
1555SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1556 return VT.bitsGT(VT: Op.getValueType()) ?
1557 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1558 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1559}
1560
1561SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1562 return VT.bitsGT(VT: Op.getValueType()) ?
1563 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1564 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1565}
1566
1567SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1568 EVT VT) {
1569 assert(!VT.isVector());
1570 auto Type = Op.getValueType();
1571 SDValue DestOp;
1572 if (Type == VT)
1573 return Op;
1574 auto Size = Op.getValueSizeInBits();
1575 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1576 if (DestOp.getValueType() == VT)
1577 return DestOp;
1578
1579 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1580}
1581
1582SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1583 EVT VT) {
1584 assert(!VT.isVector());
1585 auto Type = Op.getValueType();
1586 SDValue DestOp;
1587 if (Type == VT)
1588 return Op;
1589 auto Size = Op.getValueSizeInBits();
1590 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1591 if (DestOp.getValueType() == VT)
1592 return DestOp;
1593
1594 return getSExtOrTrunc(Op: DestOp, DL, VT);
1595}
1596
1597SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1598 EVT VT) {
1599 assert(!VT.isVector());
1600 auto Type = Op.getValueType();
1601 SDValue DestOp;
1602 if (Type == VT)
1603 return Op;
1604 auto Size = Op.getValueSizeInBits();
1605 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1606 if (DestOp.getValueType() == VT)
1607 return DestOp;
1608
1609 return getZExtOrTrunc(Op: DestOp, DL, VT);
1610}
1611
1612SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1613 EVT OpVT) {
1614 if (VT.bitsLE(VT: Op.getValueType()))
1615 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1616
1617 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1618 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1619}
1620
1621SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1622 EVT OpVT = Op.getValueType();
1623 assert(VT.isInteger() && OpVT.isInteger() &&
1624 "Cannot getZeroExtendInReg FP types");
1625 assert(VT.isVector() == OpVT.isVector() &&
1626 "getZeroExtendInReg type should be vector iff the operand "
1627 "type is vector!");
1628 assert((!VT.isVector() ||
1629 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1630 "Vector element counts must match in getZeroExtendInReg");
1631 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1632 if (OpVT == VT)
1633 return Op;
1634 // TODO: Use computeKnownBits instead of AssertZext.
1635 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1636 ->getVT()
1637 .getScalarType()
1638 .bitsLE(VT: VT.getScalarType()))
1639 return Op;
1640 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1641 loBitsSet: VT.getScalarSizeInBits());
1642 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1643}
1644
1645SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,
1646 SDValue EVL, const SDLoc &DL,
1647 EVT VT) {
1648 EVT OpVT = Op.getValueType();
1649 assert(VT.isInteger() && OpVT.isInteger() &&
1650 "Cannot getVPZeroExtendInReg FP types");
1651 assert(VT.isVector() && OpVT.isVector() &&
1652 "getVPZeroExtendInReg type and operand type should be vector!");
1653 assert(VT.getVectorElementCount() == OpVT.getVectorElementCount() &&
1654 "Vector element counts must match in getZeroExtendInReg");
1655 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1656 if (OpVT == VT)
1657 return Op;
1658 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1659 loBitsSet: VT.getScalarSizeInBits());
1660 return getNode(Opcode: ISD::VP_AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT), N3: Mask,
1661 N4: EVL);
1662}
1663
1664SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1665 // Only unsigned pointer semantics are supported right now. In the future this
1666 // might delegate to TLI to check pointer signedness.
1667 return getZExtOrTrunc(Op, DL, VT);
1668}
1669
1670SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1671 // Only unsigned pointer semantics are supported right now. In the future this
1672 // might delegate to TLI to check pointer signedness.
1673 return getZeroExtendInReg(Op, DL, VT);
1674}
1675
1676SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1677 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1678}
1679
1680/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1681SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1682 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1683}
1684
1685SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1686 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1687 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1688}
1689
1690SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1691 SDValue Mask, SDValue EVL, EVT VT) {
1692 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1693 return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1694}
1695
1696SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1697 SDValue Mask, SDValue EVL) {
1698 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1699}
1700
1701SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1702 SDValue Mask, SDValue EVL) {
1703 if (VT.bitsGT(VT: Op.getValueType()))
1704 return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1705 if (VT.bitsLT(VT: Op.getValueType()))
1706 return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1707 return Op;
1708}
1709
1710SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1711 EVT OpVT) {
1712 if (!V)
1713 return getConstant(Val: 0, DL, VT);
1714
1715 switch (TLI->getBooleanContents(Type: OpVT)) {
1716 case TargetLowering::ZeroOrOneBooleanContent:
1717 case TargetLowering::UndefinedBooleanContent:
1718 return getConstant(Val: 1, DL, VT);
1719 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1720 return getAllOnesConstant(DL, VT);
1721 }
1722 llvm_unreachable("Unexpected boolean content enum!");
1723}
1724
1725SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1726 bool isT, bool isO) {
1727 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1728 DL, VT, isTarget: isT, isOpaque: isO);
1729}
1730
1731SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1732 bool isT, bool isO) {
1733 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1734}
1735
1736SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1737 EVT VT, bool isT, bool isO) {
1738 assert(VT.isInteger() && "Cannot create FP integer constant!");
1739
1740 EVT EltVT = VT.getScalarType();
1741 const ConstantInt *Elt = &Val;
1742
1743 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1744 // to-be-splatted scalar ConstantInt.
1745 if (isa<VectorType>(Val: Elt->getType()))
1746 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1747
1748 // In some cases the vector type is legal but the element type is illegal and
1749 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1750 // inserted value (the type does not need to match the vector element type).
1751 // Any extra bits introduced will be truncated away.
1752 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1753 TargetLowering::TypePromoteInteger) {
1754 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1755 APInt NewVal;
1756 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1757 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1758 else
1759 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1760 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1761 }
1762 // In other cases the element type is illegal and needs to be expanded, for
1763 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1764 // the value into n parts and use a vector type with n-times the elements.
1765 // Then bitcast to the type requested.
1766 // Legalizing constants too early makes the DAGCombiner's job harder so we
1767 // only legalize if the DAG tells us we must produce legal types.
1768 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1769 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1770 TargetLowering::TypeExpandInteger) {
1771 const APInt &NewVal = Elt->getValue();
1772 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1773 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1774
1775 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1776 if (VT.isScalableVector() ||
1777 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1778 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1779 "Can only handle an even split!");
1780 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1781
1782 SmallVector<SDValue, 2> ScalarParts;
1783 for (unsigned i = 0; i != Parts; ++i)
1784 ScalarParts.push_back(Elt: getConstant(
1785 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1786 VT: ViaEltVT, isT, isO));
1787
1788 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1789 }
1790
1791 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1792 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1793
1794 // Check the temporary vector is the correct size. If this fails then
1795 // getTypeToTransformTo() probably returned a type whose size (in bits)
1796 // isn't a power-of-2 factor of the requested type size.
1797 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1798
1799 SmallVector<SDValue, 2> EltParts;
1800 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1801 EltParts.push_back(Elt: getConstant(
1802 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1803 VT: ViaEltVT, isT, isO));
1804
1805 // EltParts is currently in little endian order. If we actually want
1806 // big-endian order then reverse it now.
1807 if (getDataLayout().isBigEndian())
1808 std::reverse(first: EltParts.begin(), last: EltParts.end());
1809
1810 // The elements must be reversed when the element order is different
1811 // to the endianness of the elements (because the BITCAST is itself a
1812 // vector shuffle in this situation). However, we do not need any code to
1813 // perform this reversal because getConstant() is producing a vector
1814 // splat.
1815 // This situation occurs in MIPS MSA.
1816
1817 SmallVector<SDValue, 8> Ops;
1818 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1819 llvm::append_range(C&: Ops, R&: EltParts);
1820
1821 SDValue V =
1822 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1823 return V;
1824 }
1825
1826 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1827 "APInt size does not match type size!");
1828 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1829 SDVTList VTs = getVTList(VT: EltVT);
1830 FoldingSetNodeID ID;
1831 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1832 ID.AddPointer(Ptr: Elt);
1833 ID.AddBoolean(B: isO);
1834 void *IP = nullptr;
1835 SDNode *N = nullptr;
1836 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1837 if (!VT.isVector())
1838 return SDValue(N, 0);
1839
1840 if (!N) {
1841 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1842 if (!isT)
1843 N->setDebugLoc(DL.getDebugLoc());
1844 CSEMap.InsertNode(N, InsertPos: IP);
1845 InsertNode(N);
1846 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1847 }
1848
1849 SDValue Result(N, 0);
1850 if (VT.isVector())
1851 Result = getSplat(VT, DL, Op: Result);
1852 return Result;
1853}
1854
1855SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1856 bool isT, bool isO) {
1857 unsigned Size = VT.getScalarSizeInBits();
1858 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1859}
1860
1861SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1862 bool IsOpaque) {
1863 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1864 isT: IsTarget, isO: IsOpaque);
1865}
1866
1867SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1868 bool isTarget) {
1869 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1870}
1871
1872SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1873 const SDLoc &DL) {
1874 assert(VT.isInteger() && "Shift amount is not an integer type!");
1875 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1876 return getConstant(Val, DL, VT: ShiftVT);
1877}
1878
1879SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1880 const SDLoc &DL) {
1881 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1882 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1883}
1884
1885SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1886 bool isTarget) {
1887 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1888}
1889
1890SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1891 bool isTarget) {
1892 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1893}
1894
1895SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1896 EVT VT, bool isTarget) {
1897 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1898
1899 EVT EltVT = VT.getScalarType();
1900 const ConstantFP *Elt = &V;
1901
1902 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1903 // the to-be-splatted scalar ConstantFP.
1904 if (isa<VectorType>(Val: Elt->getType()))
1905 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1906
1907 // Do the map lookup using the actual bit pattern for the floating point
1908 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1909 // we don't have issues with SNANs.
1910 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1911 SDVTList VTs = getVTList(VT: EltVT);
1912 FoldingSetNodeID ID;
1913 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1914 ID.AddPointer(Ptr: Elt);
1915 void *IP = nullptr;
1916 SDNode *N = nullptr;
1917 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1918 if (!VT.isVector())
1919 return SDValue(N, 0);
1920
1921 if (!N) {
1922 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1923 CSEMap.InsertNode(N, InsertPos: IP);
1924 InsertNode(N);
1925 }
1926
1927 SDValue Result(N, 0);
1928 if (VT.isVector())
1929 Result = getSplat(VT, DL, Op: Result);
1930 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1931 return Result;
1932}
1933
1934SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1935 bool isTarget) {
1936 EVT EltVT = VT.getScalarType();
1937 if (EltVT == MVT::f32)
1938 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1939 if (EltVT == MVT::f64)
1940 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1941 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1942 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1943 bool Ignored;
1944 APFloat APF = APFloat(Val);
1945 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1946 losesInfo: &Ignored);
1947 return getConstantFP(V: APF, DL, VT, isTarget);
1948 }
1949 llvm_unreachable("Unsupported type in getConstantFP");
1950}
1951
1952SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1953 EVT VT, int64_t Offset, bool isTargetGA,
1954 unsigned TargetFlags) {
1955 assert((TargetFlags == 0 || isTargetGA) &&
1956 "Cannot set target flags on target-independent globals");
1957
1958 // Truncate (with sign-extension) the offset value to the pointer size.
1959 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1960 if (BitWidth < 64)
1961 Offset = SignExtend64(X: Offset, B: BitWidth);
1962
1963 unsigned Opc;
1964 if (GV->isThreadLocal())
1965 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1966 else
1967 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1968
1969 SDVTList VTs = getVTList(VT);
1970 FoldingSetNodeID ID;
1971 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1972 ID.AddPointer(Ptr: GV);
1973 ID.AddInteger(I: Offset);
1974 ID.AddInteger(I: TargetFlags);
1975 void *IP = nullptr;
1976 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1977 return SDValue(E, 0);
1978
1979 auto *N = newSDNode<GlobalAddressSDNode>(
1980 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1981 CSEMap.InsertNode(N, InsertPos: IP);
1982 InsertNode(N);
1983 return SDValue(N, 0);
1984}
1985
1986SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1987 SDVTList VTs = getVTList(VT: MVT::Untyped);
1988 FoldingSetNodeID ID;
1989 AddNodeIDNode(ID, OpC: ISD::DEACTIVATION_SYMBOL, VTList: VTs, OpList: {});
1990 ID.AddPointer(Ptr: GV);
1991 void *IP = nullptr;
1992 if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
1993 return SDValue(E, 0);
1994
1995 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1996 CSEMap.InsertNode(N, InsertPos: IP);
1997 InsertNode(N);
1998 return SDValue(N, 0);
1999}
2000
2001SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
2002 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
2003 SDVTList VTs = getVTList(VT);
2004 FoldingSetNodeID ID;
2005 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2006 ID.AddInteger(I: FI);
2007 void *IP = nullptr;
2008 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2009 return SDValue(E, 0);
2010
2011 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
2012 CSEMap.InsertNode(N, InsertPos: IP);
2013 InsertNode(N);
2014 return SDValue(N, 0);
2015}
2016
2017SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
2018 unsigned TargetFlags) {
2019 assert((TargetFlags == 0 || isTarget) &&
2020 "Cannot set target flags on target-independent jump tables");
2021 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
2022 SDVTList VTs = getVTList(VT);
2023 FoldingSetNodeID ID;
2024 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2025 ID.AddInteger(I: JTI);
2026 ID.AddInteger(I: TargetFlags);
2027 void *IP = nullptr;
2028 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2029 return SDValue(E, 0);
2030
2031 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
2032 CSEMap.InsertNode(N, InsertPos: IP);
2033 InsertNode(N);
2034 return SDValue(N, 0);
2035}
2036
2037SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
2038 const SDLoc &DL) {
2039 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
2040 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
2041 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
2042}
2043
2044SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2045 MaybeAlign Alignment, int Offset,
2046 bool isTarget, unsigned TargetFlags) {
2047 assert((TargetFlags == 0 || isTarget) &&
2048 "Cannot set target flags on target-independent globals");
2049 if (!Alignment)
2050 Alignment = shouldOptForSize()
2051 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2052 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2053 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2054 SDVTList VTs = getVTList(VT);
2055 FoldingSetNodeID ID;
2056 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2057 ID.AddInteger(I: Alignment->value());
2058 ID.AddInteger(I: Offset);
2059 ID.AddPointer(Ptr: C);
2060 ID.AddInteger(I: TargetFlags);
2061 void *IP = nullptr;
2062 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2063 return SDValue(E, 0);
2064
2065 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2066 Args&: TargetFlags);
2067 CSEMap.InsertNode(N, InsertPos: IP);
2068 InsertNode(N);
2069 SDValue V = SDValue(N, 0);
2070 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2071 return V;
2072}
2073
2074SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2075 MaybeAlign Alignment, int Offset,
2076 bool isTarget, unsigned TargetFlags) {
2077 assert((TargetFlags == 0 || isTarget) &&
2078 "Cannot set target flags on target-independent globals");
2079 if (!Alignment)
2080 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2081 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2082 SDVTList VTs = getVTList(VT);
2083 FoldingSetNodeID ID;
2084 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2085 ID.AddInteger(I: Alignment->value());
2086 ID.AddInteger(I: Offset);
2087 C->addSelectionDAGCSEId(ID);
2088 ID.AddInteger(I: TargetFlags);
2089 void *IP = nullptr;
2090 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2091 return SDValue(E, 0);
2092
2093 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2094 Args&: TargetFlags);
2095 CSEMap.InsertNode(N, InsertPos: IP);
2096 InsertNode(N);
2097 return SDValue(N, 0);
2098}
2099
2100SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2101 FoldingSetNodeID ID;
2102 AddNodeIDNode(ID, OpC: ISD::BasicBlock, VTList: getVTList(VT: MVT::Other), OpList: {});
2103 ID.AddPointer(Ptr: MBB);
2104 void *IP = nullptr;
2105 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2106 return SDValue(E, 0);
2107
2108 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2109 CSEMap.InsertNode(N, InsertPos: IP);
2110 InsertNode(N);
2111 return SDValue(N, 0);
2112}
2113
2114SDValue SelectionDAG::getValueType(EVT VT) {
2115 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2116 ValueTypeNodes.size())
2117 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2118
2119 SDNode *&N = VT.isExtended() ?
2120 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2121
2122 if (N) return SDValue(N, 0);
2123 N = newSDNode<VTSDNode>(Args&: VT);
2124 InsertNode(N);
2125 return SDValue(N, 0);
2126}
2127
2128SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2129 SDNode *&N = ExternalSymbols[Sym];
2130 if (N) return SDValue(N, 0);
2131 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2132 InsertNode(N);
2133 return SDValue(N, 0);
2134}
2135
2136SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2137 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2138 return getExternalSymbol(Sym: SymName.data(), VT);
2139}
2140
2141SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2142 SDNode *&N = MCSymbols[Sym];
2143 if (N)
2144 return SDValue(N, 0);
2145 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2146 InsertNode(N);
2147 return SDValue(N, 0);
2148}
2149
2150SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2151 unsigned TargetFlags) {
2152 SDNode *&N =
2153 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2154 if (N) return SDValue(N, 0);
2155 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2156 InsertNode(N);
2157 return SDValue(N, 0);
2158}
2159
2160SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2161 EVT VT, unsigned TargetFlags) {
2162 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2163 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2164}
2165
2166SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2167 if ((unsigned)Cond >= CondCodeNodes.size())
2168 CondCodeNodes.resize(new_size: Cond+1);
2169
2170 if (!CondCodeNodes[Cond]) {
2171 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2172 CondCodeNodes[Cond] = N;
2173 InsertNode(N);
2174 }
2175
2176 return SDValue(CondCodeNodes[Cond], 0);
2177}
2178
2179SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2180 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2181 "APInt size does not match type size!");
2182
2183 if (MulImm == 0)
2184 return getConstant(Val: 0, DL, VT);
2185
2186 const MachineFunction &MF = getMachineFunction();
2187 const Function &F = MF.getFunction();
2188 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2189 if (const APInt *C = CR.getSingleElement())
2190 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2191
2192 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2193}
2194
2195/// \returns a value of type \p VT that represents the runtime value of \p
2196/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2197/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2198/// or TypeSize.
2199template <typename Ty>
2200static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2201 EVT VT, Ty Quantity) {
2202 if (Quantity.isScalable())
2203 return DAG.getVScale(
2204 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2205
2206 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2207}
2208
2209SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2210 ElementCount EC) {
2211 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2212}
2213
2214SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2215 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2216}
2217
2218SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2219 ElementCount EC) {
2220 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2221 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2222 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2223 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2224}
2225
2226SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2227 APInt One(ResVT.getScalarSizeInBits(), 1);
2228 return getStepVector(DL, ResVT, StepVal: One);
2229}
2230
2231SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2232 const APInt &StepVal) {
2233 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2234 if (ResVT.isScalableVector())
2235 return getNode(
2236 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2237 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2238
2239 SmallVector<SDValue, 16> OpsStepConstants;
2240 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2241 OpsStepConstants.push_back(
2242 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2243 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2244}
2245
2246/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2247/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2248static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2249 std::swap(a&: N1, b&: N2);
2250 ShuffleVectorSDNode::commuteMask(Mask: M);
2251}
2252
2253SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2254 SDValue N2, ArrayRef<int> Mask) {
2255 assert(VT.getVectorNumElements() == Mask.size() &&
2256 "Must have the same number of vector elements as mask elements!");
2257 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2258 "Invalid VECTOR_SHUFFLE");
2259
2260 // Canonicalize shuffle undef, undef -> undef
2261 if (N1.isUndef() && N2.isUndef())
2262 return getUNDEF(VT);
2263
2264 // Validate that all indices in Mask are within the range of the elements
2265 // input to the shuffle.
2266 int NElts = Mask.size();
2267 assert(llvm::all_of(Mask,
2268 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2269 "Index out of range");
2270
2271 // Copy the mask so we can do any needed cleanup.
2272 SmallVector<int, 8> MaskVec(Mask);
2273
2274 // Canonicalize shuffle v, v -> v, undef
2275 if (N1 == N2) {
2276 N2 = getUNDEF(VT);
2277 for (int i = 0; i != NElts; ++i)
2278 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2279 }
2280
2281 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2282 if (N1.isUndef())
2283 commuteShuffle(N1, N2, M: MaskVec);
2284
2285 if (TLI->hasVectorBlend()) {
2286 // If shuffling a splat, try to blend the splat instead. We do this here so
2287 // that even when this arises during lowering we don't have to re-handle it.
2288 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2289 BitVector UndefElements;
2290 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2291 if (!Splat)
2292 return;
2293
2294 for (int i = 0; i < NElts; ++i) {
2295 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2296 continue;
2297
2298 // If this input comes from undef, mark it as such.
2299 if (UndefElements[MaskVec[i] - Offset]) {
2300 MaskVec[i] = -1;
2301 continue;
2302 }
2303
2304 // If we can blend a non-undef lane, use that instead.
2305 if (!UndefElements[i])
2306 MaskVec[i] = i + Offset;
2307 }
2308 };
2309 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2310 BlendSplat(N1BV, 0);
2311 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2312 BlendSplat(N2BV, NElts);
2313 }
2314
2315 // Canonicalize all index into lhs, -> shuffle lhs, undef
2316 // Canonicalize all index into rhs, -> shuffle rhs, undef
2317 bool AllLHS = true, AllRHS = true;
2318 bool N2Undef = N2.isUndef();
2319 for (int i = 0; i != NElts; ++i) {
2320 if (MaskVec[i] >= NElts) {
2321 if (N2Undef)
2322 MaskVec[i] = -1;
2323 else
2324 AllLHS = false;
2325 } else if (MaskVec[i] >= 0) {
2326 AllRHS = false;
2327 }
2328 }
2329 if (AllLHS && AllRHS)
2330 return getUNDEF(VT);
2331 if (AllLHS && !N2Undef)
2332 N2 = getUNDEF(VT);
2333 if (AllRHS) {
2334 N1 = getUNDEF(VT);
2335 commuteShuffle(N1, N2, M: MaskVec);
2336 }
2337 // Reset our undef status after accounting for the mask.
2338 N2Undef = N2.isUndef();
2339 // Re-check whether both sides ended up undef.
2340 if (N1.isUndef() && N2Undef)
2341 return getUNDEF(VT);
2342
2343 // If Identity shuffle return that node.
2344 bool Identity = true, AllSame = true;
2345 for (int i = 0; i != NElts; ++i) {
2346 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2347 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2348 }
2349 if (Identity && NElts)
2350 return N1;
2351
2352 // Shuffling a constant splat doesn't change the result.
2353 if (N2Undef) {
2354 SDValue V = N1;
2355
2356 // Look through any bitcasts. We check that these don't change the number
2357 // (and size) of elements and just changes their types.
2358 while (V.getOpcode() == ISD::BITCAST)
2359 V = V->getOperand(Num: 0);
2360
2361 // A splat should always show up as a build vector node.
2362 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2363 BitVector UndefElements;
2364 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2365 // If this is a splat of an undef, shuffling it is also undef.
2366 if (Splat && Splat.isUndef())
2367 return getUNDEF(VT);
2368
2369 bool SameNumElts =
2370 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2371
2372 // We only have a splat which can skip shuffles if there is a splatted
2373 // value and no undef lanes rearranged by the shuffle.
2374 if (Splat && UndefElements.none()) {
2375 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2376 // number of elements match or the value splatted is a zero constant.
2377 if (SameNumElts || isNullConstant(V: Splat))
2378 return N1;
2379 }
2380
2381 // If the shuffle itself creates a splat, build the vector directly.
2382 if (AllSame && SameNumElts) {
2383 EVT BuildVT = BV->getValueType(ResNo: 0);
2384 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2385 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2386
2387 // We may have jumped through bitcasts, so the type of the
2388 // BUILD_VECTOR may not match the type of the shuffle.
2389 if (BuildVT != VT)
2390 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2391 return NewBV;
2392 }
2393 }
2394 }
2395
2396 SDVTList VTs = getVTList(VT);
2397 FoldingSetNodeID ID;
2398 SDValue Ops[2] = { N1, N2 };
2399 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: VTs, OpList: Ops);
2400 for (int i = 0; i != NElts; ++i)
2401 ID.AddInteger(I: MaskVec[i]);
2402
2403 void* IP = nullptr;
2404 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2405 return SDValue(E, 0);
2406
2407 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2408 // SDNode doesn't have access to it. This memory will be "leaked" when
2409 // the node is deallocated, but recovered when the NodeAllocator is released.
2410 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2411 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2412
2413 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2414 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2415 createOperands(Node: N, Vals: Ops);
2416
2417 CSEMap.InsertNode(N, InsertPos: IP);
2418 InsertNode(N);
2419 SDValue V = SDValue(N, 0);
2420 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2421 return V;
2422}
2423
2424SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2425 EVT VT = SV.getValueType(ResNo: 0);
2426 SmallVector<int, 8> MaskVec(SV.getMask());
2427 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2428
2429 SDValue Op0 = SV.getOperand(Num: 0);
2430 SDValue Op1 = SV.getOperand(Num: 1);
2431 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2432}
2433
2434SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2435 SDVTList VTs = getVTList(VT);
2436 FoldingSetNodeID ID;
2437 AddNodeIDNode(ID, OpC: ISD::Register, VTList: VTs, OpList: {});
2438 ID.AddInteger(I: Reg.id());
2439 void *IP = nullptr;
2440 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2441 return SDValue(E, 0);
2442
2443 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2444 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2445 CSEMap.InsertNode(N, InsertPos: IP);
2446 InsertNode(N);
2447 return SDValue(N, 0);
2448}
2449
2450SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2451 FoldingSetNodeID ID;
2452 AddNodeIDNode(ID, OpC: ISD::RegisterMask, VTList: getVTList(VT: MVT::Untyped), OpList: {});
2453 ID.AddPointer(Ptr: RegMask);
2454 void *IP = nullptr;
2455 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2456 return SDValue(E, 0);
2457
2458 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2459 CSEMap.InsertNode(N, InsertPos: IP);
2460 InsertNode(N);
2461 return SDValue(N, 0);
2462}
2463
2464SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2465 MCSymbol *Label) {
2466 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2467}
2468
2469SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2470 SDValue Root, MCSymbol *Label) {
2471 FoldingSetNodeID ID;
2472 SDValue Ops[] = { Root };
2473 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT: MVT::Other), OpList: Ops);
2474 ID.AddPointer(Ptr: Label);
2475 void *IP = nullptr;
2476 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2477 return SDValue(E, 0);
2478
2479 auto *N =
2480 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2481 createOperands(Node: N, Vals: Ops);
2482
2483 CSEMap.InsertNode(N, InsertPos: IP);
2484 InsertNode(N);
2485 return SDValue(N, 0);
2486}
2487
2488SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2489 int64_t Offset, bool isTarget,
2490 unsigned TargetFlags) {
2491 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2492 SDVTList VTs = getVTList(VT);
2493
2494 FoldingSetNodeID ID;
2495 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2496 ID.AddPointer(Ptr: BA);
2497 ID.AddInteger(I: Offset);
2498 ID.AddInteger(I: TargetFlags);
2499 void *IP = nullptr;
2500 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2501 return SDValue(E, 0);
2502
2503 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2504 CSEMap.InsertNode(N, InsertPos: IP);
2505 InsertNode(N);
2506 return SDValue(N, 0);
2507}
2508
2509SDValue SelectionDAG::getSrcValue(const Value *V) {
2510 FoldingSetNodeID ID;
2511 AddNodeIDNode(ID, OpC: ISD::SRCVALUE, VTList: getVTList(VT: MVT::Other), OpList: {});
2512 ID.AddPointer(Ptr: V);
2513
2514 void *IP = nullptr;
2515 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2516 return SDValue(E, 0);
2517
2518 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2519 CSEMap.InsertNode(N, InsertPos: IP);
2520 InsertNode(N);
2521 return SDValue(N, 0);
2522}
2523
2524SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2525 FoldingSetNodeID ID;
2526 AddNodeIDNode(ID, OpC: ISD::MDNODE_SDNODE, VTList: getVTList(VT: MVT::Other), OpList: {});
2527 ID.AddPointer(Ptr: MD);
2528
2529 void *IP = nullptr;
2530 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2531 return SDValue(E, 0);
2532
2533 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2534 CSEMap.InsertNode(N, InsertPos: IP);
2535 InsertNode(N);
2536 return SDValue(N, 0);
2537}
2538
2539SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2540 if (VT == V.getValueType())
2541 return V;
2542
2543 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2544}
2545
2546SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2547 unsigned SrcAS, unsigned DestAS) {
2548 SDVTList VTs = getVTList(VT);
2549 SDValue Ops[] = {Ptr};
2550 FoldingSetNodeID ID;
2551 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: VTs, OpList: Ops);
2552 ID.AddInteger(I: SrcAS);
2553 ID.AddInteger(I: DestAS);
2554
2555 void *IP = nullptr;
2556 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2557 return SDValue(E, 0);
2558
2559 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2560 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2561 createOperands(Node: N, Vals: Ops);
2562
2563 CSEMap.InsertNode(N, InsertPos: IP);
2564 InsertNode(N);
2565 return SDValue(N, 0);
2566}
2567
2568SDValue SelectionDAG::getFreeze(SDValue V) {
2569 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2570}
2571
2572SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2573 UndefPoisonKind Kind) {
2574 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind))
2575 return V;
2576 return getFreeze(V);
2577}
2578
2579/// getShiftAmountOperand - Return the specified value casted to
2580/// the target's desired shift amount type.
2581SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2582 EVT OpTy = Op.getValueType();
2583 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2584 if (OpTy == ShTy || OpTy.isVector()) return Op;
2585
2586 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2587}
2588
2589SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2590 SDLoc dl(Node);
2591 const TargetLowering &TLI = getTargetLoweringInfo();
2592 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2593 EVT VT = Node->getValueType(ResNo: 0);
2594 SDValue Tmp1 = Node->getOperand(Num: 0);
2595 SDValue Tmp2 = Node->getOperand(Num: 1);
2596 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2597
2598 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2599 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2600 SDValue VAList = VAListLoad;
2601
2602 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2603 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2604 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2605
2606 VAList = getNode(
2607 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2608 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2609 }
2610
2611 // Increment the pointer, VAList, to the next vaarg
2612 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2613 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2614 Ty: VT.getTypeForEVT(Context&: *getContext())),
2615 DL: dl, VT: VAList.getValueType()));
2616 // Store the incremented VAList to the legalized pointer
2617 Tmp1 =
2618 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2619 // Load the actual argument out of the pointer VAList
2620 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2621}
2622
2623SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2624 SDLoc dl(Node);
2625 const TargetLowering &TLI = getTargetLoweringInfo();
2626 // This defaults to loading a pointer from the input and storing it to the
2627 // output, returning the chain.
2628 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2629 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2630 SDValue Tmp1 =
2631 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2632 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2633 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2634 PtrInfo: MachinePointerInfo(VD));
2635}
2636
2637Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2638 const DataLayout &DL = getDataLayout();
2639 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2640 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2641
2642 if (TLI->isTypeLegal(VT) || !VT.isVector())
2643 return RedAlign;
2644
2645 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2646 const Align StackAlign = TFI->getStackAlign();
2647
2648 // See if we can choose a smaller ABI alignment in cases where it's an
2649 // illegal vector type that will get broken down.
2650 if (RedAlign > StackAlign) {
2651 EVT IntermediateVT;
2652 MVT RegisterVT;
2653 unsigned NumIntermediates;
2654 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2655 NumIntermediates, RegisterVT);
2656 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2657 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2658 if (RedAlign2 < RedAlign)
2659 RedAlign = RedAlign2;
2660
2661 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2662 // If the stack is not realignable, the alignment should be limited to the
2663 // StackAlignment
2664 RedAlign = std::min(a: RedAlign, b: StackAlign);
2665 }
2666
2667 return RedAlign;
2668}
2669
2670SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2671 MachineFrameInfo &MFI = MF->getFrameInfo();
2672 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2673 int StackID = 0;
2674 if (Bytes.isScalable())
2675 StackID = TFI->getStackIDForScalableVectors();
2676 // The stack id gives an indication of whether the object is scalable or
2677 // not, so it's safe to pass in the minimum size here.
2678 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2679 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2680 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2681}
2682
2683SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2684 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2685 Align StackAlign =
2686 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2687 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2688}
2689
2690SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2691 TypeSize VT1Size = VT1.getStoreSize();
2692 TypeSize VT2Size = VT2.getStoreSize();
2693 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2694 "Don't know how to choose the maximum size when creating a stack "
2695 "temporary");
2696 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2697 ? VT1Size
2698 : VT2Size;
2699
2700 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2701 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2702 const DataLayout &DL = getDataLayout();
2703 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2704 return CreateStackTemporary(Bytes, Alignment: Align);
2705}
2706
2707SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2708 ISD::CondCode Cond, const SDLoc &dl,
2709 SDNodeFlags Flags) {
2710 EVT OpVT = N1.getValueType();
2711
2712 auto GetUndefBooleanConstant = [&]() {
2713 if (VT.getScalarType() == MVT::i1 ||
2714 TLI->getBooleanContents(Type: OpVT) ==
2715 TargetLowering::UndefinedBooleanContent)
2716 return getUNDEF(VT);
2717 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2718 // so we cannot use getUNDEF(). Return zero instead.
2719 return getConstant(Val: 0, DL: dl, VT);
2720 };
2721
2722 // These setcc operations always fold.
2723 switch (Cond) {
2724 default: break;
2725 case ISD::SETFALSE:
2726 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2727 case ISD::SETTRUE:
2728 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2729
2730 case ISD::SETOEQ:
2731 case ISD::SETOGT:
2732 case ISD::SETOGE:
2733 case ISD::SETOLT:
2734 case ISD::SETOLE:
2735 case ISD::SETONE:
2736 case ISD::SETO:
2737 case ISD::SETUO:
2738 case ISD::SETUEQ:
2739 case ISD::SETUNE:
2740 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2741 break;
2742 }
2743
2744 if (OpVT.isInteger()) {
2745 // For EQ and NE, we can always pick a value for the undef to make the
2746 // predicate pass or fail, so we can return undef.
2747 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2748 // icmp eq/ne X, undef -> undef.
2749 if ((N1.isUndef() || N2.isUndef()) &&
2750 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2751 return GetUndefBooleanConstant();
2752
2753 // If both operands are undef, we can return undef for int comparison.
2754 // icmp undef, undef -> undef.
2755 if (N1.isUndef() && N2.isUndef())
2756 return GetUndefBooleanConstant();
2757
2758 // icmp X, X -> true/false
2759 // icmp X, undef -> true/false because undef could be X.
2760 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2761 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2762 }
2763
2764 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2765 const APInt &C2 = N2C->getAPIntValue();
2766 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2767 const APInt &C1 = N1C->getAPIntValue();
2768
2769 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2770 DL: dl, VT, OpVT);
2771 }
2772 }
2773
2774 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2775 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2776
2777 if (N1CFP && N2CFP) {
2778 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2779 switch (Cond) {
2780 default: break;
2781 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2782 return GetUndefBooleanConstant();
2783 [[fallthrough]];
2784 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2785 OpVT);
2786 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2787 return GetUndefBooleanConstant();
2788 [[fallthrough]];
2789 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2790 R==APFloat::cmpLessThan, DL: dl, VT,
2791 OpVT);
2792 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2793 return GetUndefBooleanConstant();
2794 [[fallthrough]];
2795 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2796 OpVT);
2797 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2798 return GetUndefBooleanConstant();
2799 [[fallthrough]];
2800 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2801 VT, OpVT);
2802 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2803 return GetUndefBooleanConstant();
2804 [[fallthrough]];
2805 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2806 R==APFloat::cmpEqual, DL: dl, VT,
2807 OpVT);
2808 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2809 return GetUndefBooleanConstant();
2810 [[fallthrough]];
2811 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2812 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2813 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2814 OpVT);
2815 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2816 OpVT);
2817 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2818 R==APFloat::cmpEqual, DL: dl, VT,
2819 OpVT);
2820 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2821 OpVT);
2822 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2823 R==APFloat::cmpLessThan, DL: dl, VT,
2824 OpVT);
2825 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2826 R==APFloat::cmpUnordered, DL: dl, VT,
2827 OpVT);
2828 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2829 VT, OpVT);
2830 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2831 OpVT);
2832 }
2833 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2834 // Ensure that the constant occurs on the RHS.
2835 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2836 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2837 return SDValue();
2838 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chain=*/{},
2839 /*IsSignaling=*/false, Flags);
2840 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2841 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2842 // If an operand is known to be a nan (or undef that could be a nan), we can
2843 // fold it.
2844 // Choosing NaN for the undef will always make unordered comparison succeed
2845 // and ordered comparison fails.
2846 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2847 switch (ISD::getUnorderedFlavor(Cond)) {
2848 default:
2849 llvm_unreachable("Unknown flavor!");
2850 case 0: // Known false.
2851 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2852 case 1: // Known true.
2853 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2854 case 2: // Undefined.
2855 return GetUndefBooleanConstant();
2856 }
2857 }
2858
2859 // Could not fold it.
2860 return SDValue();
2861}
2862
2863/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2864/// use this predicate to simplify operations downstream.
2865bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2866 unsigned BitWidth = Op.getScalarValueSizeInBits();
2867 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2868}
2869
2870// TODO: Should have argument to specify if sign bit of nan is ignorable.
2871bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2872 if (Depth >= MaxRecursionDepth)
2873 return false; // Limit search depth.
2874
2875 unsigned Opc = Op.getOpcode();
2876 switch (Opc) {
2877 case ISD::FABS:
2878 return true;
2879 case ISD::AssertNoFPClass: {
2880 FPClassTest NoFPClass =
2881 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2882
2883 const FPClassTest TestMask = fcNan | fcNegative;
2884 return (NoFPClass & TestMask) == TestMask;
2885 }
2886 case ISD::ARITH_FENCE:
2887 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2888 case ISD::FEXP:
2889 case ISD::FEXP2:
2890 case ISD::FEXP10:
2891 return Op->getFlags().hasNoNaNs();
2892 case ISD::FMINNUM:
2893 case ISD::FMINNUM_IEEE:
2894 case ISD::FMINIMUM:
2895 case ISD::FMINIMUMNUM:
2896 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2897 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2898 case ISD::FMAXNUM:
2899 case ISD::FMAXNUM_IEEE:
2900 case ISD::FMAXIMUM:
2901 case ISD::FMAXIMUMNUM:
2902 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2903 // is sufficient.
2904 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2905 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2906 default:
2907 return false;
2908 }
2909
2910 llvm_unreachable("covered opcode switch");
2911}
2912
2913/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2914/// this predicate to simplify operations downstream. Mask is known to be zero
2915/// for bits that V cannot have.
2916bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2917 unsigned Depth) const {
2918 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2919}
2920
2921/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2922/// DemandedElts. We use this predicate to simplify operations downstream.
2923/// Mask is known to be zero for bits that V cannot have.
2924bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2925 const APInt &DemandedElts,
2926 unsigned Depth) const {
2927 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2928}
2929
2930/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2931/// DemandedElts. We use this predicate to simplify operations downstream.
2932bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2933 unsigned Depth /* = 0 */) const {
2934 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2935}
2936
2937/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2938bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2939 unsigned Depth) const {
2940 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2941}
2942
2943APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2944 const APInt &DemandedElts,
2945 unsigned Depth) const {
2946 EVT VT = Op.getValueType();
2947 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2948
2949 unsigned NumElts = VT.getVectorNumElements();
2950 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2951
2952 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2953 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2954 if (!DemandedElts[EltIdx])
2955 continue; // Don't query elements that are not demanded.
2956 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2957 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2958 KnownZeroElements.setBit(EltIdx);
2959 }
2960 return KnownZeroElements;
2961}
2962
2963/// isSplatValue - Return true if the vector V has the same value
2964/// across all DemandedElts. For scalable vectors, we don't know the
2965/// number of lanes at compile time. Instead, we use a 1 bit APInt
2966/// to represent a conservative value for all lanes; that is, that
2967/// one bit value is implicitly splatted across all lanes.
2968bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2969 APInt &UndefElts, unsigned Depth) const {
2970 unsigned Opcode = V.getOpcode();
2971 EVT VT = V.getValueType();
2972 assert(VT.isVector() && "Vector type expected");
2973 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2974 "scalable demanded bits are ignored");
2975
2976 if (!DemandedElts)
2977 return false; // No demanded elts, better to assume we don't know anything.
2978
2979 if (Depth >= MaxRecursionDepth)
2980 return false; // Limit search depth.
2981
2982 // Deal with some common cases here that work for both fixed and scalable
2983 // vector types.
2984 switch (Opcode) {
2985 case ISD::SPLAT_VECTOR:
2986 UndefElts = V.getOperand(i: 0).isUndef()
2987 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2988 : APInt(DemandedElts.getBitWidth(), 0);
2989 return true;
2990 case ISD::ADD:
2991 case ISD::SUB:
2992 case ISD::AND:
2993 case ISD::XOR:
2994 case ISD::OR: {
2995 APInt UndefLHS, UndefRHS;
2996 SDValue LHS = V.getOperand(i: 0);
2997 SDValue RHS = V.getOperand(i: 1);
2998 // Only recognize splats with the same demanded undef elements for both
2999 // operands, otherwise we might fail to handle binop-specific undef
3000 // handling.
3001 // e.g. (and undef, 0) -> 0 etc.
3002 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
3003 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
3004 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
3005 UndefElts = UndefLHS | UndefRHS;
3006 return true;
3007 }
3008 return false;
3009 }
3010 case ISD::ABS:
3011 case ISD::ABS_MIN_POISON:
3012 case ISD::TRUNCATE:
3013 case ISD::SIGN_EXTEND:
3014 case ISD::ZERO_EXTEND:
3015 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
3016 default:
3017 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3018 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3019 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3020 Depth);
3021 break;
3022 }
3023
3024 // We don't support other cases than those above for scalable vectors at
3025 // the moment.
3026 if (VT.isScalableVector())
3027 return false;
3028
3029 unsigned NumElts = VT.getVectorNumElements();
3030 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3031 UndefElts = APInt::getZero(numBits: NumElts);
3032
3033 switch (Opcode) {
3034 case ISD::BUILD_VECTOR: {
3035 SDValue Scl;
3036 for (unsigned i = 0; i != NumElts; ++i) {
3037 SDValue Op = V.getOperand(i);
3038 if (Op.isUndef()) {
3039 UndefElts.setBit(i);
3040 continue;
3041 }
3042 if (!DemandedElts[i])
3043 continue;
3044 if (Scl && Scl != Op)
3045 return false;
3046 Scl = Op;
3047 }
3048 return true;
3049 }
3050 case ISD::VECTOR_SHUFFLE: {
3051 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3052 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3053 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3054 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3055 for (int i = 0; i != (int)NumElts; ++i) {
3056 int M = Mask[i];
3057 if (M < 0) {
3058 UndefElts.setBit(i);
3059 continue;
3060 }
3061 if (!DemandedElts[i])
3062 continue;
3063 if (M < (int)NumElts)
3064 DemandedLHS.setBit(M);
3065 else
3066 DemandedRHS.setBit(M - NumElts);
3067 }
3068
3069 // If we aren't demanding either op, assume there's no splat.
3070 // If we are demanding both ops, assume there's no splat.
3071 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3072 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3073 return false;
3074
3075 // See if the demanded elts of the source op is a splat or we only demand
3076 // one element, which should always be a splat.
3077 // TODO: Handle source ops splats with undefs.
3078 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3079 APInt SrcUndefs;
3080 return (SrcElts.popcount() == 1) ||
3081 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3082 (SrcElts & SrcUndefs).isZero());
3083 };
3084 if (!DemandedLHS.isZero())
3085 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3086 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3087 }
3088 case ISD::EXTRACT_SUBVECTOR: {
3089 // Offset the demanded elts by the subvector index.
3090 SDValue Src = V.getOperand(i: 0);
3091 // We don't support scalable vectors at the moment.
3092 if (Src.getValueType().isScalableVector())
3093 return false;
3094 uint64_t Idx = V.getConstantOperandVal(i: 1);
3095 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3096 APInt UndefSrcElts;
3097 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3098 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3099 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3100 return true;
3101 }
3102 break;
3103 }
3104 case ISD::ANY_EXTEND_VECTOR_INREG:
3105 case ISD::SIGN_EXTEND_VECTOR_INREG:
3106 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3107 // Widen the demanded elts by the src element count.
3108 SDValue Src = V.getOperand(i: 0);
3109 // We don't support scalable vectors at the moment.
3110 if (Src.getValueType().isScalableVector())
3111 return false;
3112 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3113 APInt UndefSrcElts;
3114 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3115 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3116 UndefElts = UndefSrcElts.trunc(width: NumElts);
3117 return true;
3118 }
3119 break;
3120 }
3121 case ISD::BITCAST: {
3122 SDValue Src = V.getOperand(i: 0);
3123 EVT SrcVT = Src.getValueType();
3124 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3125 unsigned BitWidth = VT.getScalarSizeInBits();
3126
3127 // Ignore bitcasts from unsupported types.
3128 // TODO: Add fp support?
3129 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3130 break;
3131
3132 // Bitcast 'small element' vector to 'large element' vector.
3133 if ((BitWidth % SrcBitWidth) == 0) {
3134 // See if each sub element is a splat.
3135 unsigned Scale = BitWidth / SrcBitWidth;
3136 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3137 APInt ScaledDemandedElts =
3138 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3139 for (unsigned I = 0; I != Scale; ++I) {
3140 APInt SubUndefElts;
3141 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3142 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3143 SubDemandedElts &= ScaledDemandedElts;
3144 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3145 return false;
3146 // TODO: Add support for merging sub undef elements.
3147 if (!SubUndefElts.isZero())
3148 return false;
3149 }
3150 return true;
3151 }
3152 break;
3153 }
3154 }
3155
3156 return false;
3157}
3158
3159/// Helper wrapper to main isSplatValue function.
3160bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3161 EVT VT = V.getValueType();
3162 assert(VT.isVector() && "Vector type expected");
3163
3164 APInt UndefElts;
3165 // Since the number of lanes in a scalable vector is unknown at compile time,
3166 // we track one bit which is implicitly broadcast to all lanes. This means
3167 // that all lanes in a scalable vector are considered demanded.
3168 APInt DemandedElts
3169 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3170 return isSplatValue(V, DemandedElts, UndefElts) &&
3171 (AllowUndefs || !UndefElts);
3172}
3173
3174SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3175 V = peekThroughExtractSubvectors(V);
3176
3177 EVT VT = V.getValueType();
3178 unsigned Opcode = V.getOpcode();
3179 switch (Opcode) {
3180 default: {
3181 APInt UndefElts;
3182 // Since the number of lanes in a scalable vector is unknown at compile time,
3183 // we track one bit which is implicitly broadcast to all lanes. This means
3184 // that all lanes in a scalable vector are considered demanded.
3185 APInt DemandedElts
3186 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3187
3188 if (isSplatValue(V, DemandedElts, UndefElts)) {
3189 if (VT.isScalableVector()) {
3190 // DemandedElts and UndefElts are ignored for scalable vectors, since
3191 // the only supported cases are SPLAT_VECTOR nodes.
3192 SplatIdx = 0;
3193 } else {
3194 // Handle case where all demanded elements are UNDEF.
3195 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3196 SplatIdx = 0;
3197 return getUNDEF(VT);
3198 }
3199 SplatIdx = (UndefElts & DemandedElts).countr_one();
3200 }
3201 return V;
3202 }
3203 break;
3204 }
3205 case ISD::SPLAT_VECTOR:
3206 SplatIdx = 0;
3207 return V;
3208 case ISD::VECTOR_SHUFFLE: {
3209 assert(!VT.isScalableVector());
3210 // Check if this is a shuffle node doing a splat.
3211 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3212 // getTargetVShiftNode currently struggles without the splat source.
3213 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3214 if (!SVN->isSplat())
3215 break;
3216 int Idx = SVN->getSplatIndex();
3217 int NumElts = V.getValueType().getVectorNumElements();
3218 SplatIdx = Idx % NumElts;
3219 return V.getOperand(i: Idx / NumElts);
3220 }
3221 }
3222
3223 return SDValue();
3224}
3225
3226SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3227 int SplatIdx;
3228 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3229 EVT SVT = SrcVector.getValueType().getScalarType();
3230 EVT LegalSVT = SVT;
3231 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3232 if (!SVT.isInteger())
3233 return SDValue();
3234 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3235 if (LegalSVT.bitsLT(VT: SVT))
3236 return SDValue();
3237 }
3238 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3239 }
3240 return SDValue();
3241}
3242
3243std::optional<ConstantRange>
3244SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3245 unsigned Depth) const {
3246 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3247 V.getOpcode() == ISD::SRA) &&
3248 "Unknown shift node");
3249 // Shifting more than the bitwidth is not valid.
3250 unsigned BitWidth = V.getScalarValueSizeInBits();
3251
3252 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3253 const APInt &ShAmt = Cst->getAPIntValue();
3254 if (ShAmt.uge(RHS: BitWidth))
3255 return std::nullopt;
3256 return ConstantRange(ShAmt);
3257 }
3258
3259 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3260 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3261 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3262 if (!DemandedElts[i])
3263 continue;
3264 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3265 if (!SA) {
3266 MinAmt = MaxAmt = nullptr;
3267 break;
3268 }
3269 const APInt &ShAmt = SA->getAPIntValue();
3270 if (ShAmt.uge(RHS: BitWidth))
3271 return std::nullopt;
3272 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3273 MinAmt = &ShAmt;
3274 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3275 MaxAmt = &ShAmt;
3276 }
3277 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3278 "Failed to find matching min/max shift amounts");
3279 if (MinAmt && MaxAmt)
3280 return ConstantRange(*MinAmt, *MaxAmt + 1);
3281 }
3282
3283 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3284 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3285 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3286 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3287 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3288
3289 return std::nullopt;
3290}
3291
3292std::optional<unsigned>
3293SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3294 unsigned Depth) const {
3295 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3296 V.getOpcode() == ISD::SRA) &&
3297 "Unknown shift node");
3298 if (std::optional<ConstantRange> AmtRange =
3299 getValidShiftAmountRange(V, DemandedElts, Depth))
3300 if (const APInt *ShAmt = AmtRange->getSingleElement())
3301 return ShAmt->getZExtValue();
3302 return std::nullopt;
3303}
3304
3305std::optional<unsigned>
3306SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3307 APInt DemandedElts = getDemandAllEltsMask(V);
3308 return getValidShiftAmount(V, DemandedElts, Depth);
3309}
3310
3311std::optional<unsigned>
3312SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3313 unsigned Depth) const {
3314 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3315 V.getOpcode() == ISD::SRA) &&
3316 "Unknown shift node");
3317 if (std::optional<ConstantRange> AmtRange =
3318 getValidShiftAmountRange(V, DemandedElts, Depth))
3319 return AmtRange->getUnsignedMin().getZExtValue();
3320 return std::nullopt;
3321}
3322
3323std::optional<unsigned>
3324SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3325 APInt DemandedElts = getDemandAllEltsMask(V);
3326 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3327}
3328
3329std::optional<unsigned>
3330SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3331 unsigned Depth) const {
3332 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3333 V.getOpcode() == ISD::SRA) &&
3334 "Unknown shift node");
3335 if (std::optional<ConstantRange> AmtRange =
3336 getValidShiftAmountRange(V, DemandedElts, Depth))
3337 return AmtRange->getUnsignedMax().getZExtValue();
3338 return std::nullopt;
3339}
3340
3341std::optional<unsigned>
3342SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3343 APInt DemandedElts = getDemandAllEltsMask(V);
3344 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3345}
3346
3347/// Determine which bits of Op are known to be either zero or one and return
3348/// them in Known. For vectors, the known bits are those that are shared by
3349/// every vector element.
3350KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3351 APInt DemandedElts = getDemandAllEltsMask(V: Op);
3352 return computeKnownBits(Op, DemandedElts, Depth);
3353}
3354
3355/// Determine which bits of Op are known to be either zero or one and return
3356/// them in Known. The DemandedElts argument allows us to only collect the known
3357/// bits that are shared by the requested vector elements.
3358KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3359 unsigned Depth) const {
3360 unsigned BitWidth = Op.getScalarValueSizeInBits();
3361
3362 KnownBits Known(BitWidth); // Don't know anything.
3363
3364 if (auto OptAPInt = Op->bitcastToAPInt()) {
3365 // We know all of the bits for a constant!
3366 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3367 }
3368
3369 if (Depth >= MaxRecursionDepth)
3370 return Known; // Limit search depth.
3371
3372 KnownBits Known2;
3373 unsigned NumElts = DemandedElts.getBitWidth();
3374 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3375 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3376 assert((!Op.getValueType().isFixedLengthVector() ||
3377 NumElts == Op.getValueType().getVectorNumElements()) &&
3378 "Unexpected vector size");
3379
3380 if (!DemandedElts)
3381 return Known; // No demanded elts, better to assume we don't know anything.
3382
3383 unsigned Opcode = Op.getOpcode();
3384 switch (Opcode) {
3385 case ISD::MERGE_VALUES:
3386 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3387 Depth: Depth + 1);
3388 case ISD::SPLAT_VECTOR: {
3389 SDValue SrcOp = Op.getOperand(i: 0);
3390 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3391 "Expected SPLAT_VECTOR implicit truncation");
3392 // Implicitly truncate the bits to match the official semantics of
3393 // SPLAT_VECTOR.
3394 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3395 break;
3396 }
3397 case ISD::SPLAT_VECTOR_PARTS: {
3398 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3399 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3400 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3401 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3402 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3403 }
3404 break;
3405 }
3406 case ISD::STEP_VECTOR: {
3407 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3408
3409 if (Step.isPowerOf2())
3410 Known.Zero.setLowBits(Step.logBase2());
3411
3412 const Function &F = getMachineFunction().getFunction();
3413
3414 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3415 break;
3416 const APInt MinNumElts =
3417 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3418
3419 bool Overflow;
3420 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3421 .getUnsignedMax()
3422 .umul_ov(RHS: MinNumElts, Overflow);
3423 if (Overflow)
3424 break;
3425
3426 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3427 if (Overflow)
3428 break;
3429
3430 Known.Zero.setHighBits(MaxValue.countl_zero());
3431 break;
3432 }
3433 case ISD::BUILD_VECTOR:
3434 assert(!Op.getValueType().isScalableVector());
3435 // Collect the known bits that are shared by every demanded vector element.
3436 Known.setAllConflict();
3437 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3438 if (!DemandedElts[i])
3439 continue;
3440
3441 SDValue SrcOp = Op.getOperand(i);
3442 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3443
3444 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3445 if (SrcOp.getValueSizeInBits() != BitWidth) {
3446 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3447 "Expected BUILD_VECTOR implicit truncation");
3448 Known2 = Known2.trunc(BitWidth);
3449 }
3450
3451 // Known bits are the values that are shared by every demanded element.
3452 Known = Known.intersectWith(RHS: Known2);
3453
3454 // If we don't know any bits, early out.
3455 if (Known.isUnknown())
3456 break;
3457 }
3458 break;
3459 case ISD::VECTOR_COMPRESS: {
3460 SDValue Vec = Op.getOperand(i: 0);
3461 SDValue PassThru = Op.getOperand(i: 2);
3462 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3463 // If we don't know any bits, early out.
3464 if (Known.isUnknown())
3465 break;
3466 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3467 Known = Known.intersectWith(RHS: Known2);
3468 break;
3469 }
3470 case ISD::VECTOR_SHUFFLE: {
3471 assert(!Op.getValueType().isScalableVector());
3472 // Collect the known bits that are shared by every vector element referenced
3473 // by the shuffle.
3474 APInt DemandedLHS, DemandedRHS;
3475 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3476 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3477 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3478 DemandedLHS, DemandedRHS))
3479 break;
3480
3481 // Known bits are the values that are shared by every demanded element.
3482 Known.setAllConflict();
3483 if (!!DemandedLHS) {
3484 SDValue LHS = Op.getOperand(i: 0);
3485 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3486 Known = Known.intersectWith(RHS: Known2);
3487 }
3488 // If we don't know any bits, early out.
3489 if (Known.isUnknown())
3490 break;
3491 if (!!DemandedRHS) {
3492 SDValue RHS = Op.getOperand(i: 1);
3493 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3494 Known = Known.intersectWith(RHS: Known2);
3495 }
3496 break;
3497 }
3498 case ISD::VSCALE: {
3499 const Function &F = getMachineFunction().getFunction();
3500 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3501 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3502 break;
3503 }
3504 case ISD::CONCAT_VECTORS: {
3505 if (Op.getValueType().isScalableVector())
3506 break;
3507 // Split DemandedElts and test each of the demanded subvectors.
3508 Known.setAllConflict();
3509 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3510 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3511 unsigned NumSubVectors = Op.getNumOperands();
3512 for (unsigned i = 0; i != NumSubVectors; ++i) {
3513 APInt DemandedSub =
3514 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3515 if (!!DemandedSub) {
3516 SDValue Sub = Op.getOperand(i);
3517 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3518 Known = Known.intersectWith(RHS: Known2);
3519 }
3520 // If we don't know any bits, early out.
3521 if (Known.isUnknown())
3522 break;
3523 }
3524 break;
3525 }
3526 case ISD::INSERT_SUBVECTOR: {
3527 if (Op.getValueType().isScalableVector())
3528 break;
3529 // Demand any elements from the subvector and the remainder from the src its
3530 // inserted into.
3531 SDValue Src = Op.getOperand(i: 0);
3532 SDValue Sub = Op.getOperand(i: 1);
3533 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3534 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3535 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3536 APInt DemandedSrcElts = DemandedElts;
3537 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3538
3539 Known.setAllConflict();
3540 if (!!DemandedSubElts) {
3541 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3542 if (Known.isUnknown())
3543 break; // early-out.
3544 }
3545 if (!!DemandedSrcElts) {
3546 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3547 Known = Known.intersectWith(RHS: Known2);
3548 }
3549 break;
3550 }
3551 case ISD::EXTRACT_SUBVECTOR: {
3552 // Offset the demanded elts by the subvector index.
3553 SDValue Src = Op.getOperand(i: 0);
3554
3555 APInt DemandedSrcElts;
3556 if (Src.getValueType().isScalableVector())
3557 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3558 else {
3559 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3560 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3561 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3562 }
3563 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3564 break;
3565 }
3566 case ISD::SCALAR_TO_VECTOR: {
3567 if (Op.getValueType().isScalableVector())
3568 break;
3569 // We know about scalar_to_vector as much as we know about it source,
3570 // which becomes the first element of otherwise unknown vector.
3571 if (DemandedElts != 1)
3572 break;
3573
3574 SDValue N0 = Op.getOperand(i: 0);
3575 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3576 if (N0.getValueSizeInBits() != BitWidth)
3577 Known = Known.trunc(BitWidth);
3578
3579 break;
3580 }
3581 case ISD::BITCAST: {
3582 if (Op.getValueType().isScalableVector())
3583 break;
3584
3585 SDValue N0 = Op.getOperand(i: 0);
3586 EVT SubVT = N0.getValueType();
3587 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3588
3589 // Ignore bitcasts from unsupported types.
3590 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3591 break;
3592
3593 // Fast handling of 'identity' bitcasts.
3594 if (BitWidth == SubBitWidth) {
3595 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3596 break;
3597 }
3598
3599 bool IsLE = getDataLayout().isLittleEndian();
3600
3601 // Bitcast 'small element' vector to 'large element' scalar/vector.
3602 if ((BitWidth % SubBitWidth) == 0) {
3603 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3604
3605 // Collect known bits for the (larger) output by collecting the known
3606 // bits from each set of sub elements and shift these into place.
3607 // We need to separately call computeKnownBits for each set of
3608 // sub elements as the knownbits for each is likely to be different.
3609 unsigned SubScale = BitWidth / SubBitWidth;
3610 APInt SubDemandedElts(NumElts * SubScale, 0);
3611 for (unsigned i = 0; i != NumElts; ++i)
3612 if (DemandedElts[i])
3613 SubDemandedElts.setBit(i * SubScale);
3614
3615 for (unsigned i = 0; i != SubScale; ++i) {
3616 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3617 Depth: Depth + 1);
3618 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3619 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3620 }
3621 }
3622
3623 // Bitcast 'large element' scalar/vector to 'small element' vector.
3624 if ((SubBitWidth % BitWidth) == 0) {
3625 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3626
3627 // Collect known bits for the (smaller) output by collecting the known
3628 // bits from the overlapping larger input elements and extracting the
3629 // sub sections we actually care about.
3630 unsigned SubScale = SubBitWidth / BitWidth;
3631 APInt SubDemandedElts =
3632 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3633 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3634
3635 Known.setAllConflict();
3636 for (unsigned i = 0; i != NumElts; ++i)
3637 if (DemandedElts[i]) {
3638 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3639 unsigned Offset = (Shifts % SubScale) * BitWidth;
3640 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3641 // If we don't know any bits, early out.
3642 if (Known.isUnknown())
3643 break;
3644 }
3645 }
3646 break;
3647 }
3648 case ISD::AND:
3649 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3650 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3651
3652 Known &= Known2;
3653 break;
3654 case ISD::OR:
3655 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3656 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3657
3658 Known |= Known2;
3659 break;
3660 case ISD::XOR:
3661 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3662 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3663
3664 Known ^= Known2;
3665 break;
3666 case ISD::MUL: {
3667 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3668 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3669 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3670 // TODO: SelfMultiply can be poison, but not undef.
3671 if (SelfMultiply)
3672 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3673 Op: Op.getOperand(i: 0), DemandedElts, Kind: UndefPoisonKind::UndefOrPoison,
3674 Depth: Depth + 1);
3675 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3676
3677 // If the multiplication is known not to overflow, the product of a number
3678 // with itself is non-negative. Only do this if we didn't already computed
3679 // the opposite value for the sign bit.
3680 if (Op->getFlags().hasNoSignedWrap() &&
3681 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3682 !Known.isNegative())
3683 Known.makeNonNegative();
3684 break;
3685 }
3686 case ISD::MULHU: {
3687 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3688 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3689 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3690 break;
3691 }
3692 case ISD::MULHS: {
3693 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3694 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3695 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3696 break;
3697 }
3698 case ISD::ABDU: {
3699 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3700 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3701 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3702 break;
3703 }
3704 case ISD::ABDS: {
3705 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3706 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3707 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3708 unsigned SignBits1 =
3709 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3710 if (SignBits1 == 1)
3711 break;
3712 unsigned SignBits0 =
3713 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3714 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3715 break;
3716 }
3717 case ISD::UMUL_LOHI: {
3718 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3719 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3720 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3721 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3722 if (Op.getResNo() == 0)
3723 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3724 else
3725 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3726 break;
3727 }
3728 case ISD::SMUL_LOHI: {
3729 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3730 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3731 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3732 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3733 if (Op.getResNo() == 0)
3734 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3735 else
3736 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3737 break;
3738 }
3739 case ISD::AVGFLOORU: {
3740 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3741 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3742 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3743 break;
3744 }
3745 case ISD::AVGCEILU: {
3746 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3747 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3748 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3749 break;
3750 }
3751 case ISD::AVGFLOORS: {
3752 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3753 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3754 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3755 break;
3756 }
3757 case ISD::AVGCEILS: {
3758 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3759 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3760 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3761 break;
3762 }
3763 case ISD::SELECT:
3764 case ISD::VSELECT:
3765 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3766 // If we don't know any bits, early out.
3767 if (Known.isUnknown())
3768 break;
3769 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3770
3771 // Only known if known in both the LHS and RHS.
3772 Known = Known.intersectWith(RHS: Known2);
3773 break;
3774 case ISD::SELECT_CC:
3775 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3776 // If we don't know any bits, early out.
3777 if (Known.isUnknown())
3778 break;
3779 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3780
3781 // Only known if known in both the LHS and RHS.
3782 Known = Known.intersectWith(RHS: Known2);
3783 break;
3784 case ISD::SMULO:
3785 case ISD::UMULO:
3786 if (Op.getResNo() != 1)
3787 break;
3788 // The boolean result conforms to getBooleanContents.
3789 // If we know the result of a setcc has the top bits zero, use this info.
3790 // We know that we have an integer-based boolean since these operations
3791 // are only available for integer.
3792 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3793 TargetLowering::ZeroOrOneBooleanContent &&
3794 BitWidth > 1)
3795 Known.Zero.setBitsFrom(1);
3796 break;
3797 case ISD::SETCC:
3798 case ISD::SETCCCARRY:
3799 case ISD::STRICT_FSETCC:
3800 case ISD::STRICT_FSETCCS: {
3801 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3802 // If we know the result of a setcc has the top bits zero, use this info.
3803 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3804 TargetLowering::ZeroOrOneBooleanContent &&
3805 BitWidth > 1)
3806 Known.Zero.setBitsFrom(1);
3807 break;
3808 }
3809 case ISD::SHL: {
3810 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3811 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3812
3813 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3814 bool NSW = Op->getFlags().hasNoSignedWrap();
3815
3816 bool ShAmtNonZero = Known2.isNonZero();
3817
3818 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3819
3820 // Minimum shift low bits are known zero.
3821 if (std::optional<unsigned> ShMinAmt =
3822 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3823 Known.Zero.setLowBits(*ShMinAmt);
3824 break;
3825 }
3826 case ISD::SRL:
3827 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3828 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3829 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3830 Exact: Op->getFlags().hasExact());
3831
3832 // Minimum shift high bits are known zero.
3833 if (std::optional<unsigned> ShMinAmt =
3834 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3835 Known.Zero.setHighBits(*ShMinAmt);
3836 break;
3837 case ISD::SRA:
3838 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3839 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3840 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3841 Exact: Op->getFlags().hasExact());
3842 break;
3843 case ISD::ROTL:
3844 case ISD::ROTR:
3845 if (ConstantSDNode *C =
3846 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3847 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3848
3849 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3850
3851 // Canonicalize to ROTR.
3852 if (Opcode == ISD::ROTL && Amt != 0)
3853 Amt = BitWidth - Amt;
3854
3855 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3856 Known.One = Known.One.rotr(rotateAmt: Amt);
3857 }
3858 break;
3859 case ISD::FSHL:
3860 case ISD::FSHR:
3861 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3862 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3863
3864 // For fshl, 0-shift returns the 1st arg.
3865 // For fshr, 0-shift returns the 2nd arg.
3866 if (Amt == 0) {
3867 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3868 DemandedElts, Depth: Depth + 1);
3869 break;
3870 }
3871
3872 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3873 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3874 const APInt ShAmt(BitWidth, Amt);
3875 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3876 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3877 Known = Opcode == ISD::FSHL ? KnownBits::fshl(LHS: Known, RHS: Known2, Amt: ShAmt)
3878 : KnownBits::fshr(LHS: Known, RHS: Known2, Amt: ShAmt);
3879 }
3880 break;
3881 case ISD::SHL_PARTS:
3882 case ISD::SRA_PARTS:
3883 case ISD::SRL_PARTS: {
3884 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3885
3886 // Collect lo/hi source values and concatenate.
3887 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3888 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3889 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3890 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3891 Known = Known2.concat(Lo: Known);
3892
3893 // Collect shift amount.
3894 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3895
3896 if (Opcode == ISD::SHL_PARTS)
3897 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3898 else if (Opcode == ISD::SRA_PARTS)
3899 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3900 else // if (Opcode == ISD::SRL_PARTS)
3901 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3902
3903 // TODO: Minimum shift low/high bits are known zero.
3904
3905 if (Op.getResNo() == 0)
3906 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3907 else
3908 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3909 break;
3910 }
3911 case ISD::SIGN_EXTEND_INREG: {
3912 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3913 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3914 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3915 break;
3916 }
3917 case ISD::CTTZ:
3918 case ISD::CTTZ_ZERO_POISON: {
3919 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3920 // If we have a known 1, its position is our upper bound.
3921 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3922 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3923 Known.Zero.setBitsFrom(LowBits);
3924 break;
3925 }
3926 case ISD::CTLZ:
3927 case ISD::CTLZ_ZERO_POISON: {
3928 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3929 // If we have a known 1, its position is our upper bound.
3930 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3931 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3932 Known.Zero.setBitsFrom(LowBits);
3933 break;
3934 }
3935 case ISD::CTLS: {
3936 unsigned MinRedundantSignBits =
3937 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3938 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3939 APInt(BitWidth, BitWidth));
3940 Known = Range.toKnownBits();
3941 break;
3942 }
3943 case ISD::CTPOP: {
3944 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3945 // If we know some of the bits are zero, they can't be one.
3946 unsigned PossibleOnes = Known2.countMaxPopulation();
3947 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3948 break;
3949 }
3950 case ISD::PARITY: {
3951 // Parity returns 0 everywhere but the LSB.
3952 Known.Zero.setBitsFrom(1);
3953 break;
3954 }
3955 case ISD::PDEP: {
3956 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3957 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3958 Known = KnownBits::pdep(Val: Known2, Mask: Known);
3959 break;
3960 }
3961 case ISD::PEXT: {
3962 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3963 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3964 Known = KnownBits::pext(Val: Known2, Mask: Known);
3965 break;
3966 }
3967 case ISD::CLMUL: {
3968 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3969 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3970 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3971 break;
3972 }
3973 case ISD::MGATHER:
3974 case ISD::MLOAD: {
3975 ISD::LoadExtType ETy =
3976 (Opcode == ISD::MGATHER)
3977 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3978 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3979 if (ETy == ISD::ZEXTLOAD) {
3980 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3981 KnownBits Known0(MemVT.getScalarSizeInBits());
3982 return Known0.zext(BitWidth);
3983 }
3984 break;
3985 }
3986 case ISD::LOAD: {
3987 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3988 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3989 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3990 // Determine any common known bits from the loaded constant pool value.
3991 Type *CstTy = Cst->getType();
3992 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3993 !Op.getValueType().isScalableVector()) {
3994 // If its a vector splat, then we can (quickly) reuse the scalar path.
3995 // NOTE: We assume all elements match and none are UNDEF.
3996 if (CstTy->isVectorTy()) {
3997 if (const Constant *Splat = Cst->getSplatValue()) {
3998 Cst = Splat;
3999 CstTy = Cst->getType();
4000 }
4001 }
4002 // TODO - do we need to handle different bitwidths?
4003 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
4004 // Iterate across all vector elements finding common known bits.
4005 Known.setAllConflict();
4006 for (unsigned i = 0; i != NumElts; ++i) {
4007 if (!DemandedElts[i])
4008 continue;
4009 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
4010 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
4011 const APInt &Value = CInt->getValue();
4012 Known.One &= Value;
4013 Known.Zero &= ~Value;
4014 continue;
4015 }
4016 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
4017 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4018 Known.One &= Value;
4019 Known.Zero &= ~Value;
4020 continue;
4021 }
4022 }
4023 Known.One.clearAllBits();
4024 Known.Zero.clearAllBits();
4025 break;
4026 }
4027 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4028 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4029 Known = KnownBits::makeConstant(C: CInt->getValue());
4030 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4031 Known =
4032 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4033 }
4034 }
4035 }
4036 } else if (Op.getResNo() == 0) {
4037 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4038 KnownBits KnownScalarMemory(ScalarMemorySize);
4039 if (const MDNode *MD = LD->getRanges())
4040 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4041
4042 // Extend the Known bits from memory to the size of the scalar result.
4043 if (ISD::isZEXTLoad(N: Op.getNode()))
4044 Known = KnownScalarMemory.zext(BitWidth);
4045 else if (ISD::isSEXTLoad(N: Op.getNode()))
4046 Known = KnownScalarMemory.sext(BitWidth);
4047 else if (ISD::isEXTLoad(N: Op.getNode()))
4048 Known = KnownScalarMemory.anyext(BitWidth);
4049 else
4050 Known = KnownScalarMemory;
4051 assert(Known.getBitWidth() == BitWidth);
4052 return Known;
4053 }
4054 break;
4055 }
4056 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4057 if (Op.getValueType().isScalableVector())
4058 break;
4059 EVT InVT = Op.getOperand(i: 0).getValueType();
4060 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4061 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4062 Known = Known.zext(BitWidth);
4063 break;
4064 }
4065 case ISD::ZERO_EXTEND: {
4066 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4067 Known = Known.zext(BitWidth);
4068 break;
4069 }
4070 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4071 if (Op.getValueType().isScalableVector())
4072 break;
4073 EVT InVT = Op.getOperand(i: 0).getValueType();
4074 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4075 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4076 // If the sign bit is known to be zero or one, then sext will extend
4077 // it to the top bits, else it will just zext.
4078 Known = Known.sext(BitWidth);
4079 break;
4080 }
4081 case ISD::SIGN_EXTEND: {
4082 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4083 // If the sign bit is known to be zero or one, then sext will extend
4084 // it to the top bits, else it will just zext.
4085 Known = Known.sext(BitWidth);
4086 break;
4087 }
4088 case ISD::ANY_EXTEND_VECTOR_INREG: {
4089 if (Op.getValueType().isScalableVector())
4090 break;
4091 EVT InVT = Op.getOperand(i: 0).getValueType();
4092 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4093 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4094 Known = Known.anyext(BitWidth);
4095 break;
4096 }
4097 case ISD::ANY_EXTEND: {
4098 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4099 Known = Known.anyext(BitWidth);
4100 break;
4101 }
4102 case ISD::TRUNCATE: {
4103 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4104 Known = Known.trunc(BitWidth);
4105 break;
4106 }
4107 case ISD::TRUNCATE_SSAT_S: {
4108 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4109 Known = Known.truncSSat(BitWidth);
4110 break;
4111 }
4112 case ISD::TRUNCATE_SSAT_U: {
4113 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4114 Known = Known.truncSSatU(BitWidth);
4115 break;
4116 }
4117 case ISD::TRUNCATE_USAT_U: {
4118 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4119 Known = Known.truncUSat(BitWidth);
4120 break;
4121 }
4122 case ISD::AssertZext: {
4123 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4124 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4125 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4126 Known.Zero |= (~InMask);
4127 Known.One &= (~Known.Zero);
4128 break;
4129 }
4130 case ISD::AssertAlign: {
4131 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4132 assert(LogOfAlign != 0);
4133
4134 // TODO: Should use maximum with source
4135 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4136 // well as clearing one bits.
4137 Known.Zero.setLowBits(LogOfAlign);
4138 Known.One.clearLowBits(loBits: LogOfAlign);
4139 break;
4140 }
4141 case ISD::AssertNoFPClass: {
4142 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4143
4144 FPClassTest NoFPClass =
4145 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4146 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4147 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4148 // Cannot be negative.
4149 Known.makeNonNegative();
4150 }
4151
4152 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4153 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4154 // Cannot be positive.
4155 Known.makeNegative();
4156 }
4157
4158 break;
4159 }
4160 case ISD::FABS:
4161 // fabs clears the sign bit
4162 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4163 Known.makeNonNegative();
4164 break;
4165 case ISD::FGETSIGN:
4166 // All bits are zero except the low bit.
4167 Known.Zero.setBitsFrom(1);
4168 break;
4169 case ISD::ADD: {
4170 SDNodeFlags Flags = Op.getNode()->getFlags();
4171 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4172 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4173 bool SelfAdd = Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
4174 isGuaranteedNotToBeUndefOrPoison(
4175 Op: Op.getOperand(i: 0), DemandedElts,
4176 Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1);
4177 Known = KnownBits::add(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4178 NUW: Flags.hasNoUnsignedWrap(), SelfAdd);
4179 break;
4180 }
4181 case ISD::SUB: {
4182 SDNodeFlags Flags = Op.getNode()->getFlags();
4183 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4184 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4185 Known = KnownBits::sub(LHS: Known, RHS: Known2, NSW: Flags.hasNoSignedWrap(),
4186 NUW: Flags.hasNoUnsignedWrap());
4187 break;
4188 }
4189 case ISD::USUBO:
4190 case ISD::SSUBO:
4191 case ISD::USUBO_CARRY:
4192 case ISD::SSUBO_CARRY:
4193 if (Op.getResNo() == 1) {
4194 // If we know the result of a setcc has the top bits zero, use this info.
4195 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4196 TargetLowering::ZeroOrOneBooleanContent &&
4197 BitWidth > 1)
4198 Known.Zero.setBitsFrom(1);
4199 break;
4200 }
4201 [[fallthrough]];
4202 case ISD::SUBC: {
4203 assert(Op.getResNo() == 0 &&
4204 "We only compute knownbits for the difference here.");
4205
4206 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4207 KnownBits Borrow(1);
4208 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4209 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4210 // Borrow has bit width 1
4211 Borrow = Borrow.trunc(BitWidth: 1);
4212 } else {
4213 Borrow.setAllZero();
4214 }
4215
4216 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4217 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4218 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4219 break;
4220 }
4221 case ISD::UADDO:
4222 case ISD::SADDO:
4223 case ISD::UADDO_CARRY:
4224 case ISD::SADDO_CARRY:
4225 if (Op.getResNo() == 1) {
4226 // If we know the result of a setcc has the top bits zero, use this info.
4227 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4228 TargetLowering::ZeroOrOneBooleanContent &&
4229 BitWidth > 1)
4230 Known.Zero.setBitsFrom(1);
4231 break;
4232 }
4233 [[fallthrough]];
4234 case ISD::ADDC:
4235 case ISD::ADDE: {
4236 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4237
4238 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4239 KnownBits Carry(1);
4240 if (Opcode == ISD::ADDE)
4241 // Can't track carry from glue, set carry to unknown.
4242 Carry.resetAll();
4243 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4244 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4245 // Carry has bit width 1
4246 Carry = Carry.trunc(BitWidth: 1);
4247 } else {
4248 Carry.setAllZero();
4249 }
4250
4251 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4252 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4253 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4254 break;
4255 }
4256 case ISD::UDIV: {
4257 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4258 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4259 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4260 break;
4261 }
4262 case ISD::SDIV: {
4263 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4264 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4265 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4266 break;
4267 }
4268 case ISD::SREM: {
4269 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4270 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4271 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4272 break;
4273 }
4274 case ISD::UREM: {
4275 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4276 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4277 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4278 break;
4279 }
4280 case ISD::EXTRACT_ELEMENT: {
4281 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4282 const unsigned Index = Op.getConstantOperandVal(i: 1);
4283 const unsigned EltBitWidth = Op.getValueSizeInBits();
4284
4285 // Remove low part of known bits mask
4286 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4287 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4288
4289 // Remove high part of known bit mask
4290 Known = Known.trunc(BitWidth: EltBitWidth);
4291 break;
4292 }
4293 case ISD::EXTRACT_VECTOR_ELT: {
4294 SDValue InVec = Op.getOperand(i: 0);
4295 SDValue EltNo = Op.getOperand(i: 1);
4296 EVT VecVT = InVec.getValueType();
4297 // computeKnownBits not yet implemented for scalable vectors.
4298 if (VecVT.isScalableVector())
4299 break;
4300 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4301 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4302
4303 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4304 // anything about the extended bits.
4305 if (BitWidth > EltBitWidth)
4306 Known = Known.trunc(BitWidth: EltBitWidth);
4307
4308 // If we know the element index, just demand that vector element, else for
4309 // an unknown element index, ignore DemandedElts and demand them all.
4310 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4311 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4312 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4313 DemandedSrcElts =
4314 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4315
4316 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4317 if (BitWidth > EltBitWidth)
4318 Known = Known.anyext(BitWidth);
4319 break;
4320 }
4321 case ISD::INSERT_VECTOR_ELT: {
4322 if (Op.getValueType().isScalableVector())
4323 break;
4324
4325 // If we know the element index, split the demand between the
4326 // source vector and the inserted element, otherwise assume we need
4327 // the original demanded vector elements and the value.
4328 SDValue InVec = Op.getOperand(i: 0);
4329 SDValue InVal = Op.getOperand(i: 1);
4330 SDValue EltNo = Op.getOperand(i: 2);
4331 bool DemandedVal = true;
4332 APInt DemandedVecElts = DemandedElts;
4333 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4334 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4335 unsigned EltIdx = CEltNo->getZExtValue();
4336 DemandedVal = !!DemandedElts[EltIdx];
4337 DemandedVecElts.clearBit(BitPosition: EltIdx);
4338 }
4339 Known.setAllConflict();
4340 if (DemandedVal) {
4341 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4342 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4343 }
4344 if (!!DemandedVecElts) {
4345 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4346 Known = Known.intersectWith(RHS: Known2);
4347 }
4348 break;
4349 }
4350 case ISD::BITREVERSE: {
4351 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4352 Known = Known2.reverseBits();
4353 break;
4354 }
4355 case ISD::BSWAP: {
4356 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4357 Known = Known2.byteSwap();
4358 break;
4359 }
4360 case ISD::ABS:
4361 case ISD::ABS_MIN_POISON: {
4362 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4363 Known = Known2.abs();
4364 Known.Zero.setHighBits(
4365 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4366 break;
4367 }
4368 case ISD::USUBSAT: {
4369 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4370 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4371 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4372 break;
4373 }
4374 case ISD::UMIN: {
4375 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4376 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4377 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4378 break;
4379 }
4380 case ISD::UMAX: {
4381 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4382 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4383 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4384 break;
4385 }
4386 case ISD::SMIN:
4387 case ISD::SMAX: {
4388 // If we have a clamp pattern, we know that the number of sign bits will be
4389 // the minimum of the clamp min/max range.
4390 bool IsMax = (Opcode == ISD::SMAX);
4391 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4392 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4393 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4394 CstHigh =
4395 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4396 if (CstLow && CstHigh) {
4397 if (!IsMax)
4398 std::swap(a&: CstLow, b&: CstHigh);
4399
4400 const APInt &ValueLow = CstLow->getAPIntValue();
4401 const APInt &ValueHigh = CstHigh->getAPIntValue();
4402 if (ValueLow.sle(RHS: ValueHigh)) {
4403 unsigned LowSignBits = ValueLow.getNumSignBits();
4404 unsigned HighSignBits = ValueHigh.getNumSignBits();
4405 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4406 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4407 Known.One.setHighBits(MinSignBits);
4408 break;
4409 }
4410 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4411 Known.Zero.setHighBits(MinSignBits);
4412 break;
4413 }
4414 }
4415 }
4416
4417 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4418 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4419 if (IsMax)
4420 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4421 else
4422 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4423
4424 // For SMAX, if CstLow is non-negative we know the result will be
4425 // non-negative and thus all sign bits are 0.
4426 // TODO: There's an equivalent of this for smin with negative constant for
4427 // known ones.
4428 if (IsMax && CstLow) {
4429 const APInt &ValueLow = CstLow->getAPIntValue();
4430 if (ValueLow.isNonNegative()) {
4431 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4432 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4433 }
4434 }
4435
4436 break;
4437 }
4438 case ISD::UINT_TO_FP: {
4439 Known.makeNonNegative();
4440 break;
4441 }
4442 case ISD::SINT_TO_FP: {
4443 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4444 if (Known2.isNonNegative())
4445 Known.makeNonNegative();
4446 else if (Known2.isNegative())
4447 Known.makeNegative();
4448 break;
4449 }
4450 case ISD::FP_TO_UINT_SAT: {
4451 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4452 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4453 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4454 break;
4455 }
4456 case ISD::ATOMIC_LOAD: {
4457 // If we are looking at the loaded value.
4458 if (Op.getResNo() == 0) {
4459 auto *AT = cast<AtomicSDNode>(Val&: Op);
4460 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4461 KnownBits KnownScalarMemory(ScalarMemorySize);
4462 if (const MDNode *MD = AT->getRanges())
4463 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4464
4465 switch (AT->getExtensionType()) {
4466 case ISD::ZEXTLOAD:
4467 Known = KnownScalarMemory.zext(BitWidth);
4468 break;
4469 case ISD::SEXTLOAD:
4470 Known = KnownScalarMemory.sext(BitWidth);
4471 break;
4472 case ISD::EXTLOAD:
4473 switch (TLI->getExtendForAtomicOps()) {
4474 case ISD::ZERO_EXTEND:
4475 Known = KnownScalarMemory.zext(BitWidth);
4476 break;
4477 case ISD::SIGN_EXTEND:
4478 Known = KnownScalarMemory.sext(BitWidth);
4479 break;
4480 default:
4481 Known = KnownScalarMemory.anyext(BitWidth);
4482 break;
4483 }
4484 break;
4485 case ISD::NON_EXTLOAD:
4486 Known = KnownScalarMemory;
4487 break;
4488 }
4489 assert(Known.getBitWidth() == BitWidth);
4490 }
4491 break;
4492 }
4493 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4494 if (Op.getResNo() == 1) {
4495 // The boolean result conforms to getBooleanContents.
4496 // If we know the result of a setcc has the top bits zero, use this info.
4497 // We know that we have an integer-based boolean since these operations
4498 // are only available for integer.
4499 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4500 TargetLowering::ZeroOrOneBooleanContent &&
4501 BitWidth > 1)
4502 Known.Zero.setBitsFrom(1);
4503 break;
4504 }
4505 [[fallthrough]];
4506 case ISD::ATOMIC_CMP_SWAP:
4507 case ISD::ATOMIC_SWAP:
4508 case ISD::ATOMIC_LOAD_ADD:
4509 case ISD::ATOMIC_LOAD_SUB:
4510 case ISD::ATOMIC_LOAD_AND:
4511 case ISD::ATOMIC_LOAD_CLR:
4512 case ISD::ATOMIC_LOAD_OR:
4513 case ISD::ATOMIC_LOAD_XOR:
4514 case ISD::ATOMIC_LOAD_NAND:
4515 case ISD::ATOMIC_LOAD_MIN:
4516 case ISD::ATOMIC_LOAD_MAX:
4517 case ISD::ATOMIC_LOAD_UMIN:
4518 case ISD::ATOMIC_LOAD_UMAX: {
4519 // If we are looking at the loaded value.
4520 if (Op.getResNo() == 0) {
4521 auto *AT = cast<AtomicSDNode>(Val&: Op);
4522 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4523
4524 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4525 Known.Zero.setBitsFrom(MemBits);
4526 }
4527 break;
4528 }
4529 case ISD::FrameIndex:
4530 case ISD::TargetFrameIndex:
4531 TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4532 Known, MF: getMachineFunction());
4533 break;
4534
4535 default:
4536 if (Opcode < ISD::BUILTIN_OP_END)
4537 break;
4538 [[fallthrough]];
4539 case ISD::INTRINSIC_WO_CHAIN:
4540 case ISD::INTRINSIC_W_CHAIN:
4541 case ISD::INTRINSIC_VOID:
4542 // Allow the target to implement this method for its nodes.
4543 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4544 break;
4545 }
4546
4547 return Known;
4548}
4549
4550/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4551static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4552 switch (OR) {
4553 case ConstantRange::OverflowResult::MayOverflow:
4554 return SelectionDAG::OFK_Sometime;
4555 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4556 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4557 return SelectionDAG::OFK_Always;
4558 case ConstantRange::OverflowResult::NeverOverflows:
4559 return SelectionDAG::OFK_Never;
4560 }
4561 llvm_unreachable("Unknown OverflowResult");
4562}
4563
4564SelectionDAG::OverflowKind
4565SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4566 // X + 0 never overflow
4567 if (isNullConstant(V: N1))
4568 return OFK_Never;
4569
4570 // If both operands each have at least two sign bits, the addition
4571 // cannot overflow.
4572 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4573 return OFK_Never;
4574
4575 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4576 return OFK_Sometime;
4577}
4578
4579SelectionDAG::OverflowKind
4580SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4581 // X + 0 never overflow
4582 if (isNullConstant(V: N1))
4583 return OFK_Never;
4584
4585 // mulhi + 1 never overflow
4586 KnownBits N1Known = computeKnownBits(Op: N1);
4587 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4588 N1Known.getMaxValue().ult(RHS: 2))
4589 return OFK_Never;
4590
4591 KnownBits N0Known = computeKnownBits(Op: N0);
4592 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4593 N0Known.getMaxValue().ult(RHS: 2))
4594 return OFK_Never;
4595
4596 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4597 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4598 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4599 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4600}
4601
4602SelectionDAG::OverflowKind
4603SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4604 // X - 0 never overflow
4605 if (isNullConstant(V: N1))
4606 return OFK_Never;
4607
4608 // If both operands each have at least two sign bits, the subtraction
4609 // cannot overflow.
4610 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4611 return OFK_Never;
4612
4613 KnownBits N0Known = computeKnownBits(Op: N0);
4614 KnownBits N1Known = computeKnownBits(Op: N1);
4615 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4616 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4617 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4618}
4619
4620SelectionDAG::OverflowKind
4621SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4622 // X - 0 never overflow
4623 if (isNullConstant(V: N1))
4624 return OFK_Never;
4625
4626 ConstantRange N0Range =
4627 computeConstantRangeIncludingKnownBits(Op: N0, /*ForSigned=*/false);
4628 ConstantRange N1Range =
4629 computeConstantRangeIncludingKnownBits(Op: N1, /*ForSigned=*/false);
4630 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4631}
4632
4633SelectionDAG::OverflowKind
4634SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4635 // X * 0 and X * 1 never overflow.
4636 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4637 return OFK_Never;
4638
4639 ConstantRange N0Range = computeConstantRangeIncludingKnownBits(Op: N0, ForSigned: false);
4640 ConstantRange N1Range = computeConstantRangeIncludingKnownBits(Op: N1, ForSigned: false);
4641 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4642}
4643
4644SelectionDAG::OverflowKind
4645SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4646 // X * 0 and X * 1 never overflow.
4647 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4648 return OFK_Never;
4649
4650 // Get the size of the result.
4651 unsigned BitWidth = N0.getScalarValueSizeInBits();
4652
4653 // Sum of the sign bits.
4654 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4655
4656 // If we have enough sign bits, then there's no overflow.
4657 if (SignBits > BitWidth + 1)
4658 return OFK_Never;
4659
4660 if (SignBits == BitWidth + 1) {
4661 // The overflow occurs when the true multiplication of the
4662 // the operands is the minimum negative number.
4663 KnownBits N0Known = computeKnownBits(Op: N0);
4664 KnownBits N1Known = computeKnownBits(Op: N1);
4665 // If one of the operands is non-negative, then there's no
4666 // overflow.
4667 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4668 return OFK_Never;
4669 }
4670
4671 return OFK_Sometime;
4672}
4673
4674ConstantRange SelectionDAG::computeConstantRange(SDValue Op, bool ForSigned,
4675 unsigned Depth) const {
4676 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4677 return computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4678}
4679
4680ConstantRange SelectionDAG::computeConstantRange(SDValue Op,
4681 const APInt &DemandedElts,
4682 bool ForSigned,
4683 unsigned Depth) const {
4684 EVT VT = Op.getValueType();
4685 unsigned BitWidth = VT.getScalarSizeInBits();
4686
4687 if (Depth >= MaxRecursionDepth)
4688 return ConstantRange::getFull(BitWidth);
4689
4690 if (ConstantSDNode *C = isConstOrConstSplat(N: Op, DemandedElts))
4691 return ConstantRange(C->getAPIntValue());
4692
4693 unsigned Opcode = Op.getOpcode();
4694 switch (Opcode) {
4695 case ISD::VSCALE: {
4696 const Function &F = getMachineFunction().getFunction();
4697 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
4698 return getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier);
4699 }
4700 default:
4701 break;
4702 }
4703
4704 return ConstantRange::getFull(BitWidth);
4705}
4706
4707ConstantRange
4708SelectionDAG::computeConstantRangeIncludingKnownBits(SDValue Op, bool ForSigned,
4709 unsigned Depth) const {
4710 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4711 return computeConstantRangeIncludingKnownBits(Op, DemandedElts, ForSigned,
4712 Depth);
4713}
4714
4715ConstantRange SelectionDAG::computeConstantRangeIncludingKnownBits(
4716 SDValue Op, const APInt &DemandedElts, bool ForSigned,
4717 unsigned Depth) const {
4718 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4719 ConstantRange CR1 = ConstantRange::fromKnownBits(Known, IsSigned: ForSigned);
4720 ConstantRange CR2 = computeConstantRange(Op, DemandedElts, ForSigned, Depth);
4721 ConstantRange::PreferredRangeType RangeType =
4722 ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned;
4723 return CR1.intersectWith(CR: CR2, Type: RangeType);
4724}
4725
4726bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4727 unsigned Depth) const {
4728 APInt DemandedElts = getDemandAllEltsMask(V: Val);
4729 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4730}
4731
4732bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4733 const APInt &DemandedElts,
4734 bool OrZero, unsigned Depth) const {
4735 if (Depth >= MaxRecursionDepth)
4736 return false; // Limit search depth.
4737
4738 EVT OpVT = Val.getValueType();
4739 unsigned BitWidth = OpVT.getScalarSizeInBits();
4740 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4741 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4742 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4743 assert(
4744 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4745 "Unexpected vector size");
4746
4747 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4748 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4749 return (OrZero && V.isZero()) || V.isPowerOf2();
4750 };
4751
4752 // Is the constant a known power of 2 or zero?
4753 if (ISD::matchUnaryPredicate(Op: Val, Match: IsPowerOfTwoOrZero))
4754 return true;
4755
4756 switch (Val.getOpcode()) {
4757 case ISD::BUILD_VECTOR:
4758 // Are all operands of a build vector constant powers of two or zero?
4759 if (all_of(Range: enumerate(First: Val->ops()), P: [&](auto P) {
4760 auto *C = dyn_cast<ConstantSDNode>(P.value());
4761 return !DemandedElts[P.index()] || (C && IsPowerOfTwoOrZero(C));
4762 }))
4763 return true;
4764 break;
4765
4766 case ISD::SPLAT_VECTOR:
4767 // Is the operand of a splat vector a constant power of two?
4768 if (auto *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4769 if (IsPowerOfTwoOrZero(C))
4770 return true;
4771 break;
4772
4773 case ISD::EXTRACT_VECTOR_ELT: {
4774 SDValue InVec = Val.getOperand(i: 0);
4775 SDValue EltNo = Val.getOperand(i: 1);
4776 EVT VecVT = InVec.getValueType();
4777
4778 // Skip scalable vectors or implicit extensions.
4779 if (VecVT.isScalableVector() ||
4780 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4781 break;
4782
4783 // If we know the element index, just demand that vector element, else for
4784 // an unknown element index, ignore DemandedElts and demand them all.
4785 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4786 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4787 APInt DemandedSrcElts =
4788 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4789 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4790 : APInt::getAllOnes(numBits: NumSrcElts);
4791 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4792 }
4793
4794 case ISD::AND: {
4795 // Looking for `x & -x` pattern:
4796 // If x == 0:
4797 // x & -x -> 0
4798 // If x != 0:
4799 // x & -x -> non-zero pow2
4800 // so if we find the pattern return whether we know `x` is non-zero.
4801 SDValue X, Z;
4802 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))) ||
4803 (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Sub(L: m_Value(N&: Z), R: m_Deferred(V&: X)))) &&
4804 MaskedVectorIsZero(V: Z, DemandedElts, Depth: Depth + 1)))
4805 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4806 break;
4807 }
4808
4809 case ISD::SHL: {
4810 // A left-shift of a constant one will have exactly one bit set because
4811 // shifting the bit off the end is undefined.
4812 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4813 if (C && C->getAPIntValue() == 1)
4814 return true;
4815 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4816 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4817 Depth: Depth + 1);
4818 }
4819
4820 case ISD::SRL: {
4821 // A logical right-shift of a constant sign-bit will have exactly
4822 // one bit set.
4823 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4824 if (C && C->getAPIntValue().isSignMask())
4825 return true;
4826 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4827 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4828 Depth: Depth + 1);
4829 }
4830
4831 case ISD::TRUNCATE:
4832 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4833 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4834 Depth: Depth + 1);
4835
4836 case ISD::ROTL:
4837 case ISD::ROTR:
4838 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4839 Depth: Depth + 1);
4840 case ISD::BSWAP:
4841 case ISD::BITREVERSE:
4842 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4843 Depth: Depth + 1);
4844
4845 case ISD::SMIN:
4846 case ISD::SMAX:
4847 case ISD::UMIN:
4848 case ISD::UMAX:
4849 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4850 Depth: Depth + 1) &&
4851 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4852 Depth: Depth + 1);
4853
4854 case ISD::SELECT:
4855 case ISD::VSELECT:
4856 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4857 Depth: Depth + 1) &&
4858 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4859 Depth: Depth + 1);
4860
4861 case ISD::ZERO_EXTEND:
4862 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4863 Depth: Depth + 1);
4864
4865 case ISD::VSCALE:
4866 // vscale(power-of-two) is a power-of-two
4867 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4868 Depth: Depth + 1);
4869
4870 case ISD::VECTOR_SHUFFLE: {
4871 assert(!Val.getValueType().isScalableVector());
4872 // Demanded elements with undef shuffle mask elements are unknown
4873 // - we cannot guarantee they are a power of two, so return false.
4874 APInt DemandedLHS, DemandedRHS;
4875 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4876 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4877 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4878 DemandedLHS, DemandedRHS))
4879 return false;
4880
4881 // All demanded elements from LHS must be known power of two.
4882 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4883 OrZero, Depth: Depth + 1))
4884 return false;
4885
4886 // All demanded elements from RHS must be known power of two.
4887 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4888 OrZero, Depth: Depth + 1))
4889 return false;
4890
4891 return true;
4892 }
4893 }
4894
4895 // More could be done here, though the above checks are enough
4896 // to handle some common cases.
4897 return false;
4898}
4899
4900bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4901 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4902 return C1->getValueAPF().getExactLog2Abs() >= 0;
4903
4904 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4905 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4906
4907 return false;
4908}
4909
4910unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4911 APInt DemandedElts = getDemandAllEltsMask(V: Op);
4912 return ComputeNumSignBits(Op, DemandedElts, Depth);
4913}
4914
4915unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4916 unsigned Depth) const {
4917 EVT VT = Op.getValueType();
4918 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4919 unsigned VTBits = VT.getScalarSizeInBits();
4920 unsigned NumElts = DemandedElts.getBitWidth();
4921 unsigned Tmp, Tmp2;
4922 unsigned FirstAnswer = 1;
4923
4924 assert((!VT.isScalableVector() || NumElts == 1) &&
4925 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4926
4927 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4928 const APInt &Val = C->getAPIntValue();
4929 return Val.getNumSignBits();
4930 }
4931
4932 if (Depth >= MaxRecursionDepth)
4933 return 1; // Limit search depth.
4934
4935 if (!DemandedElts)
4936 return 1; // No demanded elts, better to assume we don't know anything.
4937
4938 unsigned Opcode = Op.getOpcode();
4939 switch (Opcode) {
4940 default: break;
4941 case ISD::AssertSext:
4942 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4943 return VTBits-Tmp+1;
4944 case ISD::AssertZext:
4945 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4946 return VTBits-Tmp;
4947 case ISD::FREEZE:
4948 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4949 Kind: UndefPoisonKind::UndefOrPoison))
4950 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4951 break;
4952 case ISD::MERGE_VALUES:
4953 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4954 Depth: Depth + 1);
4955 case ISD::SPLAT_VECTOR: {
4956 // Check if the sign bits of source go down as far as the truncated value.
4957 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4958 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4959 if (NumSrcSignBits > (NumSrcBits - VTBits))
4960 return NumSrcSignBits - (NumSrcBits - VTBits);
4961 break;
4962 }
4963 case ISD::BUILD_VECTOR:
4964 assert(!VT.isScalableVector());
4965 Tmp = VTBits;
4966 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4967 if (!DemandedElts[i])
4968 continue;
4969
4970 SDValue SrcOp = Op.getOperand(i);
4971 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4972 // for constant nodes to ensure we only look at the sign bits.
4973 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4974 APInt T = C->getAPIntValue().trunc(width: VTBits);
4975 Tmp2 = T.getNumSignBits();
4976 } else {
4977 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4978
4979 if (SrcOp.getValueSizeInBits() != VTBits) {
4980 assert(SrcOp.getValueSizeInBits() > VTBits &&
4981 "Expected BUILD_VECTOR implicit truncation");
4982 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4983 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4984 }
4985 }
4986 Tmp = std::min(a: Tmp, b: Tmp2);
4987 }
4988 return Tmp;
4989
4990 case ISD::VECTOR_COMPRESS: {
4991 SDValue Vec = Op.getOperand(i: 0);
4992 SDValue PassThru = Op.getOperand(i: 2);
4993 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
4994 if (Tmp == 1)
4995 return 1;
4996 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
4997 Tmp = std::min(a: Tmp, b: Tmp2);
4998 return Tmp;
4999 }
5000
5001 case ISD::VECTOR_SHUFFLE: {
5002 // Collect the minimum number of sign bits that are shared by every vector
5003 // element referenced by the shuffle.
5004 APInt DemandedLHS, DemandedRHS;
5005 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5006 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
5007 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
5008 DemandedLHS, DemandedRHS))
5009 return 1;
5010
5011 Tmp = std::numeric_limits<unsigned>::max();
5012 if (!!DemandedLHS)
5013 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
5014 if (!!DemandedRHS) {
5015 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
5016 Tmp = std::min(a: Tmp, b: Tmp2);
5017 }
5018 // If we don't know anything, early out and try computeKnownBits fall-back.
5019 if (Tmp == 1)
5020 break;
5021 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5022 return Tmp;
5023 }
5024
5025 case ISD::BITCAST: {
5026 if (VT.isScalableVector())
5027 break;
5028 SDValue N0 = Op.getOperand(i: 0);
5029 EVT SrcVT = N0.getValueType();
5030 unsigned SrcBits = SrcVT.getScalarSizeInBits();
5031
5032 // Ignore bitcasts from unsupported types..
5033 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
5034 break;
5035
5036 // Fast handling of 'identity' bitcasts.
5037 if (VTBits == SrcBits)
5038 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
5039
5040 bool IsLE = getDataLayout().isLittleEndian();
5041
5042 // Bitcast 'large element' scalar/vector to 'small element' vector.
5043 if ((SrcBits % VTBits) == 0) {
5044 assert(VT.isVector() && "Expected bitcast to vector");
5045
5046 unsigned Scale = SrcBits / VTBits;
5047 APInt SrcDemandedElts =
5048 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
5049
5050 // Fast case - sign splat can be simply split across the small elements.
5051 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
5052 if (Tmp == SrcBits)
5053 return VTBits;
5054
5055 // Slow case - determine how far the sign extends into each sub-element.
5056 Tmp2 = VTBits;
5057 for (unsigned i = 0; i != NumElts; ++i)
5058 if (DemandedElts[i]) {
5059 unsigned SubOffset = i % Scale;
5060 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
5061 SubOffset = SubOffset * VTBits;
5062 if (Tmp <= SubOffset)
5063 return 1;
5064 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
5065 }
5066 return Tmp2;
5067 }
5068 break;
5069 }
5070
5071 case ISD::FP_TO_SINT_SAT:
5072 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
5073 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5074 return VTBits - Tmp + 1;
5075 case ISD::SIGN_EXTEND:
5076 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5077 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5078 case ISD::SIGN_EXTEND_INREG:
5079 // Max of the input and what this extends.
5080 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5081 Tmp = VTBits-Tmp+1;
5082 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5083 return std::max(a: Tmp, b: Tmp2);
5084 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5085 if (VT.isScalableVector())
5086 break;
5087 SDValue Src = Op.getOperand(i: 0);
5088 EVT SrcVT = Src.getValueType();
5089 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5090 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5091 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5092 }
5093 case ISD::SRA:
5094 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5095 // SRA X, C -> adds C sign bits.
5096 if (std::optional<unsigned> ShAmt =
5097 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5098 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5099 return Tmp;
5100 case ISD::SHL:
5101 if (std::optional<ConstantRange> ShAmtRange =
5102 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5103 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5104 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5105 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5106 // shifted out, then we can compute the number of sign bits for the
5107 // operand being extended. A future improvement could be to pass along the
5108 // "shifted left by" information in the recursive calls to
5109 // ComputeKnownSignBits. Allowing us to handle this more generically.
5110 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5111 SDValue Ext = Op.getOperand(i: 0);
5112 EVT ExtVT = Ext.getValueType();
5113 SDValue Extendee = Ext.getOperand(i: 0);
5114 EVT ExtendeeVT = Extendee.getValueType();
5115 unsigned SizeDifference =
5116 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5117 if (SizeDifference <= MinShAmt) {
5118 Tmp = SizeDifference +
5119 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5120 if (MaxShAmt < Tmp)
5121 return Tmp - MaxShAmt;
5122 }
5123 }
5124 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5125 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5126 if (MaxShAmt < Tmp)
5127 return Tmp - MaxShAmt;
5128 }
5129 break;
5130 case ISD::AND:
5131 case ISD::OR:
5132 case ISD::XOR: // NOT is handled here.
5133 // Logical binary ops preserve the number of sign bits at the worst.
5134 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5135 if (Tmp != 1) {
5136 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5137 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5138 // We computed what we know about the sign bits as our first
5139 // answer. Now proceed to the generic code that uses
5140 // computeKnownBits, and pick whichever answer is better.
5141 }
5142 break;
5143
5144 case ISD::SELECT:
5145 case ISD::VSELECT:
5146 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5147 if (Tmp == 1) return 1; // Early out.
5148 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5149 return std::min(a: Tmp, b: Tmp2);
5150 case ISD::SELECT_CC:
5151 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5152 if (Tmp == 1) return 1; // Early out.
5153 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5154 return std::min(a: Tmp, b: Tmp2);
5155
5156 case ISD::SMIN:
5157 case ISD::SMAX: {
5158 // If we have a clamp pattern, we know that the number of sign bits will be
5159 // the minimum of the clamp min/max range.
5160 bool IsMax = (Opcode == ISD::SMAX);
5161 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5162 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5163 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5164 CstHigh =
5165 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5166 if (CstLow && CstHigh) {
5167 if (!IsMax)
5168 std::swap(a&: CstLow, b&: CstHigh);
5169 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5170 Tmp = CstLow->getAPIntValue().getNumSignBits();
5171 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5172 return std::min(a: Tmp, b: Tmp2);
5173 }
5174 }
5175
5176 // Fallback - just get the minimum number of sign bits of the operands.
5177 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5178 if (Tmp == 1)
5179 return 1; // Early out.
5180 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5181 return std::min(a: Tmp, b: Tmp2);
5182 }
5183 case ISD::UMIN:
5184 case ISD::UMAX:
5185 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5186 if (Tmp == 1)
5187 return 1; // Early out.
5188 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5189 return std::min(a: Tmp, b: Tmp2);
5190 case ISD::SSUBO_CARRY:
5191 case ISD::USUBO_CARRY:
5192 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5193 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5194 return VTBits;
5195 [[fallthrough]];
5196 case ISD::SADDO:
5197 case ISD::UADDO:
5198 case ISD::SADDO_CARRY:
5199 case ISD::UADDO_CARRY:
5200 case ISD::SSUBO:
5201 case ISD::USUBO:
5202 case ISD::SMULO:
5203 case ISD::UMULO:
5204 if (Op.getResNo() != 1)
5205 break;
5206 // The boolean result conforms to getBooleanContents. Fall through.
5207 // If setcc returns 0/-1, all bits are sign bits.
5208 // We know that we have an integer-based boolean since these operations
5209 // are only available for integer.
5210 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5211 TargetLowering::ZeroOrNegativeOneBooleanContent)
5212 return VTBits;
5213 break;
5214 case ISD::SETCC:
5215 case ISD::SETCCCARRY:
5216 case ISD::STRICT_FSETCC:
5217 case ISD::STRICT_FSETCCS: {
5218 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5219 // If setcc returns 0/-1, all bits are sign bits.
5220 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5221 TargetLowering::ZeroOrNegativeOneBooleanContent)
5222 return VTBits;
5223 break;
5224 }
5225 case ISD::ROTL:
5226 case ISD::ROTR:
5227 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5228
5229 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
5230 if (Tmp == VTBits)
5231 return VTBits;
5232
5233 if (ConstantSDNode *C =
5234 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
5235 unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
5236
5237 // Handle rotate right by N like a rotate left by 32-N.
5238 if (Opcode == ISD::ROTR)
5239 RotAmt = (VTBits - RotAmt) % VTBits;
5240
5241 // If we aren't rotating out all of the known-in sign bits, return the
5242 // number that are left. This handles rotl(sext(x), 1) for example.
5243 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
5244 }
5245 break;
5246 case ISD::ADD:
5247 case ISD::ADDC:
5248 // TODO: Move Operand 1 check before Operand 0 check
5249 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5250 if (Tmp == 1) return 1; // Early out.
5251
5252 // Special case decrementing a value (ADD X, -1):
5253 if (ConstantSDNode *CRHS =
5254 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5255 if (CRHS->isAllOnes()) {
5256 KnownBits Known =
5257 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5258
5259 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5260 // sign bits set.
5261 if ((Known.Zero | 1).isAllOnes())
5262 return VTBits;
5263
5264 // If we are subtracting one from a positive number, there is no carry
5265 // out of the result.
5266 if (Known.isNonNegative())
5267 return Tmp;
5268 }
5269
5270 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5271 if (Tmp2 == 1) return 1; // Early out.
5272
5273 // Add can have at most one carry bit. Thus we know that the output
5274 // is, at worst, one more bit than the inputs.
5275 return std::min(a: Tmp, b: Tmp2) - 1;
5276 case ISD::SUB:
5277 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5278 if (Tmp2 == 1) return 1; // Early out.
5279
5280 // Handle NEG.
5281 if (ConstantSDNode *CLHS =
5282 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5283 if (CLHS->isZero()) {
5284 KnownBits Known =
5285 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5286 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5287 // sign bits set.
5288 if ((Known.Zero | 1).isAllOnes())
5289 return VTBits;
5290
5291 // If the input is known to be positive (the sign bit is known clear),
5292 // the output of the NEG has the same number of sign bits as the input.
5293 if (Known.isNonNegative())
5294 return Tmp2;
5295
5296 // Otherwise, we treat this like a SUB.
5297 }
5298
5299 // Sub can have at most one carry bit. Thus we know that the output
5300 // is, at worst, one more bit than the inputs.
5301 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5302 if (Tmp == 1) return 1; // Early out.
5303 return std::min(a: Tmp, b: Tmp2) - 1;
5304 case ISD::MUL: {
5305 // The output of the Mul can be at most twice the valid bits in the inputs.
5306 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5307 if (SignBitsOp0 == 1)
5308 break;
5309 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5310 if (SignBitsOp1 == 1)
5311 break;
5312 unsigned OutValidBits =
5313 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5314 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5315 }
5316 case ISD::AVGCEILS:
5317 case ISD::AVGFLOORS:
5318 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5319 if (Tmp == 1)
5320 return 1; // Early out.
5321 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5322 return std::min(a: Tmp, b: Tmp2);
5323 case ISD::SREM:
5324 // The sign bit is the LHS's sign bit, except when the result of the
5325 // remainder is zero. The magnitude of the result should be less than or
5326 // equal to the magnitude of the LHS. Therefore, the result should have
5327 // at least as many sign bits as the left hand side.
5328 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5329 case ISD::TRUNCATE: {
5330 // Check if the sign bits of source go down as far as the truncated value.
5331 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5332 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5333 if (NumSrcSignBits > (NumSrcBits - VTBits))
5334 return NumSrcSignBits - (NumSrcBits - VTBits);
5335 break;
5336 }
5337 case ISD::EXTRACT_ELEMENT: {
5338 if (VT.isScalableVector())
5339 break;
5340 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5341 const int BitWidth = Op.getValueSizeInBits();
5342 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5343
5344 // Get reverse index (starting from 1), Op1 value indexes elements from
5345 // little end. Sign starts at big end.
5346 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5347
5348 // If the sign portion ends in our element the subtraction gives correct
5349 // result. Otherwise it gives either negative or > bitwidth result
5350 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5351 }
5352 case ISD::INSERT_VECTOR_ELT: {
5353 if (VT.isScalableVector())
5354 break;
5355 // If we know the element index, split the demand between the
5356 // source vector and the inserted element, otherwise assume we need
5357 // the original demanded vector elements and the value.
5358 SDValue InVec = Op.getOperand(i: 0);
5359 SDValue InVal = Op.getOperand(i: 1);
5360 SDValue EltNo = Op.getOperand(i: 2);
5361 bool DemandedVal = true;
5362 APInt DemandedVecElts = DemandedElts;
5363 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5364 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5365 unsigned EltIdx = CEltNo->getZExtValue();
5366 DemandedVal = !!DemandedElts[EltIdx];
5367 DemandedVecElts.clearBit(BitPosition: EltIdx);
5368 }
5369 Tmp = std::numeric_limits<unsigned>::max();
5370 if (DemandedVal) {
5371 // TODO - handle implicit truncation of inserted elements.
5372 if (InVal.getScalarValueSizeInBits() != VTBits)
5373 break;
5374 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5375 Tmp = std::min(a: Tmp, b: Tmp2);
5376 }
5377 if (!!DemandedVecElts) {
5378 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5379 Tmp = std::min(a: Tmp, b: Tmp2);
5380 }
5381 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5382 return Tmp;
5383 }
5384 case ISD::EXTRACT_VECTOR_ELT: {
5385 SDValue InVec = Op.getOperand(i: 0);
5386 SDValue EltNo = Op.getOperand(i: 1);
5387 EVT VecVT = InVec.getValueType();
5388 // ComputeNumSignBits not yet implemented for scalable vectors.
5389 if (VecVT.isScalableVector())
5390 break;
5391 const unsigned BitWidth = Op.getValueSizeInBits();
5392 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5393 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5394
5395 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5396 // anything about sign bits. But if the sizes match we can derive knowledge
5397 // about sign bits from the vector operand.
5398 if (BitWidth != EltBitWidth)
5399 break;
5400
5401 // If we know the element index, just demand that vector element, else for
5402 // an unknown element index, ignore DemandedElts and demand them all.
5403 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5404 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5405 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5406 DemandedSrcElts =
5407 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5408
5409 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5410 }
5411 case ISD::EXTRACT_SUBVECTOR: {
5412 // Offset the demanded elts by the subvector index.
5413 SDValue Src = Op.getOperand(i: 0);
5414
5415 APInt DemandedSrcElts;
5416 if (Src.getValueType().isScalableVector())
5417 DemandedSrcElts = APInt(1, 1);
5418 else {
5419 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5420 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5421 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5422 }
5423 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5424 }
5425 case ISD::CONCAT_VECTORS: {
5426 if (VT.isScalableVector())
5427 break;
5428 // Determine the minimum number of sign bits across all demanded
5429 // elts of the input vectors. Early out if the result is already 1.
5430 Tmp = std::numeric_limits<unsigned>::max();
5431 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5432 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5433 unsigned NumSubVectors = Op.getNumOperands();
5434 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5435 APInt DemandedSub =
5436 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5437 if (!DemandedSub)
5438 continue;
5439 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5440 Tmp = std::min(a: Tmp, b: Tmp2);
5441 }
5442 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5443 return Tmp;
5444 }
5445 case ISD::INSERT_SUBVECTOR: {
5446 if (VT.isScalableVector())
5447 break;
5448 // Demand any elements from the subvector and the remainder from the src its
5449 // inserted into.
5450 SDValue Src = Op.getOperand(i: 0);
5451 SDValue Sub = Op.getOperand(i: 1);
5452 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5453 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5454 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5455 APInt DemandedSrcElts = DemandedElts;
5456 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5457
5458 Tmp = std::numeric_limits<unsigned>::max();
5459 if (!!DemandedSubElts) {
5460 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5461 if (Tmp == 1)
5462 return 1; // early-out
5463 }
5464 if (!!DemandedSrcElts) {
5465 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5466 Tmp = std::min(a: Tmp, b: Tmp2);
5467 }
5468 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5469 return Tmp;
5470 }
5471 case ISD::LOAD: {
5472 // If we are looking at the loaded value of the SDNode.
5473 if (Op.getResNo() != 0)
5474 break;
5475
5476 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5477 if (const MDNode *Ranges = LD->getRanges()) {
5478 if (DemandedElts != 1)
5479 break;
5480
5481 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5482 if (VTBits > CR.getBitWidth()) {
5483 switch (LD->getExtensionType()) {
5484 case ISD::SEXTLOAD:
5485 CR = CR.signExtend(BitWidth: VTBits);
5486 break;
5487 case ISD::ZEXTLOAD:
5488 CR = CR.zeroExtend(BitWidth: VTBits);
5489 break;
5490 default:
5491 break;
5492 }
5493 }
5494
5495 if (VTBits != CR.getBitWidth())
5496 break;
5497 return std::min(a: CR.getSignedMin().getNumSignBits(),
5498 b: CR.getSignedMax().getNumSignBits());
5499 }
5500
5501 unsigned ExtType = LD->getExtensionType();
5502 switch (ExtType) {
5503 default:
5504 break;
5505 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5506 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5507 return VTBits - Tmp + 1;
5508 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5509 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5510 return VTBits - Tmp;
5511 case ISD::NON_EXTLOAD:
5512 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5513 // We only need to handle vectors - computeKnownBits should handle
5514 // scalar cases.
5515 Type *CstTy = Cst->getType();
5516 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5517 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5518 VTBits == CstTy->getScalarSizeInBits()) {
5519 Tmp = VTBits;
5520 for (unsigned i = 0; i != NumElts; ++i) {
5521 if (!DemandedElts[i])
5522 continue;
5523 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5524 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5525 const APInt &Value = CInt->getValue();
5526 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5527 continue;
5528 }
5529 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5530 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5531 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5532 continue;
5533 }
5534 }
5535 // Unknown type. Conservatively assume no bits match sign bit.
5536 return 1;
5537 }
5538 return Tmp;
5539 }
5540 }
5541 break;
5542 }
5543
5544 break;
5545 }
5546 case ISD::ATOMIC_CMP_SWAP:
5547 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5548 case ISD::ATOMIC_SWAP:
5549 case ISD::ATOMIC_LOAD_ADD:
5550 case ISD::ATOMIC_LOAD_SUB:
5551 case ISD::ATOMIC_LOAD_AND:
5552 case ISD::ATOMIC_LOAD_CLR:
5553 case ISD::ATOMIC_LOAD_OR:
5554 case ISD::ATOMIC_LOAD_XOR:
5555 case ISD::ATOMIC_LOAD_NAND:
5556 case ISD::ATOMIC_LOAD_MIN:
5557 case ISD::ATOMIC_LOAD_MAX:
5558 case ISD::ATOMIC_LOAD_UMIN:
5559 case ISD::ATOMIC_LOAD_UMAX:
5560 case ISD::ATOMIC_LOAD: {
5561 auto *AT = cast<AtomicSDNode>(Val&: Op);
5562 // If we are looking at the loaded value.
5563 if (Op.getResNo() == 0) {
5564 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5565 if (Tmp == VTBits)
5566 return 1; // early-out
5567
5568 // For atomic_load, prefer to use the extension type.
5569 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5570 switch (AT->getExtensionType()) {
5571 default:
5572 break;
5573 case ISD::SEXTLOAD:
5574 return VTBits - Tmp + 1;
5575 case ISD::ZEXTLOAD:
5576 return VTBits - Tmp;
5577 }
5578 }
5579
5580 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5581 return VTBits - Tmp + 1;
5582 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5583 return VTBits - Tmp;
5584 }
5585 break;
5586 }
5587 }
5588
5589 // Allow the target to implement this method for its nodes.
5590 if (Opcode >= ISD::BUILTIN_OP_END ||
5591 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5592 Opcode == ISD::INTRINSIC_W_CHAIN ||
5593 Opcode == ISD::INTRINSIC_VOID) {
5594 // TODO: This can probably be removed once target code is audited. This
5595 // is here purely to reduce patch size and review complexity.
5596 if (!VT.isScalableVector()) {
5597 unsigned NumBits =
5598 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5599 if (NumBits > 1)
5600 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5601 }
5602 }
5603
5604 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5605 // use this information.
5606 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5607 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5608}
5609
5610unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5611 unsigned Depth) const {
5612 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5613 return Op.getScalarValueSizeInBits() - SignBits + 1;
5614}
5615
5616unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5617 const APInt &DemandedElts,
5618 unsigned Depth) const {
5619 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5620 return Op.getScalarValueSizeInBits() - SignBits + 1;
5621}
5622
5623bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5624 UndefPoisonKind Kind,
5625 unsigned Depth) const {
5626 // Early out for FREEZE.
5627 if (Op.getOpcode() == ISD::FREEZE)
5628 return true;
5629
5630 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5631 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, Kind, Depth);
5632}
5633
5634bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5635 const APInt &DemandedElts,
5636 UndefPoisonKind Kind,
5637 unsigned Depth) const {
5638 unsigned Opcode = Op.getOpcode();
5639
5640 // Early out for FREEZE.
5641 if (Opcode == ISD::FREEZE)
5642 return true;
5643
5644 if (Depth >= MaxRecursionDepth)
5645 return false; // Limit search depth.
5646
5647 if (isIntOrFPConstant(V: Op))
5648 return true;
5649
5650 switch (Opcode) {
5651 case ISD::CONDCODE:
5652 case ISD::VALUETYPE:
5653 case ISD::FrameIndex:
5654 case ISD::TargetFrameIndex:
5655 case ISD::CopyFromReg:
5656 return true;
5657
5658 case ISD::POISON:
5659 return !includesPoison(Kind);
5660
5661 case ISD::UNDEF:
5662 return !includesUndef(Kind);
5663
5664 case ISD::BITCAST: {
5665 SDValue Src = Op.getOperand(i: 0);
5666 EVT SrcVT = Src.getValueType();
5667 EVT DstVT = Op.getValueType();
5668
5669 if (!SrcVT.isVector() || !DstVT.isVector())
5670 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5671
5672 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
5673 unsigned DstEltBits = DstVT.getScalarSizeInBits();
5674 ElementCount NumSrcElts = SrcVT.getVectorElementCount();
5675 [[maybe_unused]] ElementCount NumDstElts = DstVT.getVectorElementCount();
5676
5677 if (SrcEltBits == DstEltBits)
5678 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts, Kind,
5679 Depth: Depth + 1);
5680
5681 if (SrcEltBits < DstEltBits) {
5682 if (DstEltBits % SrcEltBits != 0)
5683 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5684
5685 assert(NumSrcElts == NumDstElts * (DstEltBits / SrcEltBits) &&
5686 "Unexpected vector bitcast");
5687 APInt DemandedSrcElts =
5688 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5689 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5690 Depth: Depth + 1);
5691 }
5692
5693 if (SrcEltBits % DstEltBits != 0)
5694 return isGuaranteedNotToBeUndefOrPoison(Op: Src, Kind, Depth: Depth + 1);
5695
5696 assert(NumDstElts == NumSrcElts * (SrcEltBits / DstEltBits) &&
5697 "Unexpected vector bitcast");
5698 APInt DemandedSrcElts =
5699 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts.getKnownMinValue());
5700 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5701 Depth: Depth + 1);
5702 }
5703
5704 case ISD::BUILD_VECTOR:
5705 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5706 // this shouldn't affect the result.
5707 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5708 if (!DemandedElts[i])
5709 continue;
5710 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), Kind, Depth: Depth + 1))
5711 return false;
5712 }
5713 return true;
5714
5715 case ISD::CONCAT_VECTORS: {
5716 EVT VT = Op.getValueType();
5717 if (!VT.isFixedLengthVector())
5718 break;
5719
5720 EVT SubVT = Op.getOperand(i: 0).getValueType();
5721 unsigned NumSubElts = SubVT.getVectorNumElements();
5722 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
5723 APInt DemandedSubElts =
5724 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: I * NumSubElts);
5725 if (!!DemandedSubElts &&
5726 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: I), DemandedElts: DemandedSubElts,
5727 Kind, Depth: Depth + 1))
5728 return false;
5729 }
5730 return true;
5731 }
5732
5733 case ISD::EXTRACT_SUBVECTOR: {
5734 SDValue Src = Op.getOperand(i: 0);
5735 if (Src.getValueType().isScalableVector())
5736 break;
5737 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5738 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5739 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5740 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5741 Depth: Depth + 1);
5742 }
5743
5744 case ISD::INSERT_SUBVECTOR: {
5745 if (Op.getValueType().isScalableVector())
5746 break;
5747 SDValue Src = Op.getOperand(i: 0);
5748 SDValue Sub = Op.getOperand(i: 1);
5749 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5750 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5751 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5752 APInt DemandedSrcElts = DemandedElts;
5753 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5754
5755 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5756 Op: Sub, DemandedElts: DemandedSubElts, Kind, Depth: Depth + 1))
5757 return false;
5758 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5759 Op: Src, DemandedElts: DemandedSrcElts, Kind, Depth: Depth + 1))
5760 return false;
5761 return true;
5762 }
5763
5764 case ISD::EXTRACT_VECTOR_ELT: {
5765 SDValue Src = Op.getOperand(i: 0);
5766 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5767 EVT SrcVT = Src.getValueType();
5768 if (SrcVT.isFixedLengthVector() && IndexC &&
5769 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5770 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5771 BitNo: IndexC->getZExtValue());
5772 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, Kind,
5773 Depth: Depth + 1);
5774 }
5775 break;
5776 }
5777
5778 case ISD::INSERT_VECTOR_ELT: {
5779 SDValue InVec = Op.getOperand(i: 0);
5780 SDValue InVal = Op.getOperand(i: 1);
5781 SDValue EltNo = Op.getOperand(i: 2);
5782 EVT VT = InVec.getValueType();
5783 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5784 if (IndexC && VT.isFixedLengthVector() &&
5785 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5786 if (DemandedElts[IndexC->getZExtValue()] &&
5787 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, Kind, Depth: Depth + 1))
5788 return false;
5789 APInt InVecDemandedElts = DemandedElts;
5790 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5791 if (!!InVecDemandedElts &&
5792 !isGuaranteedNotToBeUndefOrPoison(
5793 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5794 DemandedElts: InVecDemandedElts, Kind, Depth: Depth + 1))
5795 return false;
5796 return true;
5797 }
5798 break;
5799 }
5800
5801 case ISD::SCALAR_TO_VECTOR:
5802 // Check upper (known undef) elements.
5803 if (DemandedElts.ugt(RHS: 1) && includesUndef(Kind))
5804 return false;
5805 // Check element zero.
5806 if (DemandedElts[0] &&
5807 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1))
5808 return false;
5809 return true;
5810
5811 case ISD::SPLAT_VECTOR:
5812 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind, Depth: Depth + 1);
5813
5814 case ISD::SELECT: {
5815 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5816 /*ConsiderFlags*/ true, Depth) &&
5817 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), Kind,
5818 Depth: Depth + 1) &&
5819 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts,
5820 Kind, Depth: Depth + 1) &&
5821 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 2), DemandedElts,
5822 Kind, Depth: Depth + 1);
5823 }
5824
5825 case ISD::VECTOR_SHUFFLE: {
5826 APInt DemandedLHS, DemandedRHS;
5827 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5828 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5829 DemandedElts, DemandedLHS, DemandedRHS,
5830 /*AllowUndefElts=*/false))
5831 return false;
5832 if (!DemandedLHS.isZero() &&
5833 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Kind,
5834 Depth: Depth + 1))
5835 return false;
5836 if (!DemandedRHS.isZero() &&
5837 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Kind,
5838 Depth: Depth + 1))
5839 return false;
5840 return true;
5841 }
5842
5843 case ISD::SHL:
5844 case ISD::SRL:
5845 case ISD::SRA:
5846 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5847 // enough to check operand 0 if Op can't create undef/poison.
5848 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5849 /*ConsiderFlags*/ true, Depth) &&
5850 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5851 Kind, Depth: Depth + 1);
5852
5853 case ISD::BSWAP:
5854 case ISD::CTPOP:
5855 case ISD::BITREVERSE:
5856 case ISD::AND:
5857 case ISD::OR:
5858 case ISD::XOR:
5859 case ISD::ADD:
5860 case ISD::SUB:
5861 case ISD::MUL:
5862 case ISD::SADDSAT:
5863 case ISD::UADDSAT:
5864 case ISD::SSUBSAT:
5865 case ISD::USUBSAT:
5866 case ISD::SSHLSAT:
5867 case ISD::USHLSAT:
5868 case ISD::SMIN:
5869 case ISD::SMAX:
5870 case ISD::UMIN:
5871 case ISD::UMAX:
5872 case ISD::ZERO_EXTEND:
5873 case ISD::SIGN_EXTEND:
5874 case ISD::ANY_EXTEND:
5875 case ISD::TRUNCATE:
5876 case ISD::VSELECT: {
5877 // If Op can't create undef/poison and none of its operands are undef/poison
5878 // then Op is never undef/poison. A difference from the more common check
5879 // below, outside the switch, is that we handle elementwise operations for
5880 // which the DemandedElts mask is valid for all operands here.
5881 return !canCreateUndefOrPoison(Op, DemandedElts, Kind,
5882 /*ConsiderFlags*/ true, Depth) &&
5883 all_of(Range: Op->ops(), P: [&](SDValue V) {
5884 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, Kind,
5885 Depth: Depth + 1);
5886 });
5887 }
5888
5889 // TODO: Search for noundef attributes from library functions.
5890
5891 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5892
5893 default:
5894 // Allow the target to implement this method for its nodes.
5895 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5896 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5897 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5898 Op, DemandedElts, DAG: *this, Kind, Depth);
5899 break;
5900 }
5901
5902 // If Op can't create undef/poison and none of its operands are undef/poison
5903 // then Op is never undef/poison.
5904 // NOTE: TargetNodes can handle this in themselves in
5905 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5906 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5907 return !canCreateUndefOrPoison(Op, Kind, /*ConsiderFlags*/ true, Depth) &&
5908 all_of(Range: Op->ops(), P: [&](SDValue V) {
5909 return isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
5910 });
5911}
5912
5913bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, UndefPoisonKind Kind,
5914 bool ConsiderFlags,
5915 unsigned Depth) const {
5916 APInt DemandedElts = getDemandAllEltsMask(V: Op);
5917 return canCreateUndefOrPoison(Op, DemandedElts, Kind, ConsiderFlags, Depth);
5918}
5919
5920bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5921 UndefPoisonKind Kind,
5922 bool ConsiderFlags,
5923 unsigned Depth) const {
5924 if (ConsiderFlags && includesPoison(Kind) && Op->hasPoisonGeneratingFlags())
5925 return true;
5926
5927 unsigned Opcode = Op.getOpcode();
5928 switch (Opcode) {
5929 case ISD::AssertSext:
5930 case ISD::AssertZext:
5931 case ISD::AssertAlign:
5932 case ISD::AssertNoFPClass:
5933 // Assertion nodes can create poison if the assertion fails.
5934 return includesPoison(Kind);
5935
5936 case ISD::FREEZE:
5937 case ISD::CONCAT_VECTORS:
5938 case ISD::INSERT_SUBVECTOR:
5939 case ISD::EXTRACT_SUBVECTOR:
5940 case ISD::SADDSAT:
5941 case ISD::UADDSAT:
5942 case ISD::SSUBSAT:
5943 case ISD::USUBSAT:
5944 case ISD::MULHU:
5945 case ISD::MULHS:
5946 case ISD::AVGFLOORS:
5947 case ISD::AVGFLOORU:
5948 case ISD::AVGCEILS:
5949 case ISD::AVGCEILU:
5950 case ISD::ABDU:
5951 case ISD::ABDS:
5952 case ISD::SMIN:
5953 case ISD::SMAX:
5954 case ISD::SCMP:
5955 case ISD::UMIN:
5956 case ISD::UMAX:
5957 case ISD::UCMP:
5958 case ISD::AND:
5959 case ISD::XOR:
5960 case ISD::ROTL:
5961 case ISD::ROTR:
5962 case ISD::FSHL:
5963 case ISD::FSHR:
5964 case ISD::BSWAP:
5965 case ISD::CTTZ:
5966 case ISD::CTLZ:
5967 case ISD::CTLS:
5968 case ISD::CTPOP:
5969 case ISD::BITREVERSE:
5970 case ISD::PARITY:
5971 case ISD::SIGN_EXTEND:
5972 case ISD::TRUNCATE:
5973 case ISD::SIGN_EXTEND_INREG:
5974 case ISD::SIGN_EXTEND_VECTOR_INREG:
5975 case ISD::ZERO_EXTEND_VECTOR_INREG:
5976 case ISD::BITCAST:
5977 case ISD::BUILD_VECTOR:
5978 case ISD::BUILD_PAIR:
5979 case ISD::SPLAT_VECTOR:
5980 case ISD::FABS:
5981 case ISD::FCEIL:
5982 case ISD::FFLOOR:
5983 case ISD::FTRUNC:
5984 case ISD::FRINT:
5985 case ISD::FNEARBYINT:
5986 case ISD::FROUND:
5987 case ISD::FROUNDEVEN:
5988 return false;
5989
5990 case ISD::ABS:
5991 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
5992 // Different to Intrinsic::abs.
5993 return false;
5994 case ISD::ABS_MIN_POISON:
5995 // ABS_MIN_POISON may produce poison if the input is INT_MIN.
5996 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) <= 1;
5997
5998 case ISD::ADDC:
5999 case ISD::SUBC:
6000 case ISD::ADDE:
6001 case ISD::SUBE:
6002 case ISD::SADDO:
6003 case ISD::SSUBO:
6004 case ISD::SMULO:
6005 case ISD::SADDO_CARRY:
6006 case ISD::SSUBO_CARRY:
6007 case ISD::UADDO:
6008 case ISD::USUBO:
6009 case ISD::UMULO:
6010 case ISD::UADDO_CARRY:
6011 case ISD::USUBO_CARRY:
6012 // No poison on result or overflow flags.
6013 return false;
6014
6015 case ISD::SELECT_CC:
6016 case ISD::SETCC: {
6017 // Integer setcc cannot create undef or poison.
6018 if (Op.getOperand(i: 0).getValueType().isInteger())
6019 return false;
6020
6021 // FP compares are more complicated. They can create poison for nan/infinity
6022 // based on options and flags. The options and flags also cause special
6023 // nonan condition codes to be used. Those condition codes may be preserved
6024 // even if the nonan flag is dropped somewhere.
6025 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
6026 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
6027 return (unsigned)CCCode & 0x10U;
6028 }
6029
6030 case ISD::OR:
6031 case ISD::ZERO_EXTEND:
6032 case ISD::SELECT:
6033 case ISD::VSELECT:
6034 case ISD::ADD:
6035 case ISD::SUB:
6036 case ISD::MUL:
6037 case ISD::FNEG:
6038 case ISD::FADD:
6039 case ISD::FSUB:
6040 case ISD::FMUL:
6041 case ISD::FDIV:
6042 case ISD::FREM:
6043 case ISD::FCOPYSIGN:
6044 case ISD::FMA:
6045 case ISD::FMAD:
6046 case ISD::FMULADD:
6047 case ISD::FP_EXTEND:
6048 case ISD::FMINNUM:
6049 case ISD::FMAXNUM:
6050 case ISD::FMINNUM_IEEE:
6051 case ISD::FMAXNUM_IEEE:
6052 case ISD::FMINIMUM:
6053 case ISD::FMAXIMUM:
6054 case ISD::FMINIMUMNUM:
6055 case ISD::FMAXIMUMNUM:
6056 case ISD::FP_TO_SINT_SAT:
6057 case ISD::FP_TO_UINT_SAT:
6058 case ISD::TRUNCATE_SSAT_S:
6059 case ISD::TRUNCATE_SSAT_U:
6060 case ISD::TRUNCATE_USAT_U:
6061 // No poison except from flags (which is handled above)
6062 return false;
6063
6064 case ISD::SHL:
6065 case ISD::SRL:
6066 case ISD::SRA:
6067 // If the max shift amount isn't in range, then the shift can
6068 // create poison.
6069 return includesPoison(Kind) &&
6070 !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
6071
6072 case ISD::CTTZ_ZERO_POISON:
6073 case ISD::CTLZ_ZERO_POISON:
6074 // If the amount is zero then the result will be poison.
6075 // TODO: Add isKnownNeverZero DemandedElts handling.
6076 return includesPoison(Kind) &&
6077 !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6078
6079 case ISD::SCALAR_TO_VECTOR:
6080 // Check if we demand any upper (undef) elements.
6081 return includesUndef(Kind) && DemandedElts.ugt(RHS: 1);
6082
6083 case ISD::INSERT_VECTOR_ELT:
6084 case ISD::EXTRACT_VECTOR_ELT: {
6085 // Ensure that the element index is in bounds.
6086 if (includesPoison(Kind)) {
6087 EVT VecVT = Op.getOperand(i: 0).getValueType();
6088 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
6089 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
6090 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
6091 }
6092 return false;
6093 }
6094
6095 case ISD::VECTOR_SHUFFLE: {
6096 // Check for any demanded shuffle element that is undef.
6097 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6098 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
6099 if (Elt < 0 && DemandedElts[Idx])
6100 return true;
6101 return false;
6102 }
6103
6104 case ISD::VECTOR_COMPRESS:
6105 return false;
6106
6107 default:
6108 // Allow the target to implement this method for its nodes.
6109 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6110 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
6111 return TLI->canCreateUndefOrPoisonForTargetNode(
6112 Op, DemandedElts, DAG: *this, Kind, ConsiderFlags, Depth);
6113 break;
6114 }
6115
6116 // Be conservative and return true.
6117 return true;
6118}
6119
6120bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
6121 unsigned Opcode = Op.getOpcode();
6122 if (Opcode == ISD::OR)
6123 return Op->getFlags().hasDisjoint() ||
6124 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
6125 if (Opcode == ISD::XOR)
6126 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
6127 return false;
6128}
6129
6130bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
6131 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
6132 (Op.isAnyAdd() || isADDLike(Op));
6133}
6134
6135KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6136 FPClassTest InterestedClasses,
6137 unsigned Depth) const {
6138 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6139 return computeKnownFPClass(Op, DemandedElts, InterestedClasses, Depth);
6140}
6141
6142KnownFPClass SelectionDAG::computeKnownFPClass(SDValue Op,
6143 const APInt &DemandedElts,
6144 FPClassTest InterestedClasses,
6145 unsigned Depth) const {
6146 KnownFPClass Known;
6147
6148 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
6149 return KnownFPClass(CFP->getValueAPF());
6150
6151 if (Depth >= MaxRecursionDepth)
6152 return Known;
6153
6154 if (Op.getOpcode() == ISD::UNDEF)
6155 return Known;
6156
6157 EVT VT = Op.getValueType();
6158 assert(VT.isFloatingPoint() && "Computing KnownFPClass on non-FP op!");
6159 assert((!VT.isFixedLengthVector() ||
6160 DemandedElts.getBitWidth() == VT.getVectorNumElements()) &&
6161 "Unexpected vector size");
6162
6163 if (!DemandedElts)
6164 return Known;
6165
6166 unsigned Opcode = Op.getOpcode();
6167 switch (Opcode) {
6168 case ISD::POISON: {
6169 Known.KnownFPClasses = fcNone;
6170 Known.SignBit = false;
6171 break;
6172 }
6173 case ISD::FNEG: {
6174 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6175 InterestedClasses, Depth: Depth + 1);
6176 Known.fneg();
6177 break;
6178 }
6179 case ISD::BUILD_VECTOR: {
6180 assert(!VT.isScalableVector());
6181 bool First = true;
6182 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I) {
6183 if (!DemandedElts[I])
6184 continue;
6185
6186 if (First) {
6187 Known =
6188 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6189 First = false;
6190 } else {
6191 Known |=
6192 computeKnownFPClass(Op: Op.getOperand(i: I), InterestedClasses, Depth: Depth + 1);
6193 }
6194
6195 if (Known.isUnknown())
6196 break;
6197 }
6198 break;
6199 }
6200 case ISD::EXTRACT_VECTOR_ELT: {
6201 SDValue Src = Op.getOperand(i: 0);
6202 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6203 EVT SrcVT = Src.getValueType();
6204 if (SrcVT.isFixedLengthVector() && CIdx) {
6205 if (CIdx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6206 APInt DemandedSrcElts = APInt::getOneBitSet(
6207 numBits: SrcVT.getVectorNumElements(), BitNo: CIdx->getZExtValue());
6208 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6209 Depth: Depth + 1);
6210 } else {
6211 // Out of bounds index is poison.
6212 Known.KnownFPClasses = fcNone;
6213 }
6214 } else {
6215 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6216 }
6217 break;
6218 }
6219 case ISD::SPLAT_VECTOR: {
6220 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), InterestedClasses, Depth: Depth + 1);
6221 break;
6222 }
6223 case ISD::BITCAST: {
6224 // FIXME: It should not be necessary to check for an elementwise bitcast.
6225 // If a bitcast is not elementwise between vector / scalar types,
6226 // computeKnownBits already splices the known bits of the source elements
6227 // appropriately so as to line up with the bits of the result's demanded
6228 // elements.
6229 EVT SrcVT = Op.getOperand(i: 0).getValueType();
6230 if (VT.isScalableVector() || SrcVT.isScalableVector())
6231 break;
6232 unsigned VTNumElts = VT.isVector() ? VT.getVectorNumElements() : 1;
6233 unsigned SrcVTNumElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
6234 if (VTNumElts != SrcVTNumElts)
6235 break;
6236
6237 KnownBits Bits = computeKnownBits(Op, DemandedElts, Depth: Depth + 1);
6238 Known = KnownFPClass::bitcast(FltSemantics: VT.getFltSemantics(), Bits);
6239 break;
6240 }
6241 case ISD::FABS: {
6242 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6243 InterestedClasses, Depth: Depth + 1);
6244 Known.fabs();
6245 break;
6246 }
6247 case ISD::FCOPYSIGN: {
6248 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6249 InterestedClasses, Depth: Depth + 1);
6250 KnownFPClass KnownSign = computeKnownFPClass(Op: Op.getOperand(i: 1), DemandedElts,
6251 InterestedClasses, Depth: Depth + 1);
6252 Known.copysign(Sign: KnownSign);
6253 break;
6254 }
6255 case ISD::AssertNoFPClass: {
6256 Known = computeKnownFPClass(Op: Op.getOperand(i: 0), DemandedElts,
6257 InterestedClasses, Depth: Depth + 1);
6258 FPClassTest AssertedClasses =
6259 static_cast<FPClassTest>(Op->getConstantOperandVal(Num: 1));
6260 Known.KnownFPClasses &= ~AssertedClasses;
6261 break;
6262 }
6263 case ISD::EXTRACT_SUBVECTOR: {
6264 SDValue Src = Op.getOperand(i: 0);
6265 EVT SrcVT = Src.getValueType();
6266 if (SrcVT.isFixedLengthVector()) {
6267 unsigned Idx = Op.getConstantOperandVal(i: 1);
6268 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6269
6270 APInt DemandedSrcElts = DemandedElts.zextOrTrunc(width: NumSrcElts).shl(shiftAmt: Idx);
6271 Known = computeKnownFPClass(Op: Src, DemandedElts: DemandedSrcElts, InterestedClasses,
6272 Depth: Depth + 1);
6273 } else {
6274 Known = computeKnownFPClass(Op: Src, InterestedClasses, Depth: Depth + 1);
6275 }
6276 break;
6277 }
6278 case ISD::INSERT_SUBVECTOR: {
6279 SDValue BaseVector = Op.getOperand(i: 0);
6280 SDValue SubVector = Op.getOperand(i: 1);
6281 EVT BaseVT = BaseVector.getValueType();
6282 if (BaseVT.isFixedLengthVector()) {
6283 unsigned Idx = Op.getConstantOperandVal(i: 2);
6284 unsigned NumBaseElts = BaseVT.getVectorNumElements();
6285 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6286
6287 APInt DemandedMask =
6288 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6289 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6290 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6291
6292 if (!DemandedSrcElts.isZero())
6293 Known = computeKnownFPClass(Op: BaseVector, DemandedElts: DemandedSrcElts,
6294 InterestedClasses, Depth: Depth + 1);
6295 if (!DemandedSubElts.isZero()) {
6296 KnownFPClass SubKnown = computeKnownFPClass(
6297 Op: SubVector, DemandedElts: DemandedSubElts, InterestedClasses, Depth: Depth + 1);
6298 Known = DemandedSrcElts.isZero() ? SubKnown : (Known | SubKnown);
6299 }
6300 } else {
6301 Known = computeKnownFPClass(Op: SubVector, InterestedClasses, Depth: Depth + 1);
6302 if (!Known.isUnknown())
6303 Known |= computeKnownFPClass(Op: BaseVector, InterestedClasses, Depth: Depth + 1);
6304 }
6305 break;
6306 }
6307 case ISD::SELECT:
6308 case ISD::VSELECT: {
6309 // TODO: Add adjustKnownFPClassForSelectArm clamp recognition as in
6310 // IR-level ValueTracking.
6311 KnownFPClass KnownFalseClass = computeKnownFPClass(
6312 Op: Op.getOperand(i: 2), DemandedElts, InterestedClasses, Depth: Depth + 1);
6313 if (KnownFalseClass.isUnknown())
6314 break;
6315 KnownFPClass KnownTrueClass = computeKnownFPClass(
6316 Op: Op.getOperand(i: 1), DemandedElts, InterestedClasses, Depth: Depth + 1);
6317 Known = KnownTrueClass.intersectWith(RHS: KnownFalseClass);
6318 break;
6319 }
6320 default:
6321 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6322 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6323 TLI->computeKnownFPClassForTargetNode(Op, Known, DemandedElts, DAG: *this,
6324 Depth);
6325 }
6326 break;
6327 }
6328
6329 return Known;
6330}
6331
6332bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
6333 unsigned Depth) const {
6334 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6335 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
6336}
6337
6338bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
6339 bool SNaN, unsigned Depth) const {
6340 assert(!DemandedElts.isZero() && "No demanded elements");
6341
6342 // If we're told that NaNs won't happen, assume they won't.
6343 if (Op->getFlags().hasNoNaNs())
6344 return true;
6345
6346 if (Depth >= MaxRecursionDepth)
6347 return false; // Limit search depth.
6348
6349 unsigned Opcode = Op.getOpcode();
6350 switch (Opcode) {
6351 case ISD::FADD:
6352 case ISD::FSUB:
6353 case ISD::FMUL:
6354 case ISD::FDIV:
6355 case ISD::FREM:
6356 case ISD::FSIN:
6357 case ISD::FCOS:
6358 case ISD::FTAN:
6359 case ISD::FASIN:
6360 case ISD::FACOS:
6361 case ISD::FATAN:
6362 case ISD::FATAN2:
6363 case ISD::FSINH:
6364 case ISD::FCOSH:
6365 case ISD::FTANH:
6366 case ISD::FMA:
6367 case ISD::FMULADD:
6368 case ISD::FMAD: {
6369 if (SNaN)
6370 return true;
6371 // TODO: Need isKnownNeverInfinity
6372 return false;
6373 }
6374 case ISD::FCANONICALIZE:
6375 case ISD::FEXP:
6376 case ISD::FEXP2:
6377 case ISD::FEXP10:
6378 case ISD::FTRUNC:
6379 case ISD::FFLOOR:
6380 case ISD::FCEIL:
6381 case ISD::FROUND:
6382 case ISD::FROUNDEVEN:
6383 case ISD::LROUND:
6384 case ISD::LLROUND:
6385 case ISD::FRINT:
6386 case ISD::LRINT:
6387 case ISD::LLRINT:
6388 case ISD::FNEARBYINT:
6389 case ISD::FLDEXP: {
6390 if (SNaN)
6391 return true;
6392 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6393 }
6394 case ISD::FABS:
6395 case ISD::FNEG:
6396 case ISD::FCOPYSIGN: {
6397 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6398 }
6399 case ISD::SELECT:
6400 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6401 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6402 case ISD::FP_EXTEND:
6403 case ISD::FP_ROUND: {
6404 if (SNaN)
6405 return true;
6406 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6407 }
6408 case ISD::SINT_TO_FP:
6409 case ISD::UINT_TO_FP:
6410 return true;
6411 case ISD::FSQRT: // Need is known positive
6412 case ISD::FLOG:
6413 case ISD::FLOG2:
6414 case ISD::FLOG10:
6415 case ISD::FPOWI:
6416 case ISD::FPOW: {
6417 if (SNaN)
6418 return true;
6419 // TODO: Refine on operand
6420 return false;
6421 }
6422 case ISD::FMINNUM:
6423 case ISD::FMAXNUM:
6424 case ISD::FMINIMUMNUM:
6425 case ISD::FMAXIMUMNUM: {
6426 // Only one needs to be known not-nan, since it will be returned if the
6427 // other ends up being one.
6428 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6429 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6430 }
6431 case ISD::FMINNUM_IEEE:
6432 case ISD::FMAXNUM_IEEE: {
6433 if (SNaN)
6434 return true;
6435 // This can return a NaN if either operand is an sNaN, or if both operands
6436 // are NaN.
6437 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6438 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6439 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6440 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6441 }
6442 case ISD::FMINIMUM:
6443 case ISD::FMAXIMUM: {
6444 // TODO: Does this quiet or return the origina NaN as-is?
6445 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6446 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6447 }
6448 case ISD::EXTRACT_VECTOR_ELT: {
6449 SDValue Src = Op.getOperand(i: 0);
6450 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6451 EVT SrcVT = Src.getValueType();
6452 if (SrcVT.isFixedLengthVector() && Idx &&
6453 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6454 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6455 BitNo: Idx->getZExtValue());
6456 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6457 }
6458 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6459 }
6460 case ISD::EXTRACT_SUBVECTOR: {
6461 SDValue Src = Op.getOperand(i: 0);
6462 if (Src.getValueType().isFixedLengthVector()) {
6463 unsigned Idx = Op.getConstantOperandVal(i: 1);
6464 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6465 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6466 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6467 }
6468 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6469 }
6470 case ISD::INSERT_SUBVECTOR: {
6471 SDValue BaseVector = Op.getOperand(i: 0);
6472 SDValue SubVector = Op.getOperand(i: 1);
6473 EVT BaseVectorVT = BaseVector.getValueType();
6474 if (BaseVectorVT.isFixedLengthVector()) {
6475 unsigned Idx = Op.getConstantOperandVal(i: 2);
6476 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6477 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6478
6479 // Clear/Extract the bits at the position where the subvector will be
6480 // inserted.
6481 APInt DemandedMask =
6482 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6483 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6484 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6485
6486 bool NeverNaN = true;
6487 if (!DemandedSrcElts.isZero())
6488 NeverNaN &=
6489 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6490 if (NeverNaN && !DemandedSubElts.isZero())
6491 NeverNaN &=
6492 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6493 return NeverNaN;
6494 }
6495 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6496 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6497 }
6498 case ISD::BUILD_VECTOR: {
6499 unsigned NumElts = Op.getNumOperands();
6500 for (unsigned I = 0; I != NumElts; ++I)
6501 if (DemandedElts[I] &&
6502 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6503 return false;
6504 return true;
6505 }
6506 case ISD::SPLAT_VECTOR:
6507 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6508 case ISD::AssertNoFPClass: {
6509 FPClassTest NoFPClass =
6510 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6511 if ((NoFPClass & fcNan) == fcNan)
6512 return true;
6513 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6514 return true;
6515 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6516 }
6517 default:
6518 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6519 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6520 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6521 Depth);
6522 }
6523 break;
6524 }
6525
6526 FPClassTest NanMask = SNaN ? fcSNan : fcNan;
6527 KnownFPClass Known = computeKnownFPClass(Op, DemandedElts, InterestedClasses: NanMask, Depth);
6528 return Known.isKnownNever(Mask: NanMask);
6529}
6530
6531bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op, unsigned Depth) const {
6532 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6533 return isKnownNeverLogicalZero(Op, DemandedElts, Depth);
6534}
6535
6536bool SelectionDAG::isKnownNeverLogicalZero(SDValue Op,
6537 const APInt &DemandedElts,
6538 unsigned Depth) const {
6539 assert(!DemandedElts.isZero() && "No demanded elements");
6540 EVT VT = Op.getValueType();
6541 KnownFPClass Known =
6542 computeKnownFPClass(Op, DemandedElts, InterestedClasses: fcZero | fcSubnormal, Depth);
6543 return Known.isKnownNeverLogicalZero(Mode: getDenormalMode(VT));
6544}
6545
6546bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6547 APInt DemandedElts = getDemandAllEltsMask(V: Op);
6548 return isKnownNeverZero(Op, DemandedElts, Depth);
6549}
6550
6551bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6552 unsigned Depth) const {
6553 if (Depth >= MaxRecursionDepth)
6554 return false; // Limit search depth.
6555
6556 EVT OpVT = Op.getValueType();
6557 unsigned BitWidth = OpVT.getScalarSizeInBits();
6558
6559 assert(!Op.getValueType().isFloatingPoint() &&
6560 "Floating point types unsupported - use isKnownNeverLogicalZero");
6561
6562 // If the value is a constant, we can obviously see if it is a zero or not.
6563 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6564 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6565 return !V.isZero();
6566 };
6567
6568 if (ISD::matchUnaryPredicate(Op, Match: IsNeverZero))
6569 return true;
6570
6571 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6572 // some degree.
6573 switch (Op.getOpcode()) {
6574 default:
6575 break;
6576
6577 case ISD::BUILD_VECTOR:
6578 // Are all operands of a build vector constant non-zero?
6579 if (all_of(Range: enumerate(First: Op->ops()), P: [&](auto P) {
6580 auto *C = dyn_cast<ConstantSDNode>(P.value());
6581 return !DemandedElts[P.index()] || (C && IsNeverZero(C));
6582 }))
6583 return true;
6584 break;
6585
6586 case ISD::SPLAT_VECTOR:
6587 // Is the operand of a splat vector a constant non-zero?
6588 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 0)))
6589 if (IsNeverZero(C))
6590 return true;
6591 break;
6592
6593 case ISD::EXTRACT_VECTOR_ELT: {
6594 SDValue InVec = Op.getOperand(i: 0);
6595 SDValue EltNo = Op.getOperand(i: 1);
6596 EVT VecVT = InVec.getValueType();
6597
6598 // Skip scalable vectors or implicit extensions.
6599 if (VecVT.isScalableVector() ||
6600 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6601 break;
6602
6603 // If we know the element index, just demand that vector element, else for
6604 // an unknown element index, ignore DemandedElts and demand them all.
6605 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6606 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6607 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6608 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6609 DemandedSrcElts =
6610 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6611
6612 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6613 }
6614
6615 case ISD::OR:
6616 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6617 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6618
6619 case ISD::VSELECT:
6620 case ISD::SELECT:
6621 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6622 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6623
6624 case ISD::SHL: {
6625 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6626 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6627 KnownBits ValKnown =
6628 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6629 // 1 << X is never zero.
6630 if (ValKnown.One[0])
6631 return true;
6632 // If max shift cnt of known ones is non-zero, result is non-zero.
6633 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6634 .getMaxValue();
6635 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6636 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6637 return true;
6638 break;
6639 }
6640
6641 case ISD::VECTOR_SHUFFLE: {
6642 if (Op.getValueType().isScalableVector())
6643 return false;
6644
6645 unsigned NumElts = DemandedElts.getBitWidth();
6646
6647 // All demanded elements from LHS and RHS must be known non-zero.
6648 // Demanded elements with undef shuffle mask elements are unknown.
6649
6650 APInt DemandedLHS, DemandedRHS;
6651 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
6652 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
6653 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
6654 DemandedLHS, DemandedRHS))
6655 return false;
6656
6657 return (!DemandedLHS ||
6658 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1)) &&
6659 (!DemandedRHS ||
6660 isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1));
6661 }
6662
6663 case ISD::UADDSAT:
6664 case ISD::UMAX:
6665 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6666 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6667
6668 case ISD::UMIN:
6669 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6670 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6671
6672 // For smin/smax: If either operand is known negative/positive
6673 // respectively we don't need the other to be known at all.
6674 case ISD::SMAX: {
6675 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6676 if (Op1.isStrictlyPositive())
6677 return true;
6678
6679 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6680 if (Op0.isStrictlyPositive())
6681 return true;
6682
6683 if (Op1.isNonZero() && Op0.isNonZero())
6684 return true;
6685
6686 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6687 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6688 }
6689 case ISD::SMIN: {
6690 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6691 if (Op1.isNegative())
6692 return true;
6693
6694 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6695 if (Op0.isNegative())
6696 return true;
6697
6698 if (Op1.isNonZero() && Op0.isNonZero())
6699 return true;
6700
6701 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6702 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6703 }
6704
6705 case ISD::ROTL:
6706 case ISD::ROTR:
6707 case ISD::BITREVERSE:
6708 case ISD::BSWAP:
6709 case ISD::CTPOP:
6710 case ISD::ABS:
6711 case ISD::ABS_MIN_POISON:
6712 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6713
6714 case ISD::SRA:
6715 case ISD::SRL: {
6716 if (Op->getFlags().hasExact())
6717 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6718 KnownBits ValKnown =
6719 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6720 if (ValKnown.isNegative())
6721 return true;
6722 // If max shift cnt of known ones is non-zero, result is non-zero.
6723 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6724 .getMaxValue();
6725 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6726 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6727 return true;
6728 break;
6729 }
6730 case ISD::UDIV:
6731 case ISD::SDIV:
6732 // div exact can only produce a zero if the dividend is zero.
6733 // TODO: For udiv this is also true if Op1 u<= Op0
6734 if (Op->getFlags().hasExact())
6735 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6736 break;
6737
6738 case ISD::ADD:
6739 if (Op->getFlags().hasNoUnsignedWrap())
6740 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6741 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6742 return true;
6743 // TODO: There are a lot more cases we can prove for add.
6744 break;
6745
6746 case ISD::SUB: {
6747 if (isNullConstant(V: Op.getOperand(i: 0)))
6748 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6749
6750 std::optional<bool> ne = KnownBits::ne(
6751 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6752 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6753 return ne && *ne;
6754 }
6755
6756 case ISD::MUL:
6757 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6758 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6759 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6760 return true;
6761 break;
6762
6763 case ISD::ZERO_EXTEND:
6764 case ISD::SIGN_EXTEND:
6765 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6766 case ISD::VSCALE: {
6767 const Function &F = getMachineFunction().getFunction();
6768 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6769 ConstantRange CR =
6770 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6771 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6772 return true;
6773 break;
6774 }
6775 }
6776
6777 return computeKnownBits(Op, DemandedElts, Depth).isNonZero();
6778}
6779
6780bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6781 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6782 return !C1->isNegative();
6783
6784 switch (Op.getOpcode()) {
6785 case ISD::FABS:
6786 case ISD::FEXP:
6787 case ISD::FEXP2:
6788 case ISD::FEXP10:
6789 return true;
6790 default:
6791 return false;
6792 }
6793
6794 llvm_unreachable("covered opcode switch");
6795}
6796
6797bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6798 assert(Use.getValueType().isFloatingPoint());
6799 const SDNode *User = Use.getUser();
6800 if (User->getFlags().hasNoSignedZeros())
6801 return true;
6802
6803 unsigned OperandNo = Use.getOperandNo();
6804 // Check if this use is insensitive to the sign of zero
6805 switch (User->getOpcode()) {
6806 case ISD::SETCC:
6807 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6808 case ISD::FABS:
6809 // fabs always produces +0.0.
6810 return true;
6811 case ISD::FCOPYSIGN:
6812 // copysign overwrites the sign bit of the first operand.
6813 return OperandNo == 0;
6814 case ISD::FADD:
6815 case ISD::FSUB: {
6816 // Arithmetic with non-zero constants fixes the uncertainty around the
6817 // sign bit.
6818 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6819 return isKnownNeverLogicalZero(Op: Other);
6820 }
6821 case ISD::FP_TO_SINT:
6822 case ISD::FP_TO_UINT:
6823 // fp-to-int conversions normalize signed zeros.
6824 return true;
6825 default:
6826 return false;
6827 }
6828}
6829
6830bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6831 if (Op->getFlags().hasNoSignedZeros())
6832 return true;
6833 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6834 // regression. Ideally, this should be implemented as a demanded-bits
6835 // optimization that stems from the users.
6836 if (Op->use_size() > 2)
6837 return false;
6838 return all_of(Range: Op->uses(),
6839 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6840}
6841
6842bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6843 // Check the obvious case.
6844 if (A == B) return true;
6845
6846 // For negative and positive zero.
6847 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6848 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6849 if (CA->isZero() && CB->isZero()) return true;
6850
6851 // Otherwise they may not be equal.
6852 return false;
6853}
6854
6855// Only bits set in Mask must be negated, other bits may be arbitrary.
6856SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6857 if (isBitwiseNot(V, AllowUndefs))
6858 return V.getOperand(i: 0);
6859
6860 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6861 // bits in the non-extended part.
6862 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6863 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6864 return SDValue();
6865 SDValue ExtArg = V.getOperand(i: 0);
6866 if (ExtArg.getScalarValueSizeInBits() >=
6867 MaskC->getAPIntValue().getActiveBits() &&
6868 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6869 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6870 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6871 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6872 return SDValue();
6873}
6874
6875static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6876 // Match masked merge pattern (X & ~M) op (Y & M)
6877 // Including degenerate case (X & ~M) op M
6878 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6879 SDValue Other) {
6880 if (SDValue NotOperand =
6881 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6882 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6883 NotOperand->getOpcode() == ISD::TRUNCATE)
6884 NotOperand = NotOperand->getOperand(Num: 0);
6885
6886 if (Other == NotOperand)
6887 return true;
6888 if (Other->getOpcode() == ISD::AND)
6889 return NotOperand == Other->getOperand(Num: 0) ||
6890 NotOperand == Other->getOperand(Num: 1);
6891 }
6892 return false;
6893 };
6894
6895 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6896 A = A->getOperand(Num: 0);
6897
6898 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6899 B = B->getOperand(Num: 0);
6900
6901 if (A->getOpcode() == ISD::AND)
6902 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6903 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6904 return false;
6905}
6906
6907// FIXME: unify with llvm::haveNoCommonBitsSet.
6908bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6909 assert(A.getValueType() == B.getValueType() &&
6910 "Values must have the same type");
6911 if (haveNoCommonBitsSetCommutative(A, B) ||
6912 haveNoCommonBitsSetCommutative(A: B, B: A))
6913 return true;
6914 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6915 RHS: computeKnownBits(Op: B));
6916}
6917
6918static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6919 SelectionDAG &DAG) {
6920 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6921 return DAG.getConstant(Val: 0, DL, VT);
6922
6923 return SDValue();
6924}
6925
6926static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6927 ArrayRef<SDValue> Ops,
6928 SelectionDAG &DAG) {
6929 int NumOps = Ops.size();
6930 assert(NumOps != 0 && "Can't build an empty vector!");
6931 assert(!VT.isScalableVector() &&
6932 "BUILD_VECTOR cannot be used with scalable types");
6933 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6934 "Incorrect element count in BUILD_VECTOR!");
6935
6936 // BUILD_VECTOR of UNDEFs is UNDEF.
6937 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6938 return DAG.getUNDEF(VT);
6939
6940 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6941 SDValue IdentitySrc;
6942 bool IsIdentity = true;
6943 for (int i = 0; i != NumOps; ++i) {
6944 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6945 Ops[i].getOperand(i: 0).getValueType() != VT ||
6946 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6947 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6948 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6949 IsIdentity = false;
6950 break;
6951 }
6952 IdentitySrc = Ops[i].getOperand(i: 0);
6953 }
6954 if (IsIdentity)
6955 return IdentitySrc;
6956
6957 return SDValue();
6958}
6959
6960/// Try to simplify vector concatenation to an input value, undef, or build
6961/// vector.
6962static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6963 ArrayRef<SDValue> Ops,
6964 SelectionDAG &DAG) {
6965 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6966 assert(llvm::all_of(Ops,
6967 [Ops](SDValue Op) {
6968 return Ops[0].getValueType() == Op.getValueType();
6969 }) &&
6970 "Concatenation of vectors with inconsistent value types!");
6971 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6972 VT.getVectorElementCount() &&
6973 "Incorrect element count in vector concatenation!");
6974
6975 if (Ops.size() == 1)
6976 return Ops[0];
6977
6978 // Concat of UNDEFs is UNDEF.
6979 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6980 return DAG.getUNDEF(VT);
6981
6982 // Scan the operands and look for extract operations from a single source
6983 // that correspond to insertion at the same location via this concatenation:
6984 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6985 SDValue IdentitySrc;
6986 bool IsIdentity = true;
6987 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6988 SDValue Op = Ops[i];
6989 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6990 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6991 Op.getOperand(i: 0).getValueType() != VT ||
6992 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
6993 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
6994 IsIdentity = false;
6995 break;
6996 }
6997 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6998 "Unexpected identity source vector for concat of extracts");
6999 IdentitySrc = Op.getOperand(i: 0);
7000 }
7001 if (IsIdentity) {
7002 assert(IdentitySrc && "Failed to set source vector of extracts");
7003 return IdentitySrc;
7004 }
7005
7006 // The code below this point is only designed to work for fixed width
7007 // vectors, so we bail out for now.
7008 if (VT.isScalableVector())
7009 return SDValue();
7010
7011 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
7012 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
7013 // BUILD_VECTOR.
7014 // FIXME: Add support for SCALAR_TO_VECTOR as well.
7015 EVT SVT = VT.getScalarType();
7016 SmallVector<SDValue, 16> Elts;
7017 for (SDValue Op : Ops) {
7018 EVT OpVT = Op.getValueType();
7019 if (Op.isUndef())
7020 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
7021 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
7022 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
7023 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
7024 OpVT.getVectorNumElements() == 1 &&
7025 isNullConstant(V: Op.getOperand(i: 2)))
7026 Elts.push_back(Elt: Op.getOperand(i: 1));
7027 else
7028 return SDValue();
7029 }
7030
7031 // BUILD_VECTOR requires all inputs to be of the same type, find the
7032 // maximum type and extend them all.
7033 for (SDValue Op : Elts)
7034 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
7035
7036 if (SVT.bitsGT(VT: VT.getScalarType())) {
7037 for (SDValue &Op : Elts) {
7038 if (Op.isUndef())
7039 Op = DAG.getUNDEF(VT: SVT);
7040 else
7041 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
7042 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
7043 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
7044 }
7045 }
7046
7047 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
7048 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
7049 return V;
7050}
7051
7052/// Gets or creates the specified node.
7053SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
7054 SDVTList VTs = getVTList(VT);
7055 FoldingSetNodeID ID;
7056 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: {});
7057 void *IP = nullptr;
7058 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
7059 return SDValue(E, 0);
7060
7061 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7062 CSEMap.InsertNode(N, InsertPos: IP);
7063
7064 InsertNode(N);
7065 SDValue V = SDValue(N, 0);
7066 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7067 return V;
7068}
7069
7070SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7071 SDValue N1) {
7072 SDNodeFlags Flags;
7073 if (Inserter)
7074 Flags = Inserter->getFlags();
7075 return getNode(Opcode, DL, VT, Operand: N1, Flags);
7076}
7077
7078SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7079 SDValue N1, const SDNodeFlags Flags) {
7080 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
7081
7082 // Constant fold unary operations with a vector integer or float operand.
7083 switch (Opcode) {
7084 default:
7085 // FIXME: Entirely reasonable to perform folding of other unary
7086 // operations here as the need arises.
7087 break;
7088 case ISD::FNEG:
7089 case ISD::FABS:
7090 case ISD::FCEIL:
7091 case ISD::FTRUNC:
7092 case ISD::FFLOOR:
7093 case ISD::FP_EXTEND:
7094 case ISD::FP_TO_SINT:
7095 case ISD::FP_TO_UINT:
7096 case ISD::FP_TO_FP16:
7097 case ISD::FP_TO_BF16:
7098 case ISD::TRUNCATE:
7099 case ISD::ANY_EXTEND:
7100 case ISD::ZERO_EXTEND:
7101 case ISD::SIGN_EXTEND:
7102 case ISD::UINT_TO_FP:
7103 case ISD::SINT_TO_FP:
7104 case ISD::FP16_TO_FP:
7105 case ISD::BF16_TO_FP:
7106 case ISD::BITCAST:
7107 case ISD::ABS:
7108 case ISD::ABS_MIN_POISON:
7109 case ISD::BITREVERSE:
7110 case ISD::BSWAP:
7111 case ISD::CTLZ:
7112 case ISD::CTLZ_ZERO_POISON:
7113 case ISD::CTTZ:
7114 case ISD::CTTZ_ZERO_POISON:
7115 case ISD::CTPOP:
7116 case ISD::CTLS:
7117 case ISD::STEP_VECTOR: {
7118 SDValue Ops = {N1};
7119 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
7120 return Fold;
7121 }
7122 }
7123
7124 unsigned OpOpcode = N1.getNode()->getOpcode();
7125 switch (Opcode) {
7126 case ISD::STEP_VECTOR:
7127 assert(VT.isScalableVector() &&
7128 "STEP_VECTOR can only be used with scalable types");
7129 assert(OpOpcode == ISD::TargetConstant &&
7130 VT.getVectorElementType() == N1.getValueType() &&
7131 "Unexpected step operand");
7132 break;
7133 case ISD::FREEZE:
7134 assert(VT == N1.getValueType() && "Unexpected VT!");
7135 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, Kind: UndefPoisonKind::UndefOrPoison))
7136 return N1;
7137 break;
7138 case ISD::TokenFactor:
7139 case ISD::MERGE_VALUES:
7140 case ISD::CONCAT_VECTORS:
7141 return N1; // Factor, merge or concat of one node? No need.
7142 case ISD::BUILD_VECTOR: {
7143 // Attempt to simplify BUILD_VECTOR.
7144 SDValue Ops[] = {N1};
7145 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7146 return V;
7147 break;
7148 }
7149 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
7150 case ISD::FP_EXTEND:
7151 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7152 "Invalid FP cast!");
7153 if (N1.getValueType() == VT) return N1; // noop conversion.
7154 assert((!VT.isVector() || VT.getVectorElementCount() ==
7155 N1.getValueType().getVectorElementCount()) &&
7156 "Vector element count mismatch!");
7157 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
7158 if (N1.isUndef())
7159 return getUNDEF(VT);
7160 break;
7161 case ISD::FP_TO_SINT:
7162 case ISD::FP_TO_UINT:
7163 if (N1.isUndef())
7164 return getUNDEF(VT);
7165 break;
7166 case ISD::SINT_TO_FP:
7167 case ISD::UINT_TO_FP:
7168 // [us]itofp(undef) = 0, because the result value is bounded.
7169 if (N1.isUndef())
7170 return getConstantFP(Val: 0.0, DL, VT);
7171 break;
7172 case ISD::SIGN_EXTEND:
7173 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7174 "Invalid SIGN_EXTEND!");
7175 assert(VT.isVector() == N1.getValueType().isVector() &&
7176 "SIGN_EXTEND result type type should be vector iff the operand "
7177 "type is vector!");
7178 if (N1.getValueType() == VT) return N1; // noop extension
7179 assert((!VT.isVector() || VT.getVectorElementCount() ==
7180 N1.getValueType().getVectorElementCount()) &&
7181 "Vector element count mismatch!");
7182 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
7183 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
7184 SDNodeFlags Flags;
7185 if (OpOpcode == ISD::ZERO_EXTEND)
7186 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7187 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7188 transferDbgValues(From: N1, To: NewVal);
7189 return NewVal;
7190 }
7191
7192 if (OpOpcode == ISD::POISON)
7193 return getPOISON(VT);
7194
7195 if (N1.isUndef())
7196 // sext(undef) = 0, because the top bits will all be the same.
7197 return getConstant(Val: 0, DL, VT);
7198
7199 // Skip unnecessary sext_inreg pattern:
7200 // (sext (trunc x)) -> x iff the upper bits are all signbits.
7201 if (OpOpcode == ISD::TRUNCATE) {
7202 SDValue OpOp = N1.getOperand(i: 0);
7203 if (OpOp.getValueType() == VT) {
7204 unsigned NumSignExtBits =
7205 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
7206 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
7207 transferDbgValues(From: N1, To: OpOp);
7208 return OpOp;
7209 }
7210 }
7211 }
7212 break;
7213 case ISD::ZERO_EXTEND:
7214 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7215 "Invalid ZERO_EXTEND!");
7216 assert(VT.isVector() == N1.getValueType().isVector() &&
7217 "ZERO_EXTEND result type type should be vector iff the operand "
7218 "type is vector!");
7219 if (N1.getValueType() == VT) return N1; // noop extension
7220 assert((!VT.isVector() || VT.getVectorElementCount() ==
7221 N1.getValueType().getVectorElementCount()) &&
7222 "Vector element count mismatch!");
7223 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
7224 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
7225 SDNodeFlags Flags;
7226 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7227 SDValue NewVal =
7228 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
7229 transferDbgValues(From: N1, To: NewVal);
7230 return NewVal;
7231 }
7232
7233 if (OpOpcode == ISD::POISON)
7234 return getPOISON(VT);
7235
7236 if (N1.isUndef())
7237 // zext(undef) = 0, because the top bits will be zero.
7238 return getConstant(Val: 0, DL, VT);
7239
7240 // Skip unnecessary zext_inreg pattern:
7241 // (zext (trunc x)) -> x iff the upper bits are known zero.
7242 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
7243 // use to recognise zext_inreg patterns.
7244 if (OpOpcode == ISD::TRUNCATE) {
7245 SDValue OpOp = N1.getOperand(i: 0);
7246 if (OpOp.getValueType() == VT) {
7247 if (OpOp.getOpcode() != ISD::AND) {
7248 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
7249 loBit: N1.getScalarValueSizeInBits());
7250 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
7251 transferDbgValues(From: N1, To: OpOp);
7252 return OpOp;
7253 }
7254 }
7255 }
7256 }
7257 break;
7258 case ISD::ANY_EXTEND:
7259 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7260 "Invalid ANY_EXTEND!");
7261 assert(VT.isVector() == N1.getValueType().isVector() &&
7262 "ANY_EXTEND result type type should be vector iff the operand "
7263 "type is vector!");
7264 if (N1.getValueType() == VT) return N1; // noop extension
7265 assert((!VT.isVector() || VT.getVectorElementCount() ==
7266 N1.getValueType().getVectorElementCount()) &&
7267 "Vector element count mismatch!");
7268 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
7269
7270 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7271 OpOpcode == ISD::ANY_EXTEND) {
7272 SDNodeFlags Flags;
7273 if (OpOpcode == ISD::ZERO_EXTEND)
7274 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7275 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
7276 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7277 }
7278 if (N1.isUndef())
7279 return getUNDEF(VT);
7280
7281 // (ext (trunc x)) -> x
7282 if (OpOpcode == ISD::TRUNCATE) {
7283 SDValue OpOp = N1.getOperand(i: 0);
7284 if (OpOp.getValueType() == VT) {
7285 transferDbgValues(From: N1, To: OpOp);
7286 return OpOp;
7287 }
7288 }
7289 break;
7290 case ISD::TRUNCATE:
7291 assert(VT.isInteger() && N1.getValueType().isInteger() &&
7292 "Invalid TRUNCATE!");
7293 assert(VT.isVector() == N1.getValueType().isVector() &&
7294 "TRUNCATE result type type should be vector iff the operand "
7295 "type is vector!");
7296 if (N1.getValueType() == VT) return N1; // noop truncate
7297 assert((!VT.isVector() || VT.getVectorElementCount() ==
7298 N1.getValueType().getVectorElementCount()) &&
7299 "Vector element count mismatch!");
7300 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
7301 if (OpOpcode == ISD::TRUNCATE)
7302 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7303 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
7304 OpOpcode == ISD::ANY_EXTEND) {
7305 // If the source is smaller than the dest, we still need an extend.
7306 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
7307 VT: VT.getScalarType())) {
7308 SDNodeFlags Flags;
7309 if (OpOpcode == ISD::ZERO_EXTEND)
7310 Flags.setNonNeg(N1->getFlags().hasNonNeg());
7311 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
7312 }
7313 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
7314 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
7315 return N1.getOperand(i: 0);
7316 }
7317 if (N1.isUndef())
7318 return getUNDEF(VT);
7319 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
7320 return getVScale(DL, VT,
7321 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
7322 break;
7323 case ISD::ANY_EXTEND_VECTOR_INREG:
7324 case ISD::ZERO_EXTEND_VECTOR_INREG:
7325 case ISD::SIGN_EXTEND_VECTOR_INREG:
7326 assert(VT.isVector() && "This DAG node is restricted to vector types.");
7327 assert(N1.getValueType().bitsLE(VT) &&
7328 "The input must be the same size or smaller than the result.");
7329 assert(VT.getVectorMinNumElements() <
7330 N1.getValueType().getVectorMinNumElements() &&
7331 "The destination vector type must have fewer lanes than the input.");
7332 break;
7333 case ISD::ABS:
7334 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
7335 if (N1.isUndef())
7336 return getConstant(Val: 0, DL, VT);
7337 break;
7338 case ISD::ABS_MIN_POISON:
7339 assert(VT.isInteger() && VT == N1.getValueType() &&
7340 "Invalid ABS_MIN_POISON!");
7341 if (N1.isUndef())
7342 return getConstant(Val: 0, DL, VT);
7343 break;
7344 case ISD::BSWAP:
7345 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
7346 assert((VT.getScalarSizeInBits() % 16 == 0) &&
7347 "BSWAP types must be a multiple of 16 bits!");
7348 if (N1.isUndef())
7349 return getUNDEF(VT);
7350 // bswap(bswap(X)) -> X.
7351 if (OpOpcode == ISD::BSWAP)
7352 return N1.getOperand(i: 0);
7353 break;
7354 case ISD::BITREVERSE:
7355 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
7356 if (N1.isUndef())
7357 return getUNDEF(VT);
7358 break;
7359 case ISD::BITCAST:
7360 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
7361 "Cannot BITCAST between types of different sizes!");
7362 if (VT == N1.getValueType()) return N1; // noop conversion.
7363 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
7364 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
7365 if (N1.isUndef())
7366 return getUNDEF(VT);
7367 break;
7368 case ISD::SCALAR_TO_VECTOR:
7369 assert(VT.isVector() && !N1.getValueType().isVector() &&
7370 (VT.getVectorElementType() == N1.getValueType() ||
7371 (VT.getVectorElementType().isInteger() &&
7372 N1.getValueType().isInteger() &&
7373 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7374 "Illegal SCALAR_TO_VECTOR node!");
7375 if (N1.isUndef())
7376 return getUNDEF(VT);
7377 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7378 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7379 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7380 N1.getConstantOperandVal(i: 1) == 0 &&
7381 N1.getOperand(i: 0).getValueType() == VT)
7382 return N1.getOperand(i: 0);
7383 break;
7384 case ISD::FNEG:
7385 // Negation of an unknown bag of bits is still completely undefined.
7386 if (N1.isUndef())
7387 return getUNDEF(VT);
7388
7389 if (OpOpcode == ISD::FNEG) // --X -> X
7390 return N1.getOperand(i: 0);
7391 break;
7392 case ISD::FABS:
7393 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7394 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7395 break;
7396 case ISD::VSCALE:
7397 assert(VT == N1.getValueType() && "Unexpected VT!");
7398 break;
7399 case ISD::CTPOP:
7400 if (N1.getValueType().getScalarType() == MVT::i1)
7401 return N1;
7402 break;
7403 case ISD::CTLZ:
7404 case ISD::CTTZ:
7405 if (N1.getValueType().getScalarType() == MVT::i1)
7406 return getNOT(DL, Val: N1, VT: N1.getValueType());
7407 break;
7408 case ISD::CTLS:
7409 if (N1.getValueType().getScalarType() == MVT::i1)
7410 return getConstant(Val: 0, DL, VT);
7411 break;
7412 case ISD::VECREDUCE_ADD:
7413 if (N1.getValueType().getScalarType() == MVT::i1)
7414 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7415 break;
7416 case ISD::VECREDUCE_SMIN:
7417 case ISD::VECREDUCE_UMAX:
7418 if (N1.getValueType().getScalarType() == MVT::i1)
7419 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7420 break;
7421 case ISD::VECREDUCE_SMAX:
7422 case ISD::VECREDUCE_UMIN:
7423 if (N1.getValueType().getScalarType() == MVT::i1)
7424 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7425 break;
7426 case ISD::SPLAT_VECTOR:
7427 assert(VT.isVector() && "Wrong return type!");
7428 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7429 // that for now.
7430 assert((VT.getVectorElementType() == N1.getValueType() ||
7431 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7432 (VT.getVectorElementType().isInteger() &&
7433 N1.getValueType().isInteger() &&
7434 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7435 "Wrong operand type!");
7436 break;
7437 }
7438
7439 SDNode *N;
7440 SDVTList VTs = getVTList(VT);
7441 SDValue Ops[] = {N1};
7442 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7443 FoldingSetNodeID ID;
7444 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7445 void *IP = nullptr;
7446 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7447 E->intersectFlagsWith(Flags);
7448 return SDValue(E, 0);
7449 }
7450
7451 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7452 N->setFlags(Flags);
7453 createOperands(Node: N, Vals: Ops);
7454 CSEMap.InsertNode(N, InsertPos: IP);
7455 } else {
7456 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7457 createOperands(Node: N, Vals: Ops);
7458 }
7459
7460 InsertNode(N);
7461 SDValue V = SDValue(N, 0);
7462 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7463 return V;
7464}
7465
7466static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7467 const APInt &C2) {
7468 switch (Opcode) {
7469 case ISD::ADD: return C1 + C2;
7470 case ISD::SUB: return C1 - C2;
7471 case ISD::MUL: return C1 * C2;
7472 case ISD::AND: return C1 & C2;
7473 case ISD::OR: return C1 | C2;
7474 case ISD::XOR: return C1 ^ C2;
7475 case ISD::SHL: return C1 << C2;
7476 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7477 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7478 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7479 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7480 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7481 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7482 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7483 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7484 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7485 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7486 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7487 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7488 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7489 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7490 case ISD::UDIV:
7491 if (!C2.getBoolValue())
7492 break;
7493 return C1.udiv(RHS: C2);
7494 case ISD::UREM:
7495 if (!C2.getBoolValue())
7496 break;
7497 return C1.urem(RHS: C2);
7498 case ISD::SDIV:
7499 if (!C2.getBoolValue())
7500 break;
7501 return C1.sdiv(RHS: C2);
7502 case ISD::SREM:
7503 if (!C2.getBoolValue())
7504 break;
7505 return C1.srem(RHS: C2);
7506 case ISD::AVGFLOORS:
7507 return APIntOps::avgFloorS(C1, C2);
7508 case ISD::AVGFLOORU:
7509 return APIntOps::avgFloorU(C1, C2);
7510 case ISD::AVGCEILS:
7511 return APIntOps::avgCeilS(C1, C2);
7512 case ISD::AVGCEILU:
7513 return APIntOps::avgCeilU(C1, C2);
7514 case ISD::ABDS:
7515 return APIntOps::abds(A: C1, B: C2);
7516 case ISD::ABDU:
7517 return APIntOps::abdu(A: C1, B: C2);
7518 case ISD::MULHS:
7519 return APIntOps::mulhs(C1, C2);
7520 case ISD::MULHU:
7521 return APIntOps::mulhu(C1, C2);
7522 case ISD::CLMUL:
7523 return APIntOps::clmul(LHS: C1, RHS: C2);
7524 case ISD::CLMULR:
7525 return APIntOps::clmulr(LHS: C1, RHS: C2);
7526 case ISD::CLMULH:
7527 return APIntOps::clmulh(LHS: C1, RHS: C2);
7528 case ISD::PEXT:
7529 return APIntOps::pext(Val: C1, Mask: C2);
7530 case ISD::PDEP:
7531 return APIntOps::pdep(Val: C1, Mask: C2);
7532 }
7533 return std::nullopt;
7534}
7535// Handle constant folding with UNDEF.
7536// TODO: Handle more cases.
7537static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7538 bool IsUndef1, const APInt &C2,
7539 bool IsUndef2) {
7540 if (!(IsUndef1 || IsUndef2))
7541 return FoldValue(Opcode, C1, C2);
7542
7543 // Fold and(x, undef) -> 0
7544 // Fold mul(x, undef) -> 0
7545 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7546 return APInt::getZero(numBits: C1.getBitWidth());
7547
7548 return std::nullopt;
7549}
7550
7551SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7552 const GlobalAddressSDNode *GA,
7553 const SDNode *N2) {
7554 if (GA->getOpcode() != ISD::GlobalAddress)
7555 return SDValue();
7556 if (!TLI->isOffsetFoldingLegal(GA))
7557 return SDValue();
7558 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7559 if (!C2)
7560 return SDValue();
7561 int64_t Offset = C2->getSExtValue();
7562 switch (Opcode) {
7563 case ISD::ADD:
7564 case ISD::PTRADD:
7565 break;
7566 case ISD::SUB: Offset = -uint64_t(Offset); break;
7567 default: return SDValue();
7568 }
7569 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7570 Offset: GA->getOffset() + uint64_t(Offset));
7571}
7572
7573bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7574 switch (Opcode) {
7575 case ISD::SDIV:
7576 case ISD::UDIV:
7577 case ISD::SREM:
7578 case ISD::UREM: {
7579 // If a divisor is zero/undef or any element of a divisor vector is
7580 // zero/undef, the whole op is undef.
7581 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7582 SDValue Divisor = Ops[1];
7583 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7584 return true;
7585
7586 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7587 llvm::any_of(Range: Divisor->op_values(),
7588 P: [](SDValue V) { return V.isUndef() ||
7589 isNullConstant(V); });
7590 // TODO: Handle signed overflow.
7591 }
7592 // TODO: Handle oversized shifts.
7593 default:
7594 return false;
7595 }
7596}
7597
7598SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7599 EVT VT, ArrayRef<SDValue> Ops,
7600 SDNodeFlags Flags) {
7601 // If the opcode is a target-specific ISD node, there's nothing we can
7602 // do here and the operand rules may not line up with the below, so
7603 // bail early.
7604 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7605 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7606 // foldCONCAT_VECTORS in getNode before this is called.
7607 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7608 return SDValue();
7609
7610 unsigned NumOps = Ops.size();
7611 if (NumOps == 0)
7612 return SDValue();
7613
7614 if (isUndef(Opcode, Ops))
7615 return getUNDEF(VT);
7616
7617 // Handle unary special cases.
7618 if (NumOps == 1) {
7619 SDValue N1 = Ops[0];
7620
7621 // Constant fold unary operations with an integer constant operand. Even
7622 // opaque constant will be folded, because the folding of unary operations
7623 // doesn't create new constants with different values. Nevertheless, the
7624 // opaque flag is preserved during folding to prevent future folding with
7625 // other constants.
7626 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7627 const APInt &Val = C->getAPIntValue();
7628 switch (Opcode) {
7629 case ISD::SIGN_EXTEND:
7630 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7631 isT: C->isTargetOpcode(), isO: C->isOpaque());
7632 case ISD::TRUNCATE:
7633 if (C->isOpaque())
7634 break;
7635 [[fallthrough]];
7636 case ISD::ZERO_EXTEND:
7637 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7638 isT: C->isTargetOpcode(), isO: C->isOpaque());
7639 case ISD::ANY_EXTEND:
7640 // Some targets like RISCV prefer to sign extend some types.
7641 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7642 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7643 isT: C->isTargetOpcode(), isO: C->isOpaque());
7644 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7645 isT: C->isTargetOpcode(), isO: C->isOpaque());
7646 case ISD::ABS:
7647 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7648 isO: C->isOpaque());
7649 case ISD::ABS_MIN_POISON:
7650 if (Val.isMinSignedValue())
7651 return getPOISON(VT);
7652 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7653 isO: C->isOpaque());
7654 case ISD::BITREVERSE:
7655 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7656 isO: C->isOpaque());
7657 case ISD::BSWAP:
7658 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7659 isO: C->isOpaque());
7660 case ISD::CTPOP:
7661 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7662 isO: C->isOpaque());
7663 case ISD::CTLZ:
7664 case ISD::CTLZ_ZERO_POISON:
7665 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7666 isO: C->isOpaque());
7667 case ISD::CTTZ:
7668 case ISD::CTTZ_ZERO_POISON:
7669 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7670 isO: C->isOpaque());
7671 case ISD::CTLS:
7672 // CTLS returns the number of extra sign bits so subtract one.
7673 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7674 isT: C->isTargetOpcode(), isO: C->isOpaque());
7675 case ISD::UINT_TO_FP:
7676 case ISD::SINT_TO_FP: {
7677 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7678 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7679 RM: APFloat::rmNearestTiesToEven);
7680 return getConstantFP(V: FPV, DL, VT);
7681 }
7682 case ISD::FP16_TO_FP:
7683 case ISD::BF16_TO_FP: {
7684 bool Ignored;
7685 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7686 : APFloat::BFloat(),
7687 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7688
7689 // This can return overflow, underflow, or inexact; we don't care.
7690 // FIXME need to be more flexible about rounding mode.
7691 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7692 losesInfo: &Ignored);
7693 return getConstantFP(V: FPV, DL, VT);
7694 }
7695 case ISD::STEP_VECTOR:
7696 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7697 return V;
7698 break;
7699 case ISD::BITCAST:
7700 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7701 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7702 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7703 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7704 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7705 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7706 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7707 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7708 break;
7709 }
7710 }
7711
7712 // Constant fold unary operations with a floating point constant operand.
7713 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7714 APFloat V = C->getValueAPF(); // make copy
7715 switch (Opcode) {
7716 case ISD::FNEG:
7717 V.changeSign();
7718 return getConstantFP(V, DL, VT);
7719 case ISD::FABS:
7720 V.clearSign();
7721 return getConstantFP(V, DL, VT);
7722 case ISD::FCEIL: {
7723 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7724 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7725 return getConstantFP(V, DL, VT);
7726 return SDValue();
7727 }
7728 case ISD::FTRUNC: {
7729 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7730 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7731 return getConstantFP(V, DL, VT);
7732 return SDValue();
7733 }
7734 case ISD::FFLOOR: {
7735 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7736 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7737 return getConstantFP(V, DL, VT);
7738 return SDValue();
7739 }
7740 case ISD::FP_EXTEND: {
7741 bool ignored;
7742 // This can return overflow, underflow, or inexact; we don't care.
7743 // FIXME need to be more flexible about rounding mode.
7744 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7745 losesInfo: &ignored);
7746 return getConstantFP(V, DL, VT);
7747 }
7748 case ISD::FP_TO_SINT:
7749 case ISD::FP_TO_UINT: {
7750 bool ignored;
7751 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7752 // FIXME need to be more flexible about rounding mode.
7753 APFloat::opStatus s =
7754 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7755 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7756 break;
7757 return getConstant(Val: IntVal, DL, VT);
7758 }
7759 case ISD::FP_TO_FP16:
7760 case ISD::FP_TO_BF16: {
7761 bool Ignored;
7762 // This can return overflow, underflow, or inexact; we don't care.
7763 // FIXME need to be more flexible about rounding mode.
7764 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7765 : APFloat::BFloat(),
7766 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7767 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7768 }
7769 case ISD::BITCAST:
7770 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7771 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7772 VT);
7773 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7774 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7775 VT);
7776 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7777 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7778 VT);
7779 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7780 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7781 break;
7782 }
7783 }
7784
7785 // Early-out if we failed to constant fold a bitcast.
7786 if (Opcode == ISD::BITCAST)
7787 return SDValue();
7788 }
7789
7790 // Handle binops special cases.
7791 if (NumOps == 2) {
7792 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7793 return CFP;
7794
7795 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7796 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7797 if (C1->isOpaque() || C2->isOpaque())
7798 return SDValue();
7799
7800 std::optional<APInt> FoldAttempt =
7801 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7802 if (!FoldAttempt)
7803 return SDValue();
7804
7805 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7806 assert((!Folded || !VT.isVector()) &&
7807 "Can't fold vectors ops with scalar operands");
7808 return Folded;
7809 }
7810 }
7811
7812 // fold (add Sym, c) -> Sym+c
7813 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7814 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7815 if (TLI->isCommutativeBinOp(Opcode))
7816 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7817 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7818
7819 // fold (sext_in_reg c1) -> c2
7820 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7821 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7822
7823 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7824 unsigned FromBits = EVT.getScalarSizeInBits();
7825 Val <<= Val.getBitWidth() - FromBits;
7826 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7827 return getConstant(Val, DL, VT: ConstantVT);
7828 };
7829
7830 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7831 const APInt &Val = C1->getAPIntValue();
7832 return SignExtendInReg(Val, VT);
7833 }
7834
7835 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7836 SmallVector<SDValue, 8> ScalarOps;
7837 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7838 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7839 SDValue Op = Ops[0].getOperand(i: I);
7840 if (Op.isUndef()) {
7841 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7842 continue;
7843 }
7844 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7845 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7846 }
7847 return getBuildVector(VT, DL, Ops: ScalarOps);
7848 }
7849
7850 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7851 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7852 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7853 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7854 Ops[0].getOperand(i: 0).getValueType()));
7855 }
7856 }
7857
7858 // Handle fshl/fshr special cases.
7859 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7860 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7861 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7862 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7863
7864 if (C1 && C2 && C3) {
7865 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7866 return SDValue();
7867 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7868 &V3 = C3->getAPIntValue();
7869
7870 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7871 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7872 return getConstant(Val: FoldedVal, DL, VT);
7873 }
7874 }
7875
7876 // Handle fma/fmad special cases.
7877 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7878 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7879 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7880 Ops[2].getValueType() == VT && "FMA types must match!");
7881 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
7882 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
7883 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
7884 if (C1 && C2 && C3) {
7885 APFloat V1 = C1->getValueAPF();
7886 const APFloat &V2 = C2->getValueAPF();
7887 const APFloat &V3 = C3->getValueAPF();
7888 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7889 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7890 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7891 } else
7892 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7893 return getConstantFP(V: V1, DL, VT);
7894 }
7895 }
7896
7897 // This is for vector folding only from here on.
7898 if (!VT.isVector())
7899 return SDValue();
7900
7901 ElementCount NumElts = VT.getVectorElementCount();
7902
7903 // See if we can fold through any bitcasted integer ops.
7904 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
7905 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7906 (Ops[0].getOpcode() == ISD::BITCAST ||
7907 Ops[1].getOpcode() == ISD::BITCAST)) {
7908 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
7909 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
7910 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
7911 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
7912 if (BV1 && BV2 && N1.getValueType().isInteger() &&
7913 N2.getValueType().isInteger()) {
7914 bool IsLE = getDataLayout().isLittleEndian();
7915 unsigned EltBits = VT.getScalarSizeInBits();
7916 SmallVector<APInt> RawBits1, RawBits2;
7917 BitVector UndefElts1, UndefElts2;
7918 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
7919 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
7920 SmallVector<APInt> RawBits;
7921 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
7922 std::optional<APInt> Fold = FoldValueWithUndef(
7923 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
7924 if (!Fold)
7925 break;
7926 RawBits.push_back(Elt: *Fold);
7927 }
7928 if (RawBits.size() == NumElts.getFixedValue()) {
7929 // We have constant folded, but we might need to cast this again back
7930 // to the original (possibly legalized) type.
7931 EVT BVVT, BVEltVT;
7932 if (N1.getValueType() == VT) {
7933 BVVT = N1.getValueType();
7934 BVEltVT = BV1->getOperand(Num: 0).getValueType();
7935 } else {
7936 BVVT = N2.getValueType();
7937 BVEltVT = BV2->getOperand(Num: 0).getValueType();
7938 }
7939 unsigned BVEltBits = BVEltVT.getSizeInBits();
7940 SmallVector<APInt> DstBits;
7941 BitVector DstUndefs;
7942 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
7943 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
7944 SrcUndefElements: BitVector(RawBits.size(), false));
7945 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
7946 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
7947 if (DstUndefs[I])
7948 continue;
7949 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
7950 }
7951 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
7952 }
7953 }
7954 }
7955 // Logic ops can be folded from raw integer bits - mainly for AVX512 masks.
7956 if (ISD::isBitwiseLogicOp(Opcode) && isa<ConstantSDNode>(Val: N1) &&
7957 isa<ConstantSDNode>(Val: N2)) {
7958 if (SDValue Res = FoldConstantArithmetic(Opcode, DL, VT: N1.getValueType(),
7959 Ops: {N1, N2}, Flags))
7960 return getBitcast(VT, V: Res);
7961 }
7962 }
7963
7964 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
7965 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
7966 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
7967 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
7968 APInt RHSVal;
7969 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
7970 APInt NewStep = Opcode == ISD::MUL
7971 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
7972 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
7973 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
7974 }
7975 }
7976
7977 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
7978 return !Op.getValueType().isVector() ||
7979 Op.getValueType().getVectorElementCount() == NumElts;
7980 };
7981
7982 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
7983 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
7984 Op.getOpcode() == ISD::BUILD_VECTOR ||
7985 Op.getOpcode() == ISD::SPLAT_VECTOR;
7986 };
7987
7988 // All operands must be vector types with the same number of elements as
7989 // the result type and must be either UNDEF or a build/splat vector
7990 // or UNDEF scalars.
7991 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
7992 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
7993 return SDValue();
7994
7995 // If we are comparing vectors, then the result needs to be a i1 boolean that
7996 // is then extended back to the legal result type depending on how booleans
7997 // are represented.
7998 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
7999 ISD::NodeType ExtendCode =
8000 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
8001 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
8002 : ISD::SIGN_EXTEND;
8003
8004 // Find legal integer scalar type for constant promotion and
8005 // ensure that its scalar size is at least as large as source.
8006 EVT LegalSVT = VT.getScalarType();
8007 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
8008 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
8009 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
8010 return SDValue();
8011 }
8012
8013 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
8014 // only have one operand to check. For fixed-length vector types we may have
8015 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
8016 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
8017
8018 // Constant fold each scalar lane separately.
8019 SmallVector<SDValue, 4> ScalarResults;
8020 for (unsigned I = 0; I != NumVectorElts; I++) {
8021 SmallVector<SDValue, 4> ScalarOps;
8022 for (SDValue Op : Ops) {
8023 EVT InSVT = Op.getValueType().getScalarType();
8024 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
8025 Op.getOpcode() != ISD::SPLAT_VECTOR) {
8026 if (Op.isUndef())
8027 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
8028 else
8029 ScalarOps.push_back(Elt: Op);
8030 continue;
8031 }
8032
8033 SDValue ScalarOp =
8034 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
8035 EVT ScalarVT = ScalarOp.getValueType();
8036
8037 // Build vector (integer) scalar operands may need implicit
8038 // truncation - do this before constant folding.
8039 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
8040 // Don't create illegally-typed nodes unless they're constants or undef
8041 // - if we fail to constant fold we can't guarantee the (dead) nodes
8042 // we're creating will be cleaned up before being visited for
8043 // legalization.
8044 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
8045 !isa<ConstantSDNode>(Val: ScalarOp) &&
8046 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
8047 TargetLowering::TypeLegal)
8048 return SDValue();
8049 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
8050 }
8051
8052 ScalarOps.push_back(Elt: ScalarOp);
8053 }
8054
8055 // Constant fold the scalar operands.
8056 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
8057
8058 // Scalar folding only succeeded if the result is a constant or UNDEF.
8059 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
8060 ScalarResult.getOpcode() != ISD::ConstantFP)
8061 return SDValue();
8062
8063 // Legalize the (integer) scalar constant if necessary. We only do
8064 // this once we know the folding succeeded, since otherwise we would
8065 // get a node with illegal type which has a user.
8066 if (LegalSVT != SVT)
8067 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
8068
8069 ScalarResults.push_back(Elt: ScalarResult);
8070 }
8071
8072 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
8073 : getBuildVector(VT, DL, Ops: ScalarResults);
8074 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
8075 return V;
8076}
8077
8078SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
8079 EVT VT, ArrayRef<SDValue> Ops) {
8080 // TODO: Add support for unary/ternary fp opcodes.
8081 if (Ops.size() != 2)
8082 return SDValue();
8083
8084 // TODO: We don't do any constant folding for strict FP opcodes here, but we
8085 // should. That will require dealing with a potentially non-default
8086 // rounding mode, checking the "opStatus" return value from the APFloat
8087 // math calculations, and possibly other variations.
8088 SDValue N1 = Ops[0];
8089 SDValue N2 = Ops[1];
8090 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
8091 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
8092 if (N1CFP && N2CFP) {
8093 APFloat C1 = N1CFP->getValueAPF(); // make copy
8094 const APFloat &C2 = N2CFP->getValueAPF();
8095 switch (Opcode) {
8096 case ISD::FADD:
8097 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8098 return getConstantFP(V: C1, DL, VT);
8099 case ISD::FSUB:
8100 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8101 return getConstantFP(V: C1, DL, VT);
8102 case ISD::FMUL:
8103 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8104 return getConstantFP(V: C1, DL, VT);
8105 case ISD::FDIV:
8106 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
8107 return getConstantFP(V: C1, DL, VT);
8108 case ISD::FREM:
8109 C1.mod(RHS: C2);
8110 return getConstantFP(V: C1, DL, VT);
8111 case ISD::FCOPYSIGN:
8112 C1.copySign(RHS: C2);
8113 return getConstantFP(V: C1, DL, VT);
8114 case ISD::FMINNUM:
8115 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
8116 case ISD::FMAXNUM:
8117 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
8118 case ISD::FMINIMUM:
8119 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
8120 case ISD::FMAXIMUM:
8121 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
8122 case ISD::FMINIMUMNUM:
8123 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
8124 case ISD::FMAXIMUMNUM:
8125 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
8126 default: break;
8127 }
8128 }
8129 if (N1CFP && Opcode == ISD::FP_ROUND) {
8130 APFloat C1 = N1CFP->getValueAPF(); // make copy
8131 bool Unused;
8132 // This can return overflow, underflow, or inexact; we don't care.
8133 // FIXME need to be more flexible about rounding mode.
8134 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
8135 losesInfo: &Unused);
8136 return getConstantFP(V: C1, DL, VT);
8137 }
8138
8139 switch (Opcode) {
8140 case ISD::FSUB:
8141 // -0.0 - undef --> undef (consistent with "fneg undef")
8142 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
8143 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
8144 return getUNDEF(VT);
8145 [[fallthrough]];
8146
8147 case ISD::FADD:
8148 case ISD::FMUL:
8149 case ISD::FDIV:
8150 case ISD::FREM:
8151 // If both operands are undef, the result is undef. If 1 operand is undef,
8152 // the result is NaN. This should match the behavior of the IR optimizer.
8153 if (N1.isUndef() && N2.isUndef())
8154 return getUNDEF(VT);
8155 if (N1.isUndef() || N2.isUndef())
8156 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
8157 }
8158 return SDValue();
8159}
8160
8161SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
8162 const SDLoc &DL, EVT DstEltVT) {
8163 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
8164
8165 // If this is already the right type, we're done.
8166 if (SrcEltVT == DstEltVT)
8167 return SDValue(BV, 0);
8168
8169 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
8170 unsigned DstBitSize = DstEltVT.getSizeInBits();
8171
8172 // If this is a conversion of N elements of one type to N elements of another
8173 // type, convert each element. This handles FP<->INT cases.
8174 if (SrcBitSize == DstBitSize) {
8175 SmallVector<SDValue, 8> Ops;
8176 for (SDValue Op : BV->op_values()) {
8177 // If the vector element type is not legal, the BUILD_VECTOR operands
8178 // are promoted and implicitly truncated. Make that explicit here.
8179 if (Op.getValueType() != SrcEltVT)
8180 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
8181 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
8182 }
8183 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
8184 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
8185 return getBuildVector(VT, DL, Ops);
8186 }
8187
8188 // Otherwise, we're growing or shrinking the elements. To avoid having to
8189 // handle annoying details of growing/shrinking FP values, we convert them to
8190 // int first.
8191 if (SrcEltVT.isFloatingPoint()) {
8192 // Convert the input float vector to a int vector where the elements are the
8193 // same sizes.
8194 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
8195 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8196 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8197 DstEltVT);
8198 return SDValue();
8199 }
8200
8201 // Now we know the input is an integer vector. If the output is a FP type,
8202 // convert to integer first, then to FP of the right size.
8203 if (DstEltVT.isFloatingPoint()) {
8204 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
8205 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
8206 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
8207 DstEltVT);
8208 return SDValue();
8209 }
8210
8211 // Okay, we know the src/dst types are both integers of differing types.
8212 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
8213
8214 // Extract the constant raw bit data.
8215 BitVector UndefElements;
8216 SmallVector<APInt> RawBits;
8217 bool IsLE = getDataLayout().isLittleEndian();
8218 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
8219 return SDValue();
8220
8221 SmallVector<SDValue, 8> Ops;
8222 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
8223 if (UndefElements[I])
8224 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
8225 else
8226 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
8227 }
8228
8229 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
8230 return getBuildVector(VT, DL, Ops);
8231}
8232
8233SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
8234 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
8235
8236 // There's no need to assert on a byte-aligned pointer. All pointers are at
8237 // least byte aligned.
8238 if (A == Align(1))
8239 return Val;
8240
8241 SDVTList VTs = getVTList(VT: Val.getValueType());
8242 FoldingSetNodeID ID;
8243 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: VTs, OpList: {Val});
8244 ID.AddInteger(I: A.value());
8245
8246 void *IP = nullptr;
8247 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
8248 return SDValue(E, 0);
8249
8250 auto *N =
8251 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
8252 createOperands(Node: N, Vals: {Val});
8253
8254 CSEMap.InsertNode(N, InsertPos: IP);
8255 InsertNode(N);
8256
8257 SDValue V(N, 0);
8258 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8259 return V;
8260}
8261
8262SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8263 SDValue N1, SDValue N2) {
8264 SDNodeFlags Flags;
8265 if (Inserter)
8266 Flags = Inserter->getFlags();
8267 return getNode(Opcode, DL, VT, N1, N2, Flags);
8268}
8269
8270void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
8271 SDValue &N2) const {
8272 if (!TLI->isCommutativeBinOp(Opcode))
8273 return;
8274
8275 // Canonicalize:
8276 // binop(const, nonconst) -> binop(nonconst, const)
8277 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
8278 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
8279 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
8280 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
8281 if ((N1C && !N2C) || (N1CFP && !N2CFP))
8282 std::swap(a&: N1, b&: N2);
8283
8284 // Canonicalize:
8285 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
8286 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
8287 N2.getOpcode() == ISD::STEP_VECTOR)
8288 std::swap(a&: N1, b&: N2);
8289}
8290
8291SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8292 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
8293 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8294 N2.getOpcode() != ISD::DELETED_NODE &&
8295 "Operand is DELETED_NODE!");
8296
8297 canonicalizeCommutativeBinop(Opcode, N1, N2);
8298
8299 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
8300 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
8301
8302 // Don't allow undefs in vector splats - we might be returning N2 when folding
8303 // to zero etc.
8304 ConstantSDNode *N2CV =
8305 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
8306
8307 switch (Opcode) {
8308 default: break;
8309 case ISD::TokenFactor:
8310 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
8311 N2.getValueType() == MVT::Other && "Invalid token factor!");
8312 // Fold trivial token factors.
8313 if (N1.getOpcode() == ISD::EntryToken) return N2;
8314 if (N2.getOpcode() == ISD::EntryToken) return N1;
8315 if (N1 == N2) return N1;
8316 break;
8317 case ISD::BUILD_VECTOR: {
8318 // Attempt to simplify BUILD_VECTOR.
8319 SDValue Ops[] = {N1, N2};
8320 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8321 return V;
8322 break;
8323 }
8324 case ISD::CONCAT_VECTORS: {
8325 SDValue Ops[] = {N1, N2};
8326 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8327 return V;
8328 break;
8329 }
8330 case ISD::AND:
8331 assert(VT.isInteger() && "This operator does not apply to FP types!");
8332 assert(N1.getValueType() == N2.getValueType() &&
8333 N1.getValueType() == VT && "Binary operator types must match!");
8334 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
8335 // worth handling here.
8336 if (N2CV && N2CV->isZero())
8337 return N2;
8338 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
8339 return N1;
8340 break;
8341 case ISD::OR:
8342 case ISD::XOR:
8343 case ISD::ADD:
8344 case ISD::PTRADD:
8345 case ISD::SUB:
8346 assert(VT.isInteger() && "This operator does not apply to FP types!");
8347 assert(N1.getValueType() == N2.getValueType() &&
8348 N1.getValueType() == VT && "Binary operator types must match!");
8349 // The equal operand types requirement is unnecessarily strong for PTRADD.
8350 // However, the SelectionDAGBuilder does not generate PTRADDs with different
8351 // operand types, and we'd need to re-implement GEP's non-standard wrapping
8352 // logic everywhere where PTRADDs may be folded or combined to properly
8353 // support them. If/when we introduce pointer types to the SDAG, we will
8354 // need to relax this constraint.
8355
8356 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
8357 // it's worth handling here.
8358 if (N2CV && N2CV->isZero())
8359 return N1;
8360 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
8361 VT.getScalarType() == MVT::i1)
8362 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8363 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
8364 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
8365 N2.getOpcode() == ISD::VSCALE) {
8366 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
8367 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
8368 return getVScale(DL, VT, MulImm: C1 + C2);
8369 }
8370 break;
8371 case ISD::MUL:
8372 assert(VT.isInteger() && "This operator does not apply to FP types!");
8373 assert(N1.getValueType() == N2.getValueType() &&
8374 N1.getValueType() == VT && "Binary operator types must match!");
8375 if (VT.getScalarType() == MVT::i1)
8376 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8377 if (N2CV && N2CV->isZero())
8378 return N2;
8379 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8380 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8381 const APInt &N2CImm = N2C->getAPIntValue();
8382 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
8383 }
8384 break;
8385 case ISD::UDIV:
8386 case ISD::UREM:
8387 case ISD::MULHU:
8388 case ISD::MULHS:
8389 case ISD::SDIV:
8390 case ISD::SREM:
8391 case ISD::SADDSAT:
8392 case ISD::SSUBSAT:
8393 case ISD::UADDSAT:
8394 case ISD::USUBSAT:
8395 assert(VT.isInteger() && "This operator does not apply to FP types!");
8396 assert(N1.getValueType() == N2.getValueType() &&
8397 N1.getValueType() == VT && "Binary operator types must match!");
8398 if (VT.getScalarType() == MVT::i1) {
8399 // fold (add_sat x, y) -> (or x, y) for bool types.
8400 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8401 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8402 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8403 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8404 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8405 }
8406 break;
8407 case ISD::SCMP:
8408 case ISD::UCMP:
8409 assert(N1.getValueType() == N2.getValueType() &&
8410 "Types of operands of UCMP/SCMP must match");
8411 assert(N1.getValueType().isVector() == VT.isVector() &&
8412 "Operands and return type of must both be scalars or vectors");
8413 if (VT.isVector())
8414 assert(VT.getVectorElementCount() ==
8415 N1.getValueType().getVectorElementCount() &&
8416 "Result and operands must have the same number of elements");
8417 break;
8418 case ISD::AVGFLOORS:
8419 case ISD::AVGFLOORU:
8420 case ISD::AVGCEILS:
8421 case ISD::AVGCEILU:
8422 assert(VT.isInteger() && "This operator does not apply to FP types!");
8423 assert(N1.getValueType() == N2.getValueType() &&
8424 N1.getValueType() == VT && "Binary operator types must match!");
8425 break;
8426 case ISD::ABDS:
8427 case ISD::ABDU:
8428 assert(VT.isInteger() && "This operator does not apply to FP types!");
8429 assert(N1.getValueType() == N2.getValueType() &&
8430 N1.getValueType() == VT && "Binary operator types must match!");
8431 if (VT.getScalarType() == MVT::i1)
8432 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8433 break;
8434 case ISD::SMIN:
8435 case ISD::UMAX:
8436 assert(VT.isInteger() && "This operator does not apply to FP types!");
8437 assert(N1.getValueType() == N2.getValueType() &&
8438 N1.getValueType() == VT && "Binary operator types must match!");
8439 if (VT.getScalarType() == MVT::i1)
8440 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8441 break;
8442 case ISD::SMAX:
8443 case ISD::UMIN:
8444 assert(VT.isInteger() && "This operator does not apply to FP types!");
8445 assert(N1.getValueType() == N2.getValueType() &&
8446 N1.getValueType() == VT && "Binary operator types must match!");
8447 if (VT.getScalarType() == MVT::i1)
8448 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8449 break;
8450 case ISD::FADD:
8451 case ISD::FSUB:
8452 case ISD::FMUL:
8453 case ISD::FDIV:
8454 case ISD::FREM:
8455 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8456 assert(N1.getValueType() == N2.getValueType() &&
8457 N1.getValueType() == VT && "Binary operator types must match!");
8458 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8459 return V;
8460 break;
8461 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8462 assert(N1.getValueType() == VT &&
8463 N1.getValueType().isFloatingPoint() &&
8464 N2.getValueType().isFloatingPoint() &&
8465 "Invalid FCOPYSIGN!");
8466 break;
8467 case ISD::SHL:
8468 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8469 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8470 const APInt &ShiftImm = N2C->getAPIntValue();
8471 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8472 }
8473 [[fallthrough]];
8474 case ISD::SRA:
8475 case ISD::SRL:
8476 if (SDValue V = simplifyShift(X: N1, Y: N2))
8477 return V;
8478 [[fallthrough]];
8479 case ISD::ROTL:
8480 case ISD::ROTR:
8481 case ISD::SSHLSAT:
8482 case ISD::USHLSAT:
8483 assert(VT == N1.getValueType() &&
8484 "Shift operators return type must be the same as their first arg");
8485 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8486 "Shifts only work on integers");
8487 assert((!VT.isVector() || VT == N2.getValueType()) &&
8488 "Vector shift amounts must be in the same as their first arg");
8489 // Verify that the shift amount VT is big enough to hold valid shift
8490 // amounts. This catches things like trying to shift an i1024 value by an
8491 // i8, which is easy to fall into in generic code that uses
8492 // TLI.getShiftAmount().
8493 assert(N2.getValueType().getScalarSizeInBits() >=
8494 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8495 "Invalid use of small shift amount with oversized value!");
8496
8497 // Always fold shifts of i1 values so the code generator doesn't need to
8498 // handle them. Since we know the size of the shift has to be less than the
8499 // size of the value, the shift/rotate count is guaranteed to be zero.
8500 if (VT == MVT::i1)
8501 return N1;
8502 if (N2CV && N2CV->isZero())
8503 return N1;
8504 break;
8505 case ISD::FP_ROUND:
8506 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8507 VT.bitsLE(N1.getValueType()) && N2C &&
8508 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8509 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8510 if (N1.getValueType() == VT) return N1; // noop conversion.
8511 break;
8512 case ISD::IS_FPCLASS: {
8513 assert(N1.getValueType().isFloatingPoint() &&
8514 "IS_FPCLASS is used for a non-floating type");
8515 assert(isa<ConstantSDNode>(N2) && "FPClassTest is not Constant");
8516 // is.fpclass(poison, mask) -> poison
8517 if (N1.getOpcode() == ISD::POISON)
8518 return getPOISON(VT);
8519 FPClassTest Mask = static_cast<FPClassTest>(N2->getAsZExtVal());
8520 // If all tests are made, it doesn't matter what the value is.
8521 if ((Mask & fcAllFlags) == fcAllFlags)
8522 return getBoolConstant(V: true, DL, VT, OpVT: N1.getValueType());
8523 if ((Mask & fcAllFlags) == 0)
8524 return getBoolConstant(V: false, DL, VT, OpVT: N1.getValueType());
8525 break;
8526 }
8527 case ISD::AssertNoFPClass: {
8528 assert(N1.getValueType().isFloatingPoint() &&
8529 "AssertNoFPClass is used for a non-floating type");
8530 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8531 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8532 assert(llvm::to_underlying(NoFPClass) <=
8533 BitmaskEnumDetail::Mask<FPClassTest>() &&
8534 "FPClassTest value too large");
8535 (void)NoFPClass;
8536 break;
8537 }
8538 case ISD::AssertSext:
8539 case ISD::AssertZext: {
8540 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8541 assert(VT == N1.getValueType() && "Not an inreg extend!");
8542 assert(VT.isInteger() && EVT.isInteger() &&
8543 "Cannot *_EXTEND_INREG FP types");
8544 assert(!EVT.isVector() &&
8545 "AssertSExt/AssertZExt type should be the vector element type "
8546 "rather than the vector type!");
8547 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8548 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8549 break;
8550 }
8551 case ISD::SIGN_EXTEND_INREG: {
8552 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8553 assert(VT == N1.getValueType() && "Not an inreg extend!");
8554 assert(VT.isInteger() && EVT.isInteger() &&
8555 "Cannot *_EXTEND_INREG FP types");
8556 assert(EVT.isVector() == VT.isVector() &&
8557 "SIGN_EXTEND_INREG type should be vector iff the operand "
8558 "type is vector!");
8559 assert((!EVT.isVector() ||
8560 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8561 "Vector element counts must match in SIGN_EXTEND_INREG");
8562 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8563 if (EVT == VT) return N1; // Not actually extending
8564 break;
8565 }
8566 case ISD::FP_TO_SINT_SAT:
8567 case ISD::FP_TO_UINT_SAT: {
8568 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8569 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8570 assert(N1.getValueType().isVector() == VT.isVector() &&
8571 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8572 "vector!");
8573 assert((!VT.isVector() || VT.getVectorElementCount() ==
8574 N1.getValueType().getVectorElementCount()) &&
8575 "Vector element counts must match in FP_TO_*INT_SAT");
8576 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8577 "Type to saturate to must be a scalar.");
8578 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8579 "Not extending!");
8580 break;
8581 }
8582 case ISD::EXTRACT_VECTOR_ELT:
8583 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8584 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8585 element type of the vector.");
8586
8587 // Extract from an undefined value or using an undefined index is undefined.
8588 if (N1.isUndef() || N2.isUndef())
8589 return getUNDEF(VT);
8590
8591 // EXTRACT_VECTOR_ELT of out-of-bounds element is POISON for fixed length
8592 // vectors. For scalable vectors we will provide appropriate support for
8593 // dealing with arbitrary indices.
8594 if (N2C && N1.getValueType().isFixedLengthVector() &&
8595 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8596 return getPOISON(VT);
8597
8598 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8599 // expanding copies of large vectors from registers. This only works for
8600 // fixed length vectors, since we need to know the exact number of
8601 // elements.
8602 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8603 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8604 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8605 return getExtractVectorElt(DL, VT,
8606 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8607 Idx: N2C->getZExtValue() % Factor);
8608 }
8609
8610 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8611 // lowering is expanding large vector constants.
8612 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8613 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8614 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8615 N1.getValueType().isFixedLengthVector()) &&
8616 "BUILD_VECTOR used for scalable vectors");
8617 unsigned Index =
8618 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8619 SDValue Elt = N1.getOperand(i: Index);
8620
8621 if (VT != Elt.getValueType())
8622 // If the vector element type is not legal, the BUILD_VECTOR operands
8623 // are promoted and implicitly truncated, and the result implicitly
8624 // extended. Make that explicit here.
8625 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8626
8627 return Elt;
8628 }
8629
8630 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8631 // operations are lowered to scalars.
8632 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8633 // If the indices are the same, return the inserted element else
8634 // if the indices are known different, extract the element from
8635 // the original vector.
8636 SDValue N1Op2 = N1.getOperand(i: 2);
8637 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8638
8639 if (N1Op2C && N2C) {
8640 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8641 if (VT == N1.getOperand(i: 1).getValueType())
8642 return N1.getOperand(i: 1);
8643 if (VT.isFloatingPoint()) {
8644 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8645 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8646 }
8647 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8648 }
8649 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8650 }
8651 }
8652
8653 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8654 // when vector types are scalarized and v1iX is legal.
8655 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8656 // Here we are completely ignoring the extract element index (N2),
8657 // which is fine for fixed width vectors, since any index other than 0
8658 // is undefined anyway. However, this cannot be ignored for scalable
8659 // vectors - in theory we could support this, but we don't want to do this
8660 // without a profitability check.
8661 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8662 N1.getValueType().isFixedLengthVector() &&
8663 N1.getValueType().getVectorNumElements() == 1) {
8664 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8665 N2: N1.getOperand(i: 1));
8666 }
8667 break;
8668 case ISD::EXTRACT_ELEMENT:
8669 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8670 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8671 (N1.getValueType().isInteger() == VT.isInteger()) &&
8672 N1.getValueType() != VT &&
8673 "Wrong types for EXTRACT_ELEMENT!");
8674
8675 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8676 // 64-bit integers into 32-bit parts. Instead of building the extract of
8677 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8678 if (N1.getOpcode() == ISD::BUILD_PAIR)
8679 return N1.getOperand(i: N2C->getZExtValue());
8680
8681 // EXTRACT_ELEMENT of a constant int is also very common.
8682 if (N1C) {
8683 unsigned ElementSize = VT.getSizeInBits();
8684 unsigned Shift = ElementSize * N2C->getZExtValue();
8685 const APInt &Val = N1C->getAPIntValue();
8686 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8687 }
8688 break;
8689 case ISD::EXTRACT_SUBVECTOR: {
8690 EVT N1VT = N1.getValueType();
8691 assert(VT.isVector() && N1VT.isVector() &&
8692 "Extract subvector VTs must be vectors!");
8693 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8694 "Extract subvector VTs must have the same element type!");
8695 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8696 "Cannot extract a scalable vector from a fixed length vector!");
8697 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8698 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8699 "Extract subvector must be from larger vector to smaller vector!");
8700 assert(N2C && "Extract subvector index must be a constant");
8701 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8702 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8703 N1VT.getVectorMinNumElements()) &&
8704 "Extract subvector overflow!");
8705 assert(N2C->getAPIntValue().getBitWidth() ==
8706 TLI->getVectorIdxWidth(getDataLayout()) &&
8707 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8708 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8709 "Extract index is not a multiple of the output vector length");
8710
8711 // Trivial extraction.
8712 if (VT == N1VT)
8713 return N1;
8714
8715 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8716 if (N1.isUndef())
8717 return getUNDEF(VT);
8718
8719 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8720 // the concat have the same type as the extract.
8721 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8722 VT == N1.getOperand(i: 0).getValueType()) {
8723 unsigned Factor = VT.getVectorMinNumElements();
8724 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8725 }
8726
8727 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8728 // during shuffle legalization.
8729 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8730 VT == N1.getOperand(i: 1).getValueType())
8731 return N1.getOperand(i: 1);
8732 break;
8733 }
8734 }
8735
8736 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8737 switch (Opcode) {
8738 case ISD::XOR:
8739 case ISD::ADD:
8740 case ISD::PTRADD:
8741 case ISD::SUB:
8742 case ISD::SIGN_EXTEND_INREG:
8743 case ISD::UDIV:
8744 case ISD::SDIV:
8745 case ISD::UREM:
8746 case ISD::SREM:
8747 case ISD::MUL:
8748 case ISD::AND:
8749 case ISD::SSUBSAT:
8750 case ISD::USUBSAT:
8751 case ISD::UMIN:
8752 case ISD::OR:
8753 case ISD::SADDSAT:
8754 case ISD::UADDSAT:
8755 case ISD::UMAX:
8756 case ISD::SMAX:
8757 case ISD::SMIN:
8758 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8759 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8760 }
8761 }
8762
8763 // Canonicalize an UNDEF to the RHS, even over a constant.
8764 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8765 if (TLI->isCommutativeBinOp(Opcode)) {
8766 std::swap(a&: N1, b&: N2);
8767 } else {
8768 switch (Opcode) {
8769 case ISD::PTRADD:
8770 case ISD::SUB:
8771 // fold op(undef, non_undef_arg2) -> undef.
8772 return N1;
8773 case ISD::SIGN_EXTEND_INREG:
8774 case ISD::UDIV:
8775 case ISD::SDIV:
8776 case ISD::UREM:
8777 case ISD::SREM:
8778 case ISD::SSUBSAT:
8779 case ISD::USUBSAT:
8780 // fold op(undef, non_undef_arg2) -> 0.
8781 return getConstant(Val: 0, DL, VT);
8782 }
8783 }
8784 }
8785
8786 // Fold a bunch of operators when the RHS is undef.
8787 if (N2.getOpcode() == ISD::UNDEF) {
8788 switch (Opcode) {
8789 case ISD::XOR:
8790 if (N1.getOpcode() == ISD::UNDEF)
8791 // Handle undef ^ undef -> 0 special case. This is a common
8792 // idiom (misuse).
8793 return getConstant(Val: 0, DL, VT);
8794 [[fallthrough]];
8795 case ISD::ADD:
8796 case ISD::PTRADD:
8797 case ISD::SUB:
8798 // fold op(arg1, undef) -> undef.
8799 return N2;
8800 case ISD::UDIV:
8801 case ISD::SDIV:
8802 case ISD::UREM:
8803 case ISD::SREM:
8804 // fold op(arg1, undef) -> poison.
8805 return getPOISON(VT);
8806 case ISD::MUL:
8807 case ISD::AND:
8808 case ISD::SSUBSAT:
8809 case ISD::USUBSAT:
8810 case ISD::UMIN:
8811 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
8812 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
8813 case ISD::OR:
8814 case ISD::SADDSAT:
8815 case ISD::UADDSAT:
8816 case ISD::UMAX:
8817 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
8818 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
8819 case ISD::SMAX:
8820 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
8821 return N1.getOpcode() == ISD::UNDEF
8822 ? N2
8823 : getConstant(
8824 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
8825 VT);
8826 case ISD::SMIN:
8827 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
8828 return N1.getOpcode() == ISD::UNDEF
8829 ? N2
8830 : getConstant(
8831 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
8832 VT);
8833 }
8834 }
8835
8836 // Perform trivial constant folding.
8837 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
8838 return SV;
8839
8840 // Memoize this node if possible.
8841 SDNode *N;
8842 SDVTList VTs = getVTList(VT);
8843 SDValue Ops[] = {N1, N2};
8844 if (VT != MVT::Glue) {
8845 FoldingSetNodeID ID;
8846 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8847 void *IP = nullptr;
8848 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8849 E->intersectFlagsWith(Flags);
8850 return SDValue(E, 0);
8851 }
8852
8853 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8854 N->setFlags(Flags);
8855 createOperands(Node: N, Vals: Ops);
8856 CSEMap.InsertNode(N, InsertPos: IP);
8857 } else {
8858 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8859 createOperands(Node: N, Vals: Ops);
8860 }
8861
8862 InsertNode(N);
8863 SDValue V = SDValue(N, 0);
8864 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8865 return V;
8866}
8867
8868SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8869 SDValue N1, SDValue N2, SDValue N3) {
8870 SDNodeFlags Flags;
8871 if (Inserter)
8872 Flags = Inserter->getFlags();
8873 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
8874}
8875
8876SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8877 SDValue N1, SDValue N2, SDValue N3,
8878 const SDNodeFlags Flags) {
8879 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8880 N2.getOpcode() != ISD::DELETED_NODE &&
8881 N3.getOpcode() != ISD::DELETED_NODE &&
8882 "Operand is DELETED_NODE!");
8883 // Perform various simplifications.
8884 switch (Opcode) {
8885 case ISD::BUILD_VECTOR: {
8886 // Attempt to simplify BUILD_VECTOR.
8887 SDValue Ops[] = {N1, N2, N3};
8888 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8889 return V;
8890 break;
8891 }
8892 case ISD::CONCAT_VECTORS: {
8893 SDValue Ops[] = {N1, N2, N3};
8894 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8895 return V;
8896 break;
8897 }
8898 case ISD::SETCC: {
8899 assert(VT.isInteger() && "SETCC result type must be an integer!");
8900 assert(N1.getValueType() == N2.getValueType() &&
8901 "SETCC operands must have the same type!");
8902 assert(VT.isVector() == N1.getValueType().isVector() &&
8903 "SETCC type should be vector iff the operand type is vector!");
8904 assert((!VT.isVector() || VT.getVectorElementCount() ==
8905 N1.getValueType().getVectorElementCount()) &&
8906 "SETCC vector element counts must match!");
8907 // Use FoldSetCC to simplify SETCC's.
8908 if (SDValue V =
8909 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
8910 return V;
8911 break;
8912 }
8913 case ISD::SELECT:
8914 case ISD::VSELECT:
8915 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
8916 return V;
8917 break;
8918 case ISD::VECTOR_SHUFFLE:
8919 llvm_unreachable("should use getVectorShuffle constructor!");
8920 case ISD::VECTOR_SPLICE_LEFT:
8921 if (isNullConstant(V: N3))
8922 return N1;
8923 break;
8924 case ISD::VECTOR_SPLICE_RIGHT:
8925 if (isNullConstant(V: N3))
8926 return N2;
8927 break;
8928 case ISD::INSERT_VECTOR_ELT: {
8929 assert(VT.isVector() && VT == N1.getValueType() &&
8930 "INSERT_VECTOR_ELT vector type mismatch");
8931 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
8932 "INSERT_VECTOR_ELT scalar fp/int mismatch");
8933 assert((!VT.isFloatingPoint() ||
8934 VT.getVectorElementType() == N2.getValueType()) &&
8935 "INSERT_VECTOR_ELT fp scalar type mismatch");
8936 assert((!VT.isInteger() ||
8937 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
8938 "INSERT_VECTOR_ELT int scalar size mismatch");
8939
8940 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
8941 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
8942 // for scalable vectors where we will generate appropriate code to
8943 // deal with out-of-bounds cases correctly.
8944 if (N3C && VT.isFixedLengthVector() &&
8945 N3C->getZExtValue() >= VT.getVectorNumElements())
8946 return getUNDEF(VT);
8947
8948 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
8949 if (N3.isUndef())
8950 return getUNDEF(VT);
8951
8952 // If inserting poison, just use the input vector.
8953 if (N2.getOpcode() == ISD::POISON)
8954 return N1;
8955
8956 // Inserting undef into undef/poison is still undef.
8957 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8958 return getUNDEF(VT);
8959
8960 // If the inserted element is an UNDEF, just use the input vector.
8961 // But not if skipping the insert could make the result more poisonous.
8962 if (N2.isUndef()) {
8963 if (N3C && VT.isFixedLengthVector()) {
8964 APInt EltMask =
8965 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
8966 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8967 return N1;
8968 } else if (isGuaranteedNotToBePoison(Op: N1))
8969 return N1;
8970 }
8971 break;
8972 }
8973 case ISD::INSERT_SUBVECTOR: {
8974 // If inserting poison, just use the input vector,
8975 if (N2.getOpcode() == ISD::POISON)
8976 return N1;
8977
8978 // Inserting undef into undef/poison is still undef.
8979 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8980 return getUNDEF(VT);
8981
8982 EVT N2VT = N2.getValueType();
8983 assert(VT == N1.getValueType() &&
8984 "Dest and insert subvector source types must match!");
8985 assert(VT.isVector() && N2VT.isVector() &&
8986 "Insert subvector VTs must be vectors!");
8987 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
8988 "Insert subvector VTs must have the same element type!");
8989 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
8990 "Cannot insert a scalable vector into a fixed length vector!");
8991 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8992 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
8993 "Insert subvector must be from smaller vector to larger vector!");
8994 assert(isa<ConstantSDNode>(N3) &&
8995 "Insert subvector index must be constant");
8996 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8997 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
8998 VT.getVectorMinNumElements()) &&
8999 "Insert subvector overflow!");
9000 assert(N3->getAsAPIntVal().getBitWidth() ==
9001 TLI->getVectorIdxWidth(getDataLayout()) &&
9002 "Constant index for INSERT_SUBVECTOR has an invalid size");
9003
9004 // Trivial insertion.
9005 if (VT == N2VT)
9006 return N2;
9007
9008 // If this is an insert of an extracted vector into an undef/poison vector,
9009 // we can just use the input to the extract. But not if skipping the
9010 // extract+insert could make the result more poisonous.
9011 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
9012 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
9013 if (N1.getOpcode() == ISD::POISON)
9014 return N2.getOperand(i: 0);
9015 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
9016 unsigned LoBit = N3->getAsZExtVal();
9017 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9018 APInt EltMask =
9019 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9020 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
9021 return N2.getOperand(i: 0);
9022 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
9023 return N2.getOperand(i: 0);
9024 }
9025
9026 // If the inserted subvector is UNDEF, just use the input vector.
9027 // But not if skipping the insert could make the result more poisonous.
9028 if (N2.isUndef()) {
9029 if (VT.isFixedLengthVector()) {
9030 unsigned LoBit = N3->getAsZExtVal();
9031 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
9032 APInt EltMask =
9033 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
9034 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
9035 return N1;
9036 } else if (isGuaranteedNotToBePoison(Op: N1))
9037 return N1;
9038 }
9039 break;
9040 }
9041 case ISD::BITCAST:
9042 // Fold bit_convert nodes from a type to themselves.
9043 if (N1.getValueType() == VT)
9044 return N1;
9045 break;
9046 case ISD::VP_TRUNCATE:
9047 case ISD::VP_SIGN_EXTEND:
9048 case ISD::VP_ZERO_EXTEND:
9049 // Don't create noop casts.
9050 if (N1.getValueType() == VT)
9051 return N1;
9052 break;
9053 case ISD::VECTOR_COMPRESS: {
9054 [[maybe_unused]] EVT VecVT = N1.getValueType();
9055 [[maybe_unused]] EVT MaskVT = N2.getValueType();
9056 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
9057 assert(VT == VecVT && "Vector and result type don't match.");
9058 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
9059 "All inputs must be vectors.");
9060 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
9061 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
9062 "Vector and mask must have same number of elements.");
9063
9064 if (N1.isUndef() || N2.isUndef())
9065 return N3;
9066
9067 break;
9068 }
9069 case ISD::PARTIAL_REDUCE_UMLA:
9070 case ISD::PARTIAL_REDUCE_SMLA:
9071 case ISD::PARTIAL_REDUCE_SUMLA:
9072 case ISD::PARTIAL_REDUCE_FMLA: {
9073 [[maybe_unused]] EVT AccVT = N1.getValueType();
9074 [[maybe_unused]] EVT Input1VT = N2.getValueType();
9075 [[maybe_unused]] EVT Input2VT = N3.getValueType();
9076 assert(Input1VT.isVector() && Input1VT == Input2VT &&
9077 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9078 "node to have the same type!");
9079 assert(VT.isVector() && VT == AccVT &&
9080 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
9081 "the same type as its result!");
9082 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
9083 AccVT.getVectorElementCount()) &&
9084 "Expected the element count of the second and third operands of the "
9085 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
9086 "element count of the first operand and the result!");
9087 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
9088 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
9089 "node to have an element type which is the same as or smaller than "
9090 "the element type of the first operand and result!");
9091 break;
9092 }
9093 }
9094
9095 // Perform trivial constant folding for arithmetic operators.
9096 switch (Opcode) {
9097 case ISD::FMA:
9098 case ISD::FMAD:
9099 case ISD::SETCC:
9100 case ISD::FSHL:
9101 case ISD::FSHR:
9102 if (SDValue SV =
9103 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
9104 return SV;
9105 break;
9106 }
9107
9108 // Memoize node if it doesn't produce a glue result.
9109 SDNode *N;
9110 SDVTList VTs = getVTList(VT);
9111 SDValue Ops[] = {N1, N2, N3};
9112 if (VT != MVT::Glue) {
9113 FoldingSetNodeID ID;
9114 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9115 void *IP = nullptr;
9116 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9117 E->intersectFlagsWith(Flags);
9118 return SDValue(E, 0);
9119 }
9120
9121 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9122 N->setFlags(Flags);
9123 createOperands(Node: N, Vals: Ops);
9124 CSEMap.InsertNode(N, InsertPos: IP);
9125 } else {
9126 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
9127 createOperands(Node: N, Vals: Ops);
9128 }
9129
9130 InsertNode(N);
9131 SDValue V = SDValue(N, 0);
9132 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9133 return V;
9134}
9135
9136SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9137 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9138 const SDNodeFlags Flags) {
9139 SDValue Ops[] = { N1, N2, N3, N4 };
9140 return getNode(Opcode, DL, VT, Ops, Flags);
9141}
9142
9143SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9144 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9145 SDNodeFlags Flags;
9146 if (Inserter)
9147 Flags = Inserter->getFlags();
9148 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
9149}
9150
9151SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9152 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9153 SDValue N5, const SDNodeFlags Flags) {
9154 SDValue Ops[] = { N1, N2, N3, N4, N5 };
9155 return getNode(Opcode, DL, VT, Ops, Flags);
9156}
9157
9158SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9159 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
9160 SDValue N5) {
9161 SDNodeFlags Flags;
9162 if (Inserter)
9163 Flags = Inserter->getFlags();
9164 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
9165}
9166
9167/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
9168/// the incoming stack arguments to be loaded from the stack.
9169SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
9170 SmallVector<SDValue, 8> ArgChains;
9171
9172 // Include the original chain at the beginning of the list. When this is
9173 // used by target LowerCall hooks, this helps legalize find the
9174 // CALLSEQ_BEGIN node.
9175 ArgChains.push_back(Elt: Chain);
9176
9177 // Add a chain value for each stack argument.
9178 for (SDNode *U : getEntryNode().getNode()->users())
9179 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
9180 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
9181 if (FI->getIndex() < 0)
9182 ArgChains.push_back(Elt: SDValue(L, 1));
9183
9184 // Build a tokenfactor for all the chains.
9185 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
9186}
9187
9188/// getMemsetValue - Vectorized representation of the memset value
9189/// operand.
9190static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
9191 const SDLoc &dl) {
9192 assert(!Value.isUndef());
9193
9194 unsigned NumBits = VT.getScalarSizeInBits();
9195 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
9196 assert(C->getAPIntValue().getBitWidth() == 8);
9197 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
9198 if (VT.isInteger()) {
9199 bool IsOpaque = VT.getSizeInBits() > 64 ||
9200 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
9201 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
9202 }
9203 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
9204 }
9205
9206 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
9207 EVT IntVT = VT.getScalarType();
9208 if (!IntVT.isInteger())
9209 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
9210
9211 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
9212 if (NumBits > 8) {
9213 // Use a multiplication with 0x010101... to extend the input to the
9214 // required length.
9215 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
9216 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
9217 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
9218 }
9219
9220 if (VT != Value.getValueType() && !VT.isInteger())
9221 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
9222 if (VT != Value.getValueType())
9223 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
9224
9225 return Value;
9226}
9227
9228/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
9229/// used when a memcpy is turned into a memset when the source is a constant
9230/// string ptr.
9231static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
9232 const TargetLowering &TLI,
9233 const ConstantDataArraySlice &Slice) {
9234 // Handle vector with all elements zero.
9235 if (Slice.Array == nullptr) {
9236 if (VT.isInteger())
9237 return DAG.getConstant(Val: 0, DL: dl, VT);
9238 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
9239 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
9240 }
9241
9242 assert(!VT.isVector() && "Can't handle vector type here!");
9243 unsigned NumVTBits = VT.getSizeInBits();
9244 unsigned NumVTBytes = NumVTBits / 8;
9245 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
9246
9247 APInt Val(NumVTBits, 0);
9248 if (DAG.getDataLayout().isLittleEndian()) {
9249 for (unsigned i = 0; i != NumBytes; ++i)
9250 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
9251 } else {
9252 for (unsigned i = 0; i != NumBytes; ++i)
9253 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
9254 }
9255
9256 // If the "cost" of materializing the integer immediate is less than the cost
9257 // of a load, then it is cost effective to turn the load into the immediate.
9258 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
9259 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
9260 return DAG.getConstant(Val, DL: dl, VT);
9261 return SDValue();
9262}
9263
9264SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
9265 const SDLoc &DL,
9266 const SDNodeFlags Flags) {
9267 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
9268 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
9269}
9270
9271SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
9272 const SDLoc &DL,
9273 const SDNodeFlags Flags) {
9274 assert(Offset.getValueType().isInteger());
9275 EVT BasePtrVT = Ptr.getValueType();
9276 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
9277 PtrVT: BasePtrVT))
9278 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
9279 // InBounds only applies to PTRADD, don't set it if we generate ADD.
9280 SDNodeFlags AddFlags = Flags;
9281 AddFlags.setInBounds(false);
9282 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
9283}
9284
9285/// Returns true if memcpy source is constant data.
9286static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
9287 uint64_t SrcDelta = 0;
9288 GlobalAddressSDNode *G = nullptr;
9289 if (Src.getOpcode() == ISD::GlobalAddress)
9290 G = cast<GlobalAddressSDNode>(Val&: Src);
9291 else if (Src->isAnyAdd() &&
9292 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
9293 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
9294 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
9295 SrcDelta = Src.getConstantOperandVal(i: 1);
9296 }
9297 if (!G)
9298 return false;
9299
9300 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
9301 Offset: SrcDelta + G->getOffset());
9302}
9303
9304static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
9305 SelectionDAG &DAG) {
9306 // On Darwin, -Os means optimize for size without hurting performance, so
9307 // only really optimize for size when -Oz (MinSize) is used.
9308 if (MF.getTarget().getTargetTriple().isOSDarwin())
9309 return MF.getFunction().hasMinSize();
9310 return DAG.shouldOptForSize();
9311}
9312
9313static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
9314 SmallVector<SDValue, 32> &OutChains, unsigned From,
9315 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
9316 SmallVector<SDValue, 16> &OutStoreChains) {
9317 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
9318 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
9319 SmallVector<SDValue, 16> GluedLoadChains;
9320 for (unsigned i = From; i < To; ++i) {
9321 OutChains.push_back(Elt: OutLoadChains[i]);
9322 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
9323 }
9324
9325 // Chain for all loads.
9326 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
9327 Ops: GluedLoadChains);
9328
9329 for (unsigned i = From; i < To; ++i) {
9330 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
9331 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
9332 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
9333 MMO: ST->getMemOperand());
9334 OutChains.push_back(Elt: NewStore);
9335 }
9336}
9337
9338static SDValue
9339getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
9340 SDValue Dst, SDValue Src, uint64_t Size, Align DstAlign,
9341 Align SrcAlign, bool isVol, bool AlwaysInline,
9342 MachinePointerInfo DstPtrInfo,
9343 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9344 BatchAAResults *BatchAA) {
9345 // Turn a memcpy of undef to nop.
9346 // FIXME: We need to honor volatile even is Src is undef.
9347 if (Src.isUndef())
9348 return Chain;
9349
9350 // Expand memcpy to a series of load and store ops if the size operand falls
9351 // below a certain threshold.
9352 // TODO: In the AlwaysInline case, if the size is big then generate a loop
9353 // rather than maybe a humongous number of loads and stores.
9354 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9355 const DataLayout &DL = DAG.getDataLayout();
9356 LLVMContext &C = *DAG.getContext();
9357 std::vector<EVT> MemOps;
9358 bool DstAlignCanChange = false;
9359 MachineFunction &MF = DAG.getMachineFunction();
9360 MachineFrameInfo &MFI = MF.getFrameInfo();
9361 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9362 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9363 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9364 DstAlignCanChange = true;
9365 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9366 ConstantDataArraySlice Slice;
9367 // If marked as volatile, perform a copy even when marked as constant.
9368 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
9369 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
9370 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
9371 const MemOp Op = isZeroConstant
9372 ? MemOp::Set(Size, DstAlignCanChange, DstAlign,
9373 /*IsZeroMemset*/ true, IsVolatile: isVol)
9374 : MemOp::Copy(Size, DstAlignCanChange, DstAlign,
9375 SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
9376 if (!TLI.findOptimalMemOpLowering(
9377 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
9378 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9379 return SDValue();
9380
9381 if (DstAlignCanChange) {
9382 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9383 Align NewDstAlign = DL.getABITypeAlign(Ty);
9384
9385 // Don't promote to an alignment that would require dynamic stack
9386 // realignment which may conflict with optimizations such as tail call
9387 // optimization.
9388 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9389 if (!TRI->hasStackRealignment(MF))
9390 if (MaybeAlign StackAlign = DL.getStackAlignment())
9391 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9392
9393 if (NewDstAlign > DstAlign) {
9394 // Give the stack frame object a larger alignment if needed.
9395 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9396 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9397 DstAlign = NewDstAlign;
9398 }
9399 }
9400
9401 // Prepare AAInfo for loads/stores after lowering this memcpy.
9402 AAMDNodes NewAAInfo = AAInfo;
9403 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9404
9405 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9406 bool isConstant =
9407 BatchAA && SrcVal &&
9408 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9409
9410 MachineMemOperand::Flags MMOFlags =
9411 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9412 SmallVector<SDValue, 16> OutLoadChains;
9413 SmallVector<SDValue, 16> OutStoreChains;
9414 SmallVector<SDValue, 32> OutChains;
9415 unsigned NumMemOps = MemOps.size();
9416 uint64_t SrcOff = 0, DstOff = 0;
9417 for (unsigned i = 0; i != NumMemOps; ++i) {
9418 EVT VT = MemOps[i];
9419 unsigned VTSize = VT.getSizeInBits() / 8;
9420 SDValue Value, Store;
9421
9422 if (VTSize > Size) {
9423 // Issuing an unaligned load / store pair that overlaps with the previous
9424 // pair. Adjust the offset accordingly.
9425 assert(i == NumMemOps-1 && i != 0);
9426 SrcOff -= VTSize - Size;
9427 DstOff -= VTSize - Size;
9428 }
9429
9430 if (CopyFromConstant &&
9431 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9432 // It's unlikely a store of a vector immediate can be done in a single
9433 // instruction. It would require a load from a constantpool first.
9434 // We only handle zero vectors here.
9435 // FIXME: Handle other cases where store of vector immediate is done in
9436 // a single instruction.
9437 ConstantDataArraySlice SubSlice;
9438 if (SrcOff < Slice.Length) {
9439 SubSlice = Slice;
9440 SubSlice.move(Delta: SrcOff);
9441 } else {
9442 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9443 SubSlice.Array = nullptr;
9444 SubSlice.Offset = 0;
9445 SubSlice.Length = VTSize;
9446 }
9447 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9448 if (Value.getNode()) {
9449 Store = DAG.getStore(
9450 Chain, dl, Val: Value,
9451 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9452 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlign, MMOFlags, AAInfo: NewAAInfo);
9453 OutChains.push_back(Elt: Store);
9454 }
9455 }
9456
9457 if (!Store.getNode()) {
9458 // The type might not be legal for the target. This should only happen
9459 // if the type is smaller than a legal type, as on PPC, so the right
9460 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9461 // to Load/Store if NVT==VT.
9462 // FIXME does the case above also need this?
9463 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9464 assert(NVT.bitsGE(VT));
9465
9466 bool isDereferenceable =
9467 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9468 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9469 if (isDereferenceable)
9470 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9471 if (isConstant)
9472 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9473
9474 Value = DAG.getExtLoad(
9475 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9476 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9477 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9478 Alignment: commonAlignment(A: SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9479 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9480
9481 Store = DAG.getTruncStore(
9482 Chain, dl, Val: Value,
9483 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9484 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment: DstAlign, MMOFlags, AAInfo: NewAAInfo);
9485 OutStoreChains.push_back(Elt: Store);
9486 }
9487 SrcOff += VTSize;
9488 DstOff += VTSize;
9489 Size -= VTSize;
9490 }
9491
9492 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9493 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9494 unsigned NumLdStInMemcpy = OutStoreChains.size();
9495
9496 if (NumLdStInMemcpy) {
9497 // It may be that memcpy might be converted to memset if it's memcpy
9498 // of constants. In such a case, we won't have loads and stores, but
9499 // just stores. In the absence of loads, there is nothing to gang up.
9500 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9501 // If target does not care, just leave as it.
9502 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9503 OutChains.push_back(Elt: OutLoadChains[i]);
9504 OutChains.push_back(Elt: OutStoreChains[i]);
9505 }
9506 } else {
9507 // Ld/St less than/equal limit set by target.
9508 if (NumLdStInMemcpy <= GluedLdStLimit) {
9509 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9510 To: NumLdStInMemcpy, OutLoadChains,
9511 OutStoreChains);
9512 } else {
9513 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9514 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9515 unsigned GlueIter = 0;
9516
9517 // Residual ld/st.
9518 if (RemainingLdStInMemcpy) {
9519 chainLoadsAndStoresForMemcpy(
9520 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9521 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9522 }
9523
9524 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9525 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9526 GlueIter - GluedLdStLimit;
9527 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9528 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9529 OutLoadChains, OutStoreChains);
9530 GlueIter += GluedLdStLimit;
9531 }
9532 }
9533 }
9534 }
9535 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9536}
9537
9538static SDValue getMemmoveLoadsAndStores(
9539 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
9540 uint64_t Size, Align DstAlign, Align SrcAlign, bool isVol,
9541 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9542 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo) {
9543 // Turn a memmove of undef to nop.
9544 // FIXME: We need to honor volatile even is Src is undef.
9545 if (Src.isUndef())
9546 return Chain;
9547
9548 // Expand memmove to a series of load and store ops if the size operand falls
9549 // below a certain threshold.
9550 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9551 const DataLayout &DL = DAG.getDataLayout();
9552 LLVMContext &C = *DAG.getContext();
9553 std::vector<EVT> MemOps;
9554 bool DstAlignCanChange = false;
9555 MachineFunction &MF = DAG.getMachineFunction();
9556 MachineFrameInfo &MFI = MF.getFrameInfo();
9557 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9558 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9559 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9560 DstAlignCanChange = true;
9561 SrcAlign = std::max(a: SrcAlign, b: DAG.InferPtrAlign(Ptr: Src).valueOrOne());
9562 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9563 if (!TLI.findOptimalMemOpLowering(
9564 Context&: C, MemOps, Limit,
9565 Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign, SrcAlign, IsVolatile: isVol),
9566 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9567 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9568 return SDValue();
9569
9570 if (DstAlignCanChange) {
9571 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9572 Align NewDstAlign = DL.getABITypeAlign(Ty);
9573
9574 // Don't promote to an alignment that would require dynamic stack
9575 // realignment which may conflict with optimizations such as tail call
9576 // optimization.
9577 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9578 if (!TRI->hasStackRealignment(MF))
9579 if (MaybeAlign StackAlign = DL.getStackAlignment())
9580 NewDstAlign = std::min(a: NewDstAlign, b: *StackAlign);
9581
9582 if (NewDstAlign > DstAlign) {
9583 // Give the stack frame object a larger alignment if needed.
9584 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewDstAlign)
9585 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewDstAlign);
9586 DstAlign = NewDstAlign;
9587 }
9588 }
9589
9590 // Prepare AAInfo for loads/stores after lowering this memmove.
9591 AAMDNodes NewAAInfo = AAInfo;
9592 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9593
9594 MachineMemOperand::Flags MMOFlags =
9595 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9596 uint64_t SrcOff = 0;
9597 SmallVector<SDValue, 8> LoadValues;
9598 SmallVector<SDValue, 8> LoadChains;
9599 SmallVector<SDValue, 8> OutChains;
9600 unsigned NumMemOps = MemOps.size();
9601 for (unsigned i = 0; i < NumMemOps; i++) {
9602 EVT VT = MemOps[i];
9603 unsigned VTSize = VT.getSizeInBits() / 8;
9604 SDValue Value;
9605 bool IsOverlapping = false;
9606
9607 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - SrcOff) {
9608 // Issuing an unaligned load / store pair that overlaps with the previous
9609 // pair. Adjust the offset accordingly.
9610 SrcOff = Size - VTSize;
9611 IsOverlapping = true;
9612 }
9613
9614 // Calculate the actual alignment at the current offset. The alignment at
9615 // SrcOff may be lower than the base alignment, especially when using
9616 // overlapping loads.
9617 Align SrcAlignAtOffset = commonAlignment(A: SrcAlign, Offset: SrcOff);
9618 if (IsOverlapping) {
9619 // Verify that the target allows misaligned memory accesses at the
9620 // adjusted offset when using overlapping loads.
9621 unsigned Fast;
9622 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcPtrInfo.getAddrSpace(),
9623 Alignment: SrcAlignAtOffset, Flags: MMOFlags,
9624 &Fast) ||
9625 !Fast) {
9626 // This should have been caught by findOptimalMemOpLowering, but verify
9627 // here for safety.
9628 return SDValue();
9629 }
9630 }
9631
9632 bool isDereferenceable =
9633 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9634 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9635 if (isDereferenceable)
9636 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9637 Value =
9638 DAG.getLoad(VT, dl, Chain,
9639 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9640 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: SrcAlignAtOffset,
9641 MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9642 LoadValues.push_back(Elt: Value);
9643 LoadChains.push_back(Elt: Value.getValue(R: 1));
9644 SrcOff += VTSize;
9645 }
9646 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
9647 OutChains.clear();
9648 uint64_t DstOff = 0;
9649 for (unsigned i = 0; i < NumMemOps; i++) {
9650 EVT VT = MemOps[i];
9651 unsigned VTSize = VT.getSizeInBits() / 8;
9652 SDValue Store;
9653 bool IsOverlapping = false;
9654
9655 if (i == NumMemOps - 1 && i != 0 && VTSize > Size - DstOff) {
9656 // Issuing an unaligned load / store pair that overlaps with the previous
9657 // pair. Adjust the offset accordingly.
9658 DstOff = Size - VTSize;
9659 IsOverlapping = true;
9660 }
9661
9662 // Calculate the actual alignment at the current offset. The alignment at
9663 // DstOff may be lower than the base alignment, especially when using
9664 // overlapping stores.
9665 Align DstAlignAtOffset = commonAlignment(A: DstAlign, Offset: DstOff);
9666 if (IsOverlapping) {
9667 // Verify that the target allows misaligned memory accesses at the
9668 // adjusted offset when using overlapping stores.
9669 unsigned Fast;
9670 if (!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace: DstPtrInfo.getAddrSpace(),
9671 Alignment: DstAlignAtOffset, Flags: MMOFlags,
9672 &Fast) ||
9673 !Fast) {
9674 // This should have been caught by findOptimalMemOpLowering, but verify
9675 // here for safety.
9676 return SDValue();
9677 }
9678 }
9679 Store = DAG.getStore(
9680 Chain, dl, Val: LoadValues[i],
9681 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9682 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment: DstAlignAtOffset, MMOFlags,
9683 AAInfo: NewAAInfo);
9684 OutChains.push_back(Elt: Store);
9685 DstOff += VTSize;
9686 }
9687
9688 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9689}
9690
9691/// Lower the call to 'memset' intrinsic function into a series of store
9692/// operations.
9693///
9694/// \param DAG Selection DAG where lowered code is placed.
9695/// \param dl Link to corresponding IR location.
9696/// \param Chain Control flow dependency.
9697/// \param Dst Pointer to destination memory location.
9698/// \param Src Value of byte to write into the memory.
9699/// \param Size Number of bytes to write.
9700/// \param Alignment Alignment of the destination in bytes.
9701/// \param isVol True if destination is volatile.
9702/// \param AlwaysInline Makes sure no function call is generated.
9703/// \param DstPtrInfo IR information on the memory pointer.
9704/// \returns New head in the control flow, if lowering was successful, empty
9705/// SDValue otherwise.
9706///
9707/// The function tries to replace 'llvm.memset' intrinsic with several store
9708/// operations and value calculation code. This is usually profitable for small
9709/// memory size or when the semantic requires inlining.
9710static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9711 SDValue Chain, SDValue Dst, SDValue Src,
9712 uint64_t Size, Align Alignment, bool isVol,
9713 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9714 const AAMDNodes &AAInfo) {
9715 // Turn a memset of undef to nop.
9716 // FIXME: We need to honor volatile even is Src is undef.
9717 if (Src.isUndef())
9718 return Chain;
9719
9720 // Expand memset to a series of load/store ops if the size operand
9721 // falls below a certain threshold.
9722 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9723 std::vector<EVT> MemOps;
9724 bool DstAlignCanChange = false;
9725 LLVMContext &C = *DAG.getContext();
9726 MachineFunction &MF = DAG.getMachineFunction();
9727 MachineFrameInfo &MFI = MF.getFrameInfo();
9728 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9729 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9730 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9731 DstAlignCanChange = true;
9732 bool IsZeroVal = isNullConstant(V: Src);
9733 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9734
9735 EVT LargestVT;
9736 if (!TLI.findOptimalMemOpLowering(
9737 Context&: C, MemOps, Limit,
9738 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9739 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9740 LargestVT: &LargestVT))
9741 return SDValue();
9742
9743 if (DstAlignCanChange) {
9744 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9745 const DataLayout &DL = DAG.getDataLayout();
9746 Align NewAlign = DL.getABITypeAlign(Ty);
9747
9748 // Don't promote to an alignment that would require dynamic stack
9749 // realignment which may conflict with optimizations such as tail call
9750 // optimization.
9751 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9752 if (!TRI->hasStackRealignment(MF))
9753 if (MaybeAlign StackAlign = DL.getStackAlignment())
9754 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9755
9756 if (NewAlign > Alignment) {
9757 // Give the stack frame object a larger alignment if needed.
9758 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9759 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9760 Alignment = NewAlign;
9761 }
9762 }
9763
9764 SmallVector<SDValue, 8> OutChains;
9765 uint64_t DstOff = 0;
9766 unsigned NumMemOps = MemOps.size();
9767
9768 // Find the largest store and generate the bit pattern for it.
9769 // If target didn't set LargestVT, compute it from MemOps.
9770 if (!LargestVT.isSimple()) {
9771 LargestVT = MemOps[0];
9772 for (unsigned i = 1; i < NumMemOps; i++)
9773 if (MemOps[i].bitsGT(VT: LargestVT))
9774 LargestVT = MemOps[i];
9775 }
9776 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9777
9778 // Prepare AAInfo for loads/stores after lowering this memset.
9779 AAMDNodes NewAAInfo = AAInfo;
9780 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9781
9782 for (unsigned i = 0; i < NumMemOps; i++) {
9783 EVT VT = MemOps[i];
9784 unsigned VTSize = VT.getSizeInBits() / 8;
9785 // The target should specify store types that exactly cover the memset size
9786 // (with the last store potentially being oversized for overlapping stores).
9787 assert(Size > 0 && "Target specified more stores than needed in "
9788 "findOptimalMemOpLowering");
9789 if (VTSize > Size) {
9790 // Issuing an unaligned load / store pair that overlaps with the previous
9791 // pair. Adjust the offset accordingly.
9792 assert(i == NumMemOps-1 && i != 0);
9793 DstOff -= VTSize - Size;
9794 }
9795
9796 // If this store is smaller than the largest store see whether we can get
9797 // the smaller value for free with a truncate or extract vector element and
9798 // then store.
9799 SDValue Value = MemSetValue;
9800 if (VT.bitsLT(VT: LargestVT)) {
9801 unsigned Index;
9802 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9803 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
9804 if (!LargestVT.isVector() && !VT.isVector() &&
9805 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
9806 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
9807 else if (LargestVT.isVector() && !VT.isVector() &&
9808 TLI.shallExtractConstSplatVectorElementToStore(
9809 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
9810 ElemSizeInBits: VT.getSizeInBits(), Index) &&
9811 TLI.isTypeLegal(VT: SVT) &&
9812 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
9813 // Target which can combine store(extractelement VectorTy, Idx) can get
9814 // the smaller value for free.
9815 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
9816 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
9817 } else
9818 Value = getMemsetValue(Value: Src, VT, DAG, dl);
9819 }
9820 assert(Value.getValueType() == VT && "Value with wrong type.");
9821 SDValue Store = DAG.getStore(
9822 Chain, dl, Val: Value,
9823 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9824 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
9825 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
9826 AAInfo: NewAAInfo);
9827 OutChains.push_back(Elt: Store);
9828 DstOff += VT.getSizeInBits() / 8;
9829 // For oversized overlapping stores, only subtract the remaining bytes.
9830 // For normal stores, subtract the full store size.
9831 if (VTSize > Size) {
9832 Size = 0;
9833 } else {
9834 Size -= VTSize;
9835 }
9836 }
9837
9838 // After processing all stores, Size should be exactly 0. Any remaining bytes
9839 // indicate a bug in the target's findOptimalMemOpLowering implementation.
9840 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
9841 "stores that exactly cover the memset size");
9842
9843 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9844}
9845
9846static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
9847 unsigned AS) {
9848 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
9849 // pointer operands can be losslessly bitcasted to pointers of address space 0
9850 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
9851 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
9852 Twine(AS));
9853 }
9854}
9855
9856static bool isInTailCallPositionWrapper(const CallInst *CI,
9857 const SelectionDAG *SelDAG,
9858 bool AllowReturnsFirstArg) {
9859 if (!CI || !CI->isTailCall())
9860 return false;
9861 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
9862 // helper symbol we lower to.
9863 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
9864 ReturnsFirstArg: AllowReturnsFirstArg &&
9865 funcReturnsFirstArgOfCall(CI: *CI));
9866}
9867
9868static std::pair<SDValue, SDValue>
9869getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
9870 TargetLowering::ArgListTy &&Args,
9871 const CallInst *CI, RTLIB::Libcall Call,
9872 SelectionDAG *DAG, const TargetLowering *TLI) {
9873 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
9874
9875 if (LCImpl == RTLIB::Unsupported)
9876 return {};
9877
9878 TargetLowering::CallLoweringInfo CLI(*DAG);
9879 bool IsTailCall =
9880 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true);
9881 SDValue Callee =
9882 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
9883
9884 CLI.setDebugLoc(dl)
9885 .setChain(Chain)
9886 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9887 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
9888 .setTailCall(IsTailCall);
9889
9890 return TLI->LowerCallTo(CLI);
9891}
9892
9893std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
9894 const SDLoc &dl, SDValue S1,
9895 SDValue S2,
9896 const CallInst *CI) {
9897 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9898 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9899 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9900 Call: RTLIB::STRCMP, DAG: this, TLI);
9901}
9902
9903std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
9904 const SDLoc &dl, SDValue S1,
9905 SDValue S2,
9906 const CallInst *CI) {
9907 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9908 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9909 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9910 Call: RTLIB::STRSTR, DAG: this, TLI);
9911}
9912
9913std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
9914 const SDLoc &dl,
9915 SDValue Dst, SDValue Src,
9916 SDValue C, SDValue Size,
9917 const CallInst *CI) {
9918 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9919
9920 TargetLowering::ArgListTy Args = {
9921 {Dst, PT},
9922 {Src, PT},
9923 {C, Type::getInt32Ty(C&: *getContext())},
9924 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9925 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9926 Call: RTLIB::MEMCCPY, DAG: this, TLI);
9927}
9928
9929std::pair<SDValue, SDValue>
9930SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
9931 SDValue Mem1, SDValue Size, const CallInst *CI) {
9932 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9933 TargetLowering::ArgListTy Args = {
9934 {Mem0, PT},
9935 {Mem1, PT},
9936 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9937 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9938 Call: RTLIB::MEMCMP, DAG: this, TLI);
9939}
9940
9941std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
9942 const SDLoc &dl,
9943 SDValue Dst, SDValue Src,
9944 const CallInst *CI) {
9945 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9946 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
9947 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9948 Call: RTLIB::STRCPY, DAG: this, TLI);
9949}
9950
9951std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
9952 const SDLoc &dl,
9953 SDValue Src,
9954 const CallInst *CI) {
9955 // Emit a library call.
9956 TargetLowering::ArgListTy Args = {
9957 {Src, PointerType::getUnqual(C&: *getContext())}};
9958 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9959 Call: RTLIB::STRLEN, DAG: this, TLI);
9960}
9961
9962SDValue SelectionDAG::getMemcpy(
9963 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
9964 Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline,
9965 const CallInst *CI, std::optional<bool> OverrideTailCall,
9966 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
9967 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
9968 // Check to see if we should lower the memcpy to loads and stores first.
9969 // For cases within the target-specified limits, this is the best choice.
9970 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9971 if (ConstantSize) {
9972 // Memcpy with size zero? Just return the original chain.
9973 if (ConstantSize->isZero())
9974 return Chain;
9975
9976 SDValue Result = getMemcpyLoadsAndStores(
9977 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
9978 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9979 if (Result.getNode())
9980 return Result;
9981 }
9982
9983 // Then check to see if we should lower the memcpy with target-specific
9984 // code. If the target chooses to do this, this is the next best.
9985 if (TSI) {
9986 SDValue Result = TSI->EmitTargetCodeForMemcpy(
9987 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol,
9988 AlwaysInline, DstPtrInfo, SrcPtrInfo);
9989 if (Result.getNode())
9990 return Result;
9991 }
9992
9993 // If we really need inline code and the target declined to provide it,
9994 // use a (potentially long) sequence of loads and stores.
9995 if (AlwaysInline) {
9996 assert(ConstantSize && "AlwaysInline requires a constant size!");
9997 return getMemcpyLoadsAndStores(
9998 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
9999 SrcAlign, isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
10000 }
10001
10002 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10003 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
10004
10005 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
10006 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
10007 // respect volatile, so they may do things like read or write memory
10008 // beyond the given memory regions. But fixing this isn't easy, and most
10009 // people don't care.
10010
10011 // Emit a library call.
10012 TargetLowering::ArgListTy Args;
10013 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10014 Args.emplace_back(args&: Dst, args&: PtrTy);
10015 Args.emplace_back(args&: Src, args&: PtrTy);
10016 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10017 // FIXME: pass in SDLoc
10018 TargetLowering::CallLoweringInfo CLI(*this);
10019 bool IsTailCall = false;
10020 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
10021
10022 if (OverrideTailCall.has_value()) {
10023 IsTailCall = *OverrideTailCall;
10024 } else {
10025 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
10026 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
10027 }
10028
10029 CLI.setDebugLoc(dl)
10030 .setChain(Chain)
10031 .setLibCallee(
10032 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
10033 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10034 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10035 ArgsList: std::move(Args))
10036 .setDiscardResult()
10037 .setTailCall(IsTailCall);
10038
10039 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10040 return CallResult.second;
10041}
10042
10043SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
10044 SDValue Dst, SDValue Src, SDValue Size,
10045 Type *SizeTy, unsigned ElemSz,
10046 bool isTailCall,
10047 MachinePointerInfo DstPtrInfo,
10048 MachinePointerInfo SrcPtrInfo) {
10049 // Emit a library call.
10050 TargetLowering::ArgListTy Args;
10051 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
10052 Args.emplace_back(args&: Dst, args&: ArgTy);
10053 Args.emplace_back(args&: Src, args&: ArgTy);
10054 Args.emplace_back(args&: Size, args&: SizeTy);
10055
10056 RTLIB::Libcall LibraryCall =
10057 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10058 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10059 if (LibcallImpl == RTLIB::Unsupported)
10060 report_fatal_error(reason: "Unsupported element size");
10061
10062 TargetLowering::CallLoweringInfo CLI(*this);
10063 CLI.setDebugLoc(dl)
10064 .setChain(Chain)
10065 .setLibCallee(
10066 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10067 ResultType: Type::getVoidTy(C&: *getContext()),
10068 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10069 ArgsList: std::move(Args))
10070 .setDiscardResult()
10071 .setTailCall(isTailCall);
10072
10073 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10074 return CallResult.second;
10075}
10076
10077SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
10078 SDValue Src, SDValue Size, Align DstAlign,
10079 Align SrcAlign, bool isVol, const CallInst *CI,
10080 std::optional<bool> OverrideTailCall,
10081 MachinePointerInfo DstPtrInfo,
10082 MachinePointerInfo SrcPtrInfo,
10083 const AAMDNodes &AAInfo,
10084 BatchAAResults *BatchAA) {
10085 // Check to see if we should lower the memmove to loads and stores first.
10086 // For cases within the target-specified limits, this is the best choice.
10087 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10088 if (ConstantSize) {
10089 // Memmove with size zero? Just return the original chain.
10090 if (ConstantSize->isZero())
10091 return Chain;
10092
10093 SDValue Result = getMemmoveLoadsAndStores(
10094 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), DstAlign,
10095 SrcAlign, isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
10096 if (Result.getNode())
10097 return Result;
10098 }
10099
10100 // Then check to see if we should lower the memmove with target-specific
10101 // code. If the target chooses to do this, this is the next best.
10102 if (TSI) {
10103 SDValue Result = TSI->EmitTargetCodeForMemmove(
10104 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, DstAlign, SrcAlign, isVolatile: isVol, DstPtrInfo,
10105 SrcPtrInfo);
10106 if (Result.getNode())
10107 return Result;
10108 }
10109
10110 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10111 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
10112
10113 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
10114 // not be safe. See memcpy above for more details.
10115
10116 // Emit a library call.
10117 TargetLowering::ArgListTy Args;
10118 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
10119 Args.emplace_back(args&: Dst, args&: PtrTy);
10120 Args.emplace_back(args&: Src, args&: PtrTy);
10121 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
10122 // FIXME: pass in SDLoc
10123 TargetLowering::CallLoweringInfo CLI(*this);
10124
10125 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
10126
10127 bool IsTailCall = false;
10128 if (OverrideTailCall.has_value()) {
10129 IsTailCall = *OverrideTailCall;
10130 } else {
10131 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
10132 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
10133 }
10134
10135 CLI.setDebugLoc(dl)
10136 .setChain(Chain)
10137 .setLibCallee(
10138 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
10139 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
10140 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10141 ArgsList: std::move(Args))
10142 .setDiscardResult()
10143 .setTailCall(IsTailCall);
10144
10145 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
10146 return CallResult.second;
10147}
10148
10149SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
10150 SDValue Dst, SDValue Src, SDValue Size,
10151 Type *SizeTy, unsigned ElemSz,
10152 bool isTailCall,
10153 MachinePointerInfo DstPtrInfo,
10154 MachinePointerInfo SrcPtrInfo) {
10155 // Emit a library call.
10156 TargetLowering::ArgListTy Args;
10157 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
10158 Args.emplace_back(args&: Dst, args&: IntPtrTy);
10159 Args.emplace_back(args&: Src, args&: IntPtrTy);
10160 Args.emplace_back(args&: Size, args&: SizeTy);
10161
10162 RTLIB::Libcall LibraryCall =
10163 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10164 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10165 if (LibcallImpl == RTLIB::Unsupported)
10166 report_fatal_error(reason: "Unsupported element size");
10167
10168 TargetLowering::CallLoweringInfo CLI(*this);
10169 CLI.setDebugLoc(dl)
10170 .setChain(Chain)
10171 .setLibCallee(
10172 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10173 ResultType: Type::getVoidTy(C&: *getContext()),
10174 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10175 ArgsList: std::move(Args))
10176 .setDiscardResult()
10177 .setTailCall(isTailCall);
10178
10179 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10180 return CallResult.second;
10181}
10182
10183SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
10184 SDValue Src, SDValue Size, Align Alignment,
10185 bool isVol, bool AlwaysInline,
10186 const CallInst *CI,
10187 MachinePointerInfo DstPtrInfo,
10188 const AAMDNodes &AAInfo) {
10189 // Check to see if we should lower the memset to stores first.
10190 // For cases within the target-specified limits, this is the best choice.
10191 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
10192 if (ConstantSize) {
10193 // Memset with size zero? Just return the original chain.
10194 if (ConstantSize->isZero())
10195 return Chain;
10196
10197 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10198 Size: ConstantSize->getZExtValue(), Alignment,
10199 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
10200
10201 if (Result.getNode())
10202 return Result;
10203 }
10204
10205 // Then check to see if we should lower the memset with target-specific
10206 // code. If the target chooses to do this, this is the next best.
10207 if (TSI) {
10208 SDValue Result = TSI->EmitTargetCodeForMemset(
10209 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
10210 if (Result.getNode())
10211 return Result;
10212 }
10213
10214 // If we really need inline code and the target declined to provide it,
10215 // use a (potentially long) sequence of loads and stores.
10216 if (AlwaysInline) {
10217 assert(ConstantSize && "AlwaysInline requires a constant size!");
10218 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
10219 Size: ConstantSize->getZExtValue(), Alignment,
10220 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
10221 assert(Result &&
10222 "getMemsetStores must return a valid sequence when AlwaysInline");
10223 return Result;
10224 }
10225
10226 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
10227
10228 // Emit a library call.
10229 auto &Ctx = *getContext();
10230 const auto& DL = getDataLayout();
10231
10232 TargetLowering::CallLoweringInfo CLI(*this);
10233 // FIXME: pass in SDLoc
10234 CLI.setDebugLoc(dl).setChain(Chain);
10235
10236 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
10237 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
10238
10239 // If zeroing out and bzero is present, use it.
10240 if (UseBZero) {
10241 TargetLowering::ArgListTy Args;
10242 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10243 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10244 CLI.setLibCallee(
10245 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
10246 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
10247 } else {
10248 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10249
10250 TargetLowering::ArgListTy Args;
10251 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
10252 Args.emplace_back(args&: Src, args: Src.getValueType().getTypeForEVT(Context&: Ctx));
10253 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
10254 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
10255 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
10256 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
10257 ArgsList: std::move(Args));
10258 }
10259
10260 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
10261 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
10262
10263 // If we're going to use bzero, make sure not to tail call unless the
10264 // subsequent return doesn't need a value, as bzero doesn't return the first
10265 // arg unlike memset.
10266 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
10267 bool IsTailCall =
10268 CI && CI->isTailCall() &&
10269 isInTailCallPosition(Call: *CI, TM: getTarget(), ReturnsFirstArg: ReturnsFirstArg && LowersToMemset);
10270 CLI.setDiscardResult().setTailCall(IsTailCall);
10271
10272 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10273 return CallResult.second;
10274}
10275
10276SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
10277 SDValue Dst, SDValue Value, SDValue Size,
10278 Type *SizeTy, unsigned ElemSz,
10279 bool isTailCall,
10280 MachinePointerInfo DstPtrInfo) {
10281 // Emit a library call.
10282 TargetLowering::ArgListTy Args;
10283 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
10284 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
10285 Args.emplace_back(args&: Size, args&: SizeTy);
10286
10287 RTLIB::Libcall LibraryCall =
10288 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
10289 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
10290 if (LibcallImpl == RTLIB::Unsupported)
10291 report_fatal_error(reason: "Unsupported element size");
10292
10293 TargetLowering::CallLoweringInfo CLI(*this);
10294 CLI.setDebugLoc(dl)
10295 .setChain(Chain)
10296 .setLibCallee(
10297 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
10298 ResultType: Type::getVoidTy(C&: *getContext()),
10299 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
10300 ArgsList: std::move(Args))
10301 .setDiscardResult()
10302 .setTailCall(isTailCall);
10303
10304 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
10305 return CallResult.second;
10306}
10307
10308SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10309 SDVTList VTList, ArrayRef<SDValue> Ops,
10310 MachineMemOperand *MMO,
10311 ISD::LoadExtType ExtType) {
10312 FoldingSetNodeID ID;
10313 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10314 ID.AddInteger(I: MemVT.getRawBits());
10315 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
10316 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
10317 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10318 ID.AddInteger(I: MMO->getFlags());
10319 void* IP = nullptr;
10320 if (auto *E = cast_or_null<AtomicSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10321 E->refineAlignment(NewMMO: MMO);
10322 E->refineRanges(NewMMO: MMO);
10323 return SDValue(E, 0);
10324 }
10325
10326 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
10327 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
10328 createOperands(Node: N, Vals: Ops);
10329
10330 CSEMap.InsertNode(N, InsertPos: IP);
10331 InsertNode(N);
10332 SDValue V(N, 0);
10333 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10334 return V;
10335}
10336
10337SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
10338 EVT MemVT, SDVTList VTs, SDValue Chain,
10339 SDValue Ptr, SDValue Cmp, SDValue Swp,
10340 MachineMemOperand *MMO) {
10341 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
10342 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
10343 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
10344
10345 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
10346 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10347}
10348
10349SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
10350 SDValue Chain, SDValue Ptr, SDValue Val,
10351 MachineMemOperand *MMO) {
10352 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
10353 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
10354 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
10355 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
10356 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
10357 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
10358 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
10359 Opcode == ISD::ATOMIC_LOAD_FMIN ||
10360 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
10361 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
10362 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
10363 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
10364 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
10365 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
10366 Opcode == ISD::ATOMIC_STORE) &&
10367 "Invalid Atomic Op");
10368
10369 EVT VT = Val.getValueType();
10370
10371 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
10372 getVTList(VT1: VT, VT2: MVT::Other);
10373 SDValue Ops[] = {Chain, Ptr, Val};
10374 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
10375}
10376
10377SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10378 EVT MemVT, EVT VT, SDValue Chain,
10379 SDValue Ptr, MachineMemOperand *MMO) {
10380 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
10381 SDValue Ops[] = {Chain, Ptr};
10382 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
10383}
10384
10385/// getMergeValues - Create a MERGE_VALUES node from the given operands.
10386SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
10387 if (Ops.size() == 1)
10388 return Ops[0];
10389
10390 SmallVector<EVT, 4> VTs;
10391 VTs.reserve(N: Ops.size());
10392 for (const SDValue &Op : Ops)
10393 VTs.push_back(Elt: Op.getValueType());
10394 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
10395}
10396
10397SDValue SelectionDAG::getMemIntrinsicNode(
10398 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
10399 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
10400 MachineMemOperand::Flags Flags, LocationSize Size,
10401 const AAMDNodes &AAInfo) {
10402 if (Size.hasValue() && !Size.getValue())
10403 Size = LocationSize::precise(Value: MemVT.getStoreSize());
10404
10405 MachineFunction &MF = getMachineFunction();
10406 MachineMemOperand *MMO =
10407 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, AAInfo);
10408
10409 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
10410}
10411
10412SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10413 SDVTList VTList,
10414 ArrayRef<SDValue> Ops, EVT MemVT,
10415 MachineMemOperand *MMO) {
10416 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
10417}
10418
10419SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
10420 SDVTList VTList,
10421 ArrayRef<SDValue> Ops, EVT MemVT,
10422 ArrayRef<MachineMemOperand *> MMOs) {
10423 assert(!MMOs.empty() && "Must have at least one MMO");
10424 assert(
10425 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
10426 Opcode == ISD::PREFETCH ||
10427 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
10428 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
10429 "Opcode is not a memory-accessing opcode!");
10430
10431 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
10432 if (MMOs.size() == 1) {
10433 MemRefs = MMOs[0];
10434 } else {
10435 // Allocate: [size_t count][MMO*][MMO*]...
10436 size_t AllocSize =
10437 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
10438 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
10439 size_t *CountPtr = static_cast<size_t *>(Buffer);
10440 *CountPtr = MMOs.size();
10441 MachineMemOperand **Array =
10442 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
10443 llvm::copy(Range&: MMOs, Out: Array);
10444 MemRefs = Array;
10445 }
10446
10447 // Memoize the node unless it returns a glue result.
10448 MemIntrinsicSDNode *N;
10449 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10450 FoldingSetNodeID ID;
10451 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10452 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10453 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10454 ID.AddInteger(I: MemVT.getRawBits());
10455 for (const MachineMemOperand *MMO : MMOs) {
10456 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10457 ID.AddInteger(I: MMO->getFlags());
10458 }
10459 void *IP = nullptr;
10460 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10461 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10462 return SDValue(E, 0);
10463 }
10464
10465 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10466 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10467 createOperands(Node: N, Vals: Ops);
10468 CSEMap.InsertNode(N, InsertPos: IP);
10469 } else {
10470 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10471 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10472 createOperands(Node: N, Vals: Ops);
10473 }
10474 InsertNode(N);
10475 SDValue V(N, 0);
10476 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10477 return V;
10478}
10479
10480SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
10481 SDValue Chain, int FrameIndex) {
10482 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
10483 const auto VTs = getVTList(VT: MVT::Other);
10484 SDValue Ops[2] = {
10485 Chain,
10486 getFrameIndex(FI: FrameIndex,
10487 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10488 isTarget: true)};
10489
10490 FoldingSetNodeID ID;
10491 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10492 ID.AddInteger(I: FrameIndex);
10493 void *IP = nullptr;
10494 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10495 return SDValue(E, 0);
10496
10497 LifetimeSDNode *N =
10498 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10499 createOperands(Node: N, Vals: Ops);
10500 CSEMap.InsertNode(N, InsertPos: IP);
10501 InsertNode(N);
10502 SDValue V(N, 0);
10503 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10504 return V;
10505}
10506
10507SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10508 uint64_t Guid, uint64_t Index,
10509 uint32_t Attr) {
10510 const unsigned Opcode = ISD::PSEUDO_PROBE;
10511 const auto VTs = getVTList(VT: MVT::Other);
10512 SDValue Ops[] = {Chain};
10513 FoldingSetNodeID ID;
10514 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10515 ID.AddInteger(I: Guid);
10516 ID.AddInteger(I: Index);
10517 void *IP = nullptr;
10518 if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
10519 return SDValue(E, 0);
10520
10521 auto *N = newSDNode<PseudoProbeSDNode>(
10522 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10523 createOperands(Node: N, Vals: Ops);
10524 CSEMap.InsertNode(N, InsertPos: IP);
10525 InsertNode(N);
10526 SDValue V(N, 0);
10527 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10528 return V;
10529}
10530
10531/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10532/// MachinePointerInfo record from it. This is particularly useful because the
10533/// code generator has many cases where it doesn't bother passing in a
10534/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10535static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10536 SelectionDAG &DAG, SDValue Ptr,
10537 int64_t Offset = 0) {
10538 // If this is FI+Offset, we can model it.
10539 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10540 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10541 FI: FI->getIndex(), Offset);
10542
10543 // If this is (FI+Offset1)+Offset2, we can model it.
10544 if (Ptr.getOpcode() != ISD::ADD ||
10545 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10546 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10547 return Info;
10548
10549 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10550 return MachinePointerInfo::getFixedStack(
10551 MF&: DAG.getMachineFunction(), FI,
10552 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10553}
10554
10555/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10556/// MachinePointerInfo record from it. This is particularly useful because the
10557/// code generator has many cases where it doesn't bother passing in a
10558/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10559static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10560 SelectionDAG &DAG, SDValue Ptr,
10561 SDValue OffsetOp) {
10562 // If the 'Offset' value isn't a constant, we can't handle this.
10563 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10564 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10565 if (OffsetOp.isUndef())
10566 return InferPointerInfo(Info, DAG, Ptr);
10567 return Info;
10568}
10569
10570SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10571 EVT VT, const SDLoc &dl, SDValue Chain,
10572 SDValue Ptr, SDValue Offset,
10573 MachinePointerInfo PtrInfo, EVT MemVT,
10574 Align Alignment,
10575 MachineMemOperand::Flags MMOFlags,
10576 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10577 assert(Chain.getValueType() == MVT::Other &&
10578 "Invalid chain type");
10579
10580 MMOFlags |= MachineMemOperand::MOLoad;
10581 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10582 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10583 // clients.
10584 if (PtrInfo.V.isNull())
10585 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10586
10587 TypeSize Size = MemVT.getStoreSize();
10588 MachineFunction &MF = getMachineFunction();
10589 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10590 BaseAlignment: Alignment, AAInfo, Ranges);
10591 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10592}
10593
10594SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10595 EVT VT, const SDLoc &dl, SDValue Chain,
10596 SDValue Ptr, SDValue Offset, EVT MemVT,
10597 MachineMemOperand *MMO) {
10598 if (VT == MemVT) {
10599 ExtType = ISD::NON_EXTLOAD;
10600 } else if (ExtType == ISD::NON_EXTLOAD) {
10601 assert(VT == MemVT && "Non-extending load from different memory type!");
10602 } else {
10603 // Extending load.
10604 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10605 "Should only be an extending load, not truncating!");
10606 assert(VT.isInteger() == MemVT.isInteger() &&
10607 "Cannot convert from FP to Int or Int -> FP!");
10608 assert(VT.isVector() == MemVT.isVector() &&
10609 "Cannot use an ext load to convert to or from a vector!");
10610 assert((!VT.isVector() ||
10611 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10612 "Cannot use an ext load to change the number of vector elements!");
10613 }
10614
10615 assert((!MMO->getRanges() ||
10616 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10617 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10618 MemVT.isInteger())) &&
10619 "Range metadata and load type must match!");
10620
10621 bool Indexed = AM != ISD::UNINDEXED;
10622 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10623
10624 SDVTList VTs = Indexed ?
10625 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10626 SDValue Ops[] = { Chain, Ptr, Offset };
10627 FoldingSetNodeID ID;
10628 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
10629 ID.AddInteger(I: MemVT.getRawBits());
10630 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10631 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10632 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10633 ID.AddInteger(I: MMO->getFlags());
10634 void *IP = nullptr;
10635 if (auto *E = cast_or_null<LoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10636 E->refineAlignment(NewMMO: MMO);
10637 E->refineRanges(NewMMO: MMO);
10638 return SDValue(E, 0);
10639 }
10640 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10641 Args&: ExtType, Args&: MemVT, Args&: MMO);
10642 createOperands(Node: N, Vals: Ops);
10643
10644 CSEMap.InsertNode(N, InsertPos: IP);
10645 InsertNode(N);
10646 SDValue V(N, 0);
10647 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10648 return V;
10649}
10650
10651SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10652 SDValue Ptr, MachinePointerInfo PtrInfo,
10653 MaybeAlign Alignment,
10654 MachineMemOperand::Flags MMOFlags,
10655 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10656 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10657 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10658 PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
10659}
10660
10661SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10662 SDValue Ptr, MachineMemOperand *MMO) {
10663 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10664 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10665 MemVT: VT, MMO);
10666}
10667
10668SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10669 EVT VT, SDValue Chain, SDValue Ptr,
10670 MachinePointerInfo PtrInfo, EVT MemVT,
10671 MaybeAlign Alignment,
10672 MachineMemOperand::Flags MMOFlags,
10673 const AAMDNodes &AAInfo) {
10674 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10675 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10676 MemVT, Alignment, MMOFlags, AAInfo);
10677}
10678
10679SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10680 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10681 MachineMemOperand *MMO) {
10682 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10683 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10684 MemVT, MMO);
10685}
10686
10687SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10688 SDValue Base, SDValue Offset,
10689 ISD::MemIndexedMode AM) {
10690 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10691 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10692 // Don't propagate the invariant or dereferenceable flags.
10693 auto MMOFlags =
10694 LD->getMemOperand()->getFlags() &
10695 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10696 return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10697 Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
10698 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
10699}
10700
10701SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10702 SDValue Ptr, MachinePointerInfo PtrInfo,
10703 Align Alignment,
10704 MachineMemOperand::Flags MMOFlags,
10705 const AAMDNodes &AAInfo) {
10706 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10707
10708 MMOFlags |= MachineMemOperand::MOStore;
10709 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10710
10711 if (PtrInfo.V.isNull())
10712 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10713
10714 MachineFunction &MF = getMachineFunction();
10715 TypeSize Size = Val.getValueType().getStoreSize();
10716 MachineMemOperand *MMO =
10717 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, AAInfo);
10718 return getStore(Chain, dl, Val, Ptr, MMO);
10719}
10720
10721SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10722 SDValue Ptr, MachineMemOperand *MMO) {
10723 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10724 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10725 AM: ISD::UNINDEXED);
10726}
10727
10728SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10729 SDValue Ptr, SDValue Offset, EVT SVT,
10730 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
10731 bool IsTruncating) {
10732 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10733 EVT VT = Val.getValueType();
10734 if (VT == SVT) {
10735 IsTruncating = false;
10736 } else if (!IsTruncating) {
10737 assert(VT == SVT && "No-truncating store from different memory type!");
10738 } else {
10739 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10740 "Should only be a truncating store, not extending!");
10741 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10742 assert(VT.isVector() == SVT.isVector() &&
10743 "Cannot use trunc store to convert to or from a vector!");
10744 assert((!VT.isVector() ||
10745 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10746 "Cannot use trunc store to change the number of vector elements!");
10747 }
10748
10749 bool Indexed = AM != ISD::UNINDEXED;
10750 assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");
10751 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10752 : getVTList(VT: MVT::Other);
10753 SDValue Ops[] = {Chain, Val, Ptr, Offset};
10754 FoldingSetNodeID ID;
10755 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
10756 ID.AddInteger(I: SVT.getRawBits());
10757 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
10758 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
10759 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10760 ID.AddInteger(I: MMO->getFlags());
10761 void *IP = nullptr;
10762 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10763 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10764 return SDValue(E, 0);
10765 }
10766 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10767 Args&: IsTruncating, Args&: SVT, Args&: MMO);
10768 createOperands(Node: N, Vals: Ops);
10769
10770 CSEMap.InsertNode(N, InsertPos: IP);
10771 InsertNode(N);
10772 SDValue V(N, 0);
10773 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10774 return V;
10775}
10776
10777SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10778 SDValue Ptr, MachinePointerInfo PtrInfo,
10779 EVT SVT, Align Alignment,
10780 MachineMemOperand::Flags MMOFlags,
10781 const AAMDNodes &AAInfo) {
10782 assert(Chain.getValueType() == MVT::Other &&
10783 "Invalid chain type");
10784
10785 MMOFlags |= MachineMemOperand::MOStore;
10786 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10787
10788 if (PtrInfo.V.isNull())
10789 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10790
10791 MachineFunction &MF = getMachineFunction();
10792 MachineMemOperand *MMO = MF.getMachineMemOperand(
10793 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10794 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
10795}
10796
10797SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10798 SDValue Ptr, EVT SVT,
10799 MachineMemOperand *MMO) {
10800 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10801 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
10802}
10803
10804SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
10805 SDValue Base, SDValue Offset,
10806 ISD::MemIndexedMode AM) {
10807 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
10808 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
10809 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
10810 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
10811 IsTruncating: ST->isTruncatingStore());
10812}
10813
10814SDValue SelectionDAG::getLoadVP(
10815 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
10816 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
10817 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
10818 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
10819 const MDNode *Ranges, bool IsExpanding) {
10820 MMOFlags |= MachineMemOperand::MOLoad;
10821 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10822 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10823 // clients.
10824 if (PtrInfo.V.isNull())
10825 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10826
10827 TypeSize Size = MemVT.getStoreSize();
10828 MachineFunction &MF = getMachineFunction();
10829 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10830 BaseAlignment: Alignment, AAInfo, Ranges);
10831 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
10832 MMO, IsExpanding);
10833}
10834
10835SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
10836 ISD::LoadExtType ExtType, EVT VT,
10837 const SDLoc &dl, SDValue Chain, SDValue Ptr,
10838 SDValue Offset, SDValue Mask, SDValue EVL,
10839 EVT MemVT, MachineMemOperand *MMO,
10840 bool IsExpanding) {
10841 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10842 assert(Mask.getValueType().getVectorElementCount() ==
10843 VT.getVectorElementCount() &&
10844 "Vector width mismatch between mask and data");
10845
10846 bool Indexed = AM != ISD::UNINDEXED;
10847 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10848
10849 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10850 : getVTList(VT1: VT, VT2: MVT::Other);
10851 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
10852 FoldingSetNodeID ID;
10853 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
10854 ID.AddInteger(I: MemVT.getRawBits());
10855 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
10856 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10857 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10858 ID.AddInteger(I: MMO->getFlags());
10859 void *IP = nullptr;
10860 if (auto *E = cast_or_null<VPLoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10861 E->refineAlignment(NewMMO: MMO);
10862 E->refineRanges(NewMMO: MMO);
10863 return SDValue(E, 0);
10864 }
10865 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10866 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10867 createOperands(Node: N, Vals: Ops);
10868
10869 CSEMap.InsertNode(N, InsertPos: IP);
10870 InsertNode(N);
10871 SDValue V(N, 0);
10872 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10873 return V;
10874}
10875
10876SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10877 SDValue Ptr, SDValue Mask, SDValue EVL,
10878 MachinePointerInfo PtrInfo,
10879 MaybeAlign Alignment,
10880 MachineMemOperand::Flags MMOFlags,
10881 const AAMDNodes &AAInfo, const MDNode *Ranges,
10882 bool IsExpanding) {
10883 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10884 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10885 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
10886 IsExpanding);
10887}
10888
10889SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10890 SDValue Ptr, SDValue Mask, SDValue EVL,
10891 MachineMemOperand *MMO, bool IsExpanding) {
10892 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10893 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10894 Mask, EVL, MemVT: VT, MMO, IsExpanding);
10895}
10896
10897SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10898 EVT VT, SDValue Chain, SDValue Ptr,
10899 SDValue Mask, SDValue EVL,
10900 MachinePointerInfo PtrInfo, EVT MemVT,
10901 MaybeAlign Alignment,
10902 MachineMemOperand::Flags MMOFlags,
10903 const AAMDNodes &AAInfo, bool IsExpanding) {
10904 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10905 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10906 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
10907 IsExpanding);
10908}
10909
10910SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10911 EVT VT, SDValue Chain, SDValue Ptr,
10912 SDValue Mask, SDValue EVL, EVT MemVT,
10913 MachineMemOperand *MMO, bool IsExpanding) {
10914 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10915 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10916 EVL, MemVT, MMO, IsExpanding);
10917}
10918
10919SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
10920 SDValue Base, SDValue Offset,
10921 ISD::MemIndexedMode AM) {
10922 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
10923 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10924 // Don't propagate the invariant or dereferenceable flags.
10925 auto MMOFlags =
10926 LD->getMemOperand()->getFlags() &
10927 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10928 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10929 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
10930 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
10931 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
10932 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
10933}
10934
10935SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
10936 SDValue Ptr, SDValue Offset, SDValue Mask,
10937 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
10938 ISD::MemIndexedMode AM, bool IsTruncating,
10939 bool IsCompressing) {
10940 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10941 assert(Mask.getValueType().getVectorElementCount() ==
10942 Val.getValueType().getVectorElementCount() &&
10943 "Vector width mismatch between mask and data");
10944
10945 bool Indexed = AM != ISD::UNINDEXED;
10946 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10947 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10948 : getVTList(VT: MVT::Other);
10949 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
10950 FoldingSetNodeID ID;
10951 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10952 ID.AddInteger(I: MemVT.getRawBits());
10953 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10954 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10955 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10956 ID.AddInteger(I: MMO->getFlags());
10957 void *IP = nullptr;
10958 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10959 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10960 return SDValue(E, 0);
10961 }
10962 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10963 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10964 createOperands(Node: N, Vals: Ops);
10965
10966 CSEMap.InsertNode(N, InsertPos: IP);
10967 InsertNode(N);
10968 SDValue V(N, 0);
10969 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10970 return V;
10971}
10972
10973SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10974 SDValue Val, SDValue Ptr, SDValue Mask,
10975 SDValue EVL, MachinePointerInfo PtrInfo,
10976 EVT SVT, Align Alignment,
10977 MachineMemOperand::Flags MMOFlags,
10978 const AAMDNodes &AAInfo,
10979 bool IsCompressing) {
10980 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10981
10982 MMOFlags |= MachineMemOperand::MOStore;
10983 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10984
10985 if (PtrInfo.V.isNull())
10986 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10987
10988 MachineFunction &MF = getMachineFunction();
10989 MachineMemOperand *MMO = MF.getMachineMemOperand(
10990 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10991 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
10992 IsCompressing);
10993}
10994
10995SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10996 SDValue Val, SDValue Ptr, SDValue Mask,
10997 SDValue EVL, EVT SVT,
10998 MachineMemOperand *MMO,
10999 bool IsCompressing) {
11000 EVT VT = Val.getValueType();
11001
11002 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11003 if (VT == SVT)
11004 return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
11005 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11006 /*IsTruncating*/ false, IsCompressing);
11007
11008 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11009 "Should only be a truncating store, not extending!");
11010 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11011 assert(VT.isVector() == SVT.isVector() &&
11012 "Cannot use trunc store to convert to or from a vector!");
11013 assert((!VT.isVector() ||
11014 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11015 "Cannot use trunc store to change the number of vector elements!");
11016
11017 SDVTList VTs = getVTList(VT: MVT::Other);
11018 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
11019 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
11020 FoldingSetNodeID ID;
11021 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
11022 ID.AddInteger(I: SVT.getRawBits());
11023 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
11024 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11025 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11026 ID.AddInteger(I: MMO->getFlags());
11027 void *IP = nullptr;
11028 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11029 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11030 return SDValue(E, 0);
11031 }
11032 auto *N =
11033 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11034 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
11035 createOperands(Node: N, Vals: Ops);
11036
11037 CSEMap.InsertNode(N, InsertPos: IP);
11038 InsertNode(N);
11039 SDValue V(N, 0);
11040 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11041 return V;
11042}
11043
11044SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
11045 SDValue Base, SDValue Offset,
11046 ISD::MemIndexedMode AM) {
11047 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
11048 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
11049 SDVTList VTs = getVTList(VT1: Base.getValueType(), VT2: MVT::Other);
11050 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
11051 Offset, ST->getMask(), ST->getVectorLength()};
11052 FoldingSetNodeID ID;
11053 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
11054 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
11055 ID.AddInteger(I: ST->getRawSubclassData());
11056 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
11057 ID.AddInteger(I: ST->getMemOperand()->getFlags());
11058 void *IP = nullptr;
11059 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11060 return SDValue(E, 0);
11061
11062 auto *N = newSDNode<VPStoreSDNode>(
11063 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
11064 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
11065 createOperands(Node: N, Vals: Ops);
11066
11067 CSEMap.InsertNode(N, InsertPos: IP);
11068 InsertNode(N);
11069 SDValue V(N, 0);
11070 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11071 return V;
11072}
11073
11074SDValue SelectionDAG::getStridedLoadVP(
11075 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
11076 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
11077 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
11078 bool Indexed = AM != ISD::UNINDEXED;
11079 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
11080
11081 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
11082 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
11083 : getVTList(VT1: VT, VT2: MVT::Other);
11084 FoldingSetNodeID ID;
11085 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
11086 ID.AddInteger(I: VT.getRawBits());
11087 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
11088 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
11089 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11090
11091 void *IP = nullptr;
11092 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11093 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11094 return SDValue(E, 0);
11095 }
11096
11097 auto *N =
11098 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
11099 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
11100 createOperands(Node: N, Vals: Ops);
11101 CSEMap.InsertNode(N, InsertPos: IP);
11102 InsertNode(N);
11103 SDValue V(N, 0);
11104 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11105 return V;
11106}
11107
11108SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
11109 SDValue Ptr, SDValue Stride,
11110 SDValue Mask, SDValue EVL,
11111 MachineMemOperand *MMO,
11112 bool IsExpanding) {
11113 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
11114 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
11115 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
11116}
11117
11118SDValue SelectionDAG::getExtStridedLoadVP(
11119 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
11120 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
11121 MachineMemOperand *MMO, bool IsExpanding) {
11122 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
11123 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
11124 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
11125}
11126
11127SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
11128 SDValue Val, SDValue Ptr,
11129 SDValue Offset, SDValue Stride,
11130 SDValue Mask, SDValue EVL, EVT MemVT,
11131 MachineMemOperand *MMO,
11132 ISD::MemIndexedMode AM,
11133 bool IsTruncating, bool IsCompressing) {
11134 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11135 bool Indexed = AM != ISD::UNINDEXED;
11136 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
11137 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
11138 : getVTList(VT: MVT::Other);
11139 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
11140 FoldingSetNodeID ID;
11141 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
11142 ID.AddInteger(I: MemVT.getRawBits());
11143 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11144 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11145 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11146 void *IP = nullptr;
11147 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11148 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11149 return SDValue(E, 0);
11150 }
11151 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11152 Args&: VTs, Args&: AM, Args&: IsTruncating,
11153 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11154 createOperands(Node: N, Vals: Ops);
11155
11156 CSEMap.InsertNode(N, InsertPos: IP);
11157 InsertNode(N);
11158 SDValue V(N, 0);
11159 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11160 return V;
11161}
11162
11163SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
11164 SDValue Val, SDValue Ptr,
11165 SDValue Stride, SDValue Mask,
11166 SDValue EVL, EVT SVT,
11167 MachineMemOperand *MMO,
11168 bool IsCompressing) {
11169 EVT VT = Val.getValueType();
11170
11171 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11172 if (VT == SVT)
11173 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
11174 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
11175 /*IsTruncating*/ false, IsCompressing);
11176
11177 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
11178 "Should only be a truncating store, not extending!");
11179 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
11180 assert(VT.isVector() == SVT.isVector() &&
11181 "Cannot use trunc store to convert to or from a vector!");
11182 assert((!VT.isVector() ||
11183 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
11184 "Cannot use trunc store to change the number of vector elements!");
11185
11186 SDVTList VTs = getVTList(VT: MVT::Other);
11187 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
11188 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
11189 FoldingSetNodeID ID;
11190 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
11191 ID.AddInteger(I: SVT.getRawBits());
11192 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
11193 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
11194 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11195 void *IP = nullptr;
11196 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11197 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11198 return SDValue(E, 0);
11199 }
11200 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
11201 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
11202 Args&: IsCompressing, Args&: SVT, Args&: MMO);
11203 createOperands(Node: N, Vals: Ops);
11204
11205 CSEMap.InsertNode(N, InsertPos: IP);
11206 InsertNode(N);
11207 SDValue V(N, 0);
11208 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11209 return V;
11210}
11211
11212SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11213 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
11214 ISD::MemIndexType IndexType) {
11215 assert(Ops.size() == 6 && "Incompatible number of operands");
11216
11217 FoldingSetNodeID ID;
11218 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
11219 ID.AddInteger(I: VT.getRawBits());
11220 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
11221 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11222 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11223 ID.AddInteger(I: MMO->getFlags());
11224 void *IP = nullptr;
11225 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11226 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11227 return SDValue(E, 0);
11228 }
11229
11230 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11231 Args&: VT, Args&: MMO, Args&: IndexType);
11232 createOperands(Node: N, Vals: Ops);
11233
11234 assert(N->getMask().getValueType().getVectorElementCount() ==
11235 N->getValueType(0).getVectorElementCount() &&
11236 "Vector width mismatch between mask and data");
11237 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11238 N->getValueType(0).getVectorElementCount().isScalable() &&
11239 "Scalable flags of index and data do not match");
11240 assert(ElementCount::isKnownGE(
11241 N->getIndex().getValueType().getVectorElementCount(),
11242 N->getValueType(0).getVectorElementCount()) &&
11243 "Vector width mismatch between index and data");
11244 assert(isa<ConstantSDNode>(N->getScale()) &&
11245 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11246 "Scale should be a constant power of 2");
11247
11248 CSEMap.InsertNode(N, InsertPos: IP);
11249 InsertNode(N);
11250 SDValue V(N, 0);
11251 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11252 return V;
11253}
11254
11255SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
11256 ArrayRef<SDValue> Ops,
11257 MachineMemOperand *MMO,
11258 ISD::MemIndexType IndexType) {
11259 assert(Ops.size() == 7 && "Incompatible number of operands");
11260
11261 FoldingSetNodeID ID;
11262 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
11263 ID.AddInteger(I: VT.getRawBits());
11264 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
11265 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
11266 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11267 ID.AddInteger(I: MMO->getFlags());
11268 void *IP = nullptr;
11269 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11270 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11271 return SDValue(E, 0);
11272 }
11273 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11274 Args&: VT, Args&: MMO, Args&: IndexType);
11275 createOperands(Node: N, Vals: Ops);
11276
11277 assert(N->getMask().getValueType().getVectorElementCount() ==
11278 N->getValue().getValueType().getVectorElementCount() &&
11279 "Vector width mismatch between mask and data");
11280 assert(
11281 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11282 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11283 "Scalable flags of index and data do not match");
11284 assert(ElementCount::isKnownGE(
11285 N->getIndex().getValueType().getVectorElementCount(),
11286 N->getValue().getValueType().getVectorElementCount()) &&
11287 "Vector width mismatch between index and data");
11288 assert(isa<ConstantSDNode>(N->getScale()) &&
11289 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11290 "Scale should be a constant power of 2");
11291
11292 CSEMap.InsertNode(N, InsertPos: IP);
11293 InsertNode(N);
11294 SDValue V(N, 0);
11295 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11296 return V;
11297}
11298
11299SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
11300 SDValue Base, SDValue Offset, SDValue Mask,
11301 SDValue PassThru, EVT MemVT,
11302 MachineMemOperand *MMO,
11303 ISD::MemIndexedMode AM,
11304 ISD::LoadExtType ExtTy, bool isExpanding) {
11305 bool Indexed = AM != ISD::UNINDEXED;
11306 assert((Indexed || Offset.isUndef()) &&
11307 "Unindexed masked load with an offset!");
11308 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
11309 : getVTList(VT1: VT, VT2: MVT::Other);
11310 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
11311 FoldingSetNodeID ID;
11312 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
11313 ID.AddInteger(I: MemVT.getRawBits());
11314 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
11315 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
11316 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11317 ID.AddInteger(I: MMO->getFlags());
11318 void *IP = nullptr;
11319 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11320 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11321 return SDValue(E, 0);
11322 }
11323 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
11324 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
11325 createOperands(Node: N, Vals: Ops);
11326
11327 CSEMap.InsertNode(N, InsertPos: IP);
11328 InsertNode(N);
11329 SDValue V(N, 0);
11330 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11331 return V;
11332}
11333
11334SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
11335 SDValue Base, SDValue Offset,
11336 ISD::MemIndexedMode AM) {
11337 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
11338 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
11339 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
11340 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
11341 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
11342 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
11343}
11344
11345SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
11346 SDValue Val, SDValue Base, SDValue Offset,
11347 SDValue Mask, EVT MemVT,
11348 MachineMemOperand *MMO,
11349 ISD::MemIndexedMode AM, bool IsTruncating,
11350 bool IsCompressing) {
11351 assert(Chain.getValueType() == MVT::Other &&
11352 "Invalid chain type");
11353 bool Indexed = AM != ISD::UNINDEXED;
11354 assert((Indexed || Offset.isUndef()) &&
11355 "Unindexed masked store with an offset!");
11356 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
11357 : getVTList(VT: MVT::Other);
11358 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
11359 FoldingSetNodeID ID;
11360 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
11361 ID.AddInteger(I: MemVT.getRawBits());
11362 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
11363 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
11364 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11365 ID.AddInteger(I: MMO->getFlags());
11366 void *IP = nullptr;
11367 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11368 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11369 return SDValue(E, 0);
11370 }
11371 auto *N =
11372 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
11373 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
11374 createOperands(Node: N, Vals: Ops);
11375
11376 CSEMap.InsertNode(N, InsertPos: IP);
11377 InsertNode(N);
11378 SDValue V(N, 0);
11379 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11380 return V;
11381}
11382
11383SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
11384 SDValue Base, SDValue Offset,
11385 ISD::MemIndexedMode AM) {
11386 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
11387 assert(ST->getOffset().isUndef() &&
11388 "Masked store is already a indexed store!");
11389 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
11390 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
11391 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
11392}
11393
11394SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11395 ArrayRef<SDValue> Ops,
11396 MachineMemOperand *MMO,
11397 ISD::MemIndexType IndexType,
11398 ISD::LoadExtType ExtTy) {
11399 assert(Ops.size() == 6 && "Incompatible number of operands");
11400
11401 FoldingSetNodeID ID;
11402 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
11403 ID.AddInteger(I: MemVT.getRawBits());
11404 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
11405 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
11406 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11407 ID.AddInteger(I: MMO->getFlags());
11408 void *IP = nullptr;
11409 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11410 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11411 return SDValue(E, 0);
11412 }
11413
11414 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11415 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
11416 createOperands(Node: N, Vals: Ops);
11417
11418 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
11419 "Incompatible type of the PassThru value in MaskedGatherSDNode");
11420 assert(N->getMask().getValueType().getVectorElementCount() ==
11421 N->getValueType(0).getVectorElementCount() &&
11422 "Vector width mismatch between mask and data");
11423 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11424 N->getValueType(0).getVectorElementCount().isScalable() &&
11425 "Scalable flags of index and data do not match");
11426 assert(ElementCount::isKnownGE(
11427 N->getIndex().getValueType().getVectorElementCount(),
11428 N->getValueType(0).getVectorElementCount()) &&
11429 "Vector width mismatch between index and data");
11430 assert(isa<ConstantSDNode>(N->getScale()) &&
11431 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11432 "Scale should be a constant power of 2");
11433
11434 CSEMap.InsertNode(N, InsertPos: IP);
11435 InsertNode(N);
11436 SDValue V(N, 0);
11437 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11438 return V;
11439}
11440
11441SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
11442 ArrayRef<SDValue> Ops,
11443 MachineMemOperand *MMO,
11444 ISD::MemIndexType IndexType,
11445 bool IsTrunc) {
11446 assert(Ops.size() == 6 && "Incompatible number of operands");
11447
11448 FoldingSetNodeID ID;
11449 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
11450 ID.AddInteger(I: MemVT.getRawBits());
11451 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11452 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11453 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11454 ID.AddInteger(I: MMO->getFlags());
11455 void *IP = nullptr;
11456 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11457 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11458 return SDValue(E, 0);
11459 }
11460
11461 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11462 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11463 createOperands(Node: N, Vals: Ops);
11464
11465 assert(N->getMask().getValueType().getVectorElementCount() ==
11466 N->getValue().getValueType().getVectorElementCount() &&
11467 "Vector width mismatch between mask and data");
11468 assert(
11469 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11470 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11471 "Scalable flags of index and data do not match");
11472 assert(ElementCount::isKnownGE(
11473 N->getIndex().getValueType().getVectorElementCount(),
11474 N->getValue().getValueType().getVectorElementCount()) &&
11475 "Vector width mismatch between index and data");
11476 assert(isa<ConstantSDNode>(N->getScale()) &&
11477 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11478 "Scale should be a constant power of 2");
11479
11480 CSEMap.InsertNode(N, InsertPos: IP);
11481 InsertNode(N);
11482 SDValue V(N, 0);
11483 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11484 return V;
11485}
11486
11487SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11488 const SDLoc &dl, ArrayRef<SDValue> Ops,
11489 MachineMemOperand *MMO,
11490 ISD::MemIndexType IndexType) {
11491 assert(Ops.size() == 7 && "Incompatible number of operands");
11492
11493 FoldingSetNodeID ID;
11494 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTList: VTs, OpList: Ops);
11495 ID.AddInteger(I: MemVT.getRawBits());
11496 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11497 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11498 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11499 ID.AddInteger(I: MMO->getFlags());
11500 void *IP = nullptr;
11501 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11502 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11503 return SDValue(E, 0);
11504 }
11505
11506 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11507 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11508 createOperands(Node: N, Vals: Ops);
11509
11510 assert(N->getMask().getValueType().getVectorElementCount() ==
11511 N->getIndex().getValueType().getVectorElementCount() &&
11512 "Vector width mismatch between mask and data");
11513 assert(isa<ConstantSDNode>(N->getScale()) &&
11514 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11515 "Scale should be a constant power of 2");
11516 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11517
11518 CSEMap.InsertNode(N, InsertPos: IP);
11519 InsertNode(N);
11520 SDValue V(N, 0);
11521 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11522 return V;
11523}
11524
11525SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11526 SDValue Ptr, SDValue Mask, SDValue EVL,
11527 MachineMemOperand *MMO) {
11528 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11529 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11530 FoldingSetNodeID ID;
11531 AddNodeIDNode(ID, OpC: ISD::VP_LOAD_FF, VTList: VTs, OpList: Ops);
11532 ID.AddInteger(I: VT.getRawBits());
11533 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11534 Args&: VTs, Args&: VT, Args&: MMO));
11535 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11536 ID.AddInteger(I: MMO->getFlags());
11537 void *IP = nullptr;
11538 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11539 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11540 return SDValue(E, 0);
11541 }
11542 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11543 Args&: VT, Args&: MMO);
11544 createOperands(Node: N, Vals: Ops);
11545
11546 CSEMap.InsertNode(N, InsertPos: IP);
11547 InsertNode(N);
11548 SDValue V(N, 0);
11549 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11550 return V;
11551}
11552
11553SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11554 EVT MemVT, MachineMemOperand *MMO) {
11555 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11556 SDVTList VTs = getVTList(VT: MVT::Other);
11557 SDValue Ops[] = {Chain, Ptr};
11558 FoldingSetNodeID ID;
11559 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
11560 ID.AddInteger(I: MemVT.getRawBits());
11561 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11562 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11563 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11564 ID.AddInteger(I: MMO->getFlags());
11565 void *IP = nullptr;
11566 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11567 return SDValue(E, 0);
11568
11569 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11570 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11571 createOperands(Node: N, Vals: Ops);
11572
11573 CSEMap.InsertNode(N, InsertPos: IP);
11574 InsertNode(N);
11575 SDValue V(N, 0);
11576 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11577 return V;
11578}
11579
11580SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11581 EVT MemVT, MachineMemOperand *MMO) {
11582 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11583 SDVTList VTs = getVTList(VT: MVT::Other);
11584 SDValue Ops[] = {Chain, Ptr};
11585 FoldingSetNodeID ID;
11586 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
11587 ID.AddInteger(I: MemVT.getRawBits());
11588 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11589 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11590 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11591 ID.AddInteger(I: MMO->getFlags());
11592 void *IP = nullptr;
11593 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11594 return SDValue(E, 0);
11595
11596 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11597 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11598 createOperands(Node: N, Vals: Ops);
11599
11600 CSEMap.InsertNode(N, InsertPos: IP);
11601 InsertNode(N);
11602 SDValue V(N, 0);
11603 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11604 return V;
11605}
11606
11607SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11608 // select undef, T, F --> T (if T is a constant), otherwise F
11609 // select, ?, undef, F --> F
11610 // select, ?, T, undef --> T
11611 if (Cond.isUndef())
11612 return isConstantValueOfAnyType(N: T) ? T : F;
11613 if (T.isUndef())
11614 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11615 if (F.isUndef())
11616 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11617
11618 // select true, T, F --> T
11619 // select false, T, F --> F
11620 if (auto C = isBoolConstant(N: Cond))
11621 return *C ? T : F;
11622
11623 // select ?, T, T --> T
11624 if (T == F)
11625 return T;
11626
11627 return SDValue();
11628}
11629
11630SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11631 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11632 if (X.isUndef())
11633 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11634 // shift X, undef --> undef (because it may shift by the bitwidth)
11635 if (Y.isUndef())
11636 return getUNDEF(VT: X.getValueType());
11637
11638 // shift 0, Y --> 0
11639 // shift X, 0 --> X
11640 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11641 return X;
11642
11643 // shift X, C >= bitwidth(X) --> undef
11644 // All vector elements must be too big (or undef) to avoid partial undefs.
11645 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11646 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11647 };
11648 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11649 return getUNDEF(VT: X.getValueType());
11650
11651 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11652 if (X.getValueType().getScalarType() == MVT::i1)
11653 return X;
11654
11655 return SDValue();
11656}
11657
11658SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11659 SDNodeFlags Flags) {
11660 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11661 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11662 // operation is poison. That result can be relaxed to undef.
11663 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11664 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11665 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11666 (YC && YC->getValueAPF().isNaN());
11667 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11668 (YC && YC->getValueAPF().isInfinity());
11669
11670 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11671 return getUNDEF(VT: X.getValueType());
11672
11673 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11674 return getUNDEF(VT: X.getValueType());
11675
11676 if (!YC)
11677 return SDValue();
11678
11679 // X + -0.0 --> X
11680 if (Opcode == ISD::FADD)
11681 if (YC->getValueAPF().isNegZero())
11682 return X;
11683
11684 // X - +0.0 --> X
11685 if (Opcode == ISD::FSUB)
11686 if (YC->getValueAPF().isPosZero())
11687 return X;
11688
11689 // X * 1.0 --> X
11690 // X / 1.0 --> X
11691 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11692 if (YC->getValueAPF().isOne())
11693 return X;
11694
11695 // X * 0.0 --> 0.0
11696 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11697 if (YC->getValueAPF().isZero())
11698 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11699
11700 return SDValue();
11701}
11702
11703SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11704 SDValue Ptr, SDValue SV, unsigned Align) {
11705 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11706 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11707}
11708
11709SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11710 ArrayRef<SDUse> Ops) {
11711 switch (Ops.size()) {
11712 case 0: return getNode(Opcode, DL, VT);
11713 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11714 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11715 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11716 default: break;
11717 }
11718
11719 // Copy from an SDUse array into an SDValue array for use with
11720 // the regular getNode logic.
11721 SmallVector<SDValue, 8> NewOps(Ops);
11722 return getNode(Opcode, DL, VT, Ops: NewOps);
11723}
11724
11725SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11726 ArrayRef<SDValue> Ops) {
11727 SDNodeFlags Flags;
11728 if (Inserter)
11729 Flags = Inserter->getFlags();
11730 return getNode(Opcode, DL, VT, Ops, Flags);
11731}
11732
11733SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11734 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11735 unsigned NumOps = Ops.size();
11736 switch (NumOps) {
11737 case 0: return getNode(Opcode, DL, VT);
11738 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11739 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11740 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11741 default: break;
11742 }
11743
11744#ifndef NDEBUG
11745 for (const auto &Op : Ops)
11746 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11747 "Operand is DELETED_NODE!");
11748#endif
11749
11750 switch (Opcode) {
11751 default: break;
11752 case ISD::BUILD_VECTOR:
11753 // Attempt to simplify BUILD_VECTOR.
11754 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11755 return V;
11756 break;
11757 case ISD::CONCAT_VECTORS:
11758 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11759 return V;
11760 break;
11761 case ISD::SELECT_CC:
11762 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11763 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11764 "LHS and RHS of condition must have same type!");
11765 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11766 "True and False arms of SelectCC must have same type!");
11767 assert(Ops[2].getValueType() == VT &&
11768 "select_cc node must be of same type as true and false value!");
11769 assert((!Ops[0].getValueType().isVector() ||
11770 Ops[0].getValueType().getVectorElementCount() ==
11771 VT.getVectorElementCount()) &&
11772 "Expected select_cc with vector result to have the same sized "
11773 "comparison type!");
11774 break;
11775 case ISD::BR_CC:
11776 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11777 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11778 "LHS/RHS of comparison should match types!");
11779 break;
11780 case ISD::VP_ADD:
11781 case ISD::VP_SUB:
11782 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
11783 if (VT.getScalarType() == MVT::i1)
11784 Opcode = ISD::VP_XOR;
11785 break;
11786 case ISD::VP_MUL:
11787 // If it is VP_MUL mask operation then turn it to VP_AND
11788 if (VT.getScalarType() == MVT::i1)
11789 Opcode = ISD::VP_AND;
11790 break;
11791 case ISD::VP_REDUCE_MUL:
11792 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11793 if (VT == MVT::i1)
11794 Opcode = ISD::VP_REDUCE_AND;
11795 break;
11796 case ISD::VP_REDUCE_ADD:
11797 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11798 if (VT == MVT::i1)
11799 Opcode = ISD::VP_REDUCE_XOR;
11800 break;
11801 case ISD::VP_REDUCE_SMAX:
11802 case ISD::VP_REDUCE_UMIN:
11803 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11804 // VP_REDUCE_AND.
11805 if (VT == MVT::i1)
11806 Opcode = ISD::VP_REDUCE_AND;
11807 break;
11808 case ISD::VP_REDUCE_SMIN:
11809 case ISD::VP_REDUCE_UMAX:
11810 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11811 // VP_REDUCE_OR.
11812 if (VT == MVT::i1)
11813 Opcode = ISD::VP_REDUCE_OR;
11814 break;
11815 }
11816
11817 // Memoize nodes.
11818 SDNode *N;
11819 SDVTList VTs = getVTList(VT);
11820
11821 if (VT != MVT::Glue) {
11822 FoldingSetNodeID ID;
11823 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
11824 void *IP = nullptr;
11825
11826 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11827 E->intersectFlagsWith(Flags);
11828 return SDValue(E, 0);
11829 }
11830
11831 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11832 createOperands(Node: N, Vals: Ops);
11833
11834 CSEMap.InsertNode(N, InsertPos: IP);
11835 } else {
11836 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11837 createOperands(Node: N, Vals: Ops);
11838 }
11839
11840 N->setFlags(Flags);
11841 InsertNode(N);
11842 SDValue V(N, 0);
11843 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11844 return V;
11845}
11846
11847SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11848 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
11849 SDNodeFlags Flags;
11850 if (Inserter)
11851 Flags = Inserter->getFlags();
11852 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11853}
11854
11855SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11856 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
11857 const SDNodeFlags Flags) {
11858 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11859}
11860
11861SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11862 ArrayRef<SDValue> Ops) {
11863 SDNodeFlags Flags;
11864 if (Inserter)
11865 Flags = Inserter->getFlags();
11866 return getNode(Opcode, DL, VTList, Ops, Flags);
11867}
11868
11869SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11870 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11871 if (VTList.NumVTs == 1)
11872 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
11873
11874#ifndef NDEBUG
11875 for (const auto &Op : Ops)
11876 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11877 "Operand is DELETED_NODE!");
11878#endif
11879
11880 switch (Opcode) {
11881 case ISD::SADDO:
11882 case ISD::UADDO:
11883 case ISD::SSUBO:
11884 case ISD::USUBO: {
11885 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11886 "Invalid add/sub overflow op!");
11887 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11888 Ops[0].getValueType() == Ops[1].getValueType() &&
11889 Ops[0].getValueType() == VTList.VTs[0] &&
11890 "Binary operator types must match!");
11891 SDValue N1 = Ops[0], N2 = Ops[1];
11892 canonicalizeCommutativeBinop(Opcode, N1, N2);
11893
11894 // (X +- 0) -> X with zero-overflow.
11895 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
11896 /*AllowTruncation*/ true);
11897 if (N2CV && N2CV->isZero()) {
11898 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
11899 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
11900 }
11901
11902 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
11903 VTList.VTs[1].getScalarType() == MVT::i1) {
11904 SDValue F1 = getFreeze(V: N1);
11905 SDValue F2 = getFreeze(V: N2);
11906 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
11907 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
11908 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11909 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11910 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
11911 Flags);
11912 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
11913 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
11914 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
11915 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11916 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11917 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
11918 Flags);
11919 }
11920 }
11921 break;
11922 }
11923 case ISD::SADDO_CARRY:
11924 case ISD::UADDO_CARRY:
11925 case ISD::SSUBO_CARRY:
11926 case ISD::USUBO_CARRY:
11927 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
11928 "Invalid add/sub overflow op!");
11929 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11930 Ops[0].getValueType() == Ops[1].getValueType() &&
11931 Ops[0].getValueType() == VTList.VTs[0] &&
11932 Ops[2].getValueType() == VTList.VTs[1] &&
11933 "Binary operator types must match!");
11934 break;
11935 case ISD::SMUL_LOHI:
11936 case ISD::UMUL_LOHI: {
11937 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
11938 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
11939 VTList.VTs[0] == Ops[0].getValueType() &&
11940 VTList.VTs[0] == Ops[1].getValueType() &&
11941 "Binary operator types must match!");
11942 // Constant fold.
11943 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
11944 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
11945 if (LHS && RHS) {
11946 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
11947 unsigned OutWidth = Width * 2;
11948 APInt Val = LHS->getAPIntValue();
11949 APInt Mul = RHS->getAPIntValue();
11950 if (Opcode == ISD::SMUL_LOHI) {
11951 Val = Val.sext(width: OutWidth);
11952 Mul = Mul.sext(width: OutWidth);
11953 } else {
11954 Val = Val.zext(width: OutWidth);
11955 Mul = Mul.zext(width: OutWidth);
11956 }
11957 Val *= Mul;
11958
11959 SDValue Hi =
11960 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
11961 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
11962 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
11963 }
11964 break;
11965 }
11966 case ISD::FFREXP: {
11967 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
11968 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
11969 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
11970
11971 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
11972 int FrexpExp;
11973 APFloat FrexpMant =
11974 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
11975 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
11976 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
11977 DL, VT: VTList.VTs[1]);
11978 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
11979 }
11980
11981 break;
11982 }
11983 case ISD::STRICT_FP_EXTEND:
11984 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11985 "Invalid STRICT_FP_EXTEND!");
11986 assert(VTList.VTs[0].isFloatingPoint() &&
11987 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
11988 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11989 "STRICT_FP_EXTEND result type should be vector iff the operand "
11990 "type is vector!");
11991 assert((!VTList.VTs[0].isVector() ||
11992 VTList.VTs[0].getVectorElementCount() ==
11993 Ops[1].getValueType().getVectorElementCount()) &&
11994 "Vector element count mismatch!");
11995 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
11996 "Invalid fpext node, dst <= src!");
11997 break;
11998 case ISD::STRICT_FP_ROUND:
11999 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
12000 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
12001 "STRICT_FP_ROUND result type should be vector iff the operand "
12002 "type is vector!");
12003 assert((!VTList.VTs[0].isVector() ||
12004 VTList.VTs[0].getVectorElementCount() ==
12005 Ops[1].getValueType().getVectorElementCount()) &&
12006 "Vector element count mismatch!");
12007 assert(VTList.VTs[0].isFloatingPoint() &&
12008 Ops[1].getValueType().isFloatingPoint() &&
12009 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
12010 Ops[2].getOpcode() == ISD::TargetConstant &&
12011 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
12012 "Invalid STRICT_FP_ROUND!");
12013 break;
12014 }
12015
12016 // Memoize the node unless it returns a glue result.
12017 SDNode *N;
12018 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
12019 FoldingSetNodeID ID;
12020 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12021 void *IP = nullptr;
12022 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
12023 E->intersectFlagsWith(Flags);
12024 return SDValue(E, 0);
12025 }
12026
12027 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12028 createOperands(Node: N, Vals: Ops);
12029 CSEMap.InsertNode(N, InsertPos: IP);
12030 } else {
12031 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
12032 createOperands(Node: N, Vals: Ops);
12033 }
12034
12035 N->setFlags(Flags);
12036 InsertNode(N);
12037 SDValue V(N, 0);
12038 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
12039 return V;
12040}
12041
12042SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
12043 SDVTList VTList) {
12044 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
12045}
12046
12047SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12048 SDValue N1) {
12049 SDValue Ops[] = { N1 };
12050 return getNode(Opcode, DL, VTList, Ops);
12051}
12052
12053SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12054 SDValue N1, SDValue N2) {
12055 SDValue Ops[] = { N1, N2 };
12056 return getNode(Opcode, DL, VTList, Ops);
12057}
12058
12059SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12060 SDValue N1, SDValue N2, SDValue N3) {
12061 SDValue Ops[] = { N1, N2, N3 };
12062 return getNode(Opcode, DL, VTList, Ops);
12063}
12064
12065SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12066 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
12067 SDValue Ops[] = { N1, N2, N3, N4 };
12068 return getNode(Opcode, DL, VTList, Ops);
12069}
12070
12071SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
12072 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
12073 SDValue N5) {
12074 SDValue Ops[] = { N1, N2, N3, N4, N5 };
12075 return getNode(Opcode, DL, VTList, Ops);
12076}
12077
12078SDVTList SelectionDAG::getVTList(EVT VT) {
12079 if (!VT.isExtended())
12080 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
12081
12082 return makeVTList(VTs: &(*EVTs.insert(x: VT).first), NumVTs: 1);
12083}
12084
12085SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
12086 FoldingSetNodeID ID;
12087 ID.AddInteger(I: 2U);
12088 ID.AddInteger(I: VT1.getRawBits());
12089 ID.AddInteger(I: VT2.getRawBits());
12090
12091 void *IP = nullptr;
12092 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
12093 if (!Result) {
12094 EVT *Array = Allocator.Allocate<EVT>(Num: 2);
12095 Array[0] = VT1;
12096 Array[1] = VT2;
12097 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
12098 VTListMap.InsertNode(N: Result, InsertPos: IP);
12099 }
12100 return Result->getSDVTList();
12101}
12102
12103SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
12104 FoldingSetNodeID ID;
12105 ID.AddInteger(I: 3U);
12106 ID.AddInteger(I: VT1.getRawBits());
12107 ID.AddInteger(I: VT2.getRawBits());
12108 ID.AddInteger(I: VT3.getRawBits());
12109
12110 void *IP = nullptr;
12111 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
12112 if (!Result) {
12113 EVT *Array = Allocator.Allocate<EVT>(Num: 3);
12114 Array[0] = VT1;
12115 Array[1] = VT2;
12116 Array[2] = VT3;
12117 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
12118 VTListMap.InsertNode(N: Result, InsertPos: IP);
12119 }
12120 return Result->getSDVTList();
12121}
12122
12123SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
12124 FoldingSetNodeID ID;
12125 ID.AddInteger(I: 4U);
12126 ID.AddInteger(I: VT1.getRawBits());
12127 ID.AddInteger(I: VT2.getRawBits());
12128 ID.AddInteger(I: VT3.getRawBits());
12129 ID.AddInteger(I: VT4.getRawBits());
12130
12131 void *IP = nullptr;
12132 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
12133 if (!Result) {
12134 EVT *Array = Allocator.Allocate<EVT>(Num: 4);
12135 Array[0] = VT1;
12136 Array[1] = VT2;
12137 Array[2] = VT3;
12138 Array[3] = VT4;
12139 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
12140 VTListMap.InsertNode(N: Result, InsertPos: IP);
12141 }
12142 return Result->getSDVTList();
12143}
12144
12145SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
12146 unsigned NumVTs = VTs.size();
12147 FoldingSetNodeID ID;
12148 ID.AddInteger(I: NumVTs);
12149 for (unsigned index = 0; index < NumVTs; index++) {
12150 ID.AddInteger(I: VTs[index].getRawBits());
12151 }
12152
12153 void *IP = nullptr;
12154 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
12155 if (!Result) {
12156 EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
12157 llvm::copy(Range&: VTs, Out: Array);
12158 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
12159 VTListMap.InsertNode(N: Result, InsertPos: IP);
12160 }
12161 return Result->getSDVTList();
12162}
12163
12164
12165/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
12166/// specified operands. If the resultant node already exists in the DAG,
12167/// this does not modify the specified node, instead it returns the node that
12168/// already exists. If the resultant node does not exist in the DAG, the
12169/// input node is returned. As a degenerate case, if you specify the same
12170/// input operands as the node already has, the input node is returned.
12171SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
12172 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
12173
12174 // Check to see if there is no change.
12175 if (Op == N->getOperand(Num: 0)) return N;
12176
12177 // See if the modified node already exists.
12178 void *InsertPos = nullptr;
12179 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
12180 return Existing;
12181
12182 // Nope it doesn't. Remove the node from its current place in the maps.
12183 if (InsertPos)
12184 if (!RemoveNodeFromCSEMaps(N))
12185 InsertPos = nullptr;
12186
12187 // Now we update the operands.
12188 N->OperandList[0].set(Op);
12189
12190 updateDivergence(N);
12191 // If this gets put into a CSE map, add it.
12192 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
12193 return N;
12194}
12195
12196SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
12197 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
12198
12199 // Check to see if there is no change.
12200 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
12201 return N; // No operands changed, just return the input node.
12202
12203 // See if the modified node already exists.
12204 void *InsertPos = nullptr;
12205 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
12206 return Existing;
12207
12208 // Nope it doesn't. Remove the node from its current place in the maps.
12209 if (InsertPos)
12210 if (!RemoveNodeFromCSEMaps(N))
12211 InsertPos = nullptr;
12212
12213 // Now we update the operands.
12214 if (N->OperandList[0] != Op1)
12215 N->OperandList[0].set(Op1);
12216 if (N->OperandList[1] != Op2)
12217 N->OperandList[1].set(Op2);
12218
12219 updateDivergence(N);
12220 // If this gets put into a CSE map, add it.
12221 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
12222 return N;
12223}
12224
12225SDNode *SelectionDAG::
12226UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
12227 SDValue Ops[] = { Op1, Op2, Op3 };
12228 return UpdateNodeOperands(N, Ops);
12229}
12230
12231SDNode *SelectionDAG::
12232UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12233 SDValue Op3, SDValue Op4) {
12234 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
12235 return UpdateNodeOperands(N, Ops);
12236}
12237
12238SDNode *SelectionDAG::
12239UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
12240 SDValue Op3, SDValue Op4, SDValue Op5) {
12241 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
12242 return UpdateNodeOperands(N, Ops);
12243}
12244
12245SDNode *SelectionDAG::
12246UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
12247 unsigned NumOps = Ops.size();
12248 assert(N->getNumOperands() == NumOps &&
12249 "Update with wrong number of operands");
12250
12251 // If no operands changed just return the input node.
12252 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
12253 return N;
12254
12255 // See if the modified node already exists.
12256 void *InsertPos = nullptr;
12257 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
12258 return Existing;
12259
12260 // Nope it doesn't. Remove the node from its current place in the maps.
12261 if (InsertPos)
12262 if (!RemoveNodeFromCSEMaps(N))
12263 InsertPos = nullptr;
12264
12265 // Now we update the operands.
12266 for (unsigned i = 0; i != NumOps; ++i)
12267 if (N->OperandList[i] != Ops[i])
12268 N->OperandList[i].set(Ops[i]);
12269
12270 updateDivergence(N);
12271 // If this gets put into a CSE map, add it.
12272 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
12273 return N;
12274}
12275
12276/// DropOperands - Release the operands and set this node to have
12277/// zero operands.
12278void SDNode::DropOperands() {
12279 // Unlike the code in MorphNodeTo that does this, we don't need to
12280 // watch for dead nodes here.
12281 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
12282 SDUse &Use = *I++;
12283 Use.set(SDValue());
12284 }
12285}
12286
12287void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
12288 ArrayRef<MachineMemOperand *> NewMemRefs) {
12289 if (NewMemRefs.empty()) {
12290 N->clearMemRefs();
12291 return;
12292 }
12293
12294 // Check if we can avoid allocating by storing a single reference directly.
12295 if (NewMemRefs.size() == 1) {
12296 N->MemRefs = NewMemRefs[0];
12297 N->NumMemRefs = 1;
12298 return;
12299 }
12300
12301 MachineMemOperand **MemRefsBuffer =
12302 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
12303 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
12304 N->MemRefs = MemRefsBuffer;
12305 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
12306}
12307
12308/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
12309/// machine opcode.
12310///
12311SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12312 EVT VT) {
12313 SDVTList VTs = getVTList(VT);
12314 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12315}
12316
12317SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12318 EVT VT, SDValue Op1) {
12319 SDVTList VTs = getVTList(VT);
12320 SDValue Ops[] = { Op1 };
12321 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12322}
12323
12324SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12325 EVT VT, SDValue Op1,
12326 SDValue Op2) {
12327 SDVTList VTs = getVTList(VT);
12328 SDValue Ops[] = { Op1, Op2 };
12329 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12330}
12331
12332SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12333 EVT VT, SDValue Op1,
12334 SDValue Op2, SDValue Op3) {
12335 SDVTList VTs = getVTList(VT);
12336 SDValue Ops[] = { Op1, Op2, Op3 };
12337 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12338}
12339
12340SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12341 EVT VT, ArrayRef<SDValue> Ops) {
12342 SDVTList VTs = getVTList(VT);
12343 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12344}
12345
12346SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12347 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
12348 SDVTList VTs = getVTList(VT1, VT2);
12349 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12350}
12351
12352SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12353 EVT VT1, EVT VT2) {
12354 SDVTList VTs = getVTList(VT1, VT2);
12355 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
12356}
12357
12358SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12359 EVT VT1, EVT VT2, EVT VT3,
12360 ArrayRef<SDValue> Ops) {
12361 SDVTList VTs = getVTList(VT1, VT2, VT3);
12362 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12363}
12364
12365SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12366 EVT VT1, EVT VT2,
12367 SDValue Op1, SDValue Op2) {
12368 SDVTList VTs = getVTList(VT1, VT2);
12369 SDValue Ops[] = { Op1, Op2 };
12370 return SelectNodeTo(N, MachineOpc, VTs, Ops);
12371}
12372
12373SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
12374 SDVTList VTs,ArrayRef<SDValue> Ops) {
12375 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
12376 // Reset the NodeID to -1.
12377 New->setNodeId(-1);
12378 if (New != N) {
12379 ReplaceAllUsesWith(From: N, To: New);
12380 RemoveDeadNode(N);
12381 }
12382 return New;
12383}
12384
12385/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
12386/// the line number information on the merged node since it is not possible to
12387/// preserve the information that operation is associated with multiple lines.
12388/// This will make the debugger working better at -O0, were there is a higher
12389/// probability having other instructions associated with that line.
12390///
12391/// For IROrder, we keep the smaller of the two
12392SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
12393 DebugLoc NLoc = N->getDebugLoc();
12394 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
12395 N->setDebugLoc(DebugLoc());
12396 }
12397 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
12398 N->setIROrder(Order);
12399 return N;
12400}
12401
12402/// MorphNodeTo - This *mutates* the specified node to have the specified
12403/// return type, opcode, and operands.
12404///
12405/// Note that MorphNodeTo returns the resultant node. If there is already a
12406/// node of the specified opcode and operands, it returns that node instead of
12407/// the current one. Note that the SDLoc need not be the same.
12408///
12409/// Using MorphNodeTo is faster than creating a new node and swapping it in
12410/// with ReplaceAllUsesWith both because it often avoids allocating a new
12411/// node, and because it doesn't require CSE recalculation for any of
12412/// the node's users.
12413///
12414/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
12415/// As a consequence it isn't appropriate to use from within the DAG combiner or
12416/// the legalizer which maintain worklists that would need to be updated when
12417/// deleting things.
12418SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
12419 SDVTList VTs, ArrayRef<SDValue> Ops) {
12420 // If an identical node already exists, use it.
12421 void *IP = nullptr;
12422 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
12423 FoldingSetNodeID ID;
12424 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
12425 if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
12426 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
12427 }
12428
12429 if (!RemoveNodeFromCSEMaps(N))
12430 IP = nullptr;
12431
12432 // Start the morphing.
12433 N->NodeType = Opc;
12434 N->ValueList = VTs.VTs;
12435 N->NumValues = VTs.NumVTs;
12436
12437 // Clear the operands list, updating used nodes to remove this from their
12438 // use list. Keep track of any operands that become dead as a result.
12439 SmallPtrSet<SDNode*, 16> DeadNodeSet;
12440 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
12441 SDUse &Use = *I++;
12442 SDNode *Used = Use.getNode();
12443 Use.set(SDValue());
12444 if (Used->use_empty())
12445 DeadNodeSet.insert(Ptr: Used);
12446 }
12447
12448 // For MachineNode, initialize the memory references information.
12449 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12450 MN->clearMemRefs();
12451
12452 // Swap for an appropriately sized array from the recycler.
12453 removeOperands(Node: N);
12454 createOperands(Node: N, Vals: Ops);
12455
12456 // Delete any nodes that are still dead after adding the uses for the
12457 // new operands.
12458 if (!DeadNodeSet.empty()) {
12459 SmallVector<SDNode *, 16> DeadNodes;
12460 for (SDNode *N : DeadNodeSet)
12461 if (N->use_empty())
12462 DeadNodes.push_back(Elt: N);
12463 RemoveDeadNodes(DeadNodes);
12464 }
12465
12466 if (IP)
12467 CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
12468 return N;
12469}
12470
12471SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12472 unsigned OrigOpc = Node->getOpcode();
12473 unsigned NewOpc;
12474 switch (OrigOpc) {
12475 default:
12476 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12477#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12478 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12479#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12480 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12481#include "llvm/IR/ConstrainedOps.def"
12482 }
12483
12484 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12485
12486 // We're taking this node out of the chain, so we need to re-link things.
12487 SDValue InputChain = Node->getOperand(Num: 0);
12488 SDValue OutputChain = SDValue(Node, 1);
12489 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12490
12491 SmallVector<SDValue, 3> Ops;
12492 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12493 Ops.push_back(Elt: Node->getOperand(Num: i));
12494
12495 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12496 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12497
12498 // MorphNodeTo can operate in two ways: if an existing node with the
12499 // specified operands exists, it can just return it. Otherwise, it
12500 // updates the node in place to have the requested operands.
12501 if (Res == Node) {
12502 // If we updated the node in place, reset the node ID. To the isel,
12503 // this should be just like a newly allocated machine node.
12504 Res->setNodeId(-1);
12505 } else {
12506 ReplaceAllUsesWith(From: Node, To: Res);
12507 RemoveDeadNode(N: Node);
12508 }
12509
12510 return Res;
12511}
12512
12513/// getMachineNode - These are used for target selectors to create a new node
12514/// with specified return type(s), MachineInstr opcode, and operands.
12515///
12516/// Note that getMachineNode returns the resultant node. If there is already a
12517/// node of the specified opcode and operands, it returns that node instead of
12518/// the current one.
12519MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12520 EVT VT) {
12521 SDVTList VTs = getVTList(VT);
12522 return getMachineNode(Opcode, dl, VTs, Ops: {});
12523}
12524
12525MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12526 EVT VT, SDValue Op1) {
12527 SDVTList VTs = getVTList(VT);
12528 SDValue Ops[] = { Op1 };
12529 return getMachineNode(Opcode, dl, VTs, Ops);
12530}
12531
12532MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12533 EVT VT, SDValue Op1, SDValue Op2) {
12534 SDVTList VTs = getVTList(VT);
12535 SDValue Ops[] = { Op1, Op2 };
12536 return getMachineNode(Opcode, dl, VTs, Ops);
12537}
12538
12539MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12540 EVT VT, SDValue Op1, SDValue Op2,
12541 SDValue Op3) {
12542 SDVTList VTs = getVTList(VT);
12543 SDValue Ops[] = { Op1, Op2, Op3 };
12544 return getMachineNode(Opcode, dl, VTs, Ops);
12545}
12546
12547MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12548 EVT VT, ArrayRef<SDValue> Ops) {
12549 SDVTList VTs = getVTList(VT);
12550 return getMachineNode(Opcode, dl, VTs, Ops);
12551}
12552
12553MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12554 EVT VT1, EVT VT2, SDValue Op1,
12555 SDValue Op2) {
12556 SDVTList VTs = getVTList(VT1, VT2);
12557 SDValue Ops[] = { Op1, Op2 };
12558 return getMachineNode(Opcode, dl, VTs, Ops);
12559}
12560
12561MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12562 EVT VT1, EVT VT2, SDValue Op1,
12563 SDValue Op2, SDValue Op3) {
12564 SDVTList VTs = getVTList(VT1, VT2);
12565 SDValue Ops[] = { Op1, Op2, Op3 };
12566 return getMachineNode(Opcode, dl, VTs, Ops);
12567}
12568
12569MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12570 EVT VT1, EVT VT2,
12571 ArrayRef<SDValue> Ops) {
12572 SDVTList VTs = getVTList(VT1, VT2);
12573 return getMachineNode(Opcode, dl, VTs, Ops);
12574}
12575
12576MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12577 EVT VT1, EVT VT2, EVT VT3,
12578 SDValue Op1, SDValue Op2) {
12579 SDVTList VTs = getVTList(VT1, VT2, VT3);
12580 SDValue Ops[] = { Op1, Op2 };
12581 return getMachineNode(Opcode, dl, VTs, Ops);
12582}
12583
12584MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12585 EVT VT1, EVT VT2, EVT VT3,
12586 SDValue Op1, SDValue Op2,
12587 SDValue Op3) {
12588 SDVTList VTs = getVTList(VT1, VT2, VT3);
12589 SDValue Ops[] = { Op1, Op2, Op3 };
12590 return getMachineNode(Opcode, dl, VTs, Ops);
12591}
12592
12593MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12594 EVT VT1, EVT VT2, EVT VT3,
12595 ArrayRef<SDValue> Ops) {
12596 SDVTList VTs = getVTList(VT1, VT2, VT3);
12597 return getMachineNode(Opcode, dl, VTs, Ops);
12598}
12599
12600MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12601 ArrayRef<EVT> ResultTys,
12602 ArrayRef<SDValue> Ops) {
12603 SDVTList VTs = getVTList(VTs: ResultTys);
12604 return getMachineNode(Opcode, dl, VTs, Ops);
12605}
12606
12607MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12608 SDVTList VTs,
12609 ArrayRef<SDValue> Ops) {
12610 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12611 MachineSDNode *N;
12612 void *IP = nullptr;
12613
12614 if (DoCSE) {
12615 FoldingSetNodeID ID;
12616 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
12617 IP = nullptr;
12618 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
12619 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12620 }
12621 }
12622
12623 // Allocate a new MachineSDNode.
12624 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12625 createOperands(Node: N, Vals: Ops);
12626
12627 if (DoCSE)
12628 CSEMap.InsertNode(N, InsertPos: IP);
12629
12630 InsertNode(N);
12631 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12632 return N;
12633}
12634
12635/// getTargetExtractSubreg - A convenience function for creating
12636/// TargetOpcode::EXTRACT_SUBREG nodes.
12637SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12638 SDValue Operand) {
12639 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12640 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12641 VT, Op1: Operand, Op2: SRIdxVal);
12642 return SDValue(Subreg, 0);
12643}
12644
12645/// getTargetInsertSubreg - A convenience function for creating
12646/// TargetOpcode::INSERT_SUBREG nodes.
12647SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12648 SDValue Operand, SDValue Subreg) {
12649 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12650 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12651 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12652 return SDValue(Result, 0);
12653}
12654
12655/// getNodeIfExists - Get the specified node if it's already available, or
12656/// else return NULL.
12657SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12658 ArrayRef<SDValue> Ops,
12659 bool AllowCommute) {
12660 SDNodeFlags Flags;
12661 if (Inserter)
12662 Flags = Inserter->getFlags();
12663 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12664}
12665
12666SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12667 ArrayRef<SDValue> Ops,
12668 const SDNodeFlags Flags,
12669 bool AllowCommute) {
12670 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12671 return nullptr;
12672
12673 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12674 FoldingSetNodeID ID;
12675 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: LookupOps);
12676 void *IP = nullptr;
12677 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) {
12678 E->intersectFlagsWith(Flags);
12679 return E;
12680 }
12681 return nullptr;
12682 };
12683
12684 if (SDNode *Existing = Lookup(Ops))
12685 return Existing;
12686
12687 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12688 return Lookup({Ops[1], Ops[0]});
12689
12690 return nullptr;
12691}
12692
12693/// doesNodeExist - Check if a node exists without modifying its flags.
12694bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12695 ArrayRef<SDValue> Ops) {
12696 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12697 FoldingSetNodeID ID;
12698 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12699 void *IP = nullptr;
12700 if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
12701 return true;
12702 }
12703 return false;
12704}
12705
12706/// getDbgValue - Creates a SDDbgValue node.
12707///
12708/// SDNode
12709SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12710 SDNode *N, unsigned R, bool IsIndirect,
12711 const DebugLoc &DL, unsigned O) {
12712 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12713 "Expected inlined-at fields to agree");
12714 return new (DbgInfo->getAlloc())
12715 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12716 {}, IsIndirect, DL, O,
12717 /*IsVariadic=*/false);
12718}
12719
12720/// Constant
12721SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12722 DIExpression *Expr,
12723 const Value *C,
12724 const DebugLoc &DL, unsigned O) {
12725 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12726 "Expected inlined-at fields to agree");
12727 return new (DbgInfo->getAlloc())
12728 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12729 /*IsIndirect=*/false, DL, O,
12730 /*IsVariadic=*/false);
12731}
12732
12733/// FrameIndex
12734SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12735 DIExpression *Expr, unsigned FI,
12736 bool IsIndirect,
12737 const DebugLoc &DL,
12738 unsigned O) {
12739 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12740 "Expected inlined-at fields to agree");
12741 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12742}
12743
12744/// FrameIndex with dependencies
12745SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12746 DIExpression *Expr, unsigned FI,
12747 ArrayRef<SDNode *> Dependencies,
12748 bool IsIndirect,
12749 const DebugLoc &DL,
12750 unsigned O) {
12751 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12752 "Expected inlined-at fields to agree");
12753 return new (DbgInfo->getAlloc())
12754 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12755 Dependencies, IsIndirect, DL, O,
12756 /*IsVariadic=*/false);
12757}
12758
12759/// VReg
12760SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12761 Register VReg, bool IsIndirect,
12762 const DebugLoc &DL, unsigned O) {
12763 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12764 "Expected inlined-at fields to agree");
12765 return new (DbgInfo->getAlloc())
12766 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12767 {}, IsIndirect, DL, O,
12768 /*IsVariadic=*/false);
12769}
12770
12771SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12772 ArrayRef<SDDbgOperand> Locs,
12773 ArrayRef<SDNode *> Dependencies,
12774 bool IsIndirect, const DebugLoc &DL,
12775 unsigned O, bool IsVariadic) {
12776 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12777 "Expected inlined-at fields to agree");
12778 return new (DbgInfo->getAlloc())
12779 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12780 DL, O, IsVariadic);
12781}
12782
12783void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12784 unsigned OffsetInBits, unsigned SizeInBits,
12785 bool InvalidateDbg) {
12786 SDNode *FromNode = From.getNode();
12787 SDNode *ToNode = To.getNode();
12788 assert(FromNode && ToNode && "Can't modify dbg values");
12789
12790 // PR35338
12791 // TODO: assert(From != To && "Redundant dbg value transfer");
12792 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12793 if (From == To || FromNode == ToNode)
12794 return;
12795
12796 if (!FromNode->getHasDebugValue())
12797 return;
12798
12799 SDDbgOperand FromLocOp =
12800 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12801 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12802
12803 SmallVector<SDDbgValue *, 2> ClonedDVs;
12804 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12805 if (Dbg->isInvalidated())
12806 continue;
12807
12808 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12809
12810 // Create a new location ops vector that is equal to the old vector, but
12811 // with each instance of FromLocOp replaced with ToLocOp.
12812 bool Changed = false;
12813 auto NewLocOps = Dbg->copyLocationOps();
12814 std::replace_if(
12815 first: NewLocOps.begin(), last: NewLocOps.end(),
12816 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12817 bool Match = Op == FromLocOp;
12818 Changed |= Match;
12819 return Match;
12820 },
12821 new_value: ToLocOp);
12822 // Ignore this SDDbgValue if we didn't find a matching location.
12823 if (!Changed)
12824 continue;
12825
12826 DIVariable *Var = Dbg->getVariable();
12827 auto *Expr = Dbg->getExpression();
12828 // If a fragment is requested, update the expression.
12829 if (SizeInBits) {
12830 // When splitting a larger (e.g., sign-extended) value whose
12831 // lower bits are described with an SDDbgValue, do not attempt
12832 // to transfer the SDDbgValue to the upper bits.
12833 if (auto FI = Expr->getFragmentInfo())
12834 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12835 continue;
12836 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12837 SizeInBits);
12838 if (!Fragment)
12839 continue;
12840 Expr = *Fragment;
12841 }
12842
12843 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12844 // Clone the SDDbgValue and move it to To.
12845 SDDbgValue *Clone = getDbgValueList(
12846 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12847 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12848 IsVariadic: Dbg->isVariadic());
12849 ClonedDVs.push_back(Elt: Clone);
12850
12851 if (InvalidateDbg) {
12852 // Invalidate value and indicate the SDDbgValue should not be emitted.
12853 Dbg->setIsInvalidated();
12854 Dbg->setIsEmitted();
12855 }
12856 }
12857
12858 for (SDDbgValue *Dbg : ClonedDVs) {
12859 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12860 "Transferred DbgValues should depend on the new SDNode");
12861 AddDbgValue(DB: Dbg, isParameter: false);
12862 }
12863}
12864
12865void SelectionDAG::salvageDebugInfo(SDNode &N) {
12866 if (!N.getHasDebugValue())
12867 return;
12868
12869 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
12870 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
12871 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
12872 return SDDbgOperand::fromNode(Node, ResNo);
12873 };
12874
12875 SmallVector<SDDbgValue *, 2> ClonedDVs;
12876 for (auto *DV : GetDbgValues(SD: &N)) {
12877 if (DV->isInvalidated())
12878 continue;
12879 switch (N.getOpcode()) {
12880 default:
12881 break;
12882 case ISD::ADD: {
12883 SDValue N0 = N.getOperand(Num: 0);
12884 SDValue N1 = N.getOperand(Num: 1);
12885 if (!isa<ConstantSDNode>(Val: N0)) {
12886 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
12887 uint64_t Offset;
12888 if (RHSConstant)
12889 Offset = N.getConstantOperandVal(Num: 1);
12890 // We are not allowed to turn indirect debug values variadic, so
12891 // don't salvage those.
12892 if (!RHSConstant && DV->isIndirect())
12893 continue;
12894
12895 // Rewrite an ADD constant node into a DIExpression. Since we are
12896 // performing arithmetic to compute the variable's *value* in the
12897 // DIExpression, we need to mark the expression with a
12898 // DW_OP_stack_value.
12899 auto *DIExpr = DV->getExpression();
12900 auto NewLocOps = DV->copyLocationOps();
12901 bool Changed = false;
12902 size_t OrigLocOpsSize = NewLocOps.size();
12903 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
12904 // We're not given a ResNo to compare against because the whole
12905 // node is going away. We know that any ISD::ADD only has one
12906 // result, so we can assume any node match is using the result.
12907 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12908 NewLocOps[i].getSDNode() != &N)
12909 continue;
12910 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12911 if (RHSConstant) {
12912 SmallVector<uint64_t, 3> ExprOps;
12913 DIExpression::appendOffset(Ops&: ExprOps, Offset);
12914 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12915 } else {
12916 // Convert to a variadic expression (if not already).
12917 // convertToVariadicExpression() returns a const pointer, so we use
12918 // a temporary const variable here.
12919 const auto *TmpDIExpr =
12920 DIExpression::convertToVariadicExpression(Expr: DIExpr);
12921 SmallVector<uint64_t, 3> ExprOps;
12922 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
12923 ExprOps.push_back(Elt: NewLocOps.size());
12924 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
12925 SDDbgOperand RHS =
12926 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
12927 NewLocOps.push_back(Elt: RHS);
12928 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12929 }
12930 Changed = true;
12931 }
12932 (void)Changed;
12933 assert(Changed && "Salvage target doesn't use N");
12934
12935 bool IsVariadic =
12936 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
12937
12938 auto AdditionalDependencies = DV->getAdditionalDependencies();
12939 SDDbgValue *Clone = getDbgValueList(
12940 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
12941 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
12942 ClonedDVs.push_back(Elt: Clone);
12943 DV->setIsInvalidated();
12944 DV->setIsEmitted();
12945 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
12946 N0.getNode()->dumprFull(this);
12947 dbgs() << " into " << *DIExpr << '\n');
12948 }
12949 break;
12950 }
12951 case ISD::TRUNCATE: {
12952 SDValue N0 = N.getOperand(Num: 0);
12953 TypeSize FromSize = N0.getValueSizeInBits();
12954 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
12955
12956 DIExpression *DbgExpression = DV->getExpression();
12957 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
12958 auto NewLocOps = DV->copyLocationOps();
12959 bool Changed = false;
12960 for (size_t i = 0; i < NewLocOps.size(); ++i) {
12961 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12962 NewLocOps[i].getSDNode() != &N)
12963 continue;
12964
12965 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12966 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
12967 Changed = true;
12968 }
12969 assert(Changed && "Salvage target doesn't use N");
12970 (void)Changed;
12971
12972 SDDbgValue *Clone =
12973 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
12974 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
12975 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
12976
12977 ClonedDVs.push_back(Elt: Clone);
12978 DV->setIsInvalidated();
12979 DV->setIsEmitted();
12980 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
12981 dbgs() << " into " << *DbgExpression << '\n');
12982 break;
12983 }
12984 }
12985 }
12986
12987 for (SDDbgValue *Dbg : ClonedDVs) {
12988 assert((!Dbg->getSDNodes().empty() ||
12989 llvm::any_of(Dbg->getLocationOps(),
12990 [&](const SDDbgOperand &Op) {
12991 return Op.getKind() == SDDbgOperand::FRAMEIX;
12992 })) &&
12993 "Salvaged DbgValue should depend on a new SDNode");
12994 AddDbgValue(DB: Dbg, isParameter: false);
12995 }
12996}
12997
12998/// Creates a SDDbgLabel node.
12999SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
13000 const DebugLoc &DL, unsigned O) {
13001 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
13002 "Expected inlined-at fields to agree");
13003 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
13004}
13005
13006namespace {
13007
13008/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
13009/// pointed to by a use iterator is deleted, increment the use iterator
13010/// so that it doesn't dangle.
13011///
13012class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
13013 SDNode::use_iterator &UI;
13014 SDNode::use_iterator &UE;
13015
13016 void NodeDeleted(SDNode *N, SDNode *E) override {
13017 // Increment the iterator as needed.
13018 while (UI != UE && N == UI->getUser())
13019 ++UI;
13020 }
13021
13022public:
13023 RAUWUpdateListener(SelectionDAG &d,
13024 SDNode::use_iterator &ui,
13025 SDNode::use_iterator &ue)
13026 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
13027};
13028
13029} // end anonymous namespace
13030
13031/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13032/// This can cause recursive merging of nodes in the DAG.
13033///
13034/// This version assumes From has a single result value.
13035///
13036void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
13037 SDNode *From = FromN.getNode();
13038 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
13039 "Cannot replace with this method!");
13040 assert(From != To.getNode() && "Cannot replace uses of with self");
13041
13042 // Preserve Debug Values
13043 transferDbgValues(From: FromN, To);
13044 // Preserve extra info.
13045 copyExtraInfo(From, To: To.getNode());
13046
13047 // Iterate over all the existing uses of From. New uses will be added
13048 // to the beginning of the use list, which we avoid visiting.
13049 // This specifically avoids visiting uses of From that arise while the
13050 // replacement is happening, because any such uses would be the result
13051 // of CSE: If an existing node looks like From after one of its operands
13052 // is replaced by To, we don't want to replace of all its users with To
13053 // too. See PR3018 for more info.
13054 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13055 RAUWUpdateListener Listener(*this, UI, UE);
13056 while (UI != UE) {
13057 SDNode *User = UI->getUser();
13058
13059 // This node is about to morph, remove its old self from the CSE maps.
13060 RemoveNodeFromCSEMaps(N: User);
13061
13062 // A user can appear in a use list multiple times, and when this
13063 // happens the uses are usually next to each other in the list.
13064 // To help reduce the number of CSE recomputations, process all
13065 // the uses of this user that we can find this way.
13066 do {
13067 SDUse &Use = *UI;
13068 ++UI;
13069 Use.set(To);
13070 if (To->isDivergent() != From->isDivergent())
13071 updateDivergence(N: User);
13072 } while (UI != UE && UI->getUser() == User);
13073 // Now that we have modified User, add it back to the CSE maps. If it
13074 // already exists there, recursively merge the results together.
13075 AddModifiedNodeToCSEMaps(N: User);
13076 }
13077
13078 // If we just RAUW'd the root, take note.
13079 if (FromN == getRoot())
13080 setRoot(To);
13081}
13082
13083/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13084/// This can cause recursive merging of nodes in the DAG.
13085///
13086/// This version assumes that for each value of From, there is a
13087/// corresponding value in To in the same position with the same type.
13088///
13089void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
13090#ifndef NDEBUG
13091 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13092 assert((!From->hasAnyUseOfValue(i) ||
13093 From->getValueType(i) == To->getValueType(i)) &&
13094 "Cannot use this version of ReplaceAllUsesWith!");
13095#endif
13096
13097 // Handle the trivial case.
13098 if (From == To)
13099 return;
13100
13101 // Preserve Debug Info. Only do this if there's a use.
13102 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
13103 if (From->hasAnyUseOfValue(Value: i)) {
13104 assert((i < To->getNumValues()) && "Invalid To location");
13105 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
13106 }
13107 // Preserve extra info.
13108 copyExtraInfo(From, To);
13109
13110 // Iterate over just the existing users of From. See the comments in
13111 // the ReplaceAllUsesWith above.
13112 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13113 RAUWUpdateListener Listener(*this, UI, UE);
13114 while (UI != UE) {
13115 SDNode *User = UI->getUser();
13116
13117 // This node is about to morph, remove its old self from the CSE maps.
13118 RemoveNodeFromCSEMaps(N: User);
13119
13120 // A user can appear in a use list multiple times, and when this
13121 // happens the uses are usually next to each other in the list.
13122 // To help reduce the number of CSE recomputations, process all
13123 // the uses of this user that we can find this way.
13124 do {
13125 SDUse &Use = *UI;
13126 ++UI;
13127 Use.setNode(To);
13128 if (To->isDivergent() != From->isDivergent())
13129 updateDivergence(N: User);
13130 } while (UI != UE && UI->getUser() == User);
13131
13132 // Now that we have modified User, add it back to the CSE maps. If it
13133 // already exists there, recursively merge the results together.
13134 AddModifiedNodeToCSEMaps(N: User);
13135 }
13136
13137 // If we just RAUW'd the root, take note.
13138 if (From == getRoot().getNode())
13139 setRoot(SDValue(To, getRoot().getResNo()));
13140}
13141
13142/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
13143/// This can cause recursive merging of nodes in the DAG.
13144///
13145/// This version can replace From with any result values. To must match the
13146/// number and types of values returned by From.
13147void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
13148 if (From->getNumValues() == 1) // Handle the simple case efficiently.
13149 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
13150
13151 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
13152 // Preserve Debug Info.
13153 transferDbgValues(From: SDValue(From, i), To: To[i]);
13154 // Preserve extra info.
13155 copyExtraInfo(From, To: To[i].getNode());
13156 }
13157
13158 // Iterate over just the existing users of From. See the comments in
13159 // the ReplaceAllUsesWith above.
13160 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
13161 RAUWUpdateListener Listener(*this, UI, UE);
13162 while (UI != UE) {
13163 SDNode *User = UI->getUser();
13164
13165 // This node is about to morph, remove its old self from the CSE maps.
13166 RemoveNodeFromCSEMaps(N: User);
13167
13168 // A user can appear in a use list multiple times, and when this happens the
13169 // uses are usually next to each other in the list. To help reduce the
13170 // number of CSE and divergence recomputations, process all the uses of this
13171 // user that we can find this way.
13172 bool To_IsDivergent = false;
13173 do {
13174 SDUse &Use = *UI;
13175 const SDValue &ToOp = To[Use.getResNo()];
13176 ++UI;
13177 Use.set(ToOp);
13178 if (ToOp.getValueType() != MVT::Other)
13179 To_IsDivergent |= ToOp->isDivergent();
13180 } while (UI != UE && UI->getUser() == User);
13181
13182 if (To_IsDivergent != From->isDivergent())
13183 updateDivergence(N: User);
13184
13185 // Now that we have modified User, add it back to the CSE maps. If it
13186 // already exists there, recursively merge the results together.
13187 AddModifiedNodeToCSEMaps(N: User);
13188 }
13189
13190 // If we just RAUW'd the root, take note.
13191 if (From == getRoot().getNode())
13192 setRoot(SDValue(To[getRoot().getResNo()]));
13193}
13194
13195/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
13196/// uses of other values produced by From.getNode() alone. The Deleted
13197/// vector is handled the same way as for ReplaceAllUsesWith.
13198void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
13199 // Handle the really simple, really trivial case efficiently.
13200 if (From == To) return;
13201
13202 // Handle the simple, trivial, case efficiently.
13203 if (From.getNode()->getNumValues() == 1) {
13204 ReplaceAllUsesWith(FromN: From, To);
13205 return;
13206 }
13207
13208 // Preserve Debug Info.
13209 transferDbgValues(From, To);
13210 copyExtraInfo(From: From.getNode(), To: To.getNode());
13211
13212 // Iterate over just the existing users of From. See the comments in
13213 // the ReplaceAllUsesWith above.
13214 SDNode::use_iterator UI = From.getNode()->use_begin(),
13215 UE = From.getNode()->use_end();
13216 RAUWUpdateListener Listener(*this, UI, UE);
13217 while (UI != UE) {
13218 SDNode *User = UI->getUser();
13219 bool UserRemovedFromCSEMaps = false;
13220
13221 // A user can appear in a use list multiple times, and when this
13222 // happens the uses are usually next to each other in the list.
13223 // To help reduce the number of CSE recomputations, process all
13224 // the uses of this user that we can find this way.
13225 do {
13226 SDUse &Use = *UI;
13227
13228 // Skip uses of different values from the same node.
13229 if (Use.getResNo() != From.getResNo()) {
13230 ++UI;
13231 continue;
13232 }
13233
13234 // If this node hasn't been modified yet, it's still in the CSE maps,
13235 // so remove its old self from the CSE maps.
13236 if (!UserRemovedFromCSEMaps) {
13237 RemoveNodeFromCSEMaps(N: User);
13238 UserRemovedFromCSEMaps = true;
13239 }
13240
13241 ++UI;
13242 Use.set(To);
13243 if (To->isDivergent() != From->isDivergent())
13244 updateDivergence(N: User);
13245 } while (UI != UE && UI->getUser() == User);
13246 // We are iterating over all uses of the From node, so if a use
13247 // doesn't use the specific value, no changes are made.
13248 if (!UserRemovedFromCSEMaps)
13249 continue;
13250
13251 // Now that we have modified User, add it back to the CSE maps. If it
13252 // already exists there, recursively merge the results together.
13253 AddModifiedNodeToCSEMaps(N: User);
13254 }
13255
13256 // If we just RAUW'd the root, take note.
13257 if (From == getRoot())
13258 setRoot(To);
13259}
13260
13261namespace {
13262
13263/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
13264/// to record information about a use.
13265struct UseMemo {
13266 SDNode *User;
13267 unsigned Index;
13268 SDUse *Use;
13269};
13270
13271/// operator< - Sort Memos by User.
13272bool operator<(const UseMemo &L, const UseMemo &R) {
13273 return (intptr_t)L.User < (intptr_t)R.User;
13274}
13275
13276/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
13277/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
13278/// the node already has been taken care of recursively.
13279class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
13280 SmallVectorImpl<UseMemo> &Uses;
13281
13282 void NodeDeleted(SDNode *N, SDNode *E) override {
13283 for (UseMemo &Memo : Uses)
13284 if (Memo.User == N)
13285 Memo.User = nullptr;
13286 }
13287
13288public:
13289 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
13290 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
13291};
13292
13293} // end anonymous namespace
13294
13295/// Return true if a glue output should propagate divergence information.
13296static bool gluePropagatesDivergence(const SDNode *Node) {
13297 switch (Node->getOpcode()) {
13298 case ISD::CopyFromReg:
13299 case ISD::CopyToReg:
13300 return false;
13301 default:
13302 return true;
13303 }
13304
13305 llvm_unreachable("covered opcode switch");
13306}
13307
13308bool SelectionDAG::calculateDivergence(SDNode *N) {
13309 if (TLI->isSDNodeAlwaysUniform(N)) {
13310 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
13311 "Conflicting divergence information!");
13312 return false;
13313 }
13314 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
13315 return true;
13316 for (const auto &Op : N->ops()) {
13317 EVT VT = Op.getValueType();
13318
13319 // Skip Chain. It does not carry divergence.
13320 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
13321 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
13322 return true;
13323 }
13324 return false;
13325}
13326
13327void SelectionDAG::updateDivergence(SDNode *N) {
13328 SmallVector<SDNode *, 16> Worklist(1, N);
13329 do {
13330 N = Worklist.pop_back_val();
13331 bool IsDivergent = calculateDivergence(N);
13332 if (N->SDNodeBits.IsDivergent != IsDivergent) {
13333 N->SDNodeBits.IsDivergent = IsDivergent;
13334 llvm::append_range(C&: Worklist, R: N->users());
13335 }
13336 } while (!Worklist.empty());
13337}
13338
13339void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
13340 DenseMap<SDNode *, unsigned> Degree;
13341 Order.reserve(n: AllNodes.size());
13342 for (auto &N : allnodes()) {
13343 unsigned NOps = N.getNumOperands();
13344 Degree[&N] = NOps;
13345 if (0 == NOps)
13346 Order.push_back(x: &N);
13347 }
13348 for (size_t I = 0; I != Order.size(); ++I) {
13349 SDNode *N = Order[I];
13350 for (auto *U : N->users()) {
13351 unsigned &UnsortedOps = Degree[U];
13352 if (0 == --UnsortedOps)
13353 Order.push_back(x: U);
13354 }
13355 }
13356}
13357
13358#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
13359void SelectionDAG::VerifyDAGDivergence() {
13360 std::vector<SDNode *> TopoOrder;
13361 CreateTopologicalOrder(TopoOrder);
13362 for (auto *N : TopoOrder) {
13363 assert(calculateDivergence(N) == N->isDivergent() &&
13364 "Divergence bit inconsistency detected");
13365 }
13366}
13367#endif
13368
13369/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
13370/// uses of other values produced by From.getNode() alone. The same value
13371/// may appear in both the From and To list. The Deleted vector is
13372/// handled the same way as for ReplaceAllUsesWith.
13373void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
13374 const SDValue *To,
13375 unsigned Num){
13376 // Handle the simple, trivial case efficiently.
13377 if (Num == 1)
13378 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
13379
13380 transferDbgValues(From: *From, To: *To);
13381 copyExtraInfo(From: From->getNode(), To: To->getNode());
13382
13383 // Read up all the uses and make records of them. This helps
13384 // processing new uses that are introduced during the
13385 // replacement process.
13386 SmallVector<UseMemo, 4> Uses;
13387 for (unsigned i = 0; i != Num; ++i) {
13388 unsigned FromResNo = From[i].getResNo();
13389 SDNode *FromNode = From[i].getNode();
13390 for (SDUse &Use : FromNode->uses()) {
13391 if (Use.getResNo() == FromResNo) {
13392 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
13393 Uses.push_back(Elt: Memo);
13394 }
13395 }
13396 }
13397
13398 // Sort the uses, so that all the uses from a given User are together.
13399 llvm::sort(C&: Uses);
13400 RAUOVWUpdateListener Listener(*this, Uses);
13401
13402 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
13403 UseIndex != UseIndexEnd; ) {
13404 // We know that this user uses some value of From. If it is the right
13405 // value, update it.
13406 SDNode *User = Uses[UseIndex].User;
13407 // If the node has been deleted by recursive CSE updates when updating
13408 // another node, then just skip this entry.
13409 if (User == nullptr) {
13410 ++UseIndex;
13411 continue;
13412 }
13413
13414 // This node is about to morph, remove its old self from the CSE maps.
13415 RemoveNodeFromCSEMaps(N: User);
13416
13417 // The Uses array is sorted, so all the uses for a given User
13418 // are next to each other in the list.
13419 // To help reduce the number of CSE recomputations, process all
13420 // the uses of this user that we can find this way.
13421 do {
13422 unsigned i = Uses[UseIndex].Index;
13423 SDUse &Use = *Uses[UseIndex].Use;
13424 ++UseIndex;
13425
13426 Use.set(To[i]);
13427 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
13428
13429 // Now that we have modified User, add it back to the CSE maps. If it
13430 // already exists there, recursively merge the results together.
13431 AddModifiedNodeToCSEMaps(N: User);
13432 }
13433}
13434
13435/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
13436/// based on their topological order. It returns the maximum id and a vector
13437/// of the SDNodes* in assigned order by reference.
13438unsigned SelectionDAG::AssignTopologicalOrder() {
13439 unsigned DAGSize = 0;
13440
13441 // SortedPos tracks the progress of the algorithm. Nodes before it are
13442 // sorted, nodes after it are unsorted. When the algorithm completes
13443 // it is at the end of the list.
13444 allnodes_iterator SortedPos = allnodes_begin();
13445
13446 // Visit all the nodes. Move nodes with no operands to the front of
13447 // the list immediately. Annotate nodes that do have operands with their
13448 // operand count. Before we do this, the Node Id fields of the nodes
13449 // may contain arbitrary values. After, the Node Id fields for nodes
13450 // before SortedPos will contain the topological sort index, and the
13451 // Node Id fields for nodes At SortedPos and after will contain the
13452 // count of outstanding operands.
13453 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13454 checkForCycles(N: &N, DAG: this);
13455 unsigned Degree = N.getNumOperands();
13456 if (Degree == 0) {
13457 // A node with no uses, add it to the result array immediately.
13458 N.setNodeId(DAGSize++);
13459 allnodes_iterator Q(&N);
13460 if (Q != SortedPos)
13461 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13462 assert(SortedPos != AllNodes.end() && "Overran node list");
13463 ++SortedPos;
13464 } else {
13465 // Temporarily use the Node Id as scratch space for the degree count.
13466 N.setNodeId(Degree);
13467 }
13468 }
13469
13470 // Visit all the nodes. As we iterate, move nodes into sorted order,
13471 // such that by the time the end is reached all nodes will be sorted.
13472 for (SDNode &Node : allnodes()) {
13473 SDNode *N = &Node;
13474 checkForCycles(N, DAG: this);
13475 // N is in sorted position, so all its uses have one less operand
13476 // that needs to be sorted.
13477 for (SDNode *P : N->users()) {
13478 unsigned Degree = P->getNodeId();
13479 assert(Degree != 0 && "Invalid node degree");
13480 --Degree;
13481 if (Degree == 0) {
13482 // All of P's operands are sorted, so P may sorted now.
13483 P->setNodeId(DAGSize++);
13484 if (P->getIterator() != SortedPos)
13485 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13486 assert(SortedPos != AllNodes.end() && "Overran node list");
13487 ++SortedPos;
13488 } else {
13489 // Update P's outstanding operand count.
13490 P->setNodeId(Degree);
13491 }
13492 }
13493 if (Node.getIterator() == SortedPos) {
13494#ifndef NDEBUG
13495 allnodes_iterator I(N);
13496 SDNode *S = &*++I;
13497 dbgs() << "Overran sorted position:\n";
13498 S->dumprFull(this); dbgs() << "\n";
13499 dbgs() << "Checking if this is due to cycles\n";
13500 checkForCycles(this, true);
13501#endif
13502 llvm_unreachable(nullptr);
13503 }
13504 }
13505
13506 assert(SortedPos == AllNodes.end() &&
13507 "Topological sort incomplete!");
13508 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13509 "First node in topological sort is not the entry token!");
13510 assert(AllNodes.front().getNodeId() == 0 &&
13511 "First node in topological sort has non-zero id!");
13512 assert(AllNodes.front().getNumOperands() == 0 &&
13513 "First node in topological sort has operands!");
13514 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13515 "Last node in topologic sort has unexpected id!");
13516 assert(AllNodes.back().use_empty() &&
13517 "Last node in topologic sort has users!");
13518 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13519 return DAGSize;
13520}
13521
13522void SelectionDAG::getTopologicallyOrderedNodes(
13523 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13524 SortedNodes.clear();
13525 // Node -> remaining number of outstanding operands.
13526 DenseMap<const SDNode *, unsigned> RemainingOperands;
13527
13528 // Put nodes without any operands into SortedNodes first.
13529 for (const SDNode &N : allnodes()) {
13530 checkForCycles(N: &N, DAG: this);
13531 unsigned NumOperands = N.getNumOperands();
13532 if (NumOperands == 0)
13533 SortedNodes.push_back(Elt: &N);
13534 else
13535 // Record their total number of outstanding operands.
13536 RemainingOperands[&N] = NumOperands;
13537 }
13538
13539 // A node is pushed into SortedNodes when all of its operands (predecessors in
13540 // the graph) are also in SortedNodes.
13541 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13542 const SDNode *N = SortedNodes[i];
13543 for (const SDNode *U : N->users()) {
13544 // HandleSDNode is never part of a DAG and therefore has no entry in
13545 // RemainingOperands.
13546 if (U->getOpcode() == ISD::HANDLENODE)
13547 continue;
13548 unsigned &NumRemOperands = RemainingOperands[U];
13549 assert(NumRemOperands && "Invalid number of remaining operands");
13550 --NumRemOperands;
13551 if (!NumRemOperands)
13552 SortedNodes.push_back(Elt: U);
13553 }
13554 }
13555
13556 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13557 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13558 "First node in topological sort is not the entry token");
13559 assert(SortedNodes.front()->getNumOperands() == 0 &&
13560 "First node in topological sort has operands");
13561}
13562
13563/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13564/// value is produced by SD.
13565void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13566 for (SDNode *SD : DB->getSDNodes()) {
13567 if (!SD)
13568 continue;
13569 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13570 SD->setHasDebugValue(true);
13571 }
13572 DbgInfo->add(V: DB, isParameter);
13573}
13574
13575void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13576
13577SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13578 SDValue NewMemOpChain) {
13579 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13580 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13581 // The new memory operation must have the same position as the old load in
13582 // terms of memory dependency. Create a TokenFactor for the old load and new
13583 // memory operation and update uses of the old load's output chain to use that
13584 // TokenFactor.
13585 if (OldChain == NewMemOpChain || OldChain.use_empty())
13586 return NewMemOpChain;
13587
13588 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13589 N1: OldChain, N2: NewMemOpChain);
13590 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13591 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13592 return TokenFactor;
13593}
13594
13595SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13596 SDValue NewMemOp) {
13597 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13598 SDValue OldChain = SDValue(OldLoad, 1);
13599 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13600 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13601}
13602
13603SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13604 Function **OutFunction) {
13605 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13606
13607 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13608 auto *Module = MF->getFunction().getParent();
13609 auto *Function = Module->getFunction(Name: Symbol);
13610
13611 if (OutFunction != nullptr)
13612 *OutFunction = Function;
13613
13614 if (Function != nullptr) {
13615 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13616 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13617 }
13618
13619 std::string ErrorStr;
13620 raw_string_ostream ErrorFormatter(ErrorStr);
13621 ErrorFormatter << "Undefined external symbol ";
13622 ErrorFormatter << '"' << Symbol << '"';
13623 report_fatal_error(reason: Twine(ErrorStr));
13624}
13625
13626//===----------------------------------------------------------------------===//
13627// SDNode Class
13628//===----------------------------------------------------------------------===//
13629
13630bool llvm::isNullConstant(SDValue V) {
13631 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13632 return Const != nullptr && Const->isZero();
13633}
13634
13635bool llvm::isNullConstantOrUndef(SDValue V) {
13636 return V.isUndef() || isNullConstant(V);
13637}
13638
13639bool llvm::isNullFPConstant(SDValue V) {
13640 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13641 return Const != nullptr && Const->isZero() && !Const->isNegative();
13642}
13643
13644bool llvm::isAllOnesConstant(SDValue V) {
13645 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13646 return Const != nullptr && Const->isAllOnes();
13647}
13648
13649bool llvm::isOneConstant(SDValue V) {
13650 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13651 return Const != nullptr && Const->isOne();
13652}
13653
13654bool llvm::isMinSignedConstant(SDValue V) {
13655 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13656 return Const != nullptr && Const->isMinSignedValue();
13657}
13658
13659bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13660 SDValue V, unsigned OperandNo,
13661 unsigned Depth) const {
13662 APInt DemandedElts = getDemandAllEltsMask(V);
13663 return isIdentityElement(Opc: Opcode, Flags, V, DemandedElts, OperandNo, Depth);
13664}
13665
13666bool SelectionDAG::isIdentityElement(unsigned Opcode, SDNodeFlags Flags,
13667 SDValue V, const APInt &DemandedElts,
13668 unsigned OperandNo, unsigned Depth) const {
13669 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13670 // TODO: Target-specific opcodes could be added.
13671 if (V.getValueType().isInteger()) {
13672 KnownBits Known = computeKnownBits(Op: V, DemandedElts, Depth);
13673 if (Known.isConstant()) {
13674 const APInt &Const = Known.getConstant();
13675 switch (Opcode) {
13676 case ISD::ADD:
13677 case ISD::OR:
13678 case ISD::XOR:
13679 case ISD::UMAX:
13680 return Const.isZero();
13681 case ISD::MUL:
13682 return Const.isOne();
13683 case ISD::AND:
13684 case ISD::UMIN:
13685 return Const.isAllOnes();
13686 case ISD::SMAX:
13687 return Const.isMinSignedValue();
13688 case ISD::SMIN:
13689 return Const.isMaxSignedValue();
13690 case ISD::SUB:
13691 case ISD::SHL:
13692 case ISD::SRA:
13693 case ISD::SRL:
13694 return OperandNo == 1 && Const.isZero();
13695 case ISD::UDIV:
13696 case ISD::SDIV:
13697 return OperandNo == 1 && Const.isOne();
13698 }
13699 }
13700 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V, DemandedElts)) {
13701 switch (Opcode) {
13702 case ISD::FADD:
13703 return ConstFP->isZero() &&
13704 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13705 case ISD::FSUB:
13706 return OperandNo == 1 && ConstFP->isZero() &&
13707 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13708 case ISD::FMUL:
13709 return ConstFP->isOne();
13710 case ISD::FDIV:
13711 return OperandNo == 1 && ConstFP->isOne();
13712 case ISD::FMINNUM:
13713 case ISD::FMAXNUM: {
13714 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
13715 EVT VT = V.getValueType();
13716 const fltSemantics &Semantics = VT.getFltSemantics();
13717 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics)
13718 : !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
13719 : APFloat::getLargest(Sem: Semantics);
13720 if (Opcode == ISD::FMAXNUM)
13721 NeutralAF.changeSign();
13722
13723 return ConstFP->isExactlyValue(V: NeutralAF);
13724 }
13725 }
13726 }
13727 return false;
13728}
13729
13730SDValue llvm::peekThroughBitcasts(SDValue V) {
13731 while (V.getOpcode() == ISD::BITCAST)
13732 V = V.getOperand(i: 0);
13733 return V;
13734}
13735
13736SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13737 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13738 V = V.getOperand(i: 0);
13739 return V;
13740}
13741
13742SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13743 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13744 V = V.getOperand(i: 0);
13745 return V;
13746}
13747
13748SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13749 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13750 SDValue InVec = V.getOperand(i: 0);
13751 SDValue EltNo = V.getOperand(i: 2);
13752 EVT VT = InVec.getValueType();
13753 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13754 if (IndexC && VT.isFixedLengthVector() &&
13755 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13756 !DemandedElts[IndexC->getZExtValue()]) {
13757 V = InVec;
13758 continue;
13759 }
13760 break;
13761 }
13762 return V;
13763}
13764
13765SDValue llvm::peekThroughTruncates(SDValue V) {
13766 while (V.getOpcode() == ISD::TRUNCATE)
13767 V = V.getOperand(i: 0);
13768 return V;
13769}
13770
13771bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13772 if (V.getOpcode() != ISD::XOR)
13773 return false;
13774 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13775 unsigned NumBits = V.getScalarValueSizeInBits();
13776 ConstantSDNode *C =
13777 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13778 return C && (C->getAPIntValue().countr_one() >= NumBits);
13779}
13780
13781ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13782 bool AllowTruncation) {
13783 APInt DemandedElts = getDemandAllEltsMask(V: N);
13784 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13785}
13786
13787ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13788 bool AllowUndefs,
13789 bool AllowTruncation) {
13790 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13791 return CN;
13792
13793 // SplatVectors can truncate their operands. Ignore that case here unless
13794 // AllowTruncation is set.
13795 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13796 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13797 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13798 EVT CVT = CN->getValueType(ResNo: 0);
13799 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13800 if (AllowTruncation || CVT == VecEltVT)
13801 return CN;
13802 }
13803 }
13804
13805 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13806 BitVector UndefElements;
13807 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13808
13809 // BuildVectors can truncate their operands. Ignore that case here unless
13810 // AllowTruncation is set.
13811 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13812 if (CN && (UndefElements.none() || AllowUndefs)) {
13813 EVT CVT = CN->getValueType(ResNo: 0);
13814 EVT NSVT = N.getValueType().getScalarType();
13815 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13816 if (AllowTruncation || (CVT == NSVT))
13817 return CN;
13818 }
13819 }
13820
13821 return nullptr;
13822}
13823
13824ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13825 APInt DemandedElts = getDemandAllEltsMask(V: N);
13826 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13827}
13828
13829ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13830 const APInt &DemandedElts,
13831 bool AllowUndefs) {
13832 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13833 return CN;
13834
13835 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13836 BitVector UndefElements;
13837 ConstantFPSDNode *CN =
13838 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13839 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13840 if (CN && (UndefElements.none() || AllowUndefs))
13841 return CN;
13842 }
13843
13844 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13845 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13846 return CN;
13847
13848 return nullptr;
13849}
13850
13851bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13852 // TODO: may want to use peekThroughBitcast() here.
13853 ConstantSDNode *C =
13854 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13855 return C && C->isZero();
13856}
13857
13858bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
13859 ConstantSDNode *C =
13860 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
13861 return C && C->isOne();
13862}
13863
13864bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
13865 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13866 return C && C->isOne();
13867}
13868
13869bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
13870 N = peekThroughBitcasts(V: N);
13871 unsigned BitWidth = N.getScalarValueSizeInBits();
13872 ConstantSDNode *C =
13873 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13874 return C && C->getAPIntValue().countTrailingOnes() >= BitWidth;
13875}
13876
13877bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
13878 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13879 return C && APInt::isSameValue(I1: C->getAPIntValue(),
13880 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
13881}
13882
13883bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
13884 N = peekThroughBitcasts(V: N);
13885 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
13886 return C && C->isZero();
13887}
13888
13889bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
13890 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13891 return C && C->isZero();
13892}
13893
13894HandleSDNode::~HandleSDNode() {
13895 DropOperands();
13896}
13897
13898MemSDNode::MemSDNode(
13899 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
13900 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
13901 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
13902 bool IsVolatile = false;
13903 bool IsNonTemporal = false;
13904 bool IsDereferenceable = true;
13905 bool IsInvariant = true;
13906 for (const MachineMemOperand *MMO : memoperands()) {
13907 IsVolatile |= MMO->isVolatile();
13908 IsNonTemporal |= MMO->isNonTemporal();
13909 IsDereferenceable &= MMO->isDereferenceable();
13910 IsInvariant &= MMO->isInvariant();
13911 }
13912 MemSDNodeBits.IsVolatile = IsVolatile;
13913 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
13914 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
13915 MemSDNodeBits.IsInvariant = IsInvariant;
13916
13917 // For the single-MMO case, we check here that the size of the memory operand
13918 // fits within the size of the MMO. This is because the MMO might indicate
13919 // only a possible address range instead of specifying the affected memory
13920 // addresses precisely.
13921 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
13922 TypeSize::isKnownLE(memvt.getStoreSize(),
13923 getMemOperand()->getSize().getValue())) &&
13924 "Size mismatch!");
13925}
13926
13927/// Profile - Gather unique data for the node.
13928///
13929void SDNode::Profile(FoldingSetNodeID &ID) const {
13930 AddNodeIDNode(ID, N: this);
13931}
13932
13933namespace {
13934
13935 struct EVTArray {
13936 std::vector<EVT> VTs;
13937
13938 EVTArray() {
13939 VTs.reserve(n: MVT::VALUETYPE_SIZE);
13940 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
13941 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
13942 }
13943 };
13944
13945} // end anonymous namespace
13946
13947/// getValueTypeList - Return a pointer to the specified value type.
13948///
13949const EVT *SDNode::getValueTypeList(MVT VT) {
13950 static EVTArray SimpleVTArray;
13951
13952 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
13953 return &SimpleVTArray.VTs[VT.SimpleTy];
13954}
13955
13956/// hasAnyUseOfValue - Return true if there are any use of the indicated
13957/// value. This method ignores uses of other values defined by this operation.
13958bool SDNode::hasAnyUseOfValue(unsigned Value) const {
13959 assert(Value < getNumValues() && "Bad value!");
13960
13961 for (SDUse &U : uses())
13962 if (U.getResNo() == Value)
13963 return true;
13964
13965 return false;
13966}
13967
13968/// isOnlyUserOf - Return true if this node is the only use of N.
13969bool SDNode::isOnlyUserOf(const SDNode *N) const {
13970 bool Seen = false;
13971 for (const SDNode *User : N->users()) {
13972 if (User == this)
13973 Seen = true;
13974 else
13975 return false;
13976 }
13977
13978 return Seen;
13979}
13980
13981/// Return true if the only users of N are contained in Nodes.
13982bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
13983 bool Seen = false;
13984 for (const SDNode *User : N->users()) {
13985 if (llvm::is_contained(Range&: Nodes, Element: User))
13986 Seen = true;
13987 else
13988 return false;
13989 }
13990
13991 return Seen;
13992}
13993
13994/// Return true if the referenced return value is an operand of N.
13995bool SDValue::isOperandOf(const SDNode *N) const {
13996 return is_contained(Range: N->op_values(), Element: *this);
13997}
13998
13999bool SDNode::isOperandOf(const SDNode *N) const {
14000 return any_of(Range: N->op_values(),
14001 P: [this](SDValue Op) { return this == Op.getNode(); });
14002}
14003
14004/// reachesChainWithoutSideEffects - Return true if this operand (which must
14005/// be a chain) reaches the specified operand without crossing any
14006/// side-effecting instructions on any chain path. In practice, this looks
14007/// through token factors and non-volatile loads. In order to remain efficient,
14008/// this only looks a couple of nodes in, it does not do an exhaustive search.
14009///
14010/// Note that we only need to examine chains when we're searching for
14011/// side-effects; SelectionDAG requires that all side-effects are represented
14012/// by chains, even if another operand would force a specific ordering. This
14013/// constraint is necessary to allow transformations like splitting loads.
14014bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
14015 unsigned Depth) const {
14016 if (*this == Dest) return true;
14017
14018 // Don't search too deeply, we just want to be able to see through
14019 // TokenFactor's etc.
14020 if (Depth == 0) return false;
14021
14022 // If this is a token factor, all inputs to the TF happen in parallel.
14023 if (getOpcode() == ISD::TokenFactor) {
14024 // First, try a shallow search.
14025 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
14026 // We found the chain we want as an operand of this TokenFactor.
14027 // Essentially, we reach the chain without side-effects if we could
14028 // serialize the TokenFactor into a simple chain of operations with
14029 // Dest as the last operation. This is automatically true if the
14030 // chain has one use: there are no other ordering constraints.
14031 // If the chain has more than one use, we give up: some other
14032 // use of Dest might force a side-effect between Dest and the current
14033 // node.
14034 if (Dest.hasOneUse())
14035 return true;
14036 }
14037 // Next, try a deep search: check whether every operand of the TokenFactor
14038 // reaches Dest.
14039 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
14040 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
14041 });
14042 }
14043
14044 // Loads don't have side effects, look through them.
14045 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
14046 if (Ld->isUnordered())
14047 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
14048 }
14049 return false;
14050}
14051
14052bool SDNode::hasPredecessor(const SDNode *N) const {
14053 SmallPtrSet<const SDNode *, 32> Visited;
14054 SmallVector<const SDNode *, 16> Worklist;
14055 Worklist.push_back(Elt: this);
14056 return hasPredecessorHelper(N, Visited, Worklist);
14057}
14058
14059void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
14060 this->Flags &= Flags;
14061}
14062
14063SDValue
14064SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
14065 ArrayRef<ISD::NodeType> CandidateBinOps,
14066 bool AllowPartials) {
14067 // The pattern must end in an extract from index 0.
14068 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14069 !isNullConstant(V: Extract->getOperand(Num: 1)))
14070 return SDValue();
14071
14072 // Match against one of the candidate binary ops.
14073 SDValue Op = Extract->getOperand(Num: 0);
14074 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
14075 return Op.getOpcode() == unsigned(BinOp);
14076 }))
14077 return SDValue();
14078
14079 // Floating-point reductions may require relaxed constraints on the final step
14080 // of the reduction because they may reorder intermediate operations.
14081 unsigned CandidateBinOp = Op.getOpcode();
14082 if (Op.getValueType().isFloatingPoint()) {
14083 SDNodeFlags Flags = Op->getFlags();
14084 switch (CandidateBinOp) {
14085 case ISD::FADD:
14086 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
14087 return SDValue();
14088 break;
14089 default:
14090 llvm_unreachable("Unhandled FP opcode for binop reduction");
14091 }
14092 }
14093
14094 // Matching failed - attempt to see if we did enough stages that a partial
14095 // reduction from a subvector is possible.
14096 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
14097 if (!AllowPartials || !Op)
14098 return SDValue();
14099 EVT OpVT = Op.getValueType();
14100 EVT OpSVT = OpVT.getScalarType();
14101 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
14102 if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
14103 return SDValue();
14104 BinOp = (ISD::NodeType)CandidateBinOp;
14105 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
14106 };
14107
14108 // At each stage, we're looking for something that looks like:
14109 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
14110 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
14111 // i32 undef, i32 undef, i32 undef, i32 undef>
14112 // %a = binop <8 x i32> %op, %s
14113 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
14114 // we expect something like:
14115 // <4,5,6,7,u,u,u,u>
14116 // <2,3,u,u,u,u,u,u>
14117 // <1,u,u,u,u,u,u,u>
14118 // While a partial reduction match would be:
14119 // <2,3,u,u,u,u,u,u>
14120 // <1,u,u,u,u,u,u,u>
14121 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
14122 SDValue PrevOp;
14123 for (unsigned i = 0; i < Stages; ++i) {
14124 unsigned MaskEnd = (1 << i);
14125
14126 if (Op.getOpcode() != CandidateBinOp)
14127 return PartialReduction(PrevOp, MaskEnd);
14128
14129 SDValue Op0 = Op.getOperand(i: 0);
14130 SDValue Op1 = Op.getOperand(i: 1);
14131
14132 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
14133 if (Shuffle) {
14134 Op = Op1;
14135 } else {
14136 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
14137 Op = Op0;
14138 }
14139
14140 // The first operand of the shuffle should be the same as the other operand
14141 // of the binop.
14142 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
14143 return PartialReduction(PrevOp, MaskEnd);
14144
14145 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
14146 for (int Index = 0; Index < (int)MaskEnd; ++Index)
14147 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
14148 return PartialReduction(PrevOp, MaskEnd);
14149
14150 PrevOp = Op;
14151 }
14152
14153 // Handle subvector reductions, which tend to appear after the shuffle
14154 // reduction stages.
14155 while (Op.getOpcode() == CandidateBinOp) {
14156 unsigned NumElts = Op.getValueType().getVectorNumElements();
14157 SDValue Op0 = Op.getOperand(i: 0);
14158 SDValue Op1 = Op.getOperand(i: 1);
14159 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14160 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
14161 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
14162 break;
14163 SDValue Src = Op0.getOperand(i: 0);
14164 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
14165 if (NumSrcElts != (2 * NumElts))
14166 break;
14167 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
14168 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
14169 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
14170 Op0.getConstantOperandAPInt(i: 1) == NumElts))
14171 break;
14172 Op = Src;
14173 }
14174
14175 BinOp = (ISD::NodeType)CandidateBinOp;
14176 return Op;
14177}
14178
14179SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
14180 EVT VT = N->getValueType(ResNo: 0);
14181 EVT EltVT = VT.getVectorElementType();
14182 unsigned NE = VT.getVectorNumElements();
14183
14184 SDLoc dl(N);
14185
14186 // If ResNE is 0, fully unroll the vector op.
14187 if (ResNE == 0)
14188 ResNE = NE;
14189 else if (NE > ResNE)
14190 NE = ResNE;
14191
14192 if (N->getNumValues() == 2) {
14193 SmallVector<SDValue, 8> Scalars0, Scalars1;
14194 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14195 EVT VT1 = N->getValueType(ResNo: 1);
14196 EVT EltVT1 = VT1.getVectorElementType();
14197
14198 unsigned i;
14199 for (i = 0; i != NE; ++i) {
14200 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14201 SDValue Operand = N->getOperand(Num: j);
14202 EVT OperandVT = Operand.getValueType();
14203
14204 // A vector operand; extract a single element.
14205 EVT OperandEltVT = OperandVT.getVectorElementType();
14206 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14207 }
14208
14209 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
14210 Scalars0.push_back(Elt: EltOp);
14211 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
14212 }
14213
14214 for (; i < ResNE; ++i) {
14215 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
14216 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
14217 }
14218
14219 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14220 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, NumElements: ResNE);
14221 SDValue Vec0 = getBuildVector(VT: VecVT, DL: dl, Ops: Scalars0);
14222 SDValue Vec1 = getBuildVector(VT: VecVT1, DL: dl, Ops: Scalars1);
14223 return getMergeValues(Ops: {Vec0, Vec1}, dl);
14224 }
14225
14226 assert(N->getNumValues() == 1 &&
14227 "Can't unroll a vector with multiple results!");
14228
14229 SmallVector<SDValue, 8> Scalars;
14230 SmallVector<SDValue, 4> Operands(N->getNumOperands());
14231
14232 unsigned i;
14233 for (i= 0; i != NE; ++i) {
14234 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
14235 SDValue Operand = N->getOperand(Num: j);
14236 EVT OperandVT = Operand.getValueType();
14237 if (OperandVT.isVector()) {
14238 // A vector operand; extract a single element.
14239 EVT OperandEltVT = OperandVT.getVectorElementType();
14240 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
14241 } else {
14242 // A scalar operand; just use it as is.
14243 Operands[j] = Operand;
14244 }
14245 }
14246
14247 switch (N->getOpcode()) {
14248 default: {
14249 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
14250 Flags: N->getFlags()));
14251 break;
14252 }
14253 case ISD::VSELECT:
14254 Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
14255 break;
14256 case ISD::SHL:
14257 case ISD::SRA:
14258 case ISD::SRL:
14259 case ISD::ROTL:
14260 case ISD::ROTR:
14261 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
14262 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
14263 Op: Operands[1])));
14264 break;
14265 case ISD::SIGN_EXTEND_INREG: {
14266 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
14267 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
14268 N1: Operands[0],
14269 N2: getValueType(VT: ExtVT)));
14270 break;
14271 }
14272 case ISD::ADDRSPACECAST: {
14273 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
14274 Scalars.push_back(Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0],
14275 SrcAS: ASC->getSrcAddressSpace(),
14276 DestAS: ASC->getDestAddressSpace()));
14277 break;
14278 }
14279 }
14280 }
14281
14282 for (; i < ResNE; ++i)
14283 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
14284
14285 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
14286 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
14287}
14288
14289std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
14290 SDNode *N, unsigned ResNE) {
14291 unsigned Opcode = N->getOpcode();
14292 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
14293 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
14294 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
14295 "Expected an overflow opcode");
14296
14297 EVT ResVT = N->getValueType(ResNo: 0);
14298 EVT OvVT = N->getValueType(ResNo: 1);
14299 EVT ResEltVT = ResVT.getVectorElementType();
14300 EVT OvEltVT = OvVT.getVectorElementType();
14301 SDLoc dl(N);
14302
14303 // If ResNE is 0, fully unroll the vector op.
14304 unsigned NE = ResVT.getVectorNumElements();
14305 if (ResNE == 0)
14306 ResNE = NE;
14307 else if (NE > ResNE)
14308 NE = ResNE;
14309
14310 SmallVector<SDValue, 8> LHSScalars;
14311 SmallVector<SDValue, 8> RHSScalars;
14312 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
14313 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
14314
14315 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
14316 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
14317 SmallVector<SDValue, 8> ResScalars;
14318 SmallVector<SDValue, 8> OvScalars;
14319 for (unsigned i = 0; i < NE; ++i) {
14320 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
14321 SDValue Ov =
14322 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
14323 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
14324 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
14325
14326 ResScalars.push_back(Elt: Res);
14327 OvScalars.push_back(Elt: Ov);
14328 }
14329
14330 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
14331 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
14332
14333 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
14334 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
14335 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
14336 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
14337}
14338
14339bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
14340 LoadSDNode *Base,
14341 unsigned Bytes,
14342 int Dist) const {
14343 if (LD->isVolatile() || Base->isVolatile())
14344 return false;
14345 // TODO: probably too restrictive for atomics, revisit
14346 if (!LD->isSimple())
14347 return false;
14348 if (LD->isIndexed() || Base->isIndexed())
14349 return false;
14350 if (LD->getChain() != Base->getChain())
14351 return false;
14352 EVT VT = LD->getMemoryVT();
14353 if (VT.getSizeInBits() / 8 != Bytes)
14354 return false;
14355
14356 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
14357 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
14358
14359 int64_t Offset = 0;
14360 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
14361 return (Dist * (int64_t)Bytes == Offset);
14362 return false;
14363}
14364
14365/// InferPtrAlignment - Infer alignment of a load / store address. Return
14366/// std::nullopt if it cannot be inferred.
14367MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
14368 // If this is a GlobalAddress + cst, return the alignment.
14369 const GlobalValue *GV = nullptr;
14370 int64_t GVOffset = 0;
14371 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
14372 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
14373 KnownBits Known(PtrWidth);
14374 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
14375 unsigned AlignBits = Known.countMinTrailingZeros();
14376 if (AlignBits)
14377 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
14378 }
14379
14380 // If this is a direct reference to a stack slot, use information about the
14381 // stack slot's alignment.
14382 int FrameIdx = INT_MIN;
14383 int64_t FrameOffset = 0;
14384 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
14385 FrameIdx = FI->getIndex();
14386 } else if (isBaseWithConstantOffset(Op: Ptr) &&
14387 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
14388 // Handle FI+Cst
14389 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
14390 FrameOffset = Ptr.getConstantOperandVal(i: 1);
14391 }
14392
14393 if (FrameIdx != INT_MIN) {
14394 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
14395 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
14396 }
14397
14398 return std::nullopt;
14399}
14400
14401/// Split the scalar node with EXTRACT_ELEMENT using the provided
14402/// VTs and return the low/high part.
14403std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
14404 const SDLoc &DL,
14405 const EVT &LoVT,
14406 const EVT &HiVT) {
14407 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
14408 "Split node must be a scalar type");
14409 SDValue Lo =
14410 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
14411 SDValue Hi =
14412 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
14413 return std::make_pair(x&: Lo, y&: Hi);
14414}
14415
14416/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
14417/// which is split (or expanded) into two not necessarily identical pieces.
14418std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
14419 // Currently all types are split in half.
14420 EVT LoVT, HiVT;
14421 if (!VT.isVector())
14422 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
14423 else
14424 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
14425
14426 return std::make_pair(x&: LoVT, y&: HiVT);
14427}
14428
14429/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
14430/// type, dependent on an enveloping VT that has been split into two identical
14431/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
14432std::pair<EVT, EVT>
14433SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
14434 bool *HiIsEmpty) const {
14435 EVT EltTp = VT.getVectorElementType();
14436 // Examples:
14437 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
14438 // custom VL=9 with enveloping VL=8/8 yields 8/1
14439 // custom VL=10 with enveloping VL=8/8 yields 8/2
14440 // etc.
14441 ElementCount VTNumElts = VT.getVectorElementCount();
14442 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
14443 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
14444 "Mixing fixed width and scalable vectors when enveloping a type");
14445 EVT LoVT, HiVT;
14446 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
14447 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14448 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
14449 *HiIsEmpty = false;
14450 } else {
14451 // Flag that hi type has zero storage size, but return split envelop type
14452 // (this would be easier if vector types with zero elements were allowed).
14453 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14454 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14455 *HiIsEmpty = true;
14456 }
14457 return std::make_pair(x&: LoVT, y&: HiVT);
14458}
14459
14460/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14461/// low/high part.
14462std::pair<SDValue, SDValue>
14463SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14464 const EVT &HiVT) {
14465 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14466 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14467 "Splitting vector with an invalid mixture of fixed and scalable "
14468 "vector types");
14469 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14470 N.getValueType().getVectorMinNumElements() &&
14471 "More vector elements requested than available!");
14472 SDValue Lo, Hi;
14473 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14474 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14475 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14476 // IDX with the runtime scaling factor of the result vector type. For
14477 // fixed-width result vectors, that runtime scaling factor is 1.
14478 Hi = getExtractSubvector(DL, VT: HiVT, Vec: N, Idx: LoVT.getVectorMinNumElements());
14479 return std::make_pair(x&: Lo, y&: Hi);
14480}
14481
14482std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14483 const SDLoc &DL) {
14484 // Split the vector length parameter.
14485 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14486 EVT VT = N.getValueType();
14487 assert(VecVT.getVectorElementCount().isKnownEven() &&
14488 "Expecting the mask to be an evenly-sized vector");
14489 SDValue HalfNumElts = getElementCount(
14490 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14491 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14492 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14493 return std::make_pair(x&: Lo, y&: Hi);
14494}
14495
14496/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14497SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14498 EVT VT = N.getValueType();
14499 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14500 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14501 return getInsertSubvector(DL, Vec: getUNDEF(VT: WideVT), SubVec: N, Idx: 0);
14502}
14503
14504void SelectionDAG::ExtractVectorElements(SDValue Op,
14505 SmallVectorImpl<SDValue> &Args,
14506 unsigned Start, unsigned Count,
14507 EVT EltVT) {
14508 EVT VT = Op.getValueType();
14509 if (Count == 0)
14510 Count = VT.getVectorNumElements();
14511 if (EltVT == EVT())
14512 EltVT = VT.getVectorElementType();
14513 SDLoc SL(Op);
14514 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14515 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14516 }
14517}
14518
14519// getAddressSpace - Return the address space this GlobalAddress belongs to.
14520unsigned GlobalAddressSDNode::getAddressSpace() const {
14521 return getGlobal()->getType()->getAddressSpace();
14522}
14523
14524Type *ConstantPoolSDNode::getType() const {
14525 if (isMachineConstantPoolEntry())
14526 return Val.MachineCPVal->getType();
14527 return Val.ConstVal->getType();
14528}
14529
14530bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14531 unsigned &SplatBitSize,
14532 bool &HasAnyUndefs,
14533 unsigned MinSplatBits,
14534 bool IsBigEndian) const {
14535 EVT VT = getValueType(ResNo: 0);
14536 assert(VT.isVector() && "Expected a vector type");
14537 unsigned VecWidth = VT.getSizeInBits();
14538 if (MinSplatBits > VecWidth)
14539 return false;
14540
14541 // FIXME: The widths are based on this node's type, but build vectors can
14542 // truncate their operands.
14543 SplatValue = APInt(VecWidth, 0);
14544 SplatUndef = APInt(VecWidth, 0);
14545
14546 // Get the bits. Bits with undefined values (when the corresponding element
14547 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14548 // in SplatValue. If any of the values are not constant, give up and return
14549 // false.
14550 unsigned int NumOps = getNumOperands();
14551 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14552 unsigned EltWidth = VT.getScalarSizeInBits();
14553
14554 for (unsigned j = 0; j < NumOps; ++j) {
14555 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14556 SDValue OpVal = getOperand(Num: i);
14557 unsigned BitPos = j * EltWidth;
14558
14559 if (OpVal.isUndef())
14560 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14561 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14562 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14563 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14564 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14565 else
14566 return false;
14567 }
14568
14569 // The build_vector is all constants or undefs. Find the smallest element
14570 // size that splats the vector.
14571 HasAnyUndefs = (SplatUndef != 0);
14572
14573 // FIXME: This does not work for vectors with elements less than 8 bits.
14574 while (VecWidth > 8) {
14575 // If we can't split in half, stop here.
14576 if (VecWidth & 1)
14577 break;
14578
14579 unsigned HalfSize = VecWidth / 2;
14580 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14581 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14582 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14583 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14584
14585 // If the two halves do not match (ignoring undef bits), stop here.
14586 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14587 MinSplatBits > HalfSize)
14588 break;
14589
14590 SplatValue = HighValue | LowValue;
14591 SplatUndef = HighUndef & LowUndef;
14592
14593 VecWidth = HalfSize;
14594 }
14595
14596 // FIXME: The loop above only tries to split in halves. But if the input
14597 // vector for example is <3 x i16> it wouldn't be able to detect a
14598 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14599 // optimizations. I guess that back in the days when this helper was created
14600 // vectors normally was power-of-2 sized.
14601
14602 SplatBitSize = VecWidth;
14603 return true;
14604}
14605
14606SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14607 BitVector *UndefElements) const {
14608 unsigned NumOps = getNumOperands();
14609 if (UndefElements) {
14610 UndefElements->clear();
14611 UndefElements->resize(N: NumOps);
14612 }
14613 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14614 if (!DemandedElts)
14615 return SDValue();
14616 SDValue Splatted;
14617 for (unsigned i = 0; i != NumOps; ++i) {
14618 if (!DemandedElts[i])
14619 continue;
14620 SDValue Op = getOperand(Num: i);
14621 if (Op.isUndef()) {
14622 if (UndefElements)
14623 (*UndefElements)[i] = true;
14624 } else if (!Splatted) {
14625 Splatted = Op;
14626 } else if (Splatted != Op) {
14627 return SDValue();
14628 }
14629 }
14630
14631 if (!Splatted) {
14632 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14633 assert(getOperand(FirstDemandedIdx).isUndef() &&
14634 "Can only have a splat without a constant for all undefs.");
14635 return getOperand(Num: FirstDemandedIdx);
14636 }
14637
14638 return Splatted;
14639}
14640
14641SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14642 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14643 return getSplatValue(DemandedElts, UndefElements);
14644}
14645
14646bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14647 SmallVectorImpl<SDValue> &Sequence,
14648 BitVector *UndefElements) const {
14649 unsigned NumOps = getNumOperands();
14650 Sequence.clear();
14651 if (UndefElements) {
14652 UndefElements->clear();
14653 UndefElements->resize(N: NumOps);
14654 }
14655 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14656 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14657 return false;
14658
14659 // Set the undefs even if we don't find a sequence (like getSplatValue).
14660 if (UndefElements)
14661 for (unsigned I = 0; I != NumOps; ++I)
14662 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14663 (*UndefElements)[I] = true;
14664
14665 // Iteratively widen the sequence length looking for repetitions.
14666 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14667 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14668 for (unsigned I = 0; I != NumOps; ++I) {
14669 if (!DemandedElts[I])
14670 continue;
14671 SDValue &SeqOp = Sequence[I % SeqLen];
14672 SDValue Op = getOperand(Num: I);
14673 if (Op.isUndef()) {
14674 if (!SeqOp)
14675 SeqOp = Op;
14676 continue;
14677 }
14678 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14679 Sequence.clear();
14680 break;
14681 }
14682 SeqOp = Op;
14683 }
14684 if (!Sequence.empty())
14685 return true;
14686 }
14687
14688 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14689 return false;
14690}
14691
14692bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14693 BitVector *UndefElements) const {
14694 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14695 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14696}
14697
14698ConstantSDNode *
14699BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14700 BitVector *UndefElements) const {
14701 return dyn_cast_or_null<ConstantSDNode>(
14702 Val: getSplatValue(DemandedElts, UndefElements));
14703}
14704
14705ConstantSDNode *
14706BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14707 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14708}
14709
14710ConstantFPSDNode *
14711BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14712 BitVector *UndefElements) const {
14713 return dyn_cast_or_null<ConstantFPSDNode>(
14714 Val: getSplatValue(DemandedElts, UndefElements));
14715}
14716
14717ConstantFPSDNode *
14718BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14719 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14720}
14721
14722int32_t
14723BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14724 uint32_t BitWidth) const {
14725 if (ConstantFPSDNode *CN =
14726 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14727 bool IsExact;
14728 APSInt IntVal(BitWidth);
14729 const APFloat &APF = CN->getValueAPF();
14730 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14731 APFloat::opOK ||
14732 !IsExact)
14733 return -1;
14734
14735 return IntVal.exactLogBase2();
14736 }
14737 return -1;
14738}
14739
14740bool BuildVectorSDNode::getConstantRawBits(
14741 bool IsLittleEndian, unsigned DstEltSizeInBits,
14742 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14743 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14744 if (!isConstant())
14745 return false;
14746
14747 unsigned NumSrcOps = getNumOperands();
14748 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14749 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14750 "Invalid bitcast scale");
14751
14752 // Extract raw src bits.
14753 SmallVector<APInt> SrcBitElements(NumSrcOps,
14754 APInt::getZero(numBits: SrcEltSizeInBits));
14755 BitVector SrcUndeElements(NumSrcOps, false);
14756
14757 for (unsigned I = 0; I != NumSrcOps; ++I) {
14758 SDValue Op = getOperand(Num: I);
14759 if (Op.isUndef()) {
14760 SrcUndeElements.set(I);
14761 continue;
14762 }
14763 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14764 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14765 assert((CInt || CFP) && "Unknown constant");
14766 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14767 : CFP->getValueAPF().bitcastToAPInt();
14768 }
14769
14770 // Recast to dst width.
14771 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14772 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14773 return true;
14774}
14775
14776void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14777 unsigned DstEltSizeInBits,
14778 SmallVectorImpl<APInt> &DstBitElements,
14779 ArrayRef<APInt> SrcBitElements,
14780 BitVector &DstUndefElements,
14781 const BitVector &SrcUndefElements) {
14782 unsigned NumSrcOps = SrcBitElements.size();
14783 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14784 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14785 "Invalid bitcast scale");
14786 assert(NumSrcOps == SrcUndefElements.size() &&
14787 "Vector size mismatch");
14788
14789 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14790 DstUndefElements.clear();
14791 DstUndefElements.resize(N: NumDstOps, t: false);
14792 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14793
14794 // Concatenate src elements constant bits together into dst element.
14795 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14796 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14797 for (unsigned I = 0; I != NumDstOps; ++I) {
14798 DstUndefElements.set(I);
14799 APInt &DstBits = DstBitElements[I];
14800 for (unsigned J = 0; J != Scale; ++J) {
14801 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14802 if (SrcUndefElements[Idx])
14803 continue;
14804 DstUndefElements.reset(Idx: I);
14805 const APInt &SrcBits = SrcBitElements[Idx];
14806 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14807 "Illegal constant bitwidths");
14808 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14809 }
14810 }
14811 return;
14812 }
14813
14814 // Split src element constant bits into dst elements.
14815 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14816 for (unsigned I = 0; I != NumSrcOps; ++I) {
14817 if (SrcUndefElements[I]) {
14818 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14819 continue;
14820 }
14821 const APInt &SrcBits = SrcBitElements[I];
14822 for (unsigned J = 0; J != Scale; ++J) {
14823 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14824 APInt &DstBits = DstBitElements[Idx];
14825 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14826 }
14827 }
14828}
14829
14830bool BuildVectorSDNode::isConstant() const {
14831 for (const SDValue &Op : op_values()) {
14832 unsigned Opc = Op.getOpcode();
14833 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14834 return false;
14835 }
14836 return true;
14837}
14838
14839std::optional<std::pair<APInt, APInt>>
14840BuildVectorSDNode::isArithmeticSequence() const {
14841 unsigned NumOps = getNumOperands();
14842 if (NumOps < 2)
14843 return std::nullopt;
14844
14845 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14846 APInt Start, Stride;
14847 int FirstIdx = -1, SecondIdx = -1;
14848
14849 // Find the first two non-undef constant elements to determine Start and
14850 // Stride, then verify all remaining elements match the sequence.
14851 for (unsigned I = 0; I < NumOps; ++I) {
14852 SDValue Op = getOperand(Num: I);
14853 if (Op->isUndef())
14854 continue;
14855 if (!isa<ConstantSDNode>(Val: Op))
14856 return std::nullopt;
14857
14858 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
14859 if (FirstIdx < 0) {
14860 FirstIdx = I;
14861 Start = Val;
14862 } else if (SecondIdx < 0) {
14863 SecondIdx = I;
14864 // Compute stride using modular arithmetic. Simple division would handle
14865 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
14866 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
14867 // Note that modular arithmetic is agnostic to signed/unsigned.
14868 unsigned IdxDiff = I - FirstIdx;
14869 APInt ValDiff = Val - Start;
14870
14871 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
14872 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
14873 if (ValDiff.countr_zero() < CommonPow2Bits)
14874 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
14875 IdxDiff >>= CommonPow2Bits;
14876 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
14877
14878 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
14879 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
14880 // one, but we could try all candidates to handle more cases.
14881 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
14882 if (Stride.isZero())
14883 return std::nullopt;
14884
14885 // Step 3: Adjust Start based on the first defined element's index.
14886 Start -= Stride * FirstIdx;
14887 } else {
14888 // Verify this element matches the sequence.
14889 if (Val != Start + Stride * I)
14890 return std::nullopt;
14891 }
14892 }
14893
14894 // Need at least two defined elements.
14895 if (SecondIdx < 0)
14896 return std::nullopt;
14897
14898 return std::make_pair(x&: Start, y&: Stride);
14899}
14900
14901bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
14902 // Find the first non-undef value in the shuffle mask.
14903 unsigned i, e;
14904 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
14905 /* search */;
14906
14907 // If all elements are undefined, this shuffle can be considered a splat
14908 // (although it should eventually get simplified away completely).
14909 if (i == e)
14910 return true;
14911
14912 // Make sure all remaining elements are either undef or the same as the first
14913 // non-undef value.
14914 for (int Idx = Mask[i]; i != e; ++i)
14915 if (Mask[i] >= 0 && Mask[i] != Idx)
14916 return false;
14917 return true;
14918}
14919
14920// Returns true if it is a constant integer BuildVector or constant integer,
14921// possibly hidden by a bitcast.
14922bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
14923 SDValue N, bool AllowOpaques) const {
14924 N = peekThroughBitcasts(V: N);
14925
14926 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
14927 return AllowOpaques || !C->isOpaque();
14928
14929 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
14930 return true;
14931
14932 // Treat a GlobalAddress supporting constant offset folding as a
14933 // constant integer.
14934 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
14935 if (GA->getOpcode() == ISD::GlobalAddress &&
14936 TLI->isOffsetFoldingLegal(GA))
14937 return true;
14938
14939 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14940 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
14941 return true;
14942 return false;
14943}
14944
14945// Returns true if it is a constant float BuildVector or constant float.
14946bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
14947 if (isa<ConstantFPSDNode>(Val: N))
14948 return true;
14949
14950 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
14951 return true;
14952
14953 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14954 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
14955 return true;
14956
14957 return false;
14958}
14959
14960std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
14961 ConstantSDNode *Const =
14962 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
14963 if (!Const)
14964 return std::nullopt;
14965
14966 EVT VT = N->getValueType(ResNo: 0);
14967 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
14968 switch (TLI->getBooleanContents(Type: N.getValueType())) {
14969 case TargetLowering::ZeroOrOneBooleanContent:
14970 if (CVal.isOne())
14971 return true;
14972 if (CVal.isZero())
14973 return false;
14974 return std::nullopt;
14975 case TargetLowering::ZeroOrNegativeOneBooleanContent:
14976 if (CVal.isAllOnes())
14977 return true;
14978 if (CVal.isZero())
14979 return false;
14980 return std::nullopt;
14981 case TargetLowering::UndefinedBooleanContent:
14982 return CVal[0];
14983 }
14984 llvm_unreachable("Unknown BooleanContent enum");
14985}
14986
14987void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
14988 assert(!Node->OperandList && "Node already has operands");
14989 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
14990 "too many operands to fit into SDNode");
14991 SDUse *Ops = OperandRecycler.allocate(
14992 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
14993
14994 bool IsDivergent = false;
14995 for (unsigned I = 0; I != Vals.size(); ++I) {
14996 Ops[I].setUser(Node);
14997 Ops[I].setInitial(Vals[I]);
14998 EVT VT = Ops[I].getValueType();
14999
15000 // Skip Chain. It does not carry divergence.
15001 if (VT != MVT::Other &&
15002 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
15003 Ops[I].getNode()->isDivergent()) {
15004 IsDivergent = true;
15005 }
15006 }
15007 Node->NumOperands = Vals.size();
15008 Node->OperandList = Ops;
15009 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
15010 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
15011 Node->SDNodeBits.IsDivergent = IsDivergent;
15012 }
15013 checkForCycles(N: Node);
15014}
15015
15016SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
15017 SmallVectorImpl<SDValue> &Vals) {
15018 size_t Limit = SDNode::getMaxNumOperands();
15019 while (Vals.size() > Limit) {
15020 unsigned SliceIdx = Vals.size() - Limit;
15021 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
15022 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
15023 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
15024 Vals.emplace_back(Args&: NewTF);
15025 }
15026 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
15027}
15028
15029SDValue SelectionDAG::getIdentityElement(unsigned Opcode, const SDLoc &DL,
15030 EVT VT, SDNodeFlags Flags) {
15031 switch (Opcode) {
15032 default:
15033 return SDValue();
15034 case ISD::ADD:
15035 case ISD::OR:
15036 case ISD::XOR:
15037 case ISD::UMAX:
15038 return getConstant(Val: 0, DL, VT);
15039 case ISD::MUL:
15040 return getConstant(Val: 1, DL, VT);
15041 case ISD::AND:
15042 case ISD::UMIN:
15043 return getAllOnesConstant(DL, VT);
15044 case ISD::SMAX:
15045 return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
15046 case ISD::SMIN:
15047 return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
15048 case ISD::FADD:
15049 // If flags allow, prefer positive zero since it's generally cheaper
15050 // to materialize on most targets.
15051 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
15052 case ISD::FMUL:
15053 return getConstantFP(Val: 1.0, DL, VT);
15054 case ISD::FMINNUM:
15055 case ISD::FMAXNUM: {
15056 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
15057 const fltSemantics &Semantics = VT.getFltSemantics();
15058 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
15059 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
15060 APFloat::getLargest(Sem: Semantics);
15061 if (Opcode == ISD::FMAXNUM)
15062 NeutralAF.changeSign();
15063
15064 return getConstantFP(V: NeutralAF, DL, VT);
15065 }
15066 case ISD::FMINIMUM:
15067 case ISD::FMAXIMUM: {
15068 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
15069 const fltSemantics &Semantics = VT.getFltSemantics();
15070 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
15071 : APFloat::getLargest(Sem: Semantics);
15072 if (Opcode == ISD::FMAXIMUM)
15073 NeutralAF.changeSign();
15074
15075 return getConstantFP(V: NeutralAF, DL, VT);
15076 }
15077
15078 }
15079}
15080
15081SDValue SelectionDAG::getPartialReduceMLS(unsigned Opc, const SDLoc &DL,
15082 SDValue Acc, SDValue LHS,
15083 SDValue RHS) {
15084 EVT AccVT = Acc.getValueType();
15085 if (AccVT.isFloatingPoint()) {
15086 assert(Opc == ISD::PARTIAL_REDUCE_FMLA && "Unexpected opcode");
15087 SDValue NegRHS = getNode(Opcode: ISD::FNEG, DL, VT: RHS.getValueType(), N1: RHS);
15088 return getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: LHS, N3: NegRHS);
15089 }
15090 assert((Opc == ISD::PARTIAL_REDUCE_UMLA || Opc == ISD::PARTIAL_REDUCE_SMLA) &&
15091 "Unexpected opcode");
15092 SDValue NegAcc = getNegative(Val: Acc, DL, VT: AccVT);
15093 SDValue MLA = getNode(Opcode: Opc, DL, VT: AccVT, N1: NegAcc, N2: LHS, N3: RHS);
15094 return getNegative(Val: MLA, DL, VT: AccVT);
15095}
15096
15097/// Helper used to make a call to a library function that has one argument of
15098/// pointer type.
15099///
15100/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
15101/// used to get or set floating-point state. They have one argument of pointer
15102/// type, which points to the memory region containing bits of the
15103/// floating-point state. The value returned by such function is ignored in the
15104/// created call.
15105///
15106/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
15107/// \param Ptr Pointer used to save/load state.
15108/// \param InChain Ingoing token chain.
15109/// \returns Outgoing chain token.
15110SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
15111 SDValue InChain,
15112 const SDLoc &DLoc) {
15113 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
15114 TargetLowering::ArgListTy Args;
15115 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
15116 RTLIB::LibcallImpl LibcallImpl =
15117 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
15118 if (LibcallImpl == RTLIB::Unsupported)
15119 reportFatalUsageError(reason: "emitting call to unsupported libcall");
15120
15121 SDValue Callee =
15122 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
15123 TargetLowering::CallLoweringInfo CLI(*this);
15124 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
15125 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
15126 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
15127 return TLI->LowerCallTo(CLI).second;
15128}
15129
15130void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
15131 assert(From && To && "Invalid SDNode; empty source SDValue?");
15132 auto I = SDEI.find(Val: From);
15133 if (I == SDEI.end())
15134 return;
15135
15136 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
15137 // the iterator, hence the need to make a copy to prevent a use-after-free.
15138 NodeExtraInfo NEI = I->second;
15139 if (LLVM_LIKELY(!NEI.PCSections)) {
15140 // No deep copy required for the types of extra info set.
15141 //
15142 // FIXME: Investigate if other types of extra info also need deep copy. This
15143 // depends on the types of nodes they can be attached to: if some extra info
15144 // is only ever attached to nodes where a replacement To node is always the
15145 // node where later use and propagation of the extra info has the intended
15146 // semantics, no deep copy is required.
15147 SDEI[To] = std::move(NEI);
15148 return;
15149 }
15150
15151 const SDNode *EntrySDN = getEntryNode().getNode();
15152
15153 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
15154 // through the replacement of From with To. Otherwise, replacements of a node
15155 // (From) with more complex nodes (To and its operands) may result in lost
15156 // extra info where the root node (To) is insignificant in further propagating
15157 // and using extra info when further lowering to MIR.
15158 //
15159 // In the first step pre-populate the visited set with the nodes reachable
15160 // from the old From node. This avoids copying NodeExtraInfo to parts of the
15161 // DAG that is not new and should be left untouched.
15162 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
15163 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
15164 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
15165 if (MaxDepth == 0) {
15166 // Remember this node in case we need to increase MaxDepth and continue
15167 // populating FromReach from this node.
15168 Leafs.emplace_back(Args&: N);
15169 return;
15170 }
15171 if (!FromReach.insert(V: N).second)
15172 return;
15173 for (const SDValue &Op : N->op_values())
15174 Self(Self, Op.getNode(), MaxDepth - 1);
15175 };
15176
15177 // Copy extra info to To and all its transitive operands (that are new).
15178 SmallPtrSet<const SDNode *, 8> Visited;
15179 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
15180 if (FromReach.contains(V: N))
15181 return true;
15182 if (!Visited.insert(Ptr: N).second)
15183 return true;
15184 if (EntrySDN == N)
15185 return false;
15186 for (const SDValue &Op : N->op_values()) {
15187 if (N == To && Op.getNode() == EntrySDN) {
15188 // Special case: New node's operand is the entry node; just need to
15189 // copy extra info to new node.
15190 break;
15191 }
15192 if (!Self(Self, Op.getNode()))
15193 return false;
15194 }
15195 // Copy only if entry node was not reached.
15196 SDEI[N] = std::move(NEI);
15197 return true;
15198 };
15199
15200 // We first try with a lower MaxDepth, assuming that the path to common
15201 // operands between From and To is relatively short. This significantly
15202 // improves performance in the common case. The initial MaxDepth is big
15203 // enough to avoid retry in the common case; the last MaxDepth is large
15204 // enough to avoid having to use the fallback below (and protects from
15205 // potential stack exhaustion from recursion).
15206 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
15207 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
15208 // StartFrom is the previous (or initial) set of leafs reachable at the
15209 // previous maximum depth.
15210 SmallVector<const SDNode *> StartFrom;
15211 std::swap(LHS&: StartFrom, RHS&: Leafs);
15212 for (const SDNode *N : StartFrom)
15213 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
15214 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
15215 return;
15216 // This should happen very rarely (reached the entry node).
15217 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
15218 assert(!Leafs.empty());
15219 }
15220
15221 // This should not happen - but if it did, that means the subgraph reachable
15222 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
15223 // could not visit all reachable common operands. Consequently, we were able
15224 // to reach the entry node.
15225 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
15226 assert(false && "From subgraph too complex - increase max. MaxDepth?");
15227 // Best-effort fallback if assertions disabled.
15228 SDEI[To] = std::move(NEI);
15229}
15230
15231#ifndef NDEBUG
15232static void checkForCyclesHelper(const SDNode *N,
15233 SmallPtrSetImpl<const SDNode*> &Visited,
15234 SmallPtrSetImpl<const SDNode*> &Checked,
15235 const llvm::SelectionDAG *DAG) {
15236 // If this node has already been checked, don't check it again.
15237 if (Checked.count(N))
15238 return;
15239
15240 // If a node has already been visited on this depth-first walk, reject it as
15241 // a cycle.
15242 if (!Visited.insert(N).second) {
15243 errs() << "Detected cycle in SelectionDAG\n";
15244 dbgs() << "Offending node:\n";
15245 N->dumprFull(DAG); dbgs() << "\n";
15246 abort();
15247 }
15248
15249 for (const SDValue &Op : N->op_values())
15250 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
15251
15252 Checked.insert(N);
15253 Visited.erase(N);
15254}
15255#endif
15256
15257void llvm::checkForCycles(const llvm::SDNode *N,
15258 const llvm::SelectionDAG *DAG,
15259 bool force) {
15260#ifndef NDEBUG
15261 bool check = force;
15262#ifdef EXPENSIVE_CHECKS
15263 check = true;
15264#endif // EXPENSIVE_CHECKS
15265 if (check) {
15266 assert(N && "Checking nonexistent SDNode");
15267 SmallPtrSet<const SDNode*, 32> visited;
15268 SmallPtrSet<const SDNode*, 32> checked;
15269 checkForCyclesHelper(N, visited, checked, DAG);
15270 }
15271#endif // !NDEBUG
15272}
15273
15274void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
15275 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
15276}
15277