1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/FunctionLoweringInfo.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineConstantPool.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/RuntimeLibcallUtil.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetFrameLowering.h"
46#include "llvm/CodeGen/TargetLowering.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGen/ValueTypes.h"
50#include "llvm/CodeGenTypes/MachineValueType.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfoMetadata.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
61#include "llvm/Support/Casting.h"
62#include "llvm/Support/CodeGen.h"
63#include "llvm/Support/Compiler.h"
64#include "llvm/Support/Debug.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/MathExtras.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/TargetParser/Triple.h"
72#include "llvm/Transforms/Utils/SizeOpts.h"
73#include <algorithm>
74#include <cassert>
75#include <cstdint>
76#include <cstdlib>
77#include <limits>
78#include <optional>
79#include <string>
80#include <utility>
81#include <vector>
82
83using namespace llvm;
84using namespace llvm::SDPatternMatch;
85
86/// makeVTList - Return an instance of the SDVTList struct initialized with the
87/// specified members.
88static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
89 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
90 return Res;
91}
92
93// Default null implementations of the callbacks.
94void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
95void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
96void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
97
98void SelectionDAG::DAGNodeDeletedListener::anchor() {}
99void SelectionDAG::DAGNodeInsertedListener::anchor() {}
100
101#define DEBUG_TYPE "selectiondag"
102
103static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
104 cl::Hidden, cl::init(Val: true),
105 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
106
107static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
108 cl::desc("Number limit for gluing ld/st of memcpy."),
109 cl::Hidden, cl::init(Val: 0));
110
111static cl::opt<unsigned>
112 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
113 cl::desc("DAG combiner limit number of steps when searching DAG "
114 "for predecessor nodes"));
115
116static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
117 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
118}
119
120unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
121
122//===----------------------------------------------------------------------===//
123// ConstantFPSDNode Class
124//===----------------------------------------------------------------------===//
125
126/// isExactlyValue - We don't rely on operator== working on double values, as
127/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
128/// As such, this method can be used to do an exact bit-for-bit comparison of
129/// two floating point values.
130bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
131 return getValueAPF().bitwiseIsEqual(RHS: V);
132}
133
134bool ConstantFPSDNode::isValueValidForType(EVT VT,
135 const APFloat& Val) {
136 assert(VT.isFloatingPoint() && "Can only convert between FP types");
137
138 // convert modifies in place, so make a copy.
139 APFloat Val2 = APFloat(Val);
140 bool losesInfo;
141 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
142 losesInfo: &losesInfo);
143 return !losesInfo;
144}
145
146//===----------------------------------------------------------------------===//
147// ISD Namespace
148//===----------------------------------------------------------------------===//
149
150bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
151 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
152 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
153 unsigned EltSize =
154 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
155 SplatVal = OptAPInt->trunc(width: EltSize);
156 return true;
157 }
158 }
159
160 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
161 if (!BV)
162 return false;
163
164 APInt SplatUndef;
165 unsigned SplatBitSize;
166 bool HasUndefs;
167 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
168 // Endianness does not matter here. We are checking for a splat given the
169 // element size of the vector, and if we find such a splat for little endian
170 // layout, then that should be valid also for big endian (as the full vector
171 // size is known to be a multiple of the element size).
172 const bool IsBigEndian = false;
173 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
174 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
175 EltSize == SplatBitSize;
176}
177
178// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
179// specializations of the more general isConstantSplatVector()?
180
181bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
182 // Look through a bit convert.
183 while (N->getOpcode() == ISD::BITCAST)
184 N = N->getOperand(Num: 0).getNode();
185
186 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
187 APInt SplatVal;
188 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
189 }
190
191 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
192
193 unsigned i = 0, e = N->getNumOperands();
194
195 // Skip over all of the undef values.
196 while (i != e && N->getOperand(Num: i).isUndef())
197 ++i;
198
199 // Do not accept an all-undef vector.
200 if (i == e) return false;
201
202 // Do not accept build_vectors that aren't all constants or which have non-~0
203 // elements. We have to be a bit careful here, as the type of the constant
204 // may not be the same as the type of the vector elements due to type
205 // legalization (the elements are promoted to a legal type for the target and
206 // a vector of a type may be legal when the base element type is not).
207 // We only want to check enough bits to cover the vector elements, because
208 // we care if the resultant vector is all ones, not whether the individual
209 // constants are.
210 SDValue NotZero = N->getOperand(Num: i);
211 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
212 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
213 if (OptAPInt->countr_one() < EltSize)
214 return false;
215 } else
216 return false;
217
218 // Okay, we have at least one ~0 value, check to see if the rest match or are
219 // undefs. Even with the above element type twiddling, this should be OK, as
220 // the same type legalization should have applied to all the elements.
221 for (++i; i != e; ++i)
222 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
223 return false;
224 return true;
225}
226
227bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
228 // Look through a bit convert.
229 while (N->getOpcode() == ISD::BITCAST)
230 N = N->getOperand(Num: 0).getNode();
231
232 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
233 APInt SplatVal;
234 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
235 }
236
237 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
238
239 bool IsAllUndef = true;
240 for (const SDValue &Op : N->op_values()) {
241 if (Op.isUndef())
242 continue;
243 IsAllUndef = false;
244 // Do not accept build_vectors that aren't all constants or which have non-0
245 // elements. We have to be a bit careful here, as the type of the constant
246 // may not be the same as the type of the vector elements due to type
247 // legalization (the elements are promoted to a legal type for the target
248 // and a vector of a type may be legal when the base element type is not).
249 // We only want to check enough bits to cover the vector elements, because
250 // we care if the resultant vector is all zeros, not whether the individual
251 // constants are.
252 if (auto OptAPInt = Op->bitcastToAPInt()) {
253 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
254 if (OptAPInt->countr_zero() < EltSize)
255 return false;
256 } else
257 return false;
258 }
259
260 // Do not accept an all-undef vector.
261 if (IsAllUndef)
262 return false;
263 return true;
264}
265
266bool ISD::isBuildVectorAllOnes(const SDNode *N) {
267 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
268}
269
270bool ISD::isBuildVectorAllZeros(const SDNode *N) {
271 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
272}
273
274bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
275 if (N->getOpcode() != ISD::BUILD_VECTOR)
276 return false;
277
278 for (const SDValue &Op : N->op_values()) {
279 if (Op.isUndef())
280 continue;
281 if (!isa<ConstantSDNode>(Val: Op))
282 return false;
283 }
284 return true;
285}
286
287bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
288 if (N->getOpcode() != ISD::BUILD_VECTOR)
289 return false;
290
291 for (const SDValue &Op : N->op_values()) {
292 if (Op.isUndef())
293 continue;
294 if (!isa<ConstantFPSDNode>(Val: Op))
295 return false;
296 }
297 return true;
298}
299
300bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
301 bool Signed) {
302 assert(N->getValueType(0).isVector() && "Expected a vector!");
303
304 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
305 if (EltSize <= NewEltSize)
306 return false;
307
308 if (N->getOpcode() == ISD::ZERO_EXTEND) {
309 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
310 NewEltSize) &&
311 !Signed;
312 }
313 if (N->getOpcode() == ISD::SIGN_EXTEND) {
314 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
315 NewEltSize) &&
316 Signed;
317 }
318 if (N->getOpcode() != ISD::BUILD_VECTOR)
319 return false;
320
321 for (const SDValue &Op : N->op_values()) {
322 if (Op.isUndef())
323 continue;
324 if (!isa<ConstantSDNode>(Val: Op))
325 return false;
326
327 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
328 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
329 return false;
330 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
331 return false;
332 }
333
334 return true;
335}
336
337bool ISD::allOperandsUndef(const SDNode *N) {
338 // Return false if the node has no operands.
339 // This is "logically inconsistent" with the definition of "all" but
340 // is probably the desired behavior.
341 if (N->getNumOperands() == 0)
342 return false;
343 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
344}
345
346bool ISD::isFreezeUndef(const SDNode *N) {
347 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
348}
349
350template <typename ConstNodeType>
351bool ISD::matchUnaryPredicateImpl(SDValue Op,
352 std::function<bool(ConstNodeType *)> Match,
353 bool AllowUndefs, bool AllowTruncation) {
354 // FIXME: Add support for scalar UNDEF cases?
355 if (auto *C = dyn_cast<ConstNodeType>(Op))
356 return Match(C);
357
358 // FIXME: Add support for vector UNDEF cases?
359 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
360 ISD::SPLAT_VECTOR != Op.getOpcode())
361 return false;
362
363 EVT SVT = Op.getValueType().getScalarType();
364 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
365 if (AllowUndefs && Op.getOperand(i).isUndef()) {
366 if (!Match(nullptr))
367 return false;
368 continue;
369 }
370
371 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
372 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
373 !Match(Cst))
374 return false;
375 }
376 return true;
377}
378// Build used template types.
379template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
380 SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);
381template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
382 SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);
383
384bool ISD::matchBinaryPredicate(
385 SDValue LHS, SDValue RHS,
386 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
387 bool AllowUndefs, bool AllowTypeMismatch) {
388 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
389 return false;
390
391 // TODO: Add support for scalar UNDEF cases?
392 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
393 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
394 return Match(LHSCst, RHSCst);
395
396 // TODO: Add support for vector UNDEF cases?
397 if (LHS.getOpcode() != RHS.getOpcode() ||
398 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
399 LHS.getOpcode() != ISD::SPLAT_VECTOR))
400 return false;
401
402 EVT SVT = LHS.getValueType().getScalarType();
403 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
404 SDValue LHSOp = LHS.getOperand(i);
405 SDValue RHSOp = RHS.getOperand(i);
406 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
407 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
408 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
409 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
410 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
411 return false;
412 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
413 LHSOp.getValueType() != RHSOp.getValueType()))
414 return false;
415 if (!Match(LHSCst, RHSCst))
416 return false;
417 }
418 return true;
419}
420
421ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
422 switch (MinMaxOpc) {
423 default:
424 llvm_unreachable("unrecognized opcode");
425 case ISD::UMIN:
426 return ISD::UMAX;
427 case ISD::UMAX:
428 return ISD::UMIN;
429 case ISD::SMIN:
430 return ISD::SMAX;
431 case ISD::SMAX:
432 return ISD::SMIN;
433 }
434}
435
436ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
437 switch (MinMaxOpc) {
438 default:
439 llvm_unreachable("unrecognized min/max opcode");
440 case ISD::SMIN:
441 return ISD::UMIN;
442 case ISD::SMAX:
443 return ISD::UMAX;
444 case ISD::UMIN:
445 return ISD::SMIN;
446 case ISD::UMAX:
447 return ISD::SMAX;
448 }
449}
450
451ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
452 switch (VecReduceOpcode) {
453 default:
454 llvm_unreachable("Expected VECREDUCE opcode");
455 case ISD::VECREDUCE_FADD:
456 case ISD::VECREDUCE_SEQ_FADD:
457 case ISD::VP_REDUCE_FADD:
458 case ISD::VP_REDUCE_SEQ_FADD:
459 return ISD::FADD;
460 case ISD::VECREDUCE_FMUL:
461 case ISD::VECREDUCE_SEQ_FMUL:
462 case ISD::VP_REDUCE_FMUL:
463 case ISD::VP_REDUCE_SEQ_FMUL:
464 return ISD::FMUL;
465 case ISD::VECREDUCE_ADD:
466 case ISD::VP_REDUCE_ADD:
467 return ISD::ADD;
468 case ISD::VECREDUCE_MUL:
469 case ISD::VP_REDUCE_MUL:
470 return ISD::MUL;
471 case ISD::VECREDUCE_AND:
472 case ISD::VP_REDUCE_AND:
473 return ISD::AND;
474 case ISD::VECREDUCE_OR:
475 case ISD::VP_REDUCE_OR:
476 return ISD::OR;
477 case ISD::VECREDUCE_XOR:
478 case ISD::VP_REDUCE_XOR:
479 return ISD::XOR;
480 case ISD::VECREDUCE_SMAX:
481 case ISD::VP_REDUCE_SMAX:
482 return ISD::SMAX;
483 case ISD::VECREDUCE_SMIN:
484 case ISD::VP_REDUCE_SMIN:
485 return ISD::SMIN;
486 case ISD::VECREDUCE_UMAX:
487 case ISD::VP_REDUCE_UMAX:
488 return ISD::UMAX;
489 case ISD::VECREDUCE_UMIN:
490 case ISD::VP_REDUCE_UMIN:
491 return ISD::UMIN;
492 case ISD::VECREDUCE_FMAX:
493 case ISD::VP_REDUCE_FMAX:
494 return ISD::FMAXNUM;
495 case ISD::VECREDUCE_FMIN:
496 case ISD::VP_REDUCE_FMIN:
497 return ISD::FMINNUM;
498 case ISD::VECREDUCE_FMAXIMUM:
499 case ISD::VP_REDUCE_FMAXIMUM:
500 return ISD::FMAXIMUM;
501 case ISD::VECREDUCE_FMINIMUM:
502 case ISD::VP_REDUCE_FMINIMUM:
503 return ISD::FMINIMUM;
504 }
505}
506
507bool ISD::isVPOpcode(unsigned Opcode) {
508 switch (Opcode) {
509 default:
510 return false;
511#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
512 case ISD::VPSD: \
513 return true;
514#include "llvm/IR/VPIntrinsics.def"
515 }
516}
517
518bool ISD::isVPBinaryOp(unsigned Opcode) {
519 switch (Opcode) {
520 default:
521 break;
522#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
523#define VP_PROPERTY_BINARYOP return true;
524#define END_REGISTER_VP_SDNODE(VPSD) break;
525#include "llvm/IR/VPIntrinsics.def"
526 }
527 return false;
528}
529
530bool ISD::isVPReduction(unsigned Opcode) {
531 switch (Opcode) {
532 default:
533 return false;
534 case ISD::VP_REDUCE_ADD:
535 case ISD::VP_REDUCE_MUL:
536 case ISD::VP_REDUCE_AND:
537 case ISD::VP_REDUCE_OR:
538 case ISD::VP_REDUCE_XOR:
539 case ISD::VP_REDUCE_SMAX:
540 case ISD::VP_REDUCE_SMIN:
541 case ISD::VP_REDUCE_UMAX:
542 case ISD::VP_REDUCE_UMIN:
543 case ISD::VP_REDUCE_FMAX:
544 case ISD::VP_REDUCE_FMIN:
545 case ISD::VP_REDUCE_FMAXIMUM:
546 case ISD::VP_REDUCE_FMINIMUM:
547 case ISD::VP_REDUCE_FADD:
548 case ISD::VP_REDUCE_FMUL:
549 case ISD::VP_REDUCE_SEQ_FADD:
550 case ISD::VP_REDUCE_SEQ_FMUL:
551 return true;
552 }
553}
554
555/// The operand position of the vector mask.
556std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
557 switch (Opcode) {
558 default:
559 return std::nullopt;
560#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
561 case ISD::VPSD: \
562 return MASKPOS;
563#include "llvm/IR/VPIntrinsics.def"
564 }
565}
566
567/// The operand position of the explicit vector length parameter.
568std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
569 switch (Opcode) {
570 default:
571 return std::nullopt;
572#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
573 case ISD::VPSD: \
574 return EVLPOS;
575#include "llvm/IR/VPIntrinsics.def"
576 }
577}
578
579std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
580 bool hasFPExcept) {
581 // FIXME: Return strict opcodes in case of fp exceptions.
582 switch (VPOpcode) {
583 default:
584 return std::nullopt;
585#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
586#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
587#define END_REGISTER_VP_SDNODE(VPOPC) break;
588#include "llvm/IR/VPIntrinsics.def"
589 }
590 return std::nullopt;
591}
592
593std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
594 switch (Opcode) {
595 default:
596 return std::nullopt;
597#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
598#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
599#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
600#include "llvm/IR/VPIntrinsics.def"
601 }
602}
603
604ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
605 switch (ExtType) {
606 case ISD::EXTLOAD:
607 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
608 case ISD::SEXTLOAD:
609 return ISD::SIGN_EXTEND;
610 case ISD::ZEXTLOAD:
611 return ISD::ZERO_EXTEND;
612 default:
613 break;
614 }
615
616 llvm_unreachable("Invalid LoadExtType");
617}
618
619ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
620 // To perform this operation, we just need to swap the L and G bits of the
621 // operation.
622 unsigned OldL = (Operation >> 2) & 1;
623 unsigned OldG = (Operation >> 1) & 1;
624 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
625 (OldL << 1) | // New G bit
626 (OldG << 2)); // New L bit.
627}
628
629static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
630 unsigned Operation = Op;
631 if (isIntegerLike)
632 Operation ^= 7; // Flip L, G, E bits, but not U.
633 else
634 Operation ^= 15; // Flip all of the condition bits.
635
636 if (Operation > ISD::SETTRUE2)
637 Operation &= ~8; // Don't let N and U bits get set.
638
639 return ISD::CondCode(Operation);
640}
641
642ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
643 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
644}
645
646ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
647 bool isIntegerLike) {
648 return getSetCCInverseImpl(Op, isIntegerLike);
649}
650
651/// For an integer comparison, return 1 if the comparison is a signed operation
652/// and 2 if the result is an unsigned comparison. Return zero if the operation
653/// does not depend on the sign of the input (setne and seteq).
654static int isSignedOp(ISD::CondCode Opcode) {
655 switch (Opcode) {
656 default: llvm_unreachable("Illegal integer setcc operation!");
657 case ISD::SETEQ:
658 case ISD::SETNE: return 0;
659 case ISD::SETLT:
660 case ISD::SETLE:
661 case ISD::SETGT:
662 case ISD::SETGE: return 1;
663 case ISD::SETULT:
664 case ISD::SETULE:
665 case ISD::SETUGT:
666 case ISD::SETUGE: return 2;
667 }
668}
669
670ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
671 EVT Type) {
672 bool IsInteger = Type.isInteger();
673 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
674 // Cannot fold a signed integer setcc with an unsigned integer setcc.
675 return ISD::SETCC_INVALID;
676
677 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
678
679 // If the N and U bits get set, then the resultant comparison DOES suddenly
680 // care about orderedness, and it is true when ordered.
681 if (Op > ISD::SETTRUE2)
682 Op &= ~16; // Clear the U bit if the N bit is set.
683
684 // Canonicalize illegal integer setcc's.
685 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
686 Op = ISD::SETNE;
687
688 return ISD::CondCode(Op);
689}
690
691ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
692 EVT Type) {
693 bool IsInteger = Type.isInteger();
694 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
695 // Cannot fold a signed setcc with an unsigned setcc.
696 return ISD::SETCC_INVALID;
697
698 // Combine all of the condition bits.
699 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
700
701 // Canonicalize illegal integer setcc's.
702 if (IsInteger) {
703 switch (Result) {
704 default: break;
705 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
706 case ISD::SETOEQ: // SETEQ & SETU[LG]E
707 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
708 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
709 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
710 }
711 }
712
713 return Result;
714}
715
716//===----------------------------------------------------------------------===//
717// SDNode Profile Support
718//===----------------------------------------------------------------------===//
719
720/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
721static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
722 ID.AddInteger(I: OpC);
723}
724
725/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
726/// solely with their pointer.
727static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
728 ID.AddPointer(Ptr: VTList.VTs);
729}
730
731/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
732static void AddNodeIDOperands(FoldingSetNodeID &ID,
733 ArrayRef<SDValue> Ops) {
734 for (const auto &Op : Ops) {
735 ID.AddPointer(Ptr: Op.getNode());
736 ID.AddInteger(I: Op.getResNo());
737 }
738}
739
740/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
741static void AddNodeIDOperands(FoldingSetNodeID &ID,
742 ArrayRef<SDUse> Ops) {
743 for (const auto &Op : Ops) {
744 ID.AddPointer(Ptr: Op.getNode());
745 ID.AddInteger(I: Op.getResNo());
746 }
747}
748
749static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
750 SDVTList VTList, ArrayRef<SDValue> OpList) {
751 AddNodeIDOpcode(ID, OpC);
752 AddNodeIDValueTypes(ID, VTList);
753 AddNodeIDOperands(ID, Ops: OpList);
754}
755
756/// If this is an SDNode with special info, add this info to the NodeID data.
757static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
758 switch (N->getOpcode()) {
759 case ISD::TargetExternalSymbol:
760 case ISD::ExternalSymbol:
761 case ISD::MCSymbol:
762 llvm_unreachable("Should only be used on nodes with operands");
763 default: break; // Normal nodes don't need extra info.
764 case ISD::TargetConstant:
765 case ISD::Constant: {
766 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
767 ID.AddPointer(Ptr: C->getConstantIntValue());
768 ID.AddBoolean(B: C->isOpaque());
769 break;
770 }
771 case ISD::TargetConstantFP:
772 case ISD::ConstantFP:
773 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
774 break;
775 case ISD::TargetGlobalAddress:
776 case ISD::GlobalAddress:
777 case ISD::TargetGlobalTLSAddress:
778 case ISD::GlobalTLSAddress: {
779 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
780 ID.AddPointer(Ptr: GA->getGlobal());
781 ID.AddInteger(I: GA->getOffset());
782 ID.AddInteger(I: GA->getTargetFlags());
783 break;
784 }
785 case ISD::BasicBlock:
786 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
787 break;
788 case ISD::Register:
789 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
790 break;
791 case ISD::RegisterMask:
792 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
793 break;
794 case ISD::SRCVALUE:
795 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
796 break;
797 case ISD::FrameIndex:
798 case ISD::TargetFrameIndex:
799 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
800 break;
801 case ISD::PSEUDO_PROBE:
802 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
803 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
804 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
805 break;
806 case ISD::JumpTable:
807 case ISD::TargetJumpTable:
808 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
809 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
810 break;
811 case ISD::ConstantPool:
812 case ISD::TargetConstantPool: {
813 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
814 ID.AddInteger(I: CP->getAlign().value());
815 ID.AddInteger(I: CP->getOffset());
816 if (CP->isMachineConstantPoolEntry())
817 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
818 else
819 ID.AddPointer(Ptr: CP->getConstVal());
820 ID.AddInteger(I: CP->getTargetFlags());
821 break;
822 }
823 case ISD::TargetIndex: {
824 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
825 ID.AddInteger(I: TI->getIndex());
826 ID.AddInteger(I: TI->getOffset());
827 ID.AddInteger(I: TI->getTargetFlags());
828 break;
829 }
830 case ISD::LOAD: {
831 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
832 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
833 ID.AddInteger(I: LD->getRawSubclassData());
834 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
835 ID.AddInteger(I: LD->getMemOperand()->getFlags());
836 break;
837 }
838 case ISD::STORE: {
839 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
840 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
841 ID.AddInteger(I: ST->getRawSubclassData());
842 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
843 ID.AddInteger(I: ST->getMemOperand()->getFlags());
844 break;
845 }
846 case ISD::VP_LOAD: {
847 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
848 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
849 ID.AddInteger(I: ELD->getRawSubclassData());
850 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
851 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
852 break;
853 }
854 case ISD::VP_LOAD_FF: {
855 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
856 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
857 ID.AddInteger(I: LD->getRawSubclassData());
858 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
859 ID.AddInteger(I: LD->getMemOperand()->getFlags());
860 break;
861 }
862 case ISD::VP_STORE: {
863 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
864 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
865 ID.AddInteger(I: EST->getRawSubclassData());
866 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
867 ID.AddInteger(I: EST->getMemOperand()->getFlags());
868 break;
869 }
870 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
871 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
872 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
873 ID.AddInteger(I: SLD->getRawSubclassData());
874 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
875 break;
876 }
877 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
878 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
879 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
880 ID.AddInteger(I: SST->getRawSubclassData());
881 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
882 break;
883 }
884 case ISD::VP_GATHER: {
885 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
886 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
887 ID.AddInteger(I: EG->getRawSubclassData());
888 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
889 ID.AddInteger(I: EG->getMemOperand()->getFlags());
890 break;
891 }
892 case ISD::VP_SCATTER: {
893 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
894 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
895 ID.AddInteger(I: ES->getRawSubclassData());
896 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
897 ID.AddInteger(I: ES->getMemOperand()->getFlags());
898 break;
899 }
900 case ISD::MLOAD: {
901 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
902 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
903 ID.AddInteger(I: MLD->getRawSubclassData());
904 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
905 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
906 break;
907 }
908 case ISD::MSTORE: {
909 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
910 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
911 ID.AddInteger(I: MST->getRawSubclassData());
912 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
913 ID.AddInteger(I: MST->getMemOperand()->getFlags());
914 break;
915 }
916 case ISD::MGATHER: {
917 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
918 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
919 ID.AddInteger(I: MG->getRawSubclassData());
920 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
921 ID.AddInteger(I: MG->getMemOperand()->getFlags());
922 break;
923 }
924 case ISD::MSCATTER: {
925 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
926 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
927 ID.AddInteger(I: MS->getRawSubclassData());
928 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
929 ID.AddInteger(I: MS->getMemOperand()->getFlags());
930 break;
931 }
932 case ISD::ATOMIC_CMP_SWAP:
933 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
934 case ISD::ATOMIC_SWAP:
935 case ISD::ATOMIC_LOAD_ADD:
936 case ISD::ATOMIC_LOAD_SUB:
937 case ISD::ATOMIC_LOAD_AND:
938 case ISD::ATOMIC_LOAD_CLR:
939 case ISD::ATOMIC_LOAD_OR:
940 case ISD::ATOMIC_LOAD_XOR:
941 case ISD::ATOMIC_LOAD_NAND:
942 case ISD::ATOMIC_LOAD_MIN:
943 case ISD::ATOMIC_LOAD_MAX:
944 case ISD::ATOMIC_LOAD_UMIN:
945 case ISD::ATOMIC_LOAD_UMAX:
946 case ISD::ATOMIC_LOAD:
947 case ISD::ATOMIC_STORE: {
948 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
949 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
950 ID.AddInteger(I: AT->getRawSubclassData());
951 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
952 ID.AddInteger(I: AT->getMemOperand()->getFlags());
953 break;
954 }
955 case ISD::VECTOR_SHUFFLE: {
956 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
957 for (int M : Mask)
958 ID.AddInteger(I: M);
959 break;
960 }
961 case ISD::ADDRSPACECAST: {
962 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
963 ID.AddInteger(I: ASC->getSrcAddressSpace());
964 ID.AddInteger(I: ASC->getDestAddressSpace());
965 break;
966 }
967 case ISD::TargetBlockAddress:
968 case ISD::BlockAddress: {
969 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
970 ID.AddPointer(Ptr: BA->getBlockAddress());
971 ID.AddInteger(I: BA->getOffset());
972 ID.AddInteger(I: BA->getTargetFlags());
973 break;
974 }
975 case ISD::AssertAlign:
976 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
977 break;
978 case ISD::PREFETCH:
979 case ISD::INTRINSIC_VOID:
980 case ISD::INTRINSIC_W_CHAIN:
981 // Handled by MemIntrinsicSDNode check after the switch.
982 break;
983 case ISD::MDNODE_SDNODE:
984 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
985 break;
986 } // end switch (N->getOpcode())
987
988 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
989 // to check.
990 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
991 ID.AddInteger(I: MN->getRawSubclassData());
992 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
993 for (const MachineMemOperand *MMO : MN->memoperands()) {
994 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
995 ID.AddInteger(I: MMO->getFlags());
996 }
997 }
998}
999
1000/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
1001/// data.
1002static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1003 AddNodeIDOpcode(ID, OpC: N->getOpcode());
1004 // Add the return value info.
1005 AddNodeIDValueTypes(ID, VTList: N->getVTList());
1006 // Add the operand info.
1007 AddNodeIDOperands(ID, Ops: N->ops());
1008
1009 // Handle SDNode leafs with special info.
1010 AddNodeIDCustom(ID, N);
1011}
1012
1013//===----------------------------------------------------------------------===//
1014// SelectionDAG Class
1015//===----------------------------------------------------------------------===//
1016
1017/// doNotCSE - Return true if CSE should not be performed for this node.
1018static bool doNotCSE(SDNode *N) {
1019 if (N->getValueType(ResNo: 0) == MVT::Glue)
1020 return true; // Never CSE anything that produces a glue result.
1021
1022 switch (N->getOpcode()) {
1023 default: break;
1024 case ISD::HANDLENODE:
1025 case ISD::EH_LABEL:
1026 return true; // Never CSE these nodes.
1027 }
1028
1029 // Check that remaining values produced are not flags.
1030 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1031 if (N->getValueType(ResNo: i) == MVT::Glue)
1032 return true; // Never CSE anything that produces a glue result.
1033
1034 return false;
1035}
1036
1037/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1038/// SelectionDAG.
1039void SelectionDAG::RemoveDeadNodes() {
1040 // Create a dummy node (which is not added to allnodes), that adds a reference
1041 // to the root node, preventing it from being deleted.
1042 HandleSDNode Dummy(getRoot());
1043
1044 SmallVector<SDNode*, 128> DeadNodes;
1045
1046 // Add all obviously-dead nodes to the DeadNodes worklist.
1047 for (SDNode &Node : allnodes())
1048 if (Node.use_empty())
1049 DeadNodes.push_back(Elt: &Node);
1050
1051 RemoveDeadNodes(DeadNodes);
1052
1053 // If the root changed (e.g. it was a dead load, update the root).
1054 setRoot(Dummy.getValue());
1055}
1056
1057/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1058/// given list, and any nodes that become unreachable as a result.
1059void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1060
1061 // Process the worklist, deleting the nodes and adding their uses to the
1062 // worklist.
1063 while (!DeadNodes.empty()) {
1064 SDNode *N = DeadNodes.pop_back_val();
1065 // Skip to next node if we've already managed to delete the node. This could
1066 // happen if replacing a node causes a node previously added to the node to
1067 // be deleted.
1068 if (N->getOpcode() == ISD::DELETED_NODE)
1069 continue;
1070
1071 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1072 DUL->NodeDeleted(N, nullptr);
1073
1074 // Take the node out of the appropriate CSE map.
1075 RemoveNodeFromCSEMaps(N);
1076
1077 // Next, brutally remove the operand list. This is safe to do, as there are
1078 // no cycles in the graph.
1079 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1080 SDUse &Use = *I++;
1081 SDNode *Operand = Use.getNode();
1082 Use.set(SDValue());
1083
1084 // Now that we removed this operand, see if there are no uses of it left.
1085 if (Operand->use_empty())
1086 DeadNodes.push_back(Elt: Operand);
1087 }
1088
1089 DeallocateNode(N);
1090 }
1091}
1092
1093void SelectionDAG::RemoveDeadNode(SDNode *N){
1094 SmallVector<SDNode*, 16> DeadNodes(1, N);
1095
1096 // Create a dummy node that adds a reference to the root node, preventing
1097 // it from being deleted. (This matters if the root is an operand of the
1098 // dead node.)
1099 HandleSDNode Dummy(getRoot());
1100
1101 RemoveDeadNodes(DeadNodes);
1102}
1103
1104void SelectionDAG::DeleteNode(SDNode *N) {
1105 // First take this out of the appropriate CSE map.
1106 RemoveNodeFromCSEMaps(N);
1107
1108 // Finally, remove uses due to operands of this node, remove from the
1109 // AllNodes list, and delete the node.
1110 DeleteNodeNotInCSEMaps(N);
1111}
1112
1113void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1114 assert(N->getIterator() != AllNodes.begin() &&
1115 "Cannot delete the entry node!");
1116 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1117
1118 // Drop all of the operands and decrement used node's use counts.
1119 N->DropOperands();
1120
1121 DeallocateNode(N);
1122}
1123
1124void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1125 assert(!(V->isVariadic() && isParameter));
1126 if (isParameter)
1127 ByvalParmDbgValues.push_back(Elt: V);
1128 else
1129 DbgValues.push_back(Elt: V);
1130 for (const SDNode *Node : V->getSDNodes())
1131 if (Node)
1132 DbgValMap[Node].push_back(Elt: V);
1133}
1134
1135void SDDbgInfo::erase(const SDNode *Node) {
1136 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1137 if (I == DbgValMap.end())
1138 return;
1139 for (auto &Val: I->second)
1140 Val->setIsInvalidated();
1141 DbgValMap.erase(I);
1142}
1143
1144void SelectionDAG::DeallocateNode(SDNode *N) {
1145 // If we have operands, deallocate them.
1146 removeOperands(Node: N);
1147
1148 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1149
1150 // Set the opcode to DELETED_NODE to help catch bugs when node
1151 // memory is reallocated.
1152 // FIXME: There are places in SDag that have grown a dependency on the opcode
1153 // value in the released node.
1154 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1155 N->NodeType = ISD::DELETED_NODE;
1156
1157 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1158 // them and forget about that node.
1159 DbgInfo->erase(Node: N);
1160
1161 // Invalidate extra info.
1162 SDEI.erase(Val: N);
1163}
1164
1165#ifndef NDEBUG
1166/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1167void SelectionDAG::verifyNode(SDNode *N) const {
1168 switch (N->getOpcode()) {
1169 default:
1170 if (N->isTargetOpcode())
1171 getSelectionDAGInfo().verifyTargetNode(*this, N);
1172 break;
1173 case ISD::BUILD_PAIR: {
1174 EVT VT = N->getValueType(0);
1175 assert(N->getNumValues() == 1 && "Too many results!");
1176 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1177 "Wrong return type!");
1178 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1179 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1180 "Mismatched operand types!");
1181 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1182 "Wrong operand type!");
1183 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1184 "Wrong return type size");
1185 break;
1186 }
1187 case ISD::BUILD_VECTOR: {
1188 assert(N->getNumValues() == 1 && "Too many results!");
1189 assert(N->getValueType(0).isVector() && "Wrong return type!");
1190 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1191 "Wrong number of operands!");
1192 EVT EltVT = N->getValueType(0).getVectorElementType();
1193 for (const SDUse &Op : N->ops()) {
1194 assert((Op.getValueType() == EltVT ||
1195 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1196 EltVT.bitsLE(Op.getValueType()))) &&
1197 "Wrong operand type!");
1198 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1199 "Operands must all have the same type");
1200 }
1201 break;
1202 }
1203 case ISD::SADDO:
1204 case ISD::UADDO:
1205 case ISD::SSUBO:
1206 case ISD::USUBO:
1207 assert(N->getNumValues() == 2 && "Wrong number of results!");
1208 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1209 "Invalid add/sub overflow op!");
1210 assert(N->getVTList().VTs[0].isInteger() &&
1211 N->getVTList().VTs[1].isInteger() &&
1212 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1213 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1214 "Binary operator types must match!");
1215 break;
1216 }
1217}
1218#endif // NDEBUG
1219
1220/// Insert a newly allocated node into the DAG.
1221///
1222/// Handles insertion into the all nodes list and CSE map, as well as
1223/// verification and other common operations when a new node is allocated.
1224void SelectionDAG::InsertNode(SDNode *N) {
1225 AllNodes.push_back(val: N);
1226#ifndef NDEBUG
1227 N->PersistentId = NextPersistentId++;
1228 verifyNode(N);
1229#endif
1230 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1231 DUL->NodeInserted(N);
1232}
1233
1234/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1235/// correspond to it. This is useful when we're about to delete or repurpose
1236/// the node. We don't want future request for structurally identical nodes
1237/// to return N anymore.
1238bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1239 bool Erased = false;
1240 switch (N->getOpcode()) {
1241 case ISD::HANDLENODE: return false; // noop.
1242 case ISD::CONDCODE:
1243 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1244 "Cond code doesn't exist!");
1245 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1246 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1247 break;
1248 case ISD::ExternalSymbol:
1249 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1250 break;
1251 case ISD::TargetExternalSymbol: {
1252 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1253 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1254 ESN->getSymbol(), ESN->getTargetFlags()));
1255 break;
1256 }
1257 case ISD::MCSymbol: {
1258 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1259 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1260 break;
1261 }
1262 case ISD::VALUETYPE: {
1263 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1264 if (VT.isExtended()) {
1265 Erased = ExtendedValueTypeNodes.erase(x: VT);
1266 } else {
1267 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1268 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1269 }
1270 break;
1271 }
1272 default:
1273 // Remove it from the CSE Map.
1274 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1275 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1276 Erased = CSEMap.RemoveNode(N);
1277 break;
1278 }
1279#ifndef NDEBUG
1280 // Verify that the node was actually in one of the CSE maps, unless it has a
1281 // glue result (which cannot be CSE'd) or is one of the special cases that are
1282 // not subject to CSE.
1283 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1284 !N->isMachineOpcode() && !doNotCSE(N)) {
1285 N->dump(this);
1286 dbgs() << "\n";
1287 llvm_unreachable("Node is not in map!");
1288 }
1289#endif
1290 return Erased;
1291}
1292
1293/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1294/// maps and modified in place. Add it back to the CSE maps, unless an identical
1295/// node already exists, in which case transfer all its users to the existing
1296/// node. This transfer can potentially trigger recursive merging.
1297void
1298SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1299 // For node types that aren't CSE'd, just act as if no identical node
1300 // already exists.
1301 if (!doNotCSE(N)) {
1302 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1303 if (Existing != N) {
1304 // If there was already an existing matching node, use ReplaceAllUsesWith
1305 // to replace the dead one with the existing one. This can cause
1306 // recursive merging of other unrelated nodes down the line.
1307 Existing->intersectFlagsWith(Flags: N->getFlags());
1308 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing))
1309 MemNode->refineRanges(NewMMOs: cast<MemSDNode>(Val: N)->memoperands());
1310 ReplaceAllUsesWith(From: N, To: Existing);
1311
1312 // N is now dead. Inform the listeners and delete it.
1313 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1314 DUL->NodeDeleted(N, Existing);
1315 DeleteNodeNotInCSEMaps(N);
1316 return;
1317 }
1318 }
1319
1320 // If the node doesn't already exist, we updated it. Inform listeners.
1321 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1322 DUL->NodeUpdated(N);
1323}
1324
1325/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1326/// were replaced with those specified. If this node is never memoized,
1327/// return null, otherwise return a pointer to the slot it would take. If a
1328/// node already exists with these operands, the slot will be non-null.
1329SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1330 void *&InsertPos) {
1331 if (doNotCSE(N))
1332 return nullptr;
1333
1334 SDValue Ops[] = { Op };
1335 FoldingSetNodeID ID;
1336 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1337 AddNodeIDCustom(ID, N);
1338 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1339 if (Node)
1340 Node->intersectFlagsWith(Flags: N->getFlags());
1341 return Node;
1342}
1343
1344/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1345/// were replaced with those specified. If this node is never memoized,
1346/// return null, otherwise return a pointer to the slot it would take. If a
1347/// node already exists with these operands, the slot will be non-null.
1348SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1349 SDValue Op1, SDValue Op2,
1350 void *&InsertPos) {
1351 if (doNotCSE(N))
1352 return nullptr;
1353
1354 SDValue Ops[] = { Op1, Op2 };
1355 FoldingSetNodeID ID;
1356 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1357 AddNodeIDCustom(ID, N);
1358 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1359 if (Node)
1360 Node->intersectFlagsWith(Flags: N->getFlags());
1361 return Node;
1362}
1363
1364/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1365/// were replaced with those specified. If this node is never memoized,
1366/// return null, otherwise return a pointer to the slot it would take. If a
1367/// node already exists with these operands, the slot will be non-null.
1368SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1369 void *&InsertPos) {
1370 if (doNotCSE(N))
1371 return nullptr;
1372
1373 FoldingSetNodeID ID;
1374 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1375 AddNodeIDCustom(ID, N);
1376 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1377 if (Node)
1378 Node->intersectFlagsWith(Flags: N->getFlags());
1379 return Node;
1380}
1381
1382Align SelectionDAG::getEVTAlign(EVT VT) const {
1383 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1384 : VT.getTypeForEVT(Context&: *getContext());
1385
1386 return getDataLayout().getABITypeAlign(Ty);
1387}
1388
1389// EntryNode could meaningfully have debug info if we can find it...
1390SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1391 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1392 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1393 Root(getEntryNode()) {
1394 InsertNode(N: &EntryNode);
1395 DbgInfo = new SDDbgInfo();
1396}
1397
1398void SelectionDAG::init(MachineFunction &NewMF,
1399 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1400 const TargetLibraryInfo *LibraryInfo,
1401 const LibcallLoweringInfo *LibcallsInfo,
1402 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1403 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
1404 FunctionVarLocs const *VarLocs) {
1405 MF = &NewMF;
1406 SDAGISelPass = PassPtr;
1407 ORE = &NewORE;
1408 TLI = getSubtarget().getTargetLowering();
1409 TSI = getSubtarget().getSelectionDAGInfo();
1410 LibInfo = LibraryInfo;
1411 Libcalls = LibcallsInfo;
1412 Context = &MF->getFunction().getContext();
1413 UA = NewUA;
1414 PSI = PSIin;
1415 BFI = BFIin;
1416 MMI = &MMIin;
1417 FnVarLocs = VarLocs;
1418}
1419
1420SelectionDAG::~SelectionDAG() {
1421 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1422 allnodes_clear();
1423 OperandRecycler.clear(OperandAllocator);
1424 delete DbgInfo;
1425}
1426
1427bool SelectionDAG::shouldOptForSize() const {
1428 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1429}
1430
1431void SelectionDAG::allnodes_clear() {
1432 assert(&*AllNodes.begin() == &EntryNode);
1433 AllNodes.remove(IT: AllNodes.begin());
1434 while (!AllNodes.empty())
1435 DeallocateNode(N: &AllNodes.front());
1436#ifndef NDEBUG
1437 NextPersistentId = 0;
1438#endif
1439}
1440
1441SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1442 void *&InsertPos) {
1443 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1444 if (N) {
1445 switch (N->getOpcode()) {
1446 default: break;
1447 case ISD::Constant:
1448 case ISD::ConstantFP:
1449 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1450 "debug location. Use another overload.");
1451 }
1452 }
1453 return N;
1454}
1455
1456SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1457 const SDLoc &DL, void *&InsertPos) {
1458 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1459 if (N) {
1460 switch (N->getOpcode()) {
1461 case ISD::Constant:
1462 case ISD::ConstantFP:
1463 // Erase debug location from the node if the node is used at several
1464 // different places. Do not propagate one location to all uses as it
1465 // will cause a worse single stepping debugging experience.
1466 if (N->getDebugLoc() != DL.getDebugLoc())
1467 N->setDebugLoc(DebugLoc());
1468 break;
1469 default:
1470 // When the node's point of use is located earlier in the instruction
1471 // sequence than its prior point of use, update its debug info to the
1472 // earlier location.
1473 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1474 N->setDebugLoc(DL.getDebugLoc());
1475 break;
1476 }
1477 }
1478 return N;
1479}
1480
1481void SelectionDAG::clear() {
1482 allnodes_clear();
1483 OperandRecycler.clear(OperandAllocator);
1484 OperandAllocator.Reset();
1485 CSEMap.clear();
1486
1487 ExtendedValueTypeNodes.clear();
1488 ExternalSymbols.clear();
1489 TargetExternalSymbols.clear();
1490 MCSymbols.clear();
1491 SDEI.clear();
1492 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1493 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1494
1495 EntryNode.UseList = nullptr;
1496 InsertNode(N: &EntryNode);
1497 Root = getEntryNode();
1498 DbgInfo->clear();
1499}
1500
1501SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1502 return VT.bitsGT(VT: Op.getValueType())
1503 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1504 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1505 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1506}
1507
1508std::pair<SDValue, SDValue>
1509SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1510 const SDLoc &DL, EVT VT) {
1511 assert(!VT.bitsEq(Op.getValueType()) &&
1512 "Strict no-op FP extend/round not allowed.");
1513 SDValue Res =
1514 VT.bitsGT(VT: Op.getValueType())
1515 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1516 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1517 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1518
1519 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1520}
1521
1522SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1523 return VT.bitsGT(VT: Op.getValueType()) ?
1524 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1525 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1526}
1527
1528SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1529 return VT.bitsGT(VT: Op.getValueType()) ?
1530 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1531 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1532}
1533
1534SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1535 return VT.bitsGT(VT: Op.getValueType()) ?
1536 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1537 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1538}
1539
1540SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1541 EVT VT) {
1542 assert(!VT.isVector());
1543 auto Type = Op.getValueType();
1544 SDValue DestOp;
1545 if (Type == VT)
1546 return Op;
1547 auto Size = Op.getValueSizeInBits();
1548 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1549 if (DestOp.getValueType() == VT)
1550 return DestOp;
1551
1552 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1553}
1554
1555SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1556 EVT VT) {
1557 assert(!VT.isVector());
1558 auto Type = Op.getValueType();
1559 SDValue DestOp;
1560 if (Type == VT)
1561 return Op;
1562 auto Size = Op.getValueSizeInBits();
1563 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1564 if (DestOp.getValueType() == VT)
1565 return DestOp;
1566
1567 return getSExtOrTrunc(Op: DestOp, DL, VT);
1568}
1569
1570SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1571 EVT VT) {
1572 assert(!VT.isVector());
1573 auto Type = Op.getValueType();
1574 SDValue DestOp;
1575 if (Type == VT)
1576 return Op;
1577 auto Size = Op.getValueSizeInBits();
1578 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1579 if (DestOp.getValueType() == VT)
1580 return DestOp;
1581
1582 return getZExtOrTrunc(Op: DestOp, DL, VT);
1583}
1584
1585SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1586 EVT OpVT) {
1587 if (VT.bitsLE(VT: Op.getValueType()))
1588 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1589
1590 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1591 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1592}
1593
1594SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1595 EVT OpVT = Op.getValueType();
1596 assert(VT.isInteger() && OpVT.isInteger() &&
1597 "Cannot getZeroExtendInReg FP types");
1598 assert(VT.isVector() == OpVT.isVector() &&
1599 "getZeroExtendInReg type should be vector iff the operand "
1600 "type is vector!");
1601 assert((!VT.isVector() ||
1602 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1603 "Vector element counts must match in getZeroExtendInReg");
1604 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1605 if (OpVT == VT)
1606 return Op;
1607 // TODO: Use computeKnownBits instead of AssertZext.
1608 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1609 ->getVT()
1610 .getScalarType()
1611 .bitsLE(VT: VT.getScalarType()))
1612 return Op;
1613 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1614 loBitsSet: VT.getScalarSizeInBits());
1615 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1616}
1617
1618SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,
1619 SDValue EVL, const SDLoc &DL,
1620 EVT VT) {
1621 EVT OpVT = Op.getValueType();
1622 assert(VT.isInteger() && OpVT.isInteger() &&
1623 "Cannot getVPZeroExtendInReg FP types");
1624 assert(VT.isVector() && OpVT.isVector() &&
1625 "getVPZeroExtendInReg type and operand type should be vector!");
1626 assert(VT.getVectorElementCount() == OpVT.getVectorElementCount() &&
1627 "Vector element counts must match in getZeroExtendInReg");
1628 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1629 if (OpVT == VT)
1630 return Op;
1631 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1632 loBitsSet: VT.getScalarSizeInBits());
1633 return getNode(Opcode: ISD::VP_AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT), N3: Mask,
1634 N4: EVL);
1635}
1636
1637SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1638 // Only unsigned pointer semantics are supported right now. In the future this
1639 // might delegate to TLI to check pointer signedness.
1640 return getZExtOrTrunc(Op, DL, VT);
1641}
1642
1643SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1644 // Only unsigned pointer semantics are supported right now. In the future this
1645 // might delegate to TLI to check pointer signedness.
1646 return getZeroExtendInReg(Op, DL, VT);
1647}
1648
1649SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1650 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1651}
1652
1653/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1654SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1655 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1656}
1657
1658SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1659 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1660 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1661}
1662
1663SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1664 SDValue Mask, SDValue EVL, EVT VT) {
1665 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1666 return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1667}
1668
1669SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1670 SDValue Mask, SDValue EVL) {
1671 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1672}
1673
1674SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1675 SDValue Mask, SDValue EVL) {
1676 if (VT.bitsGT(VT: Op.getValueType()))
1677 return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1678 if (VT.bitsLT(VT: Op.getValueType()))
1679 return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1680 return Op;
1681}
1682
1683SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1684 EVT OpVT) {
1685 if (!V)
1686 return getConstant(Val: 0, DL, VT);
1687
1688 switch (TLI->getBooleanContents(Type: OpVT)) {
1689 case TargetLowering::ZeroOrOneBooleanContent:
1690 case TargetLowering::UndefinedBooleanContent:
1691 return getConstant(Val: 1, DL, VT);
1692 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1693 return getAllOnesConstant(DL, VT);
1694 }
1695 llvm_unreachable("Unexpected boolean content enum!");
1696}
1697
1698SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1699 bool isT, bool isO) {
1700 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1701 DL, VT, isTarget: isT, isOpaque: isO);
1702}
1703
1704SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1705 bool isT, bool isO) {
1706 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1707}
1708
1709SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1710 EVT VT, bool isT, bool isO) {
1711 assert(VT.isInteger() && "Cannot create FP integer constant!");
1712
1713 EVT EltVT = VT.getScalarType();
1714 const ConstantInt *Elt = &Val;
1715
1716 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1717 // to-be-splatted scalar ConstantInt.
1718 if (isa<VectorType>(Val: Elt->getType()))
1719 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1720
1721 // In some cases the vector type is legal but the element type is illegal and
1722 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1723 // inserted value (the type does not need to match the vector element type).
1724 // Any extra bits introduced will be truncated away.
1725 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1726 TargetLowering::TypePromoteInteger) {
1727 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1728 APInt NewVal;
1729 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1730 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1731 else
1732 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1733 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1734 }
1735 // In other cases the element type is illegal and needs to be expanded, for
1736 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1737 // the value into n parts and use a vector type with n-times the elements.
1738 // Then bitcast to the type requested.
1739 // Legalizing constants too early makes the DAGCombiner's job harder so we
1740 // only legalize if the DAG tells us we must produce legal types.
1741 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1742 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1743 TargetLowering::TypeExpandInteger) {
1744 const APInt &NewVal = Elt->getValue();
1745 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1746 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1747
1748 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1749 if (VT.isScalableVector() ||
1750 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1751 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1752 "Can only handle an even split!");
1753 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1754
1755 SmallVector<SDValue, 2> ScalarParts;
1756 for (unsigned i = 0; i != Parts; ++i)
1757 ScalarParts.push_back(Elt: getConstant(
1758 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1759 VT: ViaEltVT, isT, isO));
1760
1761 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1762 }
1763
1764 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1765 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1766
1767 // Check the temporary vector is the correct size. If this fails then
1768 // getTypeToTransformTo() probably returned a type whose size (in bits)
1769 // isn't a power-of-2 factor of the requested type size.
1770 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1771
1772 SmallVector<SDValue, 2> EltParts;
1773 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1774 EltParts.push_back(Elt: getConstant(
1775 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1776 VT: ViaEltVT, isT, isO));
1777
1778 // EltParts is currently in little endian order. If we actually want
1779 // big-endian order then reverse it now.
1780 if (getDataLayout().isBigEndian())
1781 std::reverse(first: EltParts.begin(), last: EltParts.end());
1782
1783 // The elements must be reversed when the element order is different
1784 // to the endianness of the elements (because the BITCAST is itself a
1785 // vector shuffle in this situation). However, we do not need any code to
1786 // perform this reversal because getConstant() is producing a vector
1787 // splat.
1788 // This situation occurs in MIPS MSA.
1789
1790 SmallVector<SDValue, 8> Ops;
1791 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1792 llvm::append_range(C&: Ops, R&: EltParts);
1793
1794 SDValue V =
1795 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1796 return V;
1797 }
1798
1799 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1800 "APInt size does not match type size!");
1801 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1802 SDVTList VTs = getVTList(VT: EltVT);
1803 FoldingSetNodeID ID;
1804 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1805 ID.AddPointer(Ptr: Elt);
1806 ID.AddBoolean(B: isO);
1807 void *IP = nullptr;
1808 SDNode *N = nullptr;
1809 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1810 if (!VT.isVector())
1811 return SDValue(N, 0);
1812
1813 if (!N) {
1814 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1815 CSEMap.InsertNode(N, InsertPos: IP);
1816 InsertNode(N);
1817 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1818 }
1819
1820 SDValue Result(N, 0);
1821 if (VT.isVector())
1822 Result = getSplat(VT, DL, Op: Result);
1823 return Result;
1824}
1825
1826SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1827 bool isT, bool isO) {
1828 unsigned Size = VT.getScalarSizeInBits();
1829 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1830}
1831
1832SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1833 bool IsOpaque) {
1834 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1835 isT: IsTarget, isO: IsOpaque);
1836}
1837
1838SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1839 bool isTarget) {
1840 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1841}
1842
1843SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1844 const SDLoc &DL) {
1845 assert(VT.isInteger() && "Shift amount is not an integer type!");
1846 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1847 return getConstant(Val, DL, VT: ShiftVT);
1848}
1849
1850SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1851 const SDLoc &DL) {
1852 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1853 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1854}
1855
1856SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1857 bool isTarget) {
1858 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1859}
1860
1861SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1862 bool isTarget) {
1863 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1864}
1865
1866SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1867 EVT VT, bool isTarget) {
1868 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1869
1870 EVT EltVT = VT.getScalarType();
1871 const ConstantFP *Elt = &V;
1872
1873 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1874 // the to-be-splatted scalar ConstantFP.
1875 if (isa<VectorType>(Val: Elt->getType()))
1876 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1877
1878 // Do the map lookup using the actual bit pattern for the floating point
1879 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1880 // we don't have issues with SNANs.
1881 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1882 SDVTList VTs = getVTList(VT: EltVT);
1883 FoldingSetNodeID ID;
1884 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1885 ID.AddPointer(Ptr: Elt);
1886 void *IP = nullptr;
1887 SDNode *N = nullptr;
1888 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1889 if (!VT.isVector())
1890 return SDValue(N, 0);
1891
1892 if (!N) {
1893 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1894 CSEMap.InsertNode(N, InsertPos: IP);
1895 InsertNode(N);
1896 }
1897
1898 SDValue Result(N, 0);
1899 if (VT.isVector())
1900 Result = getSplat(VT, DL, Op: Result);
1901 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1902 return Result;
1903}
1904
1905SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1906 bool isTarget) {
1907 EVT EltVT = VT.getScalarType();
1908 if (EltVT == MVT::f32)
1909 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1910 if (EltVT == MVT::f64)
1911 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1912 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1913 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1914 bool Ignored;
1915 APFloat APF = APFloat(Val);
1916 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1917 losesInfo: &Ignored);
1918 return getConstantFP(V: APF, DL, VT, isTarget);
1919 }
1920 llvm_unreachable("Unsupported type in getConstantFP");
1921}
1922
1923SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1924 EVT VT, int64_t Offset, bool isTargetGA,
1925 unsigned TargetFlags) {
1926 assert((TargetFlags == 0 || isTargetGA) &&
1927 "Cannot set target flags on target-independent globals");
1928
1929 // Truncate (with sign-extension) the offset value to the pointer size.
1930 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1931 if (BitWidth < 64)
1932 Offset = SignExtend64(X: Offset, B: BitWidth);
1933
1934 unsigned Opc;
1935 if (GV->isThreadLocal())
1936 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1937 else
1938 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1939
1940 SDVTList VTs = getVTList(VT);
1941 FoldingSetNodeID ID;
1942 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1943 ID.AddPointer(Ptr: GV);
1944 ID.AddInteger(I: Offset);
1945 ID.AddInteger(I: TargetFlags);
1946 void *IP = nullptr;
1947 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1948 return SDValue(E, 0);
1949
1950 auto *N = newSDNode<GlobalAddressSDNode>(
1951 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1952 CSEMap.InsertNode(N, InsertPos: IP);
1953 InsertNode(N);
1954 return SDValue(N, 0);
1955}
1956
1957SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1958 SDVTList VTs = getVTList(VT: MVT::Untyped);
1959 FoldingSetNodeID ID;
1960 AddNodeIDNode(ID, OpC: ISD::DEACTIVATION_SYMBOL, VTList: VTs, OpList: {});
1961 ID.AddPointer(Ptr: GV);
1962 void *IP = nullptr;
1963 if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
1964 return SDValue(E, 0);
1965
1966 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1967 CSEMap.InsertNode(N, InsertPos: IP);
1968 InsertNode(N);
1969 return SDValue(N, 0);
1970}
1971
1972SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1973 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1974 SDVTList VTs = getVTList(VT);
1975 FoldingSetNodeID ID;
1976 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1977 ID.AddInteger(I: FI);
1978 void *IP = nullptr;
1979 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1980 return SDValue(E, 0);
1981
1982 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1983 CSEMap.InsertNode(N, InsertPos: IP);
1984 InsertNode(N);
1985 return SDValue(N, 0);
1986}
1987
1988SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1989 unsigned TargetFlags) {
1990 assert((TargetFlags == 0 || isTarget) &&
1991 "Cannot set target flags on target-independent jump tables");
1992 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1993 SDVTList VTs = getVTList(VT);
1994 FoldingSetNodeID ID;
1995 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1996 ID.AddInteger(I: JTI);
1997 ID.AddInteger(I: TargetFlags);
1998 void *IP = nullptr;
1999 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2000 return SDValue(E, 0);
2001
2002 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
2003 CSEMap.InsertNode(N, InsertPos: IP);
2004 InsertNode(N);
2005 return SDValue(N, 0);
2006}
2007
2008SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
2009 const SDLoc &DL) {
2010 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
2011 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
2012 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
2013}
2014
2015SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2016 MaybeAlign Alignment, int Offset,
2017 bool isTarget, unsigned TargetFlags) {
2018 assert((TargetFlags == 0 || isTarget) &&
2019 "Cannot set target flags on target-independent globals");
2020 if (!Alignment)
2021 Alignment = shouldOptForSize()
2022 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2023 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2024 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2025 SDVTList VTs = getVTList(VT);
2026 FoldingSetNodeID ID;
2027 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2028 ID.AddInteger(I: Alignment->value());
2029 ID.AddInteger(I: Offset);
2030 ID.AddPointer(Ptr: C);
2031 ID.AddInteger(I: TargetFlags);
2032 void *IP = nullptr;
2033 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2034 return SDValue(E, 0);
2035
2036 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2037 Args&: TargetFlags);
2038 CSEMap.InsertNode(N, InsertPos: IP);
2039 InsertNode(N);
2040 SDValue V = SDValue(N, 0);
2041 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2042 return V;
2043}
2044
2045SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2046 MaybeAlign Alignment, int Offset,
2047 bool isTarget, unsigned TargetFlags) {
2048 assert((TargetFlags == 0 || isTarget) &&
2049 "Cannot set target flags on target-independent globals");
2050 if (!Alignment)
2051 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2052 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2053 SDVTList VTs = getVTList(VT);
2054 FoldingSetNodeID ID;
2055 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2056 ID.AddInteger(I: Alignment->value());
2057 ID.AddInteger(I: Offset);
2058 C->addSelectionDAGCSEId(ID);
2059 ID.AddInteger(I: TargetFlags);
2060 void *IP = nullptr;
2061 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2062 return SDValue(E, 0);
2063
2064 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2065 Args&: TargetFlags);
2066 CSEMap.InsertNode(N, InsertPos: IP);
2067 InsertNode(N);
2068 return SDValue(N, 0);
2069}
2070
2071SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2072 FoldingSetNodeID ID;
2073 AddNodeIDNode(ID, OpC: ISD::BasicBlock, VTList: getVTList(VT: MVT::Other), OpList: {});
2074 ID.AddPointer(Ptr: MBB);
2075 void *IP = nullptr;
2076 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2077 return SDValue(E, 0);
2078
2079 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2080 CSEMap.InsertNode(N, InsertPos: IP);
2081 InsertNode(N);
2082 return SDValue(N, 0);
2083}
2084
2085SDValue SelectionDAG::getValueType(EVT VT) {
2086 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2087 ValueTypeNodes.size())
2088 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2089
2090 SDNode *&N = VT.isExtended() ?
2091 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2092
2093 if (N) return SDValue(N, 0);
2094 N = newSDNode<VTSDNode>(Args&: VT);
2095 InsertNode(N);
2096 return SDValue(N, 0);
2097}
2098
2099SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2100 SDNode *&N = ExternalSymbols[Sym];
2101 if (N) return SDValue(N, 0);
2102 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2103 InsertNode(N);
2104 return SDValue(N, 0);
2105}
2106
2107SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2108 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2109 return getExternalSymbol(Sym: SymName.data(), VT);
2110}
2111
2112SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2113 SDNode *&N = MCSymbols[Sym];
2114 if (N)
2115 return SDValue(N, 0);
2116 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2117 InsertNode(N);
2118 return SDValue(N, 0);
2119}
2120
2121SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2122 unsigned TargetFlags) {
2123 SDNode *&N =
2124 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2125 if (N) return SDValue(N, 0);
2126 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2127 InsertNode(N);
2128 return SDValue(N, 0);
2129}
2130
2131SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2132 EVT VT, unsigned TargetFlags) {
2133 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2134 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2135}
2136
2137SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2138 if ((unsigned)Cond >= CondCodeNodes.size())
2139 CondCodeNodes.resize(new_size: Cond+1);
2140
2141 if (!CondCodeNodes[Cond]) {
2142 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2143 CondCodeNodes[Cond] = N;
2144 InsertNode(N);
2145 }
2146
2147 return SDValue(CondCodeNodes[Cond], 0);
2148}
2149
2150SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2151 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2152 "APInt size does not match type size!");
2153
2154 if (MulImm == 0)
2155 return getConstant(Val: 0, DL, VT);
2156
2157 const MachineFunction &MF = getMachineFunction();
2158 const Function &F = MF.getFunction();
2159 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2160 if (const APInt *C = CR.getSingleElement())
2161 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2162
2163 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2164}
2165
2166/// \returns a value of type \p VT that represents the runtime value of \p
2167/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2168/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2169/// or TypeSize.
2170template <typename Ty>
2171static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2172 EVT VT, Ty Quantity) {
2173 if (Quantity.isScalable())
2174 return DAG.getVScale(
2175 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2176
2177 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2178}
2179
2180SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2181 ElementCount EC) {
2182 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2183}
2184
2185SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2186 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2187}
2188
2189SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2190 ElementCount EC) {
2191 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2192 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2193 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2194 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2195}
2196
2197SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2198 APInt One(ResVT.getScalarSizeInBits(), 1);
2199 return getStepVector(DL, ResVT, StepVal: One);
2200}
2201
2202SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2203 const APInt &StepVal) {
2204 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2205 if (ResVT.isScalableVector())
2206 return getNode(
2207 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2208 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2209
2210 SmallVector<SDValue, 16> OpsStepConstants;
2211 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2212 OpsStepConstants.push_back(
2213 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2214 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2215}
2216
2217/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2218/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2219static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2220 std::swap(a&: N1, b&: N2);
2221 ShuffleVectorSDNode::commuteMask(Mask: M);
2222}
2223
2224SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2225 SDValue N2, ArrayRef<int> Mask) {
2226 assert(VT.getVectorNumElements() == Mask.size() &&
2227 "Must have the same number of vector elements as mask elements!");
2228 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2229 "Invalid VECTOR_SHUFFLE");
2230
2231 // Canonicalize shuffle undef, undef -> undef
2232 if (N1.isUndef() && N2.isUndef())
2233 return getUNDEF(VT);
2234
2235 // Validate that all indices in Mask are within the range of the elements
2236 // input to the shuffle.
2237 int NElts = Mask.size();
2238 assert(llvm::all_of(Mask,
2239 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2240 "Index out of range");
2241
2242 // Copy the mask so we can do any needed cleanup.
2243 SmallVector<int, 8> MaskVec(Mask);
2244
2245 // Canonicalize shuffle v, v -> v, undef
2246 if (N1 == N2) {
2247 N2 = getUNDEF(VT);
2248 for (int i = 0; i != NElts; ++i)
2249 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2250 }
2251
2252 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2253 if (N1.isUndef())
2254 commuteShuffle(N1, N2, M: MaskVec);
2255
2256 if (TLI->hasVectorBlend()) {
2257 // If shuffling a splat, try to blend the splat instead. We do this here so
2258 // that even when this arises during lowering we don't have to re-handle it.
2259 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2260 BitVector UndefElements;
2261 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2262 if (!Splat)
2263 return;
2264
2265 for (int i = 0; i < NElts; ++i) {
2266 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2267 continue;
2268
2269 // If this input comes from undef, mark it as such.
2270 if (UndefElements[MaskVec[i] - Offset]) {
2271 MaskVec[i] = -1;
2272 continue;
2273 }
2274
2275 // If we can blend a non-undef lane, use that instead.
2276 if (!UndefElements[i])
2277 MaskVec[i] = i + Offset;
2278 }
2279 };
2280 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2281 BlendSplat(N1BV, 0);
2282 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2283 BlendSplat(N2BV, NElts);
2284 }
2285
2286 // Canonicalize all index into lhs, -> shuffle lhs, undef
2287 // Canonicalize all index into rhs, -> shuffle rhs, undef
2288 bool AllLHS = true, AllRHS = true;
2289 bool N2Undef = N2.isUndef();
2290 for (int i = 0; i != NElts; ++i) {
2291 if (MaskVec[i] >= NElts) {
2292 if (N2Undef)
2293 MaskVec[i] = -1;
2294 else
2295 AllLHS = false;
2296 } else if (MaskVec[i] >= 0) {
2297 AllRHS = false;
2298 }
2299 }
2300 if (AllLHS && AllRHS)
2301 return getUNDEF(VT);
2302 if (AllLHS && !N2Undef)
2303 N2 = getUNDEF(VT);
2304 if (AllRHS) {
2305 N1 = getUNDEF(VT);
2306 commuteShuffle(N1, N2, M: MaskVec);
2307 }
2308 // Reset our undef status after accounting for the mask.
2309 N2Undef = N2.isUndef();
2310 // Re-check whether both sides ended up undef.
2311 if (N1.isUndef() && N2Undef)
2312 return getUNDEF(VT);
2313
2314 // If Identity shuffle return that node.
2315 bool Identity = true, AllSame = true;
2316 for (int i = 0; i != NElts; ++i) {
2317 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2318 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2319 }
2320 if (Identity && NElts)
2321 return N1;
2322
2323 // Shuffling a constant splat doesn't change the result.
2324 if (N2Undef) {
2325 SDValue V = N1;
2326
2327 // Look through any bitcasts. We check that these don't change the number
2328 // (and size) of elements and just changes their types.
2329 while (V.getOpcode() == ISD::BITCAST)
2330 V = V->getOperand(Num: 0);
2331
2332 // A splat should always show up as a build vector node.
2333 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2334 BitVector UndefElements;
2335 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2336 // If this is a splat of an undef, shuffling it is also undef.
2337 if (Splat && Splat.isUndef())
2338 return getUNDEF(VT);
2339
2340 bool SameNumElts =
2341 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2342
2343 // We only have a splat which can skip shuffles if there is a splatted
2344 // value and no undef lanes rearranged by the shuffle.
2345 if (Splat && UndefElements.none()) {
2346 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2347 // number of elements match or the value splatted is a zero constant.
2348 if (SameNumElts || isNullConstant(V: Splat))
2349 return N1;
2350 }
2351
2352 // If the shuffle itself creates a splat, build the vector directly.
2353 if (AllSame && SameNumElts) {
2354 EVT BuildVT = BV->getValueType(ResNo: 0);
2355 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2356 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2357
2358 // We may have jumped through bitcasts, so the type of the
2359 // BUILD_VECTOR may not match the type of the shuffle.
2360 if (BuildVT != VT)
2361 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2362 return NewBV;
2363 }
2364 }
2365 }
2366
2367 SDVTList VTs = getVTList(VT);
2368 FoldingSetNodeID ID;
2369 SDValue Ops[2] = { N1, N2 };
2370 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: VTs, OpList: Ops);
2371 for (int i = 0; i != NElts; ++i)
2372 ID.AddInteger(I: MaskVec[i]);
2373
2374 void* IP = nullptr;
2375 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2376 return SDValue(E, 0);
2377
2378 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2379 // SDNode doesn't have access to it. This memory will be "leaked" when
2380 // the node is deallocated, but recovered when the NodeAllocator is released.
2381 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2382 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2383
2384 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2385 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2386 createOperands(Node: N, Vals: Ops);
2387
2388 CSEMap.InsertNode(N, InsertPos: IP);
2389 InsertNode(N);
2390 SDValue V = SDValue(N, 0);
2391 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2392 return V;
2393}
2394
2395SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2396 EVT VT = SV.getValueType(ResNo: 0);
2397 SmallVector<int, 8> MaskVec(SV.getMask());
2398 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2399
2400 SDValue Op0 = SV.getOperand(Num: 0);
2401 SDValue Op1 = SV.getOperand(Num: 1);
2402 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2403}
2404
2405SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2406 SDVTList VTs = getVTList(VT);
2407 FoldingSetNodeID ID;
2408 AddNodeIDNode(ID, OpC: ISD::Register, VTList: VTs, OpList: {});
2409 ID.AddInteger(I: Reg.id());
2410 void *IP = nullptr;
2411 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2412 return SDValue(E, 0);
2413
2414 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2415 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2416 CSEMap.InsertNode(N, InsertPos: IP);
2417 InsertNode(N);
2418 return SDValue(N, 0);
2419}
2420
2421SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2422 FoldingSetNodeID ID;
2423 AddNodeIDNode(ID, OpC: ISD::RegisterMask, VTList: getVTList(VT: MVT::Untyped), OpList: {});
2424 ID.AddPointer(Ptr: RegMask);
2425 void *IP = nullptr;
2426 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2427 return SDValue(E, 0);
2428
2429 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2430 CSEMap.InsertNode(N, InsertPos: IP);
2431 InsertNode(N);
2432 return SDValue(N, 0);
2433}
2434
2435SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2436 MCSymbol *Label) {
2437 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2438}
2439
2440SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2441 SDValue Root, MCSymbol *Label) {
2442 FoldingSetNodeID ID;
2443 SDValue Ops[] = { Root };
2444 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT: MVT::Other), OpList: Ops);
2445 ID.AddPointer(Ptr: Label);
2446 void *IP = nullptr;
2447 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2448 return SDValue(E, 0);
2449
2450 auto *N =
2451 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2452 createOperands(Node: N, Vals: Ops);
2453
2454 CSEMap.InsertNode(N, InsertPos: IP);
2455 InsertNode(N);
2456 return SDValue(N, 0);
2457}
2458
2459SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2460 int64_t Offset, bool isTarget,
2461 unsigned TargetFlags) {
2462 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2463 SDVTList VTs = getVTList(VT);
2464
2465 FoldingSetNodeID ID;
2466 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2467 ID.AddPointer(Ptr: BA);
2468 ID.AddInteger(I: Offset);
2469 ID.AddInteger(I: TargetFlags);
2470 void *IP = nullptr;
2471 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2472 return SDValue(E, 0);
2473
2474 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2475 CSEMap.InsertNode(N, InsertPos: IP);
2476 InsertNode(N);
2477 return SDValue(N, 0);
2478}
2479
2480SDValue SelectionDAG::getSrcValue(const Value *V) {
2481 FoldingSetNodeID ID;
2482 AddNodeIDNode(ID, OpC: ISD::SRCVALUE, VTList: getVTList(VT: MVT::Other), OpList: {});
2483 ID.AddPointer(Ptr: V);
2484
2485 void *IP = nullptr;
2486 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2487 return SDValue(E, 0);
2488
2489 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2490 CSEMap.InsertNode(N, InsertPos: IP);
2491 InsertNode(N);
2492 return SDValue(N, 0);
2493}
2494
2495SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2496 FoldingSetNodeID ID;
2497 AddNodeIDNode(ID, OpC: ISD::MDNODE_SDNODE, VTList: getVTList(VT: MVT::Other), OpList: {});
2498 ID.AddPointer(Ptr: MD);
2499
2500 void *IP = nullptr;
2501 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2502 return SDValue(E, 0);
2503
2504 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2505 CSEMap.InsertNode(N, InsertPos: IP);
2506 InsertNode(N);
2507 return SDValue(N, 0);
2508}
2509
2510SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2511 if (VT == V.getValueType())
2512 return V;
2513
2514 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2515}
2516
2517SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2518 unsigned SrcAS, unsigned DestAS) {
2519 SDVTList VTs = getVTList(VT);
2520 SDValue Ops[] = {Ptr};
2521 FoldingSetNodeID ID;
2522 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: VTs, OpList: Ops);
2523 ID.AddInteger(I: SrcAS);
2524 ID.AddInteger(I: DestAS);
2525
2526 void *IP = nullptr;
2527 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2528 return SDValue(E, 0);
2529
2530 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2531 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2532 createOperands(Node: N, Vals: Ops);
2533
2534 CSEMap.InsertNode(N, InsertPos: IP);
2535 InsertNode(N);
2536 return SDValue(N, 0);
2537}
2538
2539SDValue SelectionDAG::getFreeze(SDValue V) {
2540 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2541}
2542
2543SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2544 bool PoisonOnly) {
2545 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, PoisonOnly))
2546 return V;
2547 return getFreeze(V);
2548}
2549
2550/// getShiftAmountOperand - Return the specified value casted to
2551/// the target's desired shift amount type.
2552SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2553 EVT OpTy = Op.getValueType();
2554 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2555 if (OpTy == ShTy || OpTy.isVector()) return Op;
2556
2557 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2558}
2559
2560SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2561 SDLoc dl(Node);
2562 const TargetLowering &TLI = getTargetLoweringInfo();
2563 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2564 EVT VT = Node->getValueType(ResNo: 0);
2565 SDValue Tmp1 = Node->getOperand(Num: 0);
2566 SDValue Tmp2 = Node->getOperand(Num: 1);
2567 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2568
2569 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2570 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2571 SDValue VAList = VAListLoad;
2572
2573 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2574 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2575 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2576
2577 VAList = getNode(
2578 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2579 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2580 }
2581
2582 // Increment the pointer, VAList, to the next vaarg
2583 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2584 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2585 Ty: VT.getTypeForEVT(Context&: *getContext())),
2586 DL: dl, VT: VAList.getValueType()));
2587 // Store the incremented VAList to the legalized pointer
2588 Tmp1 =
2589 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2590 // Load the actual argument out of the pointer VAList
2591 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2592}
2593
2594SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2595 SDLoc dl(Node);
2596 const TargetLowering &TLI = getTargetLoweringInfo();
2597 // This defaults to loading a pointer from the input and storing it to the
2598 // output, returning the chain.
2599 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2600 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2601 SDValue Tmp1 =
2602 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2603 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2604 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2605 PtrInfo: MachinePointerInfo(VD));
2606}
2607
2608Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2609 const DataLayout &DL = getDataLayout();
2610 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2611 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2612
2613 if (TLI->isTypeLegal(VT) || !VT.isVector())
2614 return RedAlign;
2615
2616 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2617 const Align StackAlign = TFI->getStackAlign();
2618
2619 // See if we can choose a smaller ABI alignment in cases where it's an
2620 // illegal vector type that will get broken down.
2621 if (RedAlign > StackAlign) {
2622 EVT IntermediateVT;
2623 MVT RegisterVT;
2624 unsigned NumIntermediates;
2625 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2626 NumIntermediates, RegisterVT);
2627 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2628 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2629 if (RedAlign2 < RedAlign)
2630 RedAlign = RedAlign2;
2631
2632 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2633 // If the stack is not realignable, the alignment should be limited to the
2634 // StackAlignment
2635 RedAlign = std::min(a: RedAlign, b: StackAlign);
2636 }
2637
2638 return RedAlign;
2639}
2640
2641SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2642 MachineFrameInfo &MFI = MF->getFrameInfo();
2643 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2644 int StackID = 0;
2645 if (Bytes.isScalable())
2646 StackID = TFI->getStackIDForScalableVectors();
2647 // The stack id gives an indication of whether the object is scalable or
2648 // not, so it's safe to pass in the minimum size here.
2649 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2650 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2651 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2652}
2653
2654SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2655 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2656 Align StackAlign =
2657 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2658 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2659}
2660
2661SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2662 TypeSize VT1Size = VT1.getStoreSize();
2663 TypeSize VT2Size = VT2.getStoreSize();
2664 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2665 "Don't know how to choose the maximum size when creating a stack "
2666 "temporary");
2667 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2668 ? VT1Size
2669 : VT2Size;
2670
2671 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2672 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2673 const DataLayout &DL = getDataLayout();
2674 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2675 return CreateStackTemporary(Bytes, Alignment: Align);
2676}
2677
2678SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2679 ISD::CondCode Cond, const SDLoc &dl,
2680 SDNodeFlags Flags) {
2681 EVT OpVT = N1.getValueType();
2682
2683 auto GetUndefBooleanConstant = [&]() {
2684 if (VT.getScalarType() == MVT::i1 ||
2685 TLI->getBooleanContents(Type: OpVT) ==
2686 TargetLowering::UndefinedBooleanContent)
2687 return getUNDEF(VT);
2688 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2689 // so we cannot use getUNDEF(). Return zero instead.
2690 return getConstant(Val: 0, DL: dl, VT);
2691 };
2692
2693 // These setcc operations always fold.
2694 switch (Cond) {
2695 default: break;
2696 case ISD::SETFALSE:
2697 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2698 case ISD::SETTRUE:
2699 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2700
2701 case ISD::SETOEQ:
2702 case ISD::SETOGT:
2703 case ISD::SETOGE:
2704 case ISD::SETOLT:
2705 case ISD::SETOLE:
2706 case ISD::SETONE:
2707 case ISD::SETO:
2708 case ISD::SETUO:
2709 case ISD::SETUEQ:
2710 case ISD::SETUNE:
2711 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2712 break;
2713 }
2714
2715 if (OpVT.isInteger()) {
2716 // For EQ and NE, we can always pick a value for the undef to make the
2717 // predicate pass or fail, so we can return undef.
2718 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2719 // icmp eq/ne X, undef -> undef.
2720 if ((N1.isUndef() || N2.isUndef()) &&
2721 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2722 return GetUndefBooleanConstant();
2723
2724 // If both operands are undef, we can return undef for int comparison.
2725 // icmp undef, undef -> undef.
2726 if (N1.isUndef() && N2.isUndef())
2727 return GetUndefBooleanConstant();
2728
2729 // icmp X, X -> true/false
2730 // icmp X, undef -> true/false because undef could be X.
2731 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2732 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2733 }
2734
2735 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2736 const APInt &C2 = N2C->getAPIntValue();
2737 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2738 const APInt &C1 = N1C->getAPIntValue();
2739
2740 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2741 DL: dl, VT, OpVT);
2742 }
2743 }
2744
2745 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2746 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2747
2748 if (N1CFP && N2CFP) {
2749 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2750 switch (Cond) {
2751 default: break;
2752 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2753 return GetUndefBooleanConstant();
2754 [[fallthrough]];
2755 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2756 OpVT);
2757 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2758 return GetUndefBooleanConstant();
2759 [[fallthrough]];
2760 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2761 R==APFloat::cmpLessThan, DL: dl, VT,
2762 OpVT);
2763 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2764 return GetUndefBooleanConstant();
2765 [[fallthrough]];
2766 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2767 OpVT);
2768 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2769 return GetUndefBooleanConstant();
2770 [[fallthrough]];
2771 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2772 VT, OpVT);
2773 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2774 return GetUndefBooleanConstant();
2775 [[fallthrough]];
2776 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2777 R==APFloat::cmpEqual, DL: dl, VT,
2778 OpVT);
2779 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2780 return GetUndefBooleanConstant();
2781 [[fallthrough]];
2782 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2783 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2784 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2785 OpVT);
2786 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2787 OpVT);
2788 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2789 R==APFloat::cmpEqual, DL: dl, VT,
2790 OpVT);
2791 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2792 OpVT);
2793 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2794 R==APFloat::cmpLessThan, DL: dl, VT,
2795 OpVT);
2796 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2797 R==APFloat::cmpUnordered, DL: dl, VT,
2798 OpVT);
2799 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2800 VT, OpVT);
2801 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2802 OpVT);
2803 }
2804 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2805 // Ensure that the constant occurs on the RHS.
2806 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2807 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2808 return SDValue();
2809 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chian=*/Chain: {},
2810 /*IsSignaling=*/false, Flags);
2811 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2812 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2813 // If an operand is known to be a nan (or undef that could be a nan), we can
2814 // fold it.
2815 // Choosing NaN for the undef will always make unordered comparison succeed
2816 // and ordered comparison fails.
2817 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2818 switch (ISD::getUnorderedFlavor(Cond)) {
2819 default:
2820 llvm_unreachable("Unknown flavor!");
2821 case 0: // Known false.
2822 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2823 case 1: // Known true.
2824 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2825 case 2: // Undefined.
2826 return GetUndefBooleanConstant();
2827 }
2828 }
2829
2830 // Could not fold it.
2831 return SDValue();
2832}
2833
2834/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2835/// use this predicate to simplify operations downstream.
2836bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2837 unsigned BitWidth = Op.getScalarValueSizeInBits();
2838 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2839}
2840
2841// TODO: Should have argument to specify if sign bit of nan is ignorable.
2842bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2843 if (Depth >= MaxRecursionDepth)
2844 return false; // Limit search depth.
2845
2846 unsigned Opc = Op.getOpcode();
2847 switch (Opc) {
2848 case ISD::FABS:
2849 return true;
2850 case ISD::AssertNoFPClass: {
2851 FPClassTest NoFPClass =
2852 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2853
2854 const FPClassTest TestMask = fcNan | fcNegative;
2855 return (NoFPClass & TestMask) == TestMask;
2856 }
2857 case ISD::ARITH_FENCE:
2858 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2859 case ISD::FEXP:
2860 case ISD::FEXP2:
2861 case ISD::FEXP10:
2862 return Op->getFlags().hasNoNaNs();
2863 case ISD::FMINNUM:
2864 case ISD::FMINNUM_IEEE:
2865 case ISD::FMINIMUM:
2866 case ISD::FMINIMUMNUM:
2867 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2868 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2869 case ISD::FMAXNUM:
2870 case ISD::FMAXNUM_IEEE:
2871 case ISD::FMAXIMUM:
2872 case ISD::FMAXIMUMNUM:
2873 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2874 // is sufficient.
2875 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2876 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2877 default:
2878 return false;
2879 }
2880
2881 llvm_unreachable("covered opcode switch");
2882}
2883
2884/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2885/// this predicate to simplify operations downstream. Mask is known to be zero
2886/// for bits that V cannot have.
2887bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2888 unsigned Depth) const {
2889 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2890}
2891
2892/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2893/// DemandedElts. We use this predicate to simplify operations downstream.
2894/// Mask is known to be zero for bits that V cannot have.
2895bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2896 const APInt &DemandedElts,
2897 unsigned Depth) const {
2898 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2899}
2900
2901/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2902/// DemandedElts. We use this predicate to simplify operations downstream.
2903bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2904 unsigned Depth /* = 0 */) const {
2905 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2906}
2907
2908/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2909bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2910 unsigned Depth) const {
2911 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2912}
2913
2914APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2915 const APInt &DemandedElts,
2916 unsigned Depth) const {
2917 EVT VT = Op.getValueType();
2918 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2919
2920 unsigned NumElts = VT.getVectorNumElements();
2921 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2922
2923 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2924 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2925 if (!DemandedElts[EltIdx])
2926 continue; // Don't query elements that are not demanded.
2927 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2928 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2929 KnownZeroElements.setBit(EltIdx);
2930 }
2931 return KnownZeroElements;
2932}
2933
2934/// isSplatValue - Return true if the vector V has the same value
2935/// across all DemandedElts. For scalable vectors, we don't know the
2936/// number of lanes at compile time. Instead, we use a 1 bit APInt
2937/// to represent a conservative value for all lanes; that is, that
2938/// one bit value is implicitly splatted across all lanes.
2939bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2940 APInt &UndefElts, unsigned Depth) const {
2941 unsigned Opcode = V.getOpcode();
2942 EVT VT = V.getValueType();
2943 assert(VT.isVector() && "Vector type expected");
2944 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2945 "scalable demanded bits are ignored");
2946
2947 if (!DemandedElts)
2948 return false; // No demanded elts, better to assume we don't know anything.
2949
2950 if (Depth >= MaxRecursionDepth)
2951 return false; // Limit search depth.
2952
2953 // Deal with some common cases here that work for both fixed and scalable
2954 // vector types.
2955 switch (Opcode) {
2956 case ISD::SPLAT_VECTOR:
2957 UndefElts = V.getOperand(i: 0).isUndef()
2958 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2959 : APInt(DemandedElts.getBitWidth(), 0);
2960 return true;
2961 case ISD::ADD:
2962 case ISD::SUB:
2963 case ISD::AND:
2964 case ISD::XOR:
2965 case ISD::OR: {
2966 APInt UndefLHS, UndefRHS;
2967 SDValue LHS = V.getOperand(i: 0);
2968 SDValue RHS = V.getOperand(i: 1);
2969 // Only recognize splats with the same demanded undef elements for both
2970 // operands, otherwise we might fail to handle binop-specific undef
2971 // handling.
2972 // e.g. (and undef, 0) -> 0 etc.
2973 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2974 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2975 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2976 UndefElts = UndefLHS | UndefRHS;
2977 return true;
2978 }
2979 return false;
2980 }
2981 case ISD::ABS:
2982 case ISD::TRUNCATE:
2983 case ISD::SIGN_EXTEND:
2984 case ISD::ZERO_EXTEND:
2985 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
2986 default:
2987 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2988 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2989 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
2990 Depth);
2991 break;
2992 }
2993
2994 // We don't support other cases than those above for scalable vectors at
2995 // the moment.
2996 if (VT.isScalableVector())
2997 return false;
2998
2999 unsigned NumElts = VT.getVectorNumElements();
3000 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3001 UndefElts = APInt::getZero(numBits: NumElts);
3002
3003 switch (Opcode) {
3004 case ISD::BUILD_VECTOR: {
3005 SDValue Scl;
3006 for (unsigned i = 0; i != NumElts; ++i) {
3007 SDValue Op = V.getOperand(i);
3008 if (Op.isUndef()) {
3009 UndefElts.setBit(i);
3010 continue;
3011 }
3012 if (!DemandedElts[i])
3013 continue;
3014 if (Scl && Scl != Op)
3015 return false;
3016 Scl = Op;
3017 }
3018 return true;
3019 }
3020 case ISD::VECTOR_SHUFFLE: {
3021 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3022 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3023 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3024 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3025 for (int i = 0; i != (int)NumElts; ++i) {
3026 int M = Mask[i];
3027 if (M < 0) {
3028 UndefElts.setBit(i);
3029 continue;
3030 }
3031 if (!DemandedElts[i])
3032 continue;
3033 if (M < (int)NumElts)
3034 DemandedLHS.setBit(M);
3035 else
3036 DemandedRHS.setBit(M - NumElts);
3037 }
3038
3039 // If we aren't demanding either op, assume there's no splat.
3040 // If we are demanding both ops, assume there's no splat.
3041 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3042 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3043 return false;
3044
3045 // See if the demanded elts of the source op is a splat or we only demand
3046 // one element, which should always be a splat.
3047 // TODO: Handle source ops splats with undefs.
3048 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3049 APInt SrcUndefs;
3050 return (SrcElts.popcount() == 1) ||
3051 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3052 (SrcElts & SrcUndefs).isZero());
3053 };
3054 if (!DemandedLHS.isZero())
3055 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3056 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3057 }
3058 case ISD::EXTRACT_SUBVECTOR: {
3059 // Offset the demanded elts by the subvector index.
3060 SDValue Src = V.getOperand(i: 0);
3061 // We don't support scalable vectors at the moment.
3062 if (Src.getValueType().isScalableVector())
3063 return false;
3064 uint64_t Idx = V.getConstantOperandVal(i: 1);
3065 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3066 APInt UndefSrcElts;
3067 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3068 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3069 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3070 return true;
3071 }
3072 break;
3073 }
3074 case ISD::ANY_EXTEND_VECTOR_INREG:
3075 case ISD::SIGN_EXTEND_VECTOR_INREG:
3076 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3077 // Widen the demanded elts by the src element count.
3078 SDValue Src = V.getOperand(i: 0);
3079 // We don't support scalable vectors at the moment.
3080 if (Src.getValueType().isScalableVector())
3081 return false;
3082 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3083 APInt UndefSrcElts;
3084 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3085 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3086 UndefElts = UndefSrcElts.trunc(width: NumElts);
3087 return true;
3088 }
3089 break;
3090 }
3091 case ISD::BITCAST: {
3092 SDValue Src = V.getOperand(i: 0);
3093 EVT SrcVT = Src.getValueType();
3094 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3095 unsigned BitWidth = VT.getScalarSizeInBits();
3096
3097 // Ignore bitcasts from unsupported types.
3098 // TODO: Add fp support?
3099 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3100 break;
3101
3102 // Bitcast 'small element' vector to 'large element' vector.
3103 if ((BitWidth % SrcBitWidth) == 0) {
3104 // See if each sub element is a splat.
3105 unsigned Scale = BitWidth / SrcBitWidth;
3106 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3107 APInt ScaledDemandedElts =
3108 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3109 for (unsigned I = 0; I != Scale; ++I) {
3110 APInt SubUndefElts;
3111 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3112 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3113 SubDemandedElts &= ScaledDemandedElts;
3114 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3115 return false;
3116 // TODO: Add support for merging sub undef elements.
3117 if (!SubUndefElts.isZero())
3118 return false;
3119 }
3120 return true;
3121 }
3122 break;
3123 }
3124 }
3125
3126 return false;
3127}
3128
3129/// Helper wrapper to main isSplatValue function.
3130bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3131 EVT VT = V.getValueType();
3132 assert(VT.isVector() && "Vector type expected");
3133
3134 APInt UndefElts;
3135 // Since the number of lanes in a scalable vector is unknown at compile time,
3136 // we track one bit which is implicitly broadcast to all lanes. This means
3137 // that all lanes in a scalable vector are considered demanded.
3138 APInt DemandedElts
3139 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3140 return isSplatValue(V, DemandedElts, UndefElts) &&
3141 (AllowUndefs || !UndefElts);
3142}
3143
3144SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3145 V = peekThroughExtractSubvectors(V);
3146
3147 EVT VT = V.getValueType();
3148 unsigned Opcode = V.getOpcode();
3149 switch (Opcode) {
3150 default: {
3151 APInt UndefElts;
3152 // Since the number of lanes in a scalable vector is unknown at compile time,
3153 // we track one bit which is implicitly broadcast to all lanes. This means
3154 // that all lanes in a scalable vector are considered demanded.
3155 APInt DemandedElts
3156 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3157
3158 if (isSplatValue(V, DemandedElts, UndefElts)) {
3159 if (VT.isScalableVector()) {
3160 // DemandedElts and UndefElts are ignored for scalable vectors, since
3161 // the only supported cases are SPLAT_VECTOR nodes.
3162 SplatIdx = 0;
3163 } else {
3164 // Handle case where all demanded elements are UNDEF.
3165 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3166 SplatIdx = 0;
3167 return getUNDEF(VT);
3168 }
3169 SplatIdx = (UndefElts & DemandedElts).countr_one();
3170 }
3171 return V;
3172 }
3173 break;
3174 }
3175 case ISD::SPLAT_VECTOR:
3176 SplatIdx = 0;
3177 return V;
3178 case ISD::VECTOR_SHUFFLE: {
3179 assert(!VT.isScalableVector());
3180 // Check if this is a shuffle node doing a splat.
3181 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3182 // getTargetVShiftNode currently struggles without the splat source.
3183 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3184 if (!SVN->isSplat())
3185 break;
3186 int Idx = SVN->getSplatIndex();
3187 int NumElts = V.getValueType().getVectorNumElements();
3188 SplatIdx = Idx % NumElts;
3189 return V.getOperand(i: Idx / NumElts);
3190 }
3191 }
3192
3193 return SDValue();
3194}
3195
3196SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3197 int SplatIdx;
3198 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3199 EVT SVT = SrcVector.getValueType().getScalarType();
3200 EVT LegalSVT = SVT;
3201 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3202 if (!SVT.isInteger())
3203 return SDValue();
3204 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3205 if (LegalSVT.bitsLT(VT: SVT))
3206 return SDValue();
3207 }
3208 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3209 }
3210 return SDValue();
3211}
3212
3213std::optional<ConstantRange>
3214SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3215 unsigned Depth) const {
3216 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3217 V.getOpcode() == ISD::SRA) &&
3218 "Unknown shift node");
3219 // Shifting more than the bitwidth is not valid.
3220 unsigned BitWidth = V.getScalarValueSizeInBits();
3221
3222 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3223 const APInt &ShAmt = Cst->getAPIntValue();
3224 if (ShAmt.uge(RHS: BitWidth))
3225 return std::nullopt;
3226 return ConstantRange(ShAmt);
3227 }
3228
3229 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3230 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3231 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3232 if (!DemandedElts[i])
3233 continue;
3234 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3235 if (!SA) {
3236 MinAmt = MaxAmt = nullptr;
3237 break;
3238 }
3239 const APInt &ShAmt = SA->getAPIntValue();
3240 if (ShAmt.uge(RHS: BitWidth))
3241 return std::nullopt;
3242 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3243 MinAmt = &ShAmt;
3244 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3245 MaxAmt = &ShAmt;
3246 }
3247 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3248 "Failed to find matching min/max shift amounts");
3249 if (MinAmt && MaxAmt)
3250 return ConstantRange(*MinAmt, *MaxAmt + 1);
3251 }
3252
3253 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3254 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3255 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3256 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3257 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3258
3259 return std::nullopt;
3260}
3261
3262std::optional<unsigned>
3263SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3264 unsigned Depth) const {
3265 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3266 V.getOpcode() == ISD::SRA) &&
3267 "Unknown shift node");
3268 if (std::optional<ConstantRange> AmtRange =
3269 getValidShiftAmountRange(V, DemandedElts, Depth))
3270 if (const APInt *ShAmt = AmtRange->getSingleElement())
3271 return ShAmt->getZExtValue();
3272 return std::nullopt;
3273}
3274
3275std::optional<unsigned>
3276SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3277 EVT VT = V.getValueType();
3278 APInt DemandedElts = VT.isFixedLengthVector()
3279 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3280 : APInt(1, 1);
3281 return getValidShiftAmount(V, DemandedElts, Depth);
3282}
3283
3284std::optional<unsigned>
3285SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3286 unsigned Depth) const {
3287 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3288 V.getOpcode() == ISD::SRA) &&
3289 "Unknown shift node");
3290 if (std::optional<ConstantRange> AmtRange =
3291 getValidShiftAmountRange(V, DemandedElts, Depth))
3292 return AmtRange->getUnsignedMin().getZExtValue();
3293 return std::nullopt;
3294}
3295
3296std::optional<unsigned>
3297SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3298 EVT VT = V.getValueType();
3299 APInt DemandedElts = VT.isFixedLengthVector()
3300 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3301 : APInt(1, 1);
3302 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3303}
3304
3305std::optional<unsigned>
3306SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3307 unsigned Depth) const {
3308 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3309 V.getOpcode() == ISD::SRA) &&
3310 "Unknown shift node");
3311 if (std::optional<ConstantRange> AmtRange =
3312 getValidShiftAmountRange(V, DemandedElts, Depth))
3313 return AmtRange->getUnsignedMax().getZExtValue();
3314 return std::nullopt;
3315}
3316
3317std::optional<unsigned>
3318SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3319 EVT VT = V.getValueType();
3320 APInt DemandedElts = VT.isFixedLengthVector()
3321 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3322 : APInt(1, 1);
3323 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3324}
3325
3326/// Determine which bits of Op are known to be either zero or one and return
3327/// them in Known. For vectors, the known bits are those that are shared by
3328/// every vector element.
3329KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3330 EVT VT = Op.getValueType();
3331
3332 // Since the number of lanes in a scalable vector is unknown at compile time,
3333 // we track one bit which is implicitly broadcast to all lanes. This means
3334 // that all lanes in a scalable vector are considered demanded.
3335 APInt DemandedElts = VT.isFixedLengthVector()
3336 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3337 : APInt(1, 1);
3338 return computeKnownBits(Op, DemandedElts, Depth);
3339}
3340
3341/// Determine which bits of Op are known to be either zero or one and return
3342/// them in Known. The DemandedElts argument allows us to only collect the known
3343/// bits that are shared by the requested vector elements.
3344KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3345 unsigned Depth) const {
3346 unsigned BitWidth = Op.getScalarValueSizeInBits();
3347
3348 KnownBits Known(BitWidth); // Don't know anything.
3349
3350 if (auto OptAPInt = Op->bitcastToAPInt()) {
3351 // We know all of the bits for a constant!
3352 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3353 }
3354
3355 if (Depth >= MaxRecursionDepth)
3356 return Known; // Limit search depth.
3357
3358 KnownBits Known2;
3359 unsigned NumElts = DemandedElts.getBitWidth();
3360 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3361 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3362 assert((!Op.getValueType().isFixedLengthVector() ||
3363 NumElts == Op.getValueType().getVectorNumElements()) &&
3364 "Unexpected vector size");
3365
3366 if (!DemandedElts)
3367 return Known; // No demanded elts, better to assume we don't know anything.
3368
3369 unsigned Opcode = Op.getOpcode();
3370 switch (Opcode) {
3371 case ISD::MERGE_VALUES:
3372 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3373 Depth: Depth + 1);
3374 case ISD::SPLAT_VECTOR: {
3375 SDValue SrcOp = Op.getOperand(i: 0);
3376 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3377 "Expected SPLAT_VECTOR implicit truncation");
3378 // Implicitly truncate the bits to match the official semantics of
3379 // SPLAT_VECTOR.
3380 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3381 break;
3382 }
3383 case ISD::SPLAT_VECTOR_PARTS: {
3384 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3385 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3386 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3387 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3388 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3389 }
3390 break;
3391 }
3392 case ISD::STEP_VECTOR: {
3393 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3394
3395 if (Step.isPowerOf2())
3396 Known.Zero.setLowBits(Step.logBase2());
3397
3398 const Function &F = getMachineFunction().getFunction();
3399
3400 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3401 break;
3402 const APInt MinNumElts =
3403 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3404
3405 bool Overflow;
3406 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3407 .getUnsignedMax()
3408 .umul_ov(RHS: MinNumElts, Overflow);
3409 if (Overflow)
3410 break;
3411
3412 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3413 if (Overflow)
3414 break;
3415
3416 Known.Zero.setHighBits(MaxValue.countl_zero());
3417 break;
3418 }
3419 case ISD::BUILD_VECTOR:
3420 assert(!Op.getValueType().isScalableVector());
3421 // Collect the known bits that are shared by every demanded vector element.
3422 Known.setAllConflict();
3423 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3424 if (!DemandedElts[i])
3425 continue;
3426
3427 SDValue SrcOp = Op.getOperand(i);
3428 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3429
3430 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3431 if (SrcOp.getValueSizeInBits() != BitWidth) {
3432 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3433 "Expected BUILD_VECTOR implicit truncation");
3434 Known2 = Known2.trunc(BitWidth);
3435 }
3436
3437 // Known bits are the values that are shared by every demanded element.
3438 Known = Known.intersectWith(RHS: Known2);
3439
3440 // If we don't know any bits, early out.
3441 if (Known.isUnknown())
3442 break;
3443 }
3444 break;
3445 case ISD::VECTOR_COMPRESS: {
3446 SDValue Vec = Op.getOperand(i: 0);
3447 SDValue PassThru = Op.getOperand(i: 2);
3448 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3449 // If we don't know any bits, early out.
3450 if (Known.isUnknown())
3451 break;
3452 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3453 Known = Known.intersectWith(RHS: Known2);
3454 break;
3455 }
3456 case ISD::VECTOR_SHUFFLE: {
3457 assert(!Op.getValueType().isScalableVector());
3458 // Collect the known bits that are shared by every vector element referenced
3459 // by the shuffle.
3460 APInt DemandedLHS, DemandedRHS;
3461 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3462 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3463 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3464 DemandedLHS, DemandedRHS))
3465 break;
3466
3467 // Known bits are the values that are shared by every demanded element.
3468 Known.setAllConflict();
3469 if (!!DemandedLHS) {
3470 SDValue LHS = Op.getOperand(i: 0);
3471 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3472 Known = Known.intersectWith(RHS: Known2);
3473 }
3474 // If we don't know any bits, early out.
3475 if (Known.isUnknown())
3476 break;
3477 if (!!DemandedRHS) {
3478 SDValue RHS = Op.getOperand(i: 1);
3479 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3480 Known = Known.intersectWith(RHS: Known2);
3481 }
3482 break;
3483 }
3484 case ISD::VSCALE: {
3485 const Function &F = getMachineFunction().getFunction();
3486 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3487 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3488 break;
3489 }
3490 case ISD::CONCAT_VECTORS: {
3491 if (Op.getValueType().isScalableVector())
3492 break;
3493 // Split DemandedElts and test each of the demanded subvectors.
3494 Known.setAllConflict();
3495 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3496 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3497 unsigned NumSubVectors = Op.getNumOperands();
3498 for (unsigned i = 0; i != NumSubVectors; ++i) {
3499 APInt DemandedSub =
3500 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3501 if (!!DemandedSub) {
3502 SDValue Sub = Op.getOperand(i);
3503 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3504 Known = Known.intersectWith(RHS: Known2);
3505 }
3506 // If we don't know any bits, early out.
3507 if (Known.isUnknown())
3508 break;
3509 }
3510 break;
3511 }
3512 case ISD::INSERT_SUBVECTOR: {
3513 if (Op.getValueType().isScalableVector())
3514 break;
3515 // Demand any elements from the subvector and the remainder from the src its
3516 // inserted into.
3517 SDValue Src = Op.getOperand(i: 0);
3518 SDValue Sub = Op.getOperand(i: 1);
3519 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3520 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3521 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3522 APInt DemandedSrcElts = DemandedElts;
3523 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3524
3525 Known.setAllConflict();
3526 if (!!DemandedSubElts) {
3527 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3528 if (Known.isUnknown())
3529 break; // early-out.
3530 }
3531 if (!!DemandedSrcElts) {
3532 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3533 Known = Known.intersectWith(RHS: Known2);
3534 }
3535 break;
3536 }
3537 case ISD::EXTRACT_SUBVECTOR: {
3538 // Offset the demanded elts by the subvector index.
3539 SDValue Src = Op.getOperand(i: 0);
3540
3541 APInt DemandedSrcElts;
3542 if (Src.getValueType().isScalableVector())
3543 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3544 else {
3545 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3546 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3547 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3548 }
3549 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3550 break;
3551 }
3552 case ISD::SCALAR_TO_VECTOR: {
3553 if (Op.getValueType().isScalableVector())
3554 break;
3555 // We know about scalar_to_vector as much as we know about it source,
3556 // which becomes the first element of otherwise unknown vector.
3557 if (DemandedElts != 1)
3558 break;
3559
3560 SDValue N0 = Op.getOperand(i: 0);
3561 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3562 if (N0.getValueSizeInBits() != BitWidth)
3563 Known = Known.trunc(BitWidth);
3564
3565 break;
3566 }
3567 case ISD::BITCAST: {
3568 if (Op.getValueType().isScalableVector())
3569 break;
3570
3571 SDValue N0 = Op.getOperand(i: 0);
3572 EVT SubVT = N0.getValueType();
3573 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3574
3575 // Ignore bitcasts from unsupported types.
3576 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3577 break;
3578
3579 // Fast handling of 'identity' bitcasts.
3580 if (BitWidth == SubBitWidth) {
3581 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3582 break;
3583 }
3584
3585 bool IsLE = getDataLayout().isLittleEndian();
3586
3587 // Bitcast 'small element' vector to 'large element' scalar/vector.
3588 if ((BitWidth % SubBitWidth) == 0) {
3589 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3590
3591 // Collect known bits for the (larger) output by collecting the known
3592 // bits from each set of sub elements and shift these into place.
3593 // We need to separately call computeKnownBits for each set of
3594 // sub elements as the knownbits for each is likely to be different.
3595 unsigned SubScale = BitWidth / SubBitWidth;
3596 APInt SubDemandedElts(NumElts * SubScale, 0);
3597 for (unsigned i = 0; i != NumElts; ++i)
3598 if (DemandedElts[i])
3599 SubDemandedElts.setBit(i * SubScale);
3600
3601 for (unsigned i = 0; i != SubScale; ++i) {
3602 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3603 Depth: Depth + 1);
3604 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3605 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3606 }
3607 }
3608
3609 // Bitcast 'large element' scalar/vector to 'small element' vector.
3610 if ((SubBitWidth % BitWidth) == 0) {
3611 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3612
3613 // Collect known bits for the (smaller) output by collecting the known
3614 // bits from the overlapping larger input elements and extracting the
3615 // sub sections we actually care about.
3616 unsigned SubScale = SubBitWidth / BitWidth;
3617 APInt SubDemandedElts =
3618 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3619 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3620
3621 Known.setAllConflict();
3622 for (unsigned i = 0; i != NumElts; ++i)
3623 if (DemandedElts[i]) {
3624 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3625 unsigned Offset = (Shifts % SubScale) * BitWidth;
3626 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3627 // If we don't know any bits, early out.
3628 if (Known.isUnknown())
3629 break;
3630 }
3631 }
3632 break;
3633 }
3634 case ISD::AND:
3635 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3636 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3637
3638 Known &= Known2;
3639 break;
3640 case ISD::OR:
3641 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3642 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3643
3644 Known |= Known2;
3645 break;
3646 case ISD::XOR:
3647 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3648 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3649
3650 Known ^= Known2;
3651 break;
3652 case ISD::MUL: {
3653 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3654 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3655 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3656 // TODO: SelfMultiply can be poison, but not undef.
3657 if (SelfMultiply)
3658 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3659 Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
3660 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3661
3662 // If the multiplication is known not to overflow, the product of a number
3663 // with itself is non-negative. Only do this if we didn't already computed
3664 // the opposite value for the sign bit.
3665 if (Op->getFlags().hasNoSignedWrap() &&
3666 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3667 !Known.isNegative())
3668 Known.makeNonNegative();
3669 break;
3670 }
3671 case ISD::MULHU: {
3672 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3673 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3674 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3675 break;
3676 }
3677 case ISD::MULHS: {
3678 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3679 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3680 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3681 break;
3682 }
3683 case ISD::ABDU: {
3684 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3685 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3686 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3687 break;
3688 }
3689 case ISD::ABDS: {
3690 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3691 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3692 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3693 unsigned SignBits1 =
3694 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3695 if (SignBits1 == 1)
3696 break;
3697 unsigned SignBits0 =
3698 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3699 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3700 break;
3701 }
3702 case ISD::UMUL_LOHI: {
3703 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3704 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3705 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3706 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3707 if (Op.getResNo() == 0)
3708 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3709 else
3710 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3711 break;
3712 }
3713 case ISD::SMUL_LOHI: {
3714 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3715 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3716 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3717 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3718 if (Op.getResNo() == 0)
3719 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3720 else
3721 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3722 break;
3723 }
3724 case ISD::AVGFLOORU: {
3725 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3726 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3727 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3728 break;
3729 }
3730 case ISD::AVGCEILU: {
3731 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3732 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3733 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3734 break;
3735 }
3736 case ISD::AVGFLOORS: {
3737 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3738 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3739 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3740 break;
3741 }
3742 case ISD::AVGCEILS: {
3743 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3744 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3745 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3746 break;
3747 }
3748 case ISD::SELECT:
3749 case ISD::VSELECT:
3750 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3751 // If we don't know any bits, early out.
3752 if (Known.isUnknown())
3753 break;
3754 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3755
3756 // Only known if known in both the LHS and RHS.
3757 Known = Known.intersectWith(RHS: Known2);
3758 break;
3759 case ISD::SELECT_CC:
3760 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3761 // If we don't know any bits, early out.
3762 if (Known.isUnknown())
3763 break;
3764 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3765
3766 // Only known if known in both the LHS and RHS.
3767 Known = Known.intersectWith(RHS: Known2);
3768 break;
3769 case ISD::SMULO:
3770 case ISD::UMULO:
3771 if (Op.getResNo() != 1)
3772 break;
3773 // The boolean result conforms to getBooleanContents.
3774 // If we know the result of a setcc has the top bits zero, use this info.
3775 // We know that we have an integer-based boolean since these operations
3776 // are only available for integer.
3777 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3778 TargetLowering::ZeroOrOneBooleanContent &&
3779 BitWidth > 1)
3780 Known.Zero.setBitsFrom(1);
3781 break;
3782 case ISD::SETCC:
3783 case ISD::SETCCCARRY:
3784 case ISD::STRICT_FSETCC:
3785 case ISD::STRICT_FSETCCS: {
3786 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3787 // If we know the result of a setcc has the top bits zero, use this info.
3788 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3789 TargetLowering::ZeroOrOneBooleanContent &&
3790 BitWidth > 1)
3791 Known.Zero.setBitsFrom(1);
3792 break;
3793 }
3794 case ISD::SHL: {
3795 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3796 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3797
3798 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3799 bool NSW = Op->getFlags().hasNoSignedWrap();
3800
3801 bool ShAmtNonZero = Known2.isNonZero();
3802
3803 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3804
3805 // Minimum shift low bits are known zero.
3806 if (std::optional<unsigned> ShMinAmt =
3807 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3808 Known.Zero.setLowBits(*ShMinAmt);
3809 break;
3810 }
3811 case ISD::SRL:
3812 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3813 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3814 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3815 Exact: Op->getFlags().hasExact());
3816
3817 // Minimum shift high bits are known zero.
3818 if (std::optional<unsigned> ShMinAmt =
3819 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3820 Known.Zero.setHighBits(*ShMinAmt);
3821 break;
3822 case ISD::SRA:
3823 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3824 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3825 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3826 Exact: Op->getFlags().hasExact());
3827 break;
3828 case ISD::ROTL:
3829 case ISD::ROTR:
3830 if (ConstantSDNode *C =
3831 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3832 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3833
3834 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3835
3836 // Canonicalize to ROTR.
3837 if (Opcode == ISD::ROTL && Amt != 0)
3838 Amt = BitWidth - Amt;
3839
3840 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3841 Known.One = Known.One.rotr(rotateAmt: Amt);
3842 }
3843 break;
3844 case ISD::FSHL:
3845 case ISD::FSHR:
3846 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3847 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3848
3849 // For fshl, 0-shift returns the 1st arg.
3850 // For fshr, 0-shift returns the 2nd arg.
3851 if (Amt == 0) {
3852 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3853 DemandedElts, Depth: Depth + 1);
3854 break;
3855 }
3856
3857 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3858 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3859 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3860 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3861 if (Opcode == ISD::FSHL) {
3862 Known <<= Amt;
3863 Known2 >>= BitWidth - Amt;
3864 } else {
3865 Known <<= BitWidth - Amt;
3866 Known2 >>= Amt;
3867 }
3868 Known = Known.unionWith(RHS: Known2);
3869 }
3870 break;
3871 case ISD::SHL_PARTS:
3872 case ISD::SRA_PARTS:
3873 case ISD::SRL_PARTS: {
3874 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3875
3876 // Collect lo/hi source values and concatenate.
3877 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3878 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3879 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3880 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3881 Known = Known2.concat(Lo: Known);
3882
3883 // Collect shift amount.
3884 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3885
3886 if (Opcode == ISD::SHL_PARTS)
3887 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3888 else if (Opcode == ISD::SRA_PARTS)
3889 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3890 else // if (Opcode == ISD::SRL_PARTS)
3891 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3892
3893 // TODO: Minimum shift low/high bits are known zero.
3894
3895 if (Op.getResNo() == 0)
3896 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3897 else
3898 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3899 break;
3900 }
3901 case ISD::SIGN_EXTEND_INREG: {
3902 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3903 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3904 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3905 break;
3906 }
3907 case ISD::CTTZ:
3908 case ISD::CTTZ_ZERO_UNDEF: {
3909 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3910 // If we have a known 1, its position is our upper bound.
3911 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3912 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3913 Known.Zero.setBitsFrom(LowBits);
3914 break;
3915 }
3916 case ISD::CTLZ:
3917 case ISD::CTLZ_ZERO_UNDEF: {
3918 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3919 // If we have a known 1, its position is our upper bound.
3920 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3921 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3922 Known.Zero.setBitsFrom(LowBits);
3923 break;
3924 }
3925 case ISD::CTLS: {
3926 unsigned MinRedundantSignBits =
3927 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3928 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3929 APInt(BitWidth, BitWidth));
3930 Known = Range.toKnownBits();
3931 break;
3932 }
3933 case ISD::CTPOP: {
3934 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3935 // If we know some of the bits are zero, they can't be one.
3936 unsigned PossibleOnes = Known2.countMaxPopulation();
3937 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3938 break;
3939 }
3940 case ISD::PARITY: {
3941 // Parity returns 0 everywhere but the LSB.
3942 Known.Zero.setBitsFrom(1);
3943 break;
3944 }
3945 case ISD::CLMUL: {
3946 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3947 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3948 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3949 break;
3950 }
3951 case ISD::MGATHER:
3952 case ISD::MLOAD: {
3953 ISD::LoadExtType ETy =
3954 (Opcode == ISD::MGATHER)
3955 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3956 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3957 if (ETy == ISD::ZEXTLOAD) {
3958 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3959 KnownBits Known0(MemVT.getScalarSizeInBits());
3960 return Known0.zext(BitWidth);
3961 }
3962 break;
3963 }
3964 case ISD::LOAD: {
3965 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3966 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3967 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3968 // Determine any common known bits from the loaded constant pool value.
3969 Type *CstTy = Cst->getType();
3970 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3971 !Op.getValueType().isScalableVector()) {
3972 // If its a vector splat, then we can (quickly) reuse the scalar path.
3973 // NOTE: We assume all elements match and none are UNDEF.
3974 if (CstTy->isVectorTy()) {
3975 if (const Constant *Splat = Cst->getSplatValue()) {
3976 Cst = Splat;
3977 CstTy = Cst->getType();
3978 }
3979 }
3980 // TODO - do we need to handle different bitwidths?
3981 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3982 // Iterate across all vector elements finding common known bits.
3983 Known.setAllConflict();
3984 for (unsigned i = 0; i != NumElts; ++i) {
3985 if (!DemandedElts[i])
3986 continue;
3987 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
3988 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
3989 const APInt &Value = CInt->getValue();
3990 Known.One &= Value;
3991 Known.Zero &= ~Value;
3992 continue;
3993 }
3994 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
3995 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3996 Known.One &= Value;
3997 Known.Zero &= ~Value;
3998 continue;
3999 }
4000 }
4001 Known.One.clearAllBits();
4002 Known.Zero.clearAllBits();
4003 break;
4004 }
4005 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4006 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4007 Known = KnownBits::makeConstant(C: CInt->getValue());
4008 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4009 Known =
4010 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4011 }
4012 }
4013 }
4014 } else if (Op.getResNo() == 0) {
4015 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4016 KnownBits KnownScalarMemory(ScalarMemorySize);
4017 if (const MDNode *MD = LD->getRanges())
4018 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4019
4020 // Extend the Known bits from memory to the size of the scalar result.
4021 if (ISD::isZEXTLoad(N: Op.getNode()))
4022 Known = KnownScalarMemory.zext(BitWidth);
4023 else if (ISD::isSEXTLoad(N: Op.getNode()))
4024 Known = KnownScalarMemory.sext(BitWidth);
4025 else if (ISD::isEXTLoad(N: Op.getNode()))
4026 Known = KnownScalarMemory.anyext(BitWidth);
4027 else
4028 Known = KnownScalarMemory;
4029 assert(Known.getBitWidth() == BitWidth);
4030 return Known;
4031 }
4032 break;
4033 }
4034 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4035 if (Op.getValueType().isScalableVector())
4036 break;
4037 EVT InVT = Op.getOperand(i: 0).getValueType();
4038 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4039 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4040 Known = Known.zext(BitWidth);
4041 break;
4042 }
4043 case ISD::ZERO_EXTEND: {
4044 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4045 Known = Known.zext(BitWidth);
4046 break;
4047 }
4048 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4049 if (Op.getValueType().isScalableVector())
4050 break;
4051 EVT InVT = Op.getOperand(i: 0).getValueType();
4052 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4053 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4054 // If the sign bit is known to be zero or one, then sext will extend
4055 // it to the top bits, else it will just zext.
4056 Known = Known.sext(BitWidth);
4057 break;
4058 }
4059 case ISD::SIGN_EXTEND: {
4060 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4061 // If the sign bit is known to be zero or one, then sext will extend
4062 // it to the top bits, else it will just zext.
4063 Known = Known.sext(BitWidth);
4064 break;
4065 }
4066 case ISD::ANY_EXTEND_VECTOR_INREG: {
4067 if (Op.getValueType().isScalableVector())
4068 break;
4069 EVT InVT = Op.getOperand(i: 0).getValueType();
4070 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4071 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4072 Known = Known.anyext(BitWidth);
4073 break;
4074 }
4075 case ISD::ANY_EXTEND: {
4076 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4077 Known = Known.anyext(BitWidth);
4078 break;
4079 }
4080 case ISD::TRUNCATE: {
4081 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4082 Known = Known.trunc(BitWidth);
4083 break;
4084 }
4085 case ISD::TRUNCATE_SSAT_S: {
4086 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4087 Known = Known.truncSSat(BitWidth);
4088 break;
4089 }
4090 case ISD::TRUNCATE_SSAT_U: {
4091 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4092 Known = Known.truncSSatU(BitWidth);
4093 break;
4094 }
4095 case ISD::TRUNCATE_USAT_U: {
4096 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4097 Known = Known.truncUSat(BitWidth);
4098 break;
4099 }
4100 case ISD::AssertZext: {
4101 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4102 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4103 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4104 Known.Zero |= (~InMask);
4105 Known.One &= (~Known.Zero);
4106 break;
4107 }
4108 case ISD::AssertAlign: {
4109 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4110 assert(LogOfAlign != 0);
4111
4112 // TODO: Should use maximum with source
4113 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4114 // well as clearing one bits.
4115 Known.Zero.setLowBits(LogOfAlign);
4116 Known.One.clearLowBits(loBits: LogOfAlign);
4117 break;
4118 }
4119 case ISD::AssertNoFPClass: {
4120 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4121
4122 FPClassTest NoFPClass =
4123 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4124 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4125 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4126 // Cannot be negative.
4127 Known.makeNonNegative();
4128 }
4129
4130 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4131 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4132 // Cannot be positive.
4133 Known.makeNegative();
4134 }
4135
4136 break;
4137 }
4138 case ISD::FGETSIGN:
4139 // All bits are zero except the low bit.
4140 Known.Zero.setBitsFrom(1);
4141 break;
4142 case ISD::ADD:
4143 case ISD::SUB: {
4144 SDNodeFlags Flags = Op.getNode()->getFlags();
4145 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4146 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4147 Known = KnownBits::computeForAddSub(
4148 Add: Op.getOpcode() == ISD::ADD, NSW: Flags.hasNoSignedWrap(),
4149 NUW: Flags.hasNoUnsignedWrap(), LHS: Known, RHS: Known2);
4150 break;
4151 }
4152 case ISD::USUBO:
4153 case ISD::SSUBO:
4154 case ISD::USUBO_CARRY:
4155 case ISD::SSUBO_CARRY:
4156 if (Op.getResNo() == 1) {
4157 // If we know the result of a setcc has the top bits zero, use this info.
4158 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4159 TargetLowering::ZeroOrOneBooleanContent &&
4160 BitWidth > 1)
4161 Known.Zero.setBitsFrom(1);
4162 break;
4163 }
4164 [[fallthrough]];
4165 case ISD::SUBC: {
4166 assert(Op.getResNo() == 0 &&
4167 "We only compute knownbits for the difference here.");
4168
4169 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4170 KnownBits Borrow(1);
4171 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4172 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4173 // Borrow has bit width 1
4174 Borrow = Borrow.trunc(BitWidth: 1);
4175 } else {
4176 Borrow.setAllZero();
4177 }
4178
4179 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4180 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4181 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4182 break;
4183 }
4184 case ISD::UADDO:
4185 case ISD::SADDO:
4186 case ISD::UADDO_CARRY:
4187 case ISD::SADDO_CARRY:
4188 if (Op.getResNo() == 1) {
4189 // If we know the result of a setcc has the top bits zero, use this info.
4190 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4191 TargetLowering::ZeroOrOneBooleanContent &&
4192 BitWidth > 1)
4193 Known.Zero.setBitsFrom(1);
4194 break;
4195 }
4196 [[fallthrough]];
4197 case ISD::ADDC:
4198 case ISD::ADDE: {
4199 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4200
4201 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4202 KnownBits Carry(1);
4203 if (Opcode == ISD::ADDE)
4204 // Can't track carry from glue, set carry to unknown.
4205 Carry.resetAll();
4206 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4207 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4208 // Carry has bit width 1
4209 Carry = Carry.trunc(BitWidth: 1);
4210 } else {
4211 Carry.setAllZero();
4212 }
4213
4214 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4215 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4216 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4217 break;
4218 }
4219 case ISD::UDIV: {
4220 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4221 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4222 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4223 break;
4224 }
4225 case ISD::SDIV: {
4226 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4227 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4228 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4229 break;
4230 }
4231 case ISD::SREM: {
4232 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4233 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4234 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4235 break;
4236 }
4237 case ISD::UREM: {
4238 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4239 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4240 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4241 break;
4242 }
4243 case ISD::EXTRACT_ELEMENT: {
4244 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4245 const unsigned Index = Op.getConstantOperandVal(i: 1);
4246 const unsigned EltBitWidth = Op.getValueSizeInBits();
4247
4248 // Remove low part of known bits mask
4249 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4250 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4251
4252 // Remove high part of known bit mask
4253 Known = Known.trunc(BitWidth: EltBitWidth);
4254 break;
4255 }
4256 case ISD::EXTRACT_VECTOR_ELT: {
4257 SDValue InVec = Op.getOperand(i: 0);
4258 SDValue EltNo = Op.getOperand(i: 1);
4259 EVT VecVT = InVec.getValueType();
4260 // computeKnownBits not yet implemented for scalable vectors.
4261 if (VecVT.isScalableVector())
4262 break;
4263 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4264 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4265
4266 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4267 // anything about the extended bits.
4268 if (BitWidth > EltBitWidth)
4269 Known = Known.trunc(BitWidth: EltBitWidth);
4270
4271 // If we know the element index, just demand that vector element, else for
4272 // an unknown element index, ignore DemandedElts and demand them all.
4273 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4274 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4275 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4276 DemandedSrcElts =
4277 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4278
4279 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4280 if (BitWidth > EltBitWidth)
4281 Known = Known.anyext(BitWidth);
4282 break;
4283 }
4284 case ISD::INSERT_VECTOR_ELT: {
4285 if (Op.getValueType().isScalableVector())
4286 break;
4287
4288 // If we know the element index, split the demand between the
4289 // source vector and the inserted element, otherwise assume we need
4290 // the original demanded vector elements and the value.
4291 SDValue InVec = Op.getOperand(i: 0);
4292 SDValue InVal = Op.getOperand(i: 1);
4293 SDValue EltNo = Op.getOperand(i: 2);
4294 bool DemandedVal = true;
4295 APInt DemandedVecElts = DemandedElts;
4296 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4297 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4298 unsigned EltIdx = CEltNo->getZExtValue();
4299 DemandedVal = !!DemandedElts[EltIdx];
4300 DemandedVecElts.clearBit(BitPosition: EltIdx);
4301 }
4302 Known.setAllConflict();
4303 if (DemandedVal) {
4304 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4305 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4306 }
4307 if (!!DemandedVecElts) {
4308 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4309 Known = Known.intersectWith(RHS: Known2);
4310 }
4311 break;
4312 }
4313 case ISD::BITREVERSE: {
4314 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4315 Known = Known2.reverseBits();
4316 break;
4317 }
4318 case ISD::BSWAP: {
4319 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4320 Known = Known2.byteSwap();
4321 break;
4322 }
4323 case ISD::ABS: {
4324 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4325 Known = Known2.abs();
4326 Known.Zero.setHighBits(
4327 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4328 break;
4329 }
4330 case ISD::USUBSAT: {
4331 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4332 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4333 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4334 break;
4335 }
4336 case ISD::UMIN: {
4337 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4338 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4339 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4340 break;
4341 }
4342 case ISD::UMAX: {
4343 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4344 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4345 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4346 break;
4347 }
4348 case ISD::SMIN:
4349 case ISD::SMAX: {
4350 // If we have a clamp pattern, we know that the number of sign bits will be
4351 // the minimum of the clamp min/max range.
4352 bool IsMax = (Opcode == ISD::SMAX);
4353 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4354 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4355 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4356 CstHigh =
4357 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4358 if (CstLow && CstHigh) {
4359 if (!IsMax)
4360 std::swap(a&: CstLow, b&: CstHigh);
4361
4362 const APInt &ValueLow = CstLow->getAPIntValue();
4363 const APInt &ValueHigh = CstHigh->getAPIntValue();
4364 if (ValueLow.sle(RHS: ValueHigh)) {
4365 unsigned LowSignBits = ValueLow.getNumSignBits();
4366 unsigned HighSignBits = ValueHigh.getNumSignBits();
4367 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4368 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4369 Known.One.setHighBits(MinSignBits);
4370 break;
4371 }
4372 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4373 Known.Zero.setHighBits(MinSignBits);
4374 break;
4375 }
4376 }
4377 }
4378
4379 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4380 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4381 if (IsMax)
4382 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4383 else
4384 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4385
4386 // For SMAX, if CstLow is non-negative we know the result will be
4387 // non-negative and thus all sign bits are 0.
4388 // TODO: There's an equivalent of this for smin with negative constant for
4389 // known ones.
4390 if (IsMax && CstLow) {
4391 const APInt &ValueLow = CstLow->getAPIntValue();
4392 if (ValueLow.isNonNegative()) {
4393 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4394 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4395 }
4396 }
4397
4398 break;
4399 }
4400 case ISD::UINT_TO_FP: {
4401 Known.makeNonNegative();
4402 break;
4403 }
4404 case ISD::SINT_TO_FP: {
4405 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4406 if (Known2.isNonNegative())
4407 Known.makeNonNegative();
4408 else if (Known2.isNegative())
4409 Known.makeNegative();
4410 break;
4411 }
4412 case ISD::FP_TO_UINT_SAT: {
4413 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4414 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4415 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4416 break;
4417 }
4418 case ISD::ATOMIC_LOAD: {
4419 // If we are looking at the loaded value.
4420 if (Op.getResNo() == 0) {
4421 auto *AT = cast<AtomicSDNode>(Val&: Op);
4422 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4423 KnownBits KnownScalarMemory(ScalarMemorySize);
4424 if (const MDNode *MD = AT->getRanges())
4425 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4426
4427 switch (AT->getExtensionType()) {
4428 case ISD::ZEXTLOAD:
4429 Known = KnownScalarMemory.zext(BitWidth);
4430 break;
4431 case ISD::SEXTLOAD:
4432 Known = KnownScalarMemory.sext(BitWidth);
4433 break;
4434 case ISD::EXTLOAD:
4435 switch (TLI->getExtendForAtomicOps()) {
4436 case ISD::ZERO_EXTEND:
4437 Known = KnownScalarMemory.zext(BitWidth);
4438 break;
4439 case ISD::SIGN_EXTEND:
4440 Known = KnownScalarMemory.sext(BitWidth);
4441 break;
4442 default:
4443 Known = KnownScalarMemory.anyext(BitWidth);
4444 break;
4445 }
4446 break;
4447 case ISD::NON_EXTLOAD:
4448 Known = KnownScalarMemory;
4449 break;
4450 }
4451 assert(Known.getBitWidth() == BitWidth);
4452 }
4453 break;
4454 }
4455 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4456 if (Op.getResNo() == 1) {
4457 // The boolean result conforms to getBooleanContents.
4458 // If we know the result of a setcc has the top bits zero, use this info.
4459 // We know that we have an integer-based boolean since these operations
4460 // are only available for integer.
4461 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4462 TargetLowering::ZeroOrOneBooleanContent &&
4463 BitWidth > 1)
4464 Known.Zero.setBitsFrom(1);
4465 break;
4466 }
4467 [[fallthrough]];
4468 case ISD::ATOMIC_CMP_SWAP:
4469 case ISD::ATOMIC_SWAP:
4470 case ISD::ATOMIC_LOAD_ADD:
4471 case ISD::ATOMIC_LOAD_SUB:
4472 case ISD::ATOMIC_LOAD_AND:
4473 case ISD::ATOMIC_LOAD_CLR:
4474 case ISD::ATOMIC_LOAD_OR:
4475 case ISD::ATOMIC_LOAD_XOR:
4476 case ISD::ATOMIC_LOAD_NAND:
4477 case ISD::ATOMIC_LOAD_MIN:
4478 case ISD::ATOMIC_LOAD_MAX:
4479 case ISD::ATOMIC_LOAD_UMIN:
4480 case ISD::ATOMIC_LOAD_UMAX: {
4481 // If we are looking at the loaded value.
4482 if (Op.getResNo() == 0) {
4483 auto *AT = cast<AtomicSDNode>(Val&: Op);
4484 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4485
4486 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4487 Known.Zero.setBitsFrom(MemBits);
4488 }
4489 break;
4490 }
4491 case ISD::FrameIndex:
4492 case ISD::TargetFrameIndex:
4493 TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4494 Known, MF: getMachineFunction());
4495 break;
4496
4497 default:
4498 if (Opcode < ISD::BUILTIN_OP_END)
4499 break;
4500 [[fallthrough]];
4501 case ISD::INTRINSIC_WO_CHAIN:
4502 case ISD::INTRINSIC_W_CHAIN:
4503 case ISD::INTRINSIC_VOID:
4504 // TODO: Probably okay to remove after audit; here to reduce change size
4505 // in initial enablement patch for scalable vectors
4506 if (Op.getValueType().isScalableVector())
4507 break;
4508
4509 // Allow the target to implement this method for its nodes.
4510 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4511 break;
4512 }
4513
4514 return Known;
4515}
4516
4517/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4518static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4519 switch (OR) {
4520 case ConstantRange::OverflowResult::MayOverflow:
4521 return SelectionDAG::OFK_Sometime;
4522 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4523 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4524 return SelectionDAG::OFK_Always;
4525 case ConstantRange::OverflowResult::NeverOverflows:
4526 return SelectionDAG::OFK_Never;
4527 }
4528 llvm_unreachable("Unknown OverflowResult");
4529}
4530
4531SelectionDAG::OverflowKind
4532SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4533 // X + 0 never overflow
4534 if (isNullConstant(V: N1))
4535 return OFK_Never;
4536
4537 // If both operands each have at least two sign bits, the addition
4538 // cannot overflow.
4539 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4540 return OFK_Never;
4541
4542 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4543 return OFK_Sometime;
4544}
4545
4546SelectionDAG::OverflowKind
4547SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4548 // X + 0 never overflow
4549 if (isNullConstant(V: N1))
4550 return OFK_Never;
4551
4552 // mulhi + 1 never overflow
4553 KnownBits N1Known = computeKnownBits(Op: N1);
4554 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4555 N1Known.getMaxValue().ult(RHS: 2))
4556 return OFK_Never;
4557
4558 KnownBits N0Known = computeKnownBits(Op: N0);
4559 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4560 N0Known.getMaxValue().ult(RHS: 2))
4561 return OFK_Never;
4562
4563 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4564 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4565 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4566 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4567}
4568
4569SelectionDAG::OverflowKind
4570SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4571 // X - 0 never overflow
4572 if (isNullConstant(V: N1))
4573 return OFK_Never;
4574
4575 // If both operands each have at least two sign bits, the subtraction
4576 // cannot overflow.
4577 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4578 return OFK_Never;
4579
4580 KnownBits N0Known = computeKnownBits(Op: N0);
4581 KnownBits N1Known = computeKnownBits(Op: N1);
4582 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4583 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4584 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4585}
4586
4587SelectionDAG::OverflowKind
4588SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4589 // X - 0 never overflow
4590 if (isNullConstant(V: N1))
4591 return OFK_Never;
4592
4593 KnownBits N0Known = computeKnownBits(Op: N0);
4594 KnownBits N1Known = computeKnownBits(Op: N1);
4595 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4596 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4597 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4598}
4599
4600SelectionDAG::OverflowKind
4601SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4602 // X * 0 and X * 1 never overflow.
4603 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4604 return OFK_Never;
4605
4606 KnownBits N0Known = computeKnownBits(Op: N0);
4607 KnownBits N1Known = computeKnownBits(Op: N1);
4608 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4609 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4610 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4611}
4612
4613SelectionDAG::OverflowKind
4614SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4615 // X * 0 and X * 1 never overflow.
4616 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4617 return OFK_Never;
4618
4619 // Get the size of the result.
4620 unsigned BitWidth = N0.getScalarValueSizeInBits();
4621
4622 // Sum of the sign bits.
4623 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4624
4625 // If we have enough sign bits, then there's no overflow.
4626 if (SignBits > BitWidth + 1)
4627 return OFK_Never;
4628
4629 if (SignBits == BitWidth + 1) {
4630 // The overflow occurs when the true multiplication of the
4631 // the operands is the minimum negative number.
4632 KnownBits N0Known = computeKnownBits(Op: N0);
4633 KnownBits N1Known = computeKnownBits(Op: N1);
4634 // If one of the operands is non-negative, then there's no
4635 // overflow.
4636 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4637 return OFK_Never;
4638 }
4639
4640 return OFK_Sometime;
4641}
4642
4643bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4644 unsigned Depth) const {
4645 EVT VT = Val.getValueType();
4646
4647 // Since the number of lanes in a scalable vector is unknown at compile time,
4648 // we track one bit which is implicitly broadcast to all lanes. This means
4649 // that all lanes in a scalable vector are considered demanded.
4650 APInt DemandedElts = VT.isFixedLengthVector()
4651 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4652 : APInt(1, 1);
4653
4654 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4655}
4656
4657bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4658 const APInt &DemandedElts,
4659 bool OrZero, unsigned Depth) const {
4660 if (Depth >= MaxRecursionDepth)
4661 return false; // Limit search depth.
4662
4663 EVT OpVT = Val.getValueType();
4664 unsigned BitWidth = OpVT.getScalarSizeInBits();
4665 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4666 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4667 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4668 assert(
4669 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4670 "Unexpected vector size");
4671
4672 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4673 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4674 return (OrZero && V.isZero()) || V.isPowerOf2();
4675 };
4676
4677 // Is the constant a known power of 2 or zero?
4678 if (ISD::matchUnaryPredicate(Op: Val, Match: IsPowerOfTwoOrZero))
4679 return true;
4680
4681 switch (Val.getOpcode()) {
4682 case ISD::BUILD_VECTOR:
4683 // Are all operands of a build vector constant powers of two or zero?
4684 if (all_of(Range: enumerate(First: Val->ops()), P: [&](auto P) {
4685 auto *C = dyn_cast<ConstantSDNode>(P.value());
4686 return !DemandedElts[P.index()] || (C && IsPowerOfTwoOrZero(C));
4687 }))
4688 return true;
4689 break;
4690
4691 case ISD::SPLAT_VECTOR:
4692 // Is the operand of a splat vector a constant power of two?
4693 if (auto *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4694 if (IsPowerOfTwoOrZero(C))
4695 return true;
4696 break;
4697
4698 case ISD::AND: {
4699 // Looking for `x & -x` pattern:
4700 // If x == 0:
4701 // x & -x -> 0
4702 // If x != 0:
4703 // x & -x -> non-zero pow2
4704 // so if we find the pattern return whether we know `x` is non-zero.
4705 // TODO OrZero handling
4706 SDValue X;
4707 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))))
4708 return isKnownNeverZero(Op: X, Depth);
4709 break;
4710 }
4711
4712 case ISD::SHL: {
4713 // A left-shift of a constant one will have exactly one bit set because
4714 // shifting the bit off the end is undefined.
4715 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4716 if (C && C->getAPIntValue() == 1)
4717 return true;
4718 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4719 Depth: Depth + 1) &&
4720 isKnownNeverZero(Op: Val, Depth);
4721 }
4722
4723 case ISD::SRL: {
4724 // A logical right-shift of a constant sign-bit will have exactly
4725 // one bit set.
4726 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4727 if (C && C->getAPIntValue().isSignMask())
4728 return true;
4729 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4730 Depth: Depth + 1) &&
4731 isKnownNeverZero(Op: Val, Depth);
4732 }
4733
4734 case ISD::ROTL:
4735 case ISD::ROTR:
4736 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4737 Depth: Depth + 1);
4738
4739 case ISD::SMIN:
4740 case ISD::SMAX:
4741 case ISD::UMIN:
4742 case ISD::UMAX:
4743 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), /*OrZero=*/false,
4744 Depth: Depth + 1) &&
4745 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4746 Depth: Depth + 1);
4747
4748 case ISD::SELECT:
4749 case ISD::VSELECT:
4750 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4751 Depth: Depth + 1) &&
4752 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4753 Depth: Depth + 1);
4754
4755 case ISD::ZERO_EXTEND:
4756 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4757 Depth: Depth + 1);
4758
4759 case ISD::VSCALE:
4760 // vscale(power-of-two) is a power-of-two for some targets
4761 if (getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4762 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false, Depth: Depth + 1))
4763 return true;
4764 break;
4765 }
4766
4767 // More could be done here, though the above checks are enough
4768 // to handle some common cases.
4769 return false;
4770}
4771
4772bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4773 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4774 return C1->getValueAPF().getExactLog2Abs() >= 0;
4775
4776 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4777 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4778
4779 return false;
4780}
4781
4782unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4783 EVT VT = Op.getValueType();
4784
4785 // Since the number of lanes in a scalable vector is unknown at compile time,
4786 // we track one bit which is implicitly broadcast to all lanes. This means
4787 // that all lanes in a scalable vector are considered demanded.
4788 APInt DemandedElts = VT.isFixedLengthVector()
4789 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4790 : APInt(1, 1);
4791 return ComputeNumSignBits(Op, DemandedElts, Depth);
4792}
4793
4794unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4795 unsigned Depth) const {
4796 EVT VT = Op.getValueType();
4797 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4798 unsigned VTBits = VT.getScalarSizeInBits();
4799 unsigned NumElts = DemandedElts.getBitWidth();
4800 unsigned Tmp, Tmp2;
4801 unsigned FirstAnswer = 1;
4802
4803 assert((!VT.isScalableVector() || NumElts == 1) &&
4804 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4805
4806 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4807 const APInt &Val = C->getAPIntValue();
4808 return Val.getNumSignBits();
4809 }
4810
4811 if (Depth >= MaxRecursionDepth)
4812 return 1; // Limit search depth.
4813
4814 if (!DemandedElts)
4815 return 1; // No demanded elts, better to assume we don't know anything.
4816
4817 unsigned Opcode = Op.getOpcode();
4818 switch (Opcode) {
4819 default: break;
4820 case ISD::AssertSext:
4821 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4822 return VTBits-Tmp+1;
4823 case ISD::AssertZext:
4824 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4825 return VTBits-Tmp;
4826 case ISD::FREEZE:
4827 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4828 /*PoisonOnly=*/false))
4829 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4830 break;
4831 case ISD::MERGE_VALUES:
4832 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4833 Depth: Depth + 1);
4834 case ISD::SPLAT_VECTOR: {
4835 // Check if the sign bits of source go down as far as the truncated value.
4836 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4837 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4838 if (NumSrcSignBits > (NumSrcBits - VTBits))
4839 return NumSrcSignBits - (NumSrcBits - VTBits);
4840 break;
4841 }
4842 case ISD::BUILD_VECTOR:
4843 assert(!VT.isScalableVector());
4844 Tmp = VTBits;
4845 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4846 if (!DemandedElts[i])
4847 continue;
4848
4849 SDValue SrcOp = Op.getOperand(i);
4850 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4851 // for constant nodes to ensure we only look at the sign bits.
4852 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4853 APInt T = C->getAPIntValue().trunc(width: VTBits);
4854 Tmp2 = T.getNumSignBits();
4855 } else {
4856 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4857
4858 if (SrcOp.getValueSizeInBits() != VTBits) {
4859 assert(SrcOp.getValueSizeInBits() > VTBits &&
4860 "Expected BUILD_VECTOR implicit truncation");
4861 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4862 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4863 }
4864 }
4865 Tmp = std::min(a: Tmp, b: Tmp2);
4866 }
4867 return Tmp;
4868
4869 case ISD::VECTOR_COMPRESS: {
4870 SDValue Vec = Op.getOperand(i: 0);
4871 SDValue PassThru = Op.getOperand(i: 2);
4872 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
4873 if (Tmp == 1)
4874 return 1;
4875 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
4876 Tmp = std::min(a: Tmp, b: Tmp2);
4877 return Tmp;
4878 }
4879
4880 case ISD::VECTOR_SHUFFLE: {
4881 // Collect the minimum number of sign bits that are shared by every vector
4882 // element referenced by the shuffle.
4883 APInt DemandedLHS, DemandedRHS;
4884 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4885 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4886 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4887 DemandedLHS, DemandedRHS))
4888 return 1;
4889
4890 Tmp = std::numeric_limits<unsigned>::max();
4891 if (!!DemandedLHS)
4892 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
4893 if (!!DemandedRHS) {
4894 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
4895 Tmp = std::min(a: Tmp, b: Tmp2);
4896 }
4897 // If we don't know anything, early out and try computeKnownBits fall-back.
4898 if (Tmp == 1)
4899 break;
4900 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4901 return Tmp;
4902 }
4903
4904 case ISD::BITCAST: {
4905 if (VT.isScalableVector())
4906 break;
4907 SDValue N0 = Op.getOperand(i: 0);
4908 EVT SrcVT = N0.getValueType();
4909 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4910
4911 // Ignore bitcasts from unsupported types..
4912 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4913 break;
4914
4915 // Fast handling of 'identity' bitcasts.
4916 if (VTBits == SrcBits)
4917 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
4918
4919 bool IsLE = getDataLayout().isLittleEndian();
4920
4921 // Bitcast 'large element' scalar/vector to 'small element' vector.
4922 if ((SrcBits % VTBits) == 0) {
4923 assert(VT.isVector() && "Expected bitcast to vector");
4924
4925 unsigned Scale = SrcBits / VTBits;
4926 APInt SrcDemandedElts =
4927 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
4928
4929 // Fast case - sign splat can be simply split across the small elements.
4930 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
4931 if (Tmp == SrcBits)
4932 return VTBits;
4933
4934 // Slow case - determine how far the sign extends into each sub-element.
4935 Tmp2 = VTBits;
4936 for (unsigned i = 0; i != NumElts; ++i)
4937 if (DemandedElts[i]) {
4938 unsigned SubOffset = i % Scale;
4939 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4940 SubOffset = SubOffset * VTBits;
4941 if (Tmp <= SubOffset)
4942 return 1;
4943 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
4944 }
4945 return Tmp2;
4946 }
4947 break;
4948 }
4949
4950 case ISD::FP_TO_SINT_SAT:
4951 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4952 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4953 return VTBits - Tmp + 1;
4954 case ISD::SIGN_EXTEND:
4955 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
4956 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
4957 case ISD::SIGN_EXTEND_INREG:
4958 // Max of the input and what this extends.
4959 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4960 Tmp = VTBits-Tmp+1;
4961 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4962 return std::max(a: Tmp, b: Tmp2);
4963 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4964 if (VT.isScalableVector())
4965 break;
4966 SDValue Src = Op.getOperand(i: 0);
4967 EVT SrcVT = Src.getValueType();
4968 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
4969 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4970 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
4971 }
4972 case ISD::SRA:
4973 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4974 // SRA X, C -> adds C sign bits.
4975 if (std::optional<unsigned> ShAmt =
4976 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
4977 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
4978 return Tmp;
4979 case ISD::SHL:
4980 if (std::optional<ConstantRange> ShAmtRange =
4981 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
4982 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
4983 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
4984 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
4985 // shifted out, then we can compute the number of sign bits for the
4986 // operand being extended. A future improvement could be to pass along the
4987 // "shifted left by" information in the recursive calls to
4988 // ComputeKnownSignBits. Allowing us to handle this more generically.
4989 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
4990 SDValue Ext = Op.getOperand(i: 0);
4991 EVT ExtVT = Ext.getValueType();
4992 SDValue Extendee = Ext.getOperand(i: 0);
4993 EVT ExtendeeVT = Extendee.getValueType();
4994 unsigned SizeDifference =
4995 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
4996 if (SizeDifference <= MinShAmt) {
4997 Tmp = SizeDifference +
4998 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
4999 if (MaxShAmt < Tmp)
5000 return Tmp - MaxShAmt;
5001 }
5002 }
5003 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5004 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5005 if (MaxShAmt < Tmp)
5006 return Tmp - MaxShAmt;
5007 }
5008 break;
5009 case ISD::AND:
5010 case ISD::OR:
5011 case ISD::XOR: // NOT is handled here.
5012 // Logical binary ops preserve the number of sign bits at the worst.
5013 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5014 if (Tmp != 1) {
5015 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5016 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5017 // We computed what we know about the sign bits as our first
5018 // answer. Now proceed to the generic code that uses
5019 // computeKnownBits, and pick whichever answer is better.
5020 }
5021 break;
5022
5023 case ISD::SELECT:
5024 case ISD::VSELECT:
5025 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5026 if (Tmp == 1) return 1; // Early out.
5027 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5028 return std::min(a: Tmp, b: Tmp2);
5029 case ISD::SELECT_CC:
5030 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5031 if (Tmp == 1) return 1; // Early out.
5032 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5033 return std::min(a: Tmp, b: Tmp2);
5034
5035 case ISD::SMIN:
5036 case ISD::SMAX: {
5037 // If we have a clamp pattern, we know that the number of sign bits will be
5038 // the minimum of the clamp min/max range.
5039 bool IsMax = (Opcode == ISD::SMAX);
5040 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5041 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5042 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5043 CstHigh =
5044 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5045 if (CstLow && CstHigh) {
5046 if (!IsMax)
5047 std::swap(a&: CstLow, b&: CstHigh);
5048 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5049 Tmp = CstLow->getAPIntValue().getNumSignBits();
5050 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5051 return std::min(a: Tmp, b: Tmp2);
5052 }
5053 }
5054
5055 // Fallback - just get the minimum number of sign bits of the operands.
5056 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5057 if (Tmp == 1)
5058 return 1; // Early out.
5059 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5060 return std::min(a: Tmp, b: Tmp2);
5061 }
5062 case ISD::UMIN:
5063 case ISD::UMAX:
5064 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5065 if (Tmp == 1)
5066 return 1; // Early out.
5067 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5068 return std::min(a: Tmp, b: Tmp2);
5069 case ISD::SSUBO_CARRY:
5070 case ISD::USUBO_CARRY:
5071 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5072 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5073 return VTBits;
5074 [[fallthrough]];
5075 case ISD::SADDO:
5076 case ISD::UADDO:
5077 case ISD::SADDO_CARRY:
5078 case ISD::UADDO_CARRY:
5079 case ISD::SSUBO:
5080 case ISD::USUBO:
5081 case ISD::SMULO:
5082 case ISD::UMULO:
5083 if (Op.getResNo() != 1)
5084 break;
5085 // The boolean result conforms to getBooleanContents. Fall through.
5086 // If setcc returns 0/-1, all bits are sign bits.
5087 // We know that we have an integer-based boolean since these operations
5088 // are only available for integer.
5089 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5090 TargetLowering::ZeroOrNegativeOneBooleanContent)
5091 return VTBits;
5092 break;
5093 case ISD::SETCC:
5094 case ISD::SETCCCARRY:
5095 case ISD::STRICT_FSETCC:
5096 case ISD::STRICT_FSETCCS: {
5097 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5098 // If setcc returns 0/-1, all bits are sign bits.
5099 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5100 TargetLowering::ZeroOrNegativeOneBooleanContent)
5101 return VTBits;
5102 break;
5103 }
5104 case ISD::ROTL:
5105 case ISD::ROTR:
5106 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5107
5108 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
5109 if (Tmp == VTBits)
5110 return VTBits;
5111
5112 if (ConstantSDNode *C =
5113 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
5114 unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
5115
5116 // Handle rotate right by N like a rotate left by 32-N.
5117 if (Opcode == ISD::ROTR)
5118 RotAmt = (VTBits - RotAmt) % VTBits;
5119
5120 // If we aren't rotating out all of the known-in sign bits, return the
5121 // number that are left. This handles rotl(sext(x), 1) for example.
5122 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
5123 }
5124 break;
5125 case ISD::ADD:
5126 case ISD::ADDC:
5127 // TODO: Move Operand 1 check before Operand 0 check
5128 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5129 if (Tmp == 1) return 1; // Early out.
5130
5131 // Special case decrementing a value (ADD X, -1):
5132 if (ConstantSDNode *CRHS =
5133 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5134 if (CRHS->isAllOnes()) {
5135 KnownBits Known =
5136 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5137
5138 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5139 // sign bits set.
5140 if ((Known.Zero | 1).isAllOnes())
5141 return VTBits;
5142
5143 // If we are subtracting one from a positive number, there is no carry
5144 // out of the result.
5145 if (Known.isNonNegative())
5146 return Tmp;
5147 }
5148
5149 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5150 if (Tmp2 == 1) return 1; // Early out.
5151
5152 // Add can have at most one carry bit. Thus we know that the output
5153 // is, at worst, one more bit than the inputs.
5154 return std::min(a: Tmp, b: Tmp2) - 1;
5155 case ISD::SUB:
5156 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5157 if (Tmp2 == 1) return 1; // Early out.
5158
5159 // Handle NEG.
5160 if (ConstantSDNode *CLHS =
5161 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5162 if (CLHS->isZero()) {
5163 KnownBits Known =
5164 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5165 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5166 // sign bits set.
5167 if ((Known.Zero | 1).isAllOnes())
5168 return VTBits;
5169
5170 // If the input is known to be positive (the sign bit is known clear),
5171 // the output of the NEG has the same number of sign bits as the input.
5172 if (Known.isNonNegative())
5173 return Tmp2;
5174
5175 // Otherwise, we treat this like a SUB.
5176 }
5177
5178 // Sub can have at most one carry bit. Thus we know that the output
5179 // is, at worst, one more bit than the inputs.
5180 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5181 if (Tmp == 1) return 1; // Early out.
5182 return std::min(a: Tmp, b: Tmp2) - 1;
5183 case ISD::MUL: {
5184 // The output of the Mul can be at most twice the valid bits in the inputs.
5185 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5186 if (SignBitsOp0 == 1)
5187 break;
5188 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5189 if (SignBitsOp1 == 1)
5190 break;
5191 unsigned OutValidBits =
5192 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5193 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5194 }
5195 case ISD::AVGCEILS:
5196 case ISD::AVGFLOORS:
5197 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5198 if (Tmp == 1)
5199 return 1; // Early out.
5200 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5201 return std::min(a: Tmp, b: Tmp2);
5202 case ISD::SREM:
5203 // The sign bit is the LHS's sign bit, except when the result of the
5204 // remainder is zero. The magnitude of the result should be less than or
5205 // equal to the magnitude of the LHS. Therefore, the result should have
5206 // at least as many sign bits as the left hand side.
5207 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5208 case ISD::TRUNCATE: {
5209 // Check if the sign bits of source go down as far as the truncated value.
5210 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5211 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5212 if (NumSrcSignBits > (NumSrcBits - VTBits))
5213 return NumSrcSignBits - (NumSrcBits - VTBits);
5214 break;
5215 }
5216 case ISD::EXTRACT_ELEMENT: {
5217 if (VT.isScalableVector())
5218 break;
5219 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5220 const int BitWidth = Op.getValueSizeInBits();
5221 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5222
5223 // Get reverse index (starting from 1), Op1 value indexes elements from
5224 // little end. Sign starts at big end.
5225 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5226
5227 // If the sign portion ends in our element the subtraction gives correct
5228 // result. Otherwise it gives either negative or > bitwidth result
5229 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5230 }
5231 case ISD::INSERT_VECTOR_ELT: {
5232 if (VT.isScalableVector())
5233 break;
5234 // If we know the element index, split the demand between the
5235 // source vector and the inserted element, otherwise assume we need
5236 // the original demanded vector elements and the value.
5237 SDValue InVec = Op.getOperand(i: 0);
5238 SDValue InVal = Op.getOperand(i: 1);
5239 SDValue EltNo = Op.getOperand(i: 2);
5240 bool DemandedVal = true;
5241 APInt DemandedVecElts = DemandedElts;
5242 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5243 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5244 unsigned EltIdx = CEltNo->getZExtValue();
5245 DemandedVal = !!DemandedElts[EltIdx];
5246 DemandedVecElts.clearBit(BitPosition: EltIdx);
5247 }
5248 Tmp = std::numeric_limits<unsigned>::max();
5249 if (DemandedVal) {
5250 // TODO - handle implicit truncation of inserted elements.
5251 if (InVal.getScalarValueSizeInBits() != VTBits)
5252 break;
5253 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5254 Tmp = std::min(a: Tmp, b: Tmp2);
5255 }
5256 if (!!DemandedVecElts) {
5257 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5258 Tmp = std::min(a: Tmp, b: Tmp2);
5259 }
5260 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5261 return Tmp;
5262 }
5263 case ISD::EXTRACT_VECTOR_ELT: {
5264 SDValue InVec = Op.getOperand(i: 0);
5265 SDValue EltNo = Op.getOperand(i: 1);
5266 EVT VecVT = InVec.getValueType();
5267 // ComputeNumSignBits not yet implemented for scalable vectors.
5268 if (VecVT.isScalableVector())
5269 break;
5270 const unsigned BitWidth = Op.getValueSizeInBits();
5271 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5272 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5273
5274 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5275 // anything about sign bits. But if the sizes match we can derive knowledge
5276 // about sign bits from the vector operand.
5277 if (BitWidth != EltBitWidth)
5278 break;
5279
5280 // If we know the element index, just demand that vector element, else for
5281 // an unknown element index, ignore DemandedElts and demand them all.
5282 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5283 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5284 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5285 DemandedSrcElts =
5286 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5287
5288 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5289 }
5290 case ISD::EXTRACT_SUBVECTOR: {
5291 // Offset the demanded elts by the subvector index.
5292 SDValue Src = Op.getOperand(i: 0);
5293
5294 APInt DemandedSrcElts;
5295 if (Src.getValueType().isScalableVector())
5296 DemandedSrcElts = APInt(1, 1);
5297 else {
5298 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5299 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5300 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5301 }
5302 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5303 }
5304 case ISD::CONCAT_VECTORS: {
5305 if (VT.isScalableVector())
5306 break;
5307 // Determine the minimum number of sign bits across all demanded
5308 // elts of the input vectors. Early out if the result is already 1.
5309 Tmp = std::numeric_limits<unsigned>::max();
5310 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5311 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5312 unsigned NumSubVectors = Op.getNumOperands();
5313 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5314 APInt DemandedSub =
5315 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5316 if (!DemandedSub)
5317 continue;
5318 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5319 Tmp = std::min(a: Tmp, b: Tmp2);
5320 }
5321 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5322 return Tmp;
5323 }
5324 case ISD::INSERT_SUBVECTOR: {
5325 if (VT.isScalableVector())
5326 break;
5327 // Demand any elements from the subvector and the remainder from the src its
5328 // inserted into.
5329 SDValue Src = Op.getOperand(i: 0);
5330 SDValue Sub = Op.getOperand(i: 1);
5331 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5332 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5333 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5334 APInt DemandedSrcElts = DemandedElts;
5335 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5336
5337 Tmp = std::numeric_limits<unsigned>::max();
5338 if (!!DemandedSubElts) {
5339 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5340 if (Tmp == 1)
5341 return 1; // early-out
5342 }
5343 if (!!DemandedSrcElts) {
5344 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5345 Tmp = std::min(a: Tmp, b: Tmp2);
5346 }
5347 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5348 return Tmp;
5349 }
5350 case ISD::LOAD: {
5351 // If we are looking at the loaded value of the SDNode.
5352 if (Op.getResNo() != 0)
5353 break;
5354
5355 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5356 if (const MDNode *Ranges = LD->getRanges()) {
5357 if (DemandedElts != 1)
5358 break;
5359
5360 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5361 if (VTBits > CR.getBitWidth()) {
5362 switch (LD->getExtensionType()) {
5363 case ISD::SEXTLOAD:
5364 CR = CR.signExtend(BitWidth: VTBits);
5365 break;
5366 case ISD::ZEXTLOAD:
5367 CR = CR.zeroExtend(BitWidth: VTBits);
5368 break;
5369 default:
5370 break;
5371 }
5372 }
5373
5374 if (VTBits != CR.getBitWidth())
5375 break;
5376 return std::min(a: CR.getSignedMin().getNumSignBits(),
5377 b: CR.getSignedMax().getNumSignBits());
5378 }
5379
5380 unsigned ExtType = LD->getExtensionType();
5381 switch (ExtType) {
5382 default:
5383 break;
5384 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5385 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5386 return VTBits - Tmp + 1;
5387 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5388 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5389 return VTBits - Tmp;
5390 case ISD::NON_EXTLOAD:
5391 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5392 // We only need to handle vectors - computeKnownBits should handle
5393 // scalar cases.
5394 Type *CstTy = Cst->getType();
5395 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5396 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5397 VTBits == CstTy->getScalarSizeInBits()) {
5398 Tmp = VTBits;
5399 for (unsigned i = 0; i != NumElts; ++i) {
5400 if (!DemandedElts[i])
5401 continue;
5402 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5403 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5404 const APInt &Value = CInt->getValue();
5405 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5406 continue;
5407 }
5408 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5409 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5410 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5411 continue;
5412 }
5413 }
5414 // Unknown type. Conservatively assume no bits match sign bit.
5415 return 1;
5416 }
5417 return Tmp;
5418 }
5419 }
5420 break;
5421 }
5422
5423 break;
5424 }
5425 case ISD::ATOMIC_CMP_SWAP:
5426 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5427 case ISD::ATOMIC_SWAP:
5428 case ISD::ATOMIC_LOAD_ADD:
5429 case ISD::ATOMIC_LOAD_SUB:
5430 case ISD::ATOMIC_LOAD_AND:
5431 case ISD::ATOMIC_LOAD_CLR:
5432 case ISD::ATOMIC_LOAD_OR:
5433 case ISD::ATOMIC_LOAD_XOR:
5434 case ISD::ATOMIC_LOAD_NAND:
5435 case ISD::ATOMIC_LOAD_MIN:
5436 case ISD::ATOMIC_LOAD_MAX:
5437 case ISD::ATOMIC_LOAD_UMIN:
5438 case ISD::ATOMIC_LOAD_UMAX:
5439 case ISD::ATOMIC_LOAD: {
5440 auto *AT = cast<AtomicSDNode>(Val&: Op);
5441 // If we are looking at the loaded value.
5442 if (Op.getResNo() == 0) {
5443 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5444 if (Tmp == VTBits)
5445 return 1; // early-out
5446
5447 // For atomic_load, prefer to use the extension type.
5448 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5449 switch (AT->getExtensionType()) {
5450 default:
5451 break;
5452 case ISD::SEXTLOAD:
5453 return VTBits - Tmp + 1;
5454 case ISD::ZEXTLOAD:
5455 return VTBits - Tmp;
5456 }
5457 }
5458
5459 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5460 return VTBits - Tmp + 1;
5461 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5462 return VTBits - Tmp;
5463 }
5464 break;
5465 }
5466 }
5467
5468 // Allow the target to implement this method for its nodes.
5469 if (Opcode >= ISD::BUILTIN_OP_END ||
5470 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5471 Opcode == ISD::INTRINSIC_W_CHAIN ||
5472 Opcode == ISD::INTRINSIC_VOID) {
5473 // TODO: This can probably be removed once target code is audited. This
5474 // is here purely to reduce patch size and review complexity.
5475 if (!VT.isScalableVector()) {
5476 unsigned NumBits =
5477 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5478 if (NumBits > 1)
5479 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5480 }
5481 }
5482
5483 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5484 // use this information.
5485 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5486 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5487}
5488
5489unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5490 unsigned Depth) const {
5491 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5492 return Op.getScalarValueSizeInBits() - SignBits + 1;
5493}
5494
5495unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5496 const APInt &DemandedElts,
5497 unsigned Depth) const {
5498 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5499 return Op.getScalarValueSizeInBits() - SignBits + 1;
5500}
5501
5502bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
5503 unsigned Depth) const {
5504 // Early out for FREEZE.
5505 if (Op.getOpcode() == ISD::FREEZE)
5506 return true;
5507
5508 EVT VT = Op.getValueType();
5509 APInt DemandedElts = VT.isFixedLengthVector()
5510 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5511 : APInt(1, 1);
5512 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
5513}
5514
5515bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5516 const APInt &DemandedElts,
5517 bool PoisonOnly,
5518 unsigned Depth) const {
5519 unsigned Opcode = Op.getOpcode();
5520
5521 // Early out for FREEZE.
5522 if (Opcode == ISD::FREEZE)
5523 return true;
5524
5525 if (Depth >= MaxRecursionDepth)
5526 return false; // Limit search depth.
5527
5528 if (isIntOrFPConstant(V: Op))
5529 return true;
5530
5531 switch (Opcode) {
5532 case ISD::CONDCODE:
5533 case ISD::VALUETYPE:
5534 case ISD::FrameIndex:
5535 case ISD::TargetFrameIndex:
5536 case ISD::CopyFromReg:
5537 return true;
5538
5539 case ISD::POISON:
5540 return false;
5541
5542 case ISD::UNDEF:
5543 return PoisonOnly;
5544
5545 case ISD::BUILD_VECTOR:
5546 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5547 // this shouldn't affect the result.
5548 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5549 if (!DemandedElts[i])
5550 continue;
5551 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly,
5552 Depth: Depth + 1))
5553 return false;
5554 }
5555 return true;
5556
5557 case ISD::EXTRACT_SUBVECTOR: {
5558 SDValue Src = Op.getOperand(i: 0);
5559 if (Src.getValueType().isScalableVector())
5560 break;
5561 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5562 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5563 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5564 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5565 Depth: Depth + 1);
5566 }
5567
5568 case ISD::INSERT_SUBVECTOR: {
5569 if (Op.getValueType().isScalableVector())
5570 break;
5571 SDValue Src = Op.getOperand(i: 0);
5572 SDValue Sub = Op.getOperand(i: 1);
5573 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5574 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5575 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5576 APInt DemandedSrcElts = DemandedElts;
5577 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5578
5579 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5580 Op: Sub, DemandedElts: DemandedSubElts, PoisonOnly, Depth: Depth + 1))
5581 return false;
5582 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5583 Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly, Depth: Depth + 1))
5584 return false;
5585 return true;
5586 }
5587
5588 case ISD::EXTRACT_VECTOR_ELT: {
5589 SDValue Src = Op.getOperand(i: 0);
5590 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5591 EVT SrcVT = Src.getValueType();
5592 if (SrcVT.isFixedLengthVector() && IndexC &&
5593 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5594 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5595 BitNo: IndexC->getZExtValue());
5596 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5597 Depth: Depth + 1);
5598 }
5599 break;
5600 }
5601
5602 case ISD::INSERT_VECTOR_ELT: {
5603 SDValue InVec = Op.getOperand(i: 0);
5604 SDValue InVal = Op.getOperand(i: 1);
5605 SDValue EltNo = Op.getOperand(i: 2);
5606 EVT VT = InVec.getValueType();
5607 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5608 if (IndexC && VT.isFixedLengthVector() &&
5609 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5610 if (DemandedElts[IndexC->getZExtValue()] &&
5611 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, PoisonOnly, Depth: Depth + 1))
5612 return false;
5613 APInt InVecDemandedElts = DemandedElts;
5614 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5615 if (!!InVecDemandedElts &&
5616 !isGuaranteedNotToBeUndefOrPoison(
5617 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5618 DemandedElts: InVecDemandedElts, PoisonOnly, Depth: Depth + 1))
5619 return false;
5620 return true;
5621 }
5622 break;
5623 }
5624
5625 case ISD::SCALAR_TO_VECTOR:
5626 // Check upper (known undef) elements.
5627 if (DemandedElts.ugt(RHS: 1) && !PoisonOnly)
5628 return false;
5629 // Check element zero.
5630 if (DemandedElts[0] && !isGuaranteedNotToBeUndefOrPoison(
5631 Op: Op.getOperand(i: 0), PoisonOnly, Depth: Depth + 1))
5632 return false;
5633 return true;
5634
5635 case ISD::SPLAT_VECTOR:
5636 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), PoisonOnly,
5637 Depth: Depth + 1);
5638
5639 case ISD::VECTOR_SHUFFLE: {
5640 APInt DemandedLHS, DemandedRHS;
5641 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5642 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5643 DemandedElts, DemandedLHS, DemandedRHS,
5644 /*AllowUndefElts=*/false))
5645 return false;
5646 if (!DemandedLHS.isZero() &&
5647 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS,
5648 PoisonOnly, Depth: Depth + 1))
5649 return false;
5650 if (!DemandedRHS.isZero() &&
5651 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS,
5652 PoisonOnly, Depth: Depth + 1))
5653 return false;
5654 return true;
5655 }
5656
5657 case ISD::SHL:
5658 case ISD::SRL:
5659 case ISD::SRA:
5660 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5661 // enough to check operand 0 if Op can't create undef/poison.
5662 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5663 /*ConsiderFlags*/ true, Depth) &&
5664 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5665 PoisonOnly, Depth: Depth + 1);
5666
5667 case ISD::BSWAP:
5668 case ISD::CTPOP:
5669 case ISD::BITREVERSE:
5670 case ISD::AND:
5671 case ISD::OR:
5672 case ISD::XOR:
5673 case ISD::ADD:
5674 case ISD::SUB:
5675 case ISD::MUL:
5676 case ISD::SADDSAT:
5677 case ISD::UADDSAT:
5678 case ISD::SSUBSAT:
5679 case ISD::USUBSAT:
5680 case ISD::SSHLSAT:
5681 case ISD::USHLSAT:
5682 case ISD::SMIN:
5683 case ISD::SMAX:
5684 case ISD::UMIN:
5685 case ISD::UMAX:
5686 case ISD::ZERO_EXTEND:
5687 case ISD::SIGN_EXTEND:
5688 case ISD::ANY_EXTEND:
5689 case ISD::TRUNCATE:
5690 case ISD::VSELECT: {
5691 // If Op can't create undef/poison and none of its operands are undef/poison
5692 // then Op is never undef/poison. A difference from the more common check
5693 // below, outside the switch, is that we handle elementwise operations for
5694 // which the DemandedElts mask is valid for all operands here.
5695 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5696 /*ConsiderFlags*/ true, Depth) &&
5697 all_of(Range: Op->ops(), P: [&](SDValue V) {
5698 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts,
5699 PoisonOnly, Depth: Depth + 1);
5700 });
5701 }
5702
5703 // TODO: Search for noundef attributes from library functions.
5704
5705 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5706
5707 default:
5708 // Allow the target to implement this method for its nodes.
5709 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5710 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5711 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5712 Op, DemandedElts, DAG: *this, PoisonOnly, Depth);
5713 break;
5714 }
5715
5716 // If Op can't create undef/poison and none of its operands are undef/poison
5717 // then Op is never undef/poison.
5718 // NOTE: TargetNodes can handle this in themselves in
5719 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5720 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5721 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5722 Depth) &&
5723 all_of(Range: Op->ops(), P: [&](SDValue V) {
5724 return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1);
5725 });
5726}
5727
5728bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
5729 bool ConsiderFlags,
5730 unsigned Depth) const {
5731 EVT VT = Op.getValueType();
5732 APInt DemandedElts = VT.isFixedLengthVector()
5733 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5734 : APInt(1, 1);
5735 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5736 Depth);
5737}
5738
5739bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5740 bool PoisonOnly, bool ConsiderFlags,
5741 unsigned Depth) const {
5742 if (ConsiderFlags && Op->hasPoisonGeneratingFlags())
5743 return true;
5744
5745 unsigned Opcode = Op.getOpcode();
5746 switch (Opcode) {
5747 case ISD::AssertSext:
5748 case ISD::AssertZext:
5749 case ISD::AssertAlign:
5750 case ISD::AssertNoFPClass:
5751 // Assertion nodes can create poison if the assertion fails.
5752 return true;
5753
5754 case ISD::FREEZE:
5755 case ISD::CONCAT_VECTORS:
5756 case ISD::INSERT_SUBVECTOR:
5757 case ISD::EXTRACT_SUBVECTOR:
5758 case ISD::SADDSAT:
5759 case ISD::UADDSAT:
5760 case ISD::SSUBSAT:
5761 case ISD::USUBSAT:
5762 case ISD::MULHU:
5763 case ISD::MULHS:
5764 case ISD::AVGFLOORS:
5765 case ISD::AVGFLOORU:
5766 case ISD::AVGCEILS:
5767 case ISD::AVGCEILU:
5768 case ISD::ABDU:
5769 case ISD::ABDS:
5770 case ISD::SMIN:
5771 case ISD::SMAX:
5772 case ISD::SCMP:
5773 case ISD::UMIN:
5774 case ISD::UMAX:
5775 case ISD::UCMP:
5776 case ISD::AND:
5777 case ISD::XOR:
5778 case ISD::ROTL:
5779 case ISD::ROTR:
5780 case ISD::FSHL:
5781 case ISD::FSHR:
5782 case ISD::BSWAP:
5783 case ISD::CTTZ:
5784 case ISD::CTLZ:
5785 case ISD::CTLS:
5786 case ISD::CTPOP:
5787 case ISD::BITREVERSE:
5788 case ISD::PARITY:
5789 case ISD::SIGN_EXTEND:
5790 case ISD::TRUNCATE:
5791 case ISD::SIGN_EXTEND_INREG:
5792 case ISD::SIGN_EXTEND_VECTOR_INREG:
5793 case ISD::ZERO_EXTEND_VECTOR_INREG:
5794 case ISD::BITCAST:
5795 case ISD::BUILD_VECTOR:
5796 case ISD::BUILD_PAIR:
5797 case ISD::SPLAT_VECTOR:
5798 case ISD::FABS:
5799 return false;
5800
5801 case ISD::ABS:
5802 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
5803 // Different to Intrinsic::abs.
5804 return false;
5805
5806 case ISD::ADDC:
5807 case ISD::SUBC:
5808 case ISD::ADDE:
5809 case ISD::SUBE:
5810 case ISD::SADDO:
5811 case ISD::SSUBO:
5812 case ISD::SMULO:
5813 case ISD::SADDO_CARRY:
5814 case ISD::SSUBO_CARRY:
5815 case ISD::UADDO:
5816 case ISD::USUBO:
5817 case ISD::UMULO:
5818 case ISD::UADDO_CARRY:
5819 case ISD::USUBO_CARRY:
5820 // No poison on result or overflow flags.
5821 return false;
5822
5823 case ISD::SELECT_CC:
5824 case ISD::SETCC: {
5825 // Integer setcc cannot create undef or poison.
5826 if (Op.getOperand(i: 0).getValueType().isInteger())
5827 return false;
5828
5829 // FP compares are more complicated. They can create poison for nan/infinity
5830 // based on options and flags. The options and flags also cause special
5831 // nonan condition codes to be used. Those condition codes may be preserved
5832 // even if the nonan flag is dropped somewhere.
5833 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
5834 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
5835 return (unsigned)CCCode & 0x10U;
5836 }
5837
5838 case ISD::OR:
5839 case ISD::ZERO_EXTEND:
5840 case ISD::SELECT:
5841 case ISD::VSELECT:
5842 case ISD::ADD:
5843 case ISD::SUB:
5844 case ISD::MUL:
5845 case ISD::FNEG:
5846 case ISD::FADD:
5847 case ISD::FSUB:
5848 case ISD::FMUL:
5849 case ISD::FDIV:
5850 case ISD::FREM:
5851 case ISD::FCOPYSIGN:
5852 case ISD::FMA:
5853 case ISD::FMAD:
5854 case ISD::FMULADD:
5855 case ISD::FP_EXTEND:
5856 case ISD::FP_TO_SINT_SAT:
5857 case ISD::FP_TO_UINT_SAT:
5858 case ISD::TRUNCATE_SSAT_S:
5859 case ISD::TRUNCATE_SSAT_U:
5860 case ISD::TRUNCATE_USAT_U:
5861 // No poison except from flags (which is handled above)
5862 return false;
5863
5864 case ISD::SHL:
5865 case ISD::SRL:
5866 case ISD::SRA:
5867 // If the max shift amount isn't in range, then the shift can
5868 // create poison.
5869 return !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
5870
5871 case ISD::CTTZ_ZERO_UNDEF:
5872 case ISD::CTLZ_ZERO_UNDEF:
5873 // If the amount is zero then the result will be poison.
5874 // TODO: Add isKnownNeverZero DemandedElts handling.
5875 return !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5876
5877 case ISD::SCALAR_TO_VECTOR:
5878 // Check if we demand any upper (undef) elements.
5879 return !PoisonOnly && DemandedElts.ugt(RHS: 1);
5880
5881 case ISD::INSERT_VECTOR_ELT:
5882 case ISD::EXTRACT_VECTOR_ELT: {
5883 // Ensure that the element index is in bounds.
5884 EVT VecVT = Op.getOperand(i: 0).getValueType();
5885 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
5886 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
5887 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5888 }
5889
5890 case ISD::VECTOR_SHUFFLE: {
5891 // Check for any demanded shuffle element that is undef.
5892 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5893 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
5894 if (Elt < 0 && DemandedElts[Idx])
5895 return true;
5896 return false;
5897 }
5898
5899 case ISD::VECTOR_COMPRESS:
5900 return false;
5901
5902 default:
5903 // Allow the target to implement this method for its nodes.
5904 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5905 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5906 return TLI->canCreateUndefOrPoisonForTargetNode(
5907 Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth);
5908 break;
5909 }
5910
5911 // Be conservative and return true.
5912 return true;
5913}
5914
5915bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
5916 unsigned Opcode = Op.getOpcode();
5917 if (Opcode == ISD::OR)
5918 return Op->getFlags().hasDisjoint() ||
5919 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
5920 if (Opcode == ISD::XOR)
5921 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
5922 return false;
5923}
5924
5925bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5926 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
5927 (Op.isAnyAdd() || isADDLike(Op));
5928}
5929
5930bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
5931 unsigned Depth) const {
5932 EVT VT = Op.getValueType();
5933
5934 // Since the number of lanes in a scalable vector is unknown at compile time,
5935 // we track one bit which is implicitly broadcast to all lanes. This means
5936 // that all lanes in a scalable vector are considered demanded.
5937 APInt DemandedElts = VT.isFixedLengthVector()
5938 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5939 : APInt(1, 1);
5940
5941 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
5942}
5943
5944bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
5945 bool SNaN, unsigned Depth) const {
5946 assert(!DemandedElts.isZero() && "No demanded elements");
5947
5948 // If we're told that NaNs won't happen, assume they won't.
5949 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5950 return true;
5951
5952 if (Depth >= MaxRecursionDepth)
5953 return false; // Limit search depth.
5954
5955 // If the value is a constant, we can obviously see if it is a NaN or not.
5956 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
5957 return !C->getValueAPF().isNaN() ||
5958 (SNaN && !C->getValueAPF().isSignaling());
5959 }
5960
5961 unsigned Opcode = Op.getOpcode();
5962 switch (Opcode) {
5963 case ISD::FADD:
5964 case ISD::FSUB:
5965 case ISD::FMUL:
5966 case ISD::FDIV:
5967 case ISD::FREM:
5968 case ISD::FSIN:
5969 case ISD::FCOS:
5970 case ISD::FTAN:
5971 case ISD::FASIN:
5972 case ISD::FACOS:
5973 case ISD::FATAN:
5974 case ISD::FATAN2:
5975 case ISD::FSINH:
5976 case ISD::FCOSH:
5977 case ISD::FTANH:
5978 case ISD::FMA:
5979 case ISD::FMULADD:
5980 case ISD::FMAD: {
5981 if (SNaN)
5982 return true;
5983 // TODO: Need isKnownNeverInfinity
5984 return false;
5985 }
5986 case ISD::FCANONICALIZE:
5987 case ISD::FEXP:
5988 case ISD::FEXP2:
5989 case ISD::FEXP10:
5990 case ISD::FTRUNC:
5991 case ISD::FFLOOR:
5992 case ISD::FCEIL:
5993 case ISD::FROUND:
5994 case ISD::FROUNDEVEN:
5995 case ISD::LROUND:
5996 case ISD::LLROUND:
5997 case ISD::FRINT:
5998 case ISD::LRINT:
5999 case ISD::LLRINT:
6000 case ISD::FNEARBYINT:
6001 case ISD::FLDEXP: {
6002 if (SNaN)
6003 return true;
6004 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6005 }
6006 case ISD::FABS:
6007 case ISD::FNEG:
6008 case ISD::FCOPYSIGN: {
6009 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6010 }
6011 case ISD::SELECT:
6012 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6013 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6014 case ISD::FP_EXTEND:
6015 case ISD::FP_ROUND: {
6016 if (SNaN)
6017 return true;
6018 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6019 }
6020 case ISD::SINT_TO_FP:
6021 case ISD::UINT_TO_FP:
6022 return true;
6023 case ISD::FSQRT: // Need is known positive
6024 case ISD::FLOG:
6025 case ISD::FLOG2:
6026 case ISD::FLOG10:
6027 case ISD::FPOWI:
6028 case ISD::FPOW: {
6029 if (SNaN)
6030 return true;
6031 // TODO: Refine on operand
6032 return false;
6033 }
6034 case ISD::FMINNUM:
6035 case ISD::FMAXNUM:
6036 case ISD::FMINIMUMNUM:
6037 case ISD::FMAXIMUMNUM: {
6038 // Only one needs to be known not-nan, since it will be returned if the
6039 // other ends up being one.
6040 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6041 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6042 }
6043 case ISD::FMINNUM_IEEE:
6044 case ISD::FMAXNUM_IEEE: {
6045 if (SNaN)
6046 return true;
6047 // This can return a NaN if either operand is an sNaN, or if both operands
6048 // are NaN.
6049 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6050 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6051 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6052 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6053 }
6054 case ISD::FMINIMUM:
6055 case ISD::FMAXIMUM: {
6056 // TODO: Does this quiet or return the origina NaN as-is?
6057 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6058 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6059 }
6060 case ISD::EXTRACT_VECTOR_ELT: {
6061 SDValue Src = Op.getOperand(i: 0);
6062 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6063 EVT SrcVT = Src.getValueType();
6064 if (SrcVT.isFixedLengthVector() && Idx &&
6065 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6066 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6067 BitNo: Idx->getZExtValue());
6068 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6069 }
6070 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6071 }
6072 case ISD::EXTRACT_SUBVECTOR: {
6073 SDValue Src = Op.getOperand(i: 0);
6074 if (Src.getValueType().isFixedLengthVector()) {
6075 unsigned Idx = Op.getConstantOperandVal(i: 1);
6076 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6077 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6078 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6079 }
6080 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6081 }
6082 case ISD::INSERT_SUBVECTOR: {
6083 SDValue BaseVector = Op.getOperand(i: 0);
6084 SDValue SubVector = Op.getOperand(i: 1);
6085 EVT BaseVectorVT = BaseVector.getValueType();
6086 if (BaseVectorVT.isFixedLengthVector()) {
6087 unsigned Idx = Op.getConstantOperandVal(i: 2);
6088 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6089 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6090
6091 // Clear/Extract the bits at the position where the subvector will be
6092 // inserted.
6093 APInt DemandedMask =
6094 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6095 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6096 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6097
6098 bool NeverNaN = true;
6099 if (!DemandedSrcElts.isZero())
6100 NeverNaN &=
6101 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6102 if (NeverNaN && !DemandedSubElts.isZero())
6103 NeverNaN &=
6104 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6105 return NeverNaN;
6106 }
6107 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6108 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6109 }
6110 case ISD::BUILD_VECTOR: {
6111 unsigned NumElts = Op.getNumOperands();
6112 for (unsigned I = 0; I != NumElts; ++I)
6113 if (DemandedElts[I] &&
6114 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6115 return false;
6116 return true;
6117 }
6118 case ISD::SPLAT_VECTOR:
6119 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6120 case ISD::AssertNoFPClass: {
6121 FPClassTest NoFPClass =
6122 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6123 if ((NoFPClass & fcNan) == fcNan)
6124 return true;
6125 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6126 return true;
6127 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6128 }
6129 default:
6130 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6131 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6132 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6133 Depth);
6134 }
6135
6136 return false;
6137 }
6138}
6139
6140bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
6141 assert(Op.getValueType().isFloatingPoint() &&
6142 "Floating point type expected");
6143
6144 // If the value is a constant, we can obviously see if it is a zero or not.
6145 return ISD::matchUnaryFpPredicate(
6146 Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); });
6147}
6148
6149bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6150 EVT VT = Op.getValueType();
6151
6152 // Since the number of lanes in a scalable vector is unknown at compile time,
6153 // we track one bit which is implicitly broadcast to all lanes. This means
6154 // that all lanes in a scalable vector are considered demanded.
6155 APInt DemandedElts = VT.isFixedLengthVector()
6156 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
6157 : APInt(1, 1);
6158
6159 return isKnownNeverZero(Op, DemandedElts, Depth);
6160}
6161
6162bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6163 unsigned Depth) const {
6164 if (Depth >= MaxRecursionDepth)
6165 return false; // Limit search depth.
6166
6167 EVT OpVT = Op.getValueType();
6168 unsigned BitWidth = OpVT.getScalarSizeInBits();
6169
6170 assert(!Op.getValueType().isFloatingPoint() &&
6171 "Floating point types unsupported - use isKnownNeverZeroFloat");
6172
6173 // If the value is a constant, we can obviously see if it is a zero or not.
6174 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6175 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6176 return !V.isZero();
6177 };
6178
6179 if (ISD::matchUnaryPredicate(Op, Match: IsNeverZero))
6180 return true;
6181
6182 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6183 // some degree.
6184 switch (Op.getOpcode()) {
6185 default:
6186 break;
6187
6188 case ISD::BUILD_VECTOR:
6189 // Are all operands of a build vector constant non-zero?
6190 if (all_of(Range: enumerate(First: Op->ops()), P: [&](auto P) {
6191 auto *C = dyn_cast<ConstantSDNode>(P.value());
6192 return !DemandedElts[P.index()] || (C && IsNeverZero(C));
6193 }))
6194 return true;
6195 break;
6196
6197 case ISD::SPLAT_VECTOR:
6198 // Is the operand of a splat vector a constant non-zero?
6199 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 0)))
6200 if (IsNeverZero(C))
6201 return true;
6202 break;
6203
6204 case ISD::OR:
6205 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
6206 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6207
6208 case ISD::VSELECT:
6209 case ISD::SELECT:
6210 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6211 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6212
6213 case ISD::SHL: {
6214 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6215 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6216 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6217 // 1 << X is never zero.
6218 if (ValKnown.One[0])
6219 return true;
6220 // If max shift cnt of known ones is non-zero, result is non-zero.
6221 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
6222 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6223 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6224 return true;
6225 break;
6226 }
6227 case ISD::UADDSAT:
6228 case ISD::UMAX:
6229 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
6230 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6231
6232 // For smin/smax: If either operand is known negative/positive
6233 // respectively we don't need the other to be known at all.
6234 case ISD::SMAX: {
6235 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6236 if (Op1.isStrictlyPositive())
6237 return true;
6238
6239 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6240 if (Op0.isStrictlyPositive())
6241 return true;
6242
6243 if (Op1.isNonZero() && Op0.isNonZero())
6244 return true;
6245
6246 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6247 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6248 }
6249 case ISD::SMIN: {
6250 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6251 if (Op1.isNegative())
6252 return true;
6253
6254 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6255 if (Op0.isNegative())
6256 return true;
6257
6258 if (Op1.isNonZero() && Op0.isNonZero())
6259 return true;
6260
6261 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6262 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6263 }
6264 case ISD::UMIN:
6265 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6266 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6267
6268 case ISD::ROTL:
6269 case ISD::ROTR:
6270 case ISD::BITREVERSE:
6271 case ISD::BSWAP:
6272 case ISD::CTPOP:
6273 case ISD::ABS:
6274 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6275
6276 case ISD::SRA:
6277 case ISD::SRL: {
6278 if (Op->getFlags().hasExact())
6279 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6280 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6281 if (ValKnown.isNegative())
6282 return true;
6283 // If max shift cnt of known ones is non-zero, result is non-zero.
6284 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
6285 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6286 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6287 return true;
6288 break;
6289 }
6290 case ISD::UDIV:
6291 case ISD::SDIV:
6292 // div exact can only produce a zero if the dividend is zero.
6293 // TODO: For udiv this is also true if Op1 u<= Op0
6294 if (Op->getFlags().hasExact())
6295 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6296 break;
6297
6298 case ISD::ADD:
6299 if (Op->getFlags().hasNoUnsignedWrap())
6300 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
6301 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6302 return true;
6303 // TODO: There are a lot more cases we can prove for add.
6304 break;
6305
6306 case ISD::SUB: {
6307 if (isNullConstant(V: Op.getOperand(i: 0)))
6308 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6309
6310 std::optional<bool> ne =
6311 KnownBits::ne(LHS: computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1),
6312 RHS: computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1));
6313 return ne && *ne;
6314 }
6315
6316 case ISD::MUL:
6317 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6318 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6319 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6320 return true;
6321 break;
6322
6323 case ISD::ZERO_EXTEND:
6324 case ISD::SIGN_EXTEND:
6325 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6326 case ISD::VSCALE: {
6327 const Function &F = getMachineFunction().getFunction();
6328 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6329 ConstantRange CR =
6330 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6331 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6332 return true;
6333 break;
6334 }
6335 }
6336
6337 return computeKnownBits(Op, Depth).isNonZero();
6338}
6339
6340bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6341 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6342 return !C1->isNegative();
6343
6344 switch (Op.getOpcode()) {
6345 case ISD::FABS:
6346 case ISD::FEXP:
6347 case ISD::FEXP2:
6348 case ISD::FEXP10:
6349 return true;
6350 default:
6351 return false;
6352 }
6353
6354 llvm_unreachable("covered opcode switch");
6355}
6356
6357bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6358 assert(Use.getValueType().isFloatingPoint());
6359 const SDNode *User = Use.getUser();
6360 if (User->getFlags().hasNoSignedZeros())
6361 return true;
6362
6363 unsigned OperandNo = Use.getOperandNo();
6364 // Check if this use is insensitive to the sign of zero
6365 switch (User->getOpcode()) {
6366 case ISD::SETCC:
6367 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6368 case ISD::FABS:
6369 // fabs always produces +0.0.
6370 return true;
6371 case ISD::FCOPYSIGN:
6372 // copysign overwrites the sign bit of the first operand.
6373 return OperandNo == 0;
6374 case ISD::FADD:
6375 case ISD::FSUB: {
6376 // Arithmetic with non-zero constants fixes the uncertainty around the
6377 // sign bit.
6378 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6379 return isKnownNeverZeroFloat(Op: Other);
6380 }
6381 case ISD::FP_TO_SINT:
6382 case ISD::FP_TO_UINT:
6383 // fp-to-int conversions normalize signed zeros.
6384 return true;
6385 default:
6386 return false;
6387 }
6388}
6389
6390bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6391 if (Op->getFlags().hasNoSignedZeros())
6392 return true;
6393 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6394 // regression. Ideally, this should be implemented as a demanded-bits
6395 // optimization that stems from the users.
6396 if (Op->use_size() > 2)
6397 return false;
6398 return all_of(Range: Op->uses(),
6399 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6400}
6401
6402bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6403 // Check the obvious case.
6404 if (A == B) return true;
6405
6406 // For negative and positive zero.
6407 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6408 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6409 if (CA->isZero() && CB->isZero()) return true;
6410
6411 // Otherwise they may not be equal.
6412 return false;
6413}
6414
6415// Only bits set in Mask must be negated, other bits may be arbitrary.
6416SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6417 if (isBitwiseNot(V, AllowUndefs))
6418 return V.getOperand(i: 0);
6419
6420 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6421 // bits in the non-extended part.
6422 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6423 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6424 return SDValue();
6425 SDValue ExtArg = V.getOperand(i: 0);
6426 if (ExtArg.getScalarValueSizeInBits() >=
6427 MaskC->getAPIntValue().getActiveBits() &&
6428 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6429 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6430 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6431 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6432 return SDValue();
6433}
6434
6435static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6436 // Match masked merge pattern (X & ~M) op (Y & M)
6437 // Including degenerate case (X & ~M) op M
6438 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6439 SDValue Other) {
6440 if (SDValue NotOperand =
6441 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6442 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6443 NotOperand->getOpcode() == ISD::TRUNCATE)
6444 NotOperand = NotOperand->getOperand(Num: 0);
6445
6446 if (Other == NotOperand)
6447 return true;
6448 if (Other->getOpcode() == ISD::AND)
6449 return NotOperand == Other->getOperand(Num: 0) ||
6450 NotOperand == Other->getOperand(Num: 1);
6451 }
6452 return false;
6453 };
6454
6455 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6456 A = A->getOperand(Num: 0);
6457
6458 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6459 B = B->getOperand(Num: 0);
6460
6461 if (A->getOpcode() == ISD::AND)
6462 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6463 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6464 return false;
6465}
6466
6467// FIXME: unify with llvm::haveNoCommonBitsSet.
6468bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6469 assert(A.getValueType() == B.getValueType() &&
6470 "Values must have the same type");
6471 if (haveNoCommonBitsSetCommutative(A, B) ||
6472 haveNoCommonBitsSetCommutative(A: B, B: A))
6473 return true;
6474 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6475 RHS: computeKnownBits(Op: B));
6476}
6477
6478static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6479 SelectionDAG &DAG) {
6480 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6481 return DAG.getConstant(Val: 0, DL, VT);
6482
6483 return SDValue();
6484}
6485
6486static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6487 ArrayRef<SDValue> Ops,
6488 SelectionDAG &DAG) {
6489 int NumOps = Ops.size();
6490 assert(NumOps != 0 && "Can't build an empty vector!");
6491 assert(!VT.isScalableVector() &&
6492 "BUILD_VECTOR cannot be used with scalable types");
6493 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6494 "Incorrect element count in BUILD_VECTOR!");
6495
6496 // BUILD_VECTOR of UNDEFs is UNDEF.
6497 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6498 return DAG.getUNDEF(VT);
6499
6500 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6501 SDValue IdentitySrc;
6502 bool IsIdentity = true;
6503 for (int i = 0; i != NumOps; ++i) {
6504 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6505 Ops[i].getOperand(i: 0).getValueType() != VT ||
6506 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6507 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6508 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6509 IsIdentity = false;
6510 break;
6511 }
6512 IdentitySrc = Ops[i].getOperand(i: 0);
6513 }
6514 if (IsIdentity)
6515 return IdentitySrc;
6516
6517 return SDValue();
6518}
6519
6520/// Try to simplify vector concatenation to an input value, undef, or build
6521/// vector.
6522static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6523 ArrayRef<SDValue> Ops,
6524 SelectionDAG &DAG) {
6525 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6526 assert(llvm::all_of(Ops,
6527 [Ops](SDValue Op) {
6528 return Ops[0].getValueType() == Op.getValueType();
6529 }) &&
6530 "Concatenation of vectors with inconsistent value types!");
6531 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6532 VT.getVectorElementCount() &&
6533 "Incorrect element count in vector concatenation!");
6534
6535 if (Ops.size() == 1)
6536 return Ops[0];
6537
6538 // Concat of UNDEFs is UNDEF.
6539 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6540 return DAG.getUNDEF(VT);
6541
6542 // Scan the operands and look for extract operations from a single source
6543 // that correspond to insertion at the same location via this concatenation:
6544 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6545 SDValue IdentitySrc;
6546 bool IsIdentity = true;
6547 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6548 SDValue Op = Ops[i];
6549 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6550 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6551 Op.getOperand(i: 0).getValueType() != VT ||
6552 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
6553 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
6554 IsIdentity = false;
6555 break;
6556 }
6557 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6558 "Unexpected identity source vector for concat of extracts");
6559 IdentitySrc = Op.getOperand(i: 0);
6560 }
6561 if (IsIdentity) {
6562 assert(IdentitySrc && "Failed to set source vector of extracts");
6563 return IdentitySrc;
6564 }
6565
6566 // The code below this point is only designed to work for fixed width
6567 // vectors, so we bail out for now.
6568 if (VT.isScalableVector())
6569 return SDValue();
6570
6571 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
6572 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
6573 // BUILD_VECTOR.
6574 // FIXME: Add support for SCALAR_TO_VECTOR as well.
6575 EVT SVT = VT.getScalarType();
6576 SmallVector<SDValue, 16> Elts;
6577 for (SDValue Op : Ops) {
6578 EVT OpVT = Op.getValueType();
6579 if (Op.isUndef())
6580 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
6581 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
6582 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
6583 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
6584 OpVT.getVectorNumElements() == 1 &&
6585 isNullConstant(V: Op.getOperand(i: 2)))
6586 Elts.push_back(Elt: Op.getOperand(i: 1));
6587 else
6588 return SDValue();
6589 }
6590
6591 // BUILD_VECTOR requires all inputs to be of the same type, find the
6592 // maximum type and extend them all.
6593 for (SDValue Op : Elts)
6594 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
6595
6596 if (SVT.bitsGT(VT: VT.getScalarType())) {
6597 for (SDValue &Op : Elts) {
6598 if (Op.isUndef())
6599 Op = DAG.getUNDEF(VT: SVT);
6600 else
6601 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
6602 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
6603 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
6604 }
6605 }
6606
6607 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
6608 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
6609 return V;
6610}
6611
6612/// Gets or creates the specified node.
6613SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
6614 SDVTList VTs = getVTList(VT);
6615 FoldingSetNodeID ID;
6616 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: {});
6617 void *IP = nullptr;
6618 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
6619 return SDValue(E, 0);
6620
6621 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6622 CSEMap.InsertNode(N, InsertPos: IP);
6623
6624 InsertNode(N);
6625 SDValue V = SDValue(N, 0);
6626 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6627 return V;
6628}
6629
6630SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6631 SDValue N1) {
6632 SDNodeFlags Flags;
6633 if (Inserter)
6634 Flags = Inserter->getFlags();
6635 return getNode(Opcode, DL, VT, Operand: N1, Flags);
6636}
6637
6638SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6639 SDValue N1, const SDNodeFlags Flags) {
6640 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
6641
6642 // Constant fold unary operations with a vector integer or float operand.
6643 switch (Opcode) {
6644 default:
6645 // FIXME: Entirely reasonable to perform folding of other unary
6646 // operations here as the need arises.
6647 break;
6648 case ISD::FNEG:
6649 case ISD::FABS:
6650 case ISD::FCEIL:
6651 case ISD::FTRUNC:
6652 case ISD::FFLOOR:
6653 case ISD::FP_EXTEND:
6654 case ISD::FP_TO_SINT:
6655 case ISD::FP_TO_UINT:
6656 case ISD::FP_TO_FP16:
6657 case ISD::FP_TO_BF16:
6658 case ISD::TRUNCATE:
6659 case ISD::ANY_EXTEND:
6660 case ISD::ZERO_EXTEND:
6661 case ISD::SIGN_EXTEND:
6662 case ISD::UINT_TO_FP:
6663 case ISD::SINT_TO_FP:
6664 case ISD::FP16_TO_FP:
6665 case ISD::BF16_TO_FP:
6666 case ISD::BITCAST:
6667 case ISD::ABS:
6668 case ISD::BITREVERSE:
6669 case ISD::BSWAP:
6670 case ISD::CTLZ:
6671 case ISD::CTLZ_ZERO_UNDEF:
6672 case ISD::CTTZ:
6673 case ISD::CTTZ_ZERO_UNDEF:
6674 case ISD::CTPOP:
6675 case ISD::CTLS:
6676 case ISD::STEP_VECTOR: {
6677 SDValue Ops = {N1};
6678 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
6679 return Fold;
6680 }
6681 }
6682
6683 unsigned OpOpcode = N1.getNode()->getOpcode();
6684 switch (Opcode) {
6685 case ISD::STEP_VECTOR:
6686 assert(VT.isScalableVector() &&
6687 "STEP_VECTOR can only be used with scalable types");
6688 assert(OpOpcode == ISD::TargetConstant &&
6689 VT.getVectorElementType() == N1.getValueType() &&
6690 "Unexpected step operand");
6691 break;
6692 case ISD::FREEZE:
6693 assert(VT == N1.getValueType() && "Unexpected VT!");
6694 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly=*/false))
6695 return N1;
6696 break;
6697 case ISD::TokenFactor:
6698 case ISD::MERGE_VALUES:
6699 case ISD::CONCAT_VECTORS:
6700 return N1; // Factor, merge or concat of one node? No need.
6701 case ISD::BUILD_VECTOR: {
6702 // Attempt to simplify BUILD_VECTOR.
6703 SDValue Ops[] = {N1};
6704 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
6705 return V;
6706 break;
6707 }
6708 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
6709 case ISD::FP_EXTEND:
6710 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
6711 "Invalid FP cast!");
6712 if (N1.getValueType() == VT) return N1; // noop conversion.
6713 assert((!VT.isVector() || VT.getVectorElementCount() ==
6714 N1.getValueType().getVectorElementCount()) &&
6715 "Vector element count mismatch!");
6716 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
6717 if (N1.isUndef())
6718 return getUNDEF(VT);
6719 break;
6720 case ISD::FP_TO_SINT:
6721 case ISD::FP_TO_UINT:
6722 if (N1.isUndef())
6723 return getUNDEF(VT);
6724 break;
6725 case ISD::SINT_TO_FP:
6726 case ISD::UINT_TO_FP:
6727 // [us]itofp(undef) = 0, because the result value is bounded.
6728 if (N1.isUndef())
6729 return getConstantFP(Val: 0.0, DL, VT);
6730 break;
6731 case ISD::SIGN_EXTEND:
6732 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6733 "Invalid SIGN_EXTEND!");
6734 assert(VT.isVector() == N1.getValueType().isVector() &&
6735 "SIGN_EXTEND result type type should be vector iff the operand "
6736 "type is vector!");
6737 if (N1.getValueType() == VT) return N1; // noop extension
6738 assert((!VT.isVector() || VT.getVectorElementCount() ==
6739 N1.getValueType().getVectorElementCount()) &&
6740 "Vector element count mismatch!");
6741 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
6742 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
6743 SDNodeFlags Flags;
6744 if (OpOpcode == ISD::ZERO_EXTEND)
6745 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6746 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6747 transferDbgValues(From: N1, To: NewVal);
6748 return NewVal;
6749 }
6750
6751 if (OpOpcode == ISD::POISON)
6752 return getPOISON(VT);
6753
6754 if (N1.isUndef())
6755 // sext(undef) = 0, because the top bits will all be the same.
6756 return getConstant(Val: 0, DL, VT);
6757
6758 // Skip unnecessary sext_inreg pattern:
6759 // (sext (trunc x)) -> x iff the upper bits are all signbits.
6760 if (OpOpcode == ISD::TRUNCATE) {
6761 SDValue OpOp = N1.getOperand(i: 0);
6762 if (OpOp.getValueType() == VT) {
6763 unsigned NumSignExtBits =
6764 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
6765 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
6766 transferDbgValues(From: N1, To: OpOp);
6767 return OpOp;
6768 }
6769 }
6770 }
6771 break;
6772 case ISD::ZERO_EXTEND:
6773 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6774 "Invalid ZERO_EXTEND!");
6775 assert(VT.isVector() == N1.getValueType().isVector() &&
6776 "ZERO_EXTEND result type type should be vector iff the operand "
6777 "type is vector!");
6778 if (N1.getValueType() == VT) return N1; // noop extension
6779 assert((!VT.isVector() || VT.getVectorElementCount() ==
6780 N1.getValueType().getVectorElementCount()) &&
6781 "Vector element count mismatch!");
6782 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
6783 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
6784 SDNodeFlags Flags;
6785 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6786 SDValue NewVal =
6787 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
6788 transferDbgValues(From: N1, To: NewVal);
6789 return NewVal;
6790 }
6791
6792 if (OpOpcode == ISD::POISON)
6793 return getPOISON(VT);
6794
6795 if (N1.isUndef())
6796 // zext(undef) = 0, because the top bits will be zero.
6797 return getConstant(Val: 0, DL, VT);
6798
6799 // Skip unnecessary zext_inreg pattern:
6800 // (zext (trunc x)) -> x iff the upper bits are known zero.
6801 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
6802 // use to recognise zext_inreg patterns.
6803 if (OpOpcode == ISD::TRUNCATE) {
6804 SDValue OpOp = N1.getOperand(i: 0);
6805 if (OpOp.getValueType() == VT) {
6806 if (OpOp.getOpcode() != ISD::AND) {
6807 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
6808 loBit: N1.getScalarValueSizeInBits());
6809 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
6810 transferDbgValues(From: N1, To: OpOp);
6811 return OpOp;
6812 }
6813 }
6814 }
6815 }
6816 break;
6817 case ISD::ANY_EXTEND:
6818 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6819 "Invalid ANY_EXTEND!");
6820 assert(VT.isVector() == N1.getValueType().isVector() &&
6821 "ANY_EXTEND result type type should be vector iff the operand "
6822 "type is vector!");
6823 if (N1.getValueType() == VT) return N1; // noop extension
6824 assert((!VT.isVector() || VT.getVectorElementCount() ==
6825 N1.getValueType().getVectorElementCount()) &&
6826 "Vector element count mismatch!");
6827 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
6828
6829 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6830 OpOpcode == ISD::ANY_EXTEND) {
6831 SDNodeFlags Flags;
6832 if (OpOpcode == ISD::ZERO_EXTEND)
6833 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6834 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
6835 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6836 }
6837 if (N1.isUndef())
6838 return getUNDEF(VT);
6839
6840 // (ext (trunc x)) -> x
6841 if (OpOpcode == ISD::TRUNCATE) {
6842 SDValue OpOp = N1.getOperand(i: 0);
6843 if (OpOp.getValueType() == VT) {
6844 transferDbgValues(From: N1, To: OpOp);
6845 return OpOp;
6846 }
6847 }
6848 break;
6849 case ISD::TRUNCATE:
6850 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6851 "Invalid TRUNCATE!");
6852 assert(VT.isVector() == N1.getValueType().isVector() &&
6853 "TRUNCATE result type type should be vector iff the operand "
6854 "type is vector!");
6855 if (N1.getValueType() == VT) return N1; // noop truncate
6856 assert((!VT.isVector() || VT.getVectorElementCount() ==
6857 N1.getValueType().getVectorElementCount()) &&
6858 "Vector element count mismatch!");
6859 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
6860 if (OpOpcode == ISD::TRUNCATE)
6861 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6862 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6863 OpOpcode == ISD::ANY_EXTEND) {
6864 // If the source is smaller than the dest, we still need an extend.
6865 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
6866 VT: VT.getScalarType())) {
6867 SDNodeFlags Flags;
6868 if (OpOpcode == ISD::ZERO_EXTEND)
6869 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6870 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6871 }
6872 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
6873 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6874 return N1.getOperand(i: 0);
6875 }
6876 if (N1.isUndef())
6877 return getUNDEF(VT);
6878 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
6879 return getVScale(DL, VT,
6880 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
6881 break;
6882 case ISD::ANY_EXTEND_VECTOR_INREG:
6883 case ISD::ZERO_EXTEND_VECTOR_INREG:
6884 case ISD::SIGN_EXTEND_VECTOR_INREG:
6885 assert(VT.isVector() && "This DAG node is restricted to vector types.");
6886 assert(N1.getValueType().bitsLE(VT) &&
6887 "The input must be the same size or smaller than the result.");
6888 assert(VT.getVectorMinNumElements() <
6889 N1.getValueType().getVectorMinNumElements() &&
6890 "The destination vector type must have fewer lanes than the input.");
6891 break;
6892 case ISD::ABS:
6893 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
6894 if (N1.isUndef())
6895 return getConstant(Val: 0, DL, VT);
6896 break;
6897 case ISD::BSWAP:
6898 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
6899 assert((VT.getScalarSizeInBits() % 16 == 0) &&
6900 "BSWAP types must be a multiple of 16 bits!");
6901 if (N1.isUndef())
6902 return getUNDEF(VT);
6903 // bswap(bswap(X)) -> X.
6904 if (OpOpcode == ISD::BSWAP)
6905 return N1.getOperand(i: 0);
6906 break;
6907 case ISD::BITREVERSE:
6908 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
6909 if (N1.isUndef())
6910 return getUNDEF(VT);
6911 break;
6912 case ISD::BITCAST:
6913 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
6914 "Cannot BITCAST between types of different sizes!");
6915 if (VT == N1.getValueType()) return N1; // noop conversion.
6916 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
6917 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
6918 if (N1.isUndef())
6919 return getUNDEF(VT);
6920 break;
6921 case ISD::SCALAR_TO_VECTOR:
6922 assert(VT.isVector() && !N1.getValueType().isVector() &&
6923 (VT.getVectorElementType() == N1.getValueType() ||
6924 (VT.getVectorElementType().isInteger() &&
6925 N1.getValueType().isInteger() &&
6926 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
6927 "Illegal SCALAR_TO_VECTOR node!");
6928 if (N1.isUndef())
6929 return getUNDEF(VT);
6930 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
6931 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
6932 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
6933 N1.getConstantOperandVal(i: 1) == 0 &&
6934 N1.getOperand(i: 0).getValueType() == VT)
6935 return N1.getOperand(i: 0);
6936 break;
6937 case ISD::FNEG:
6938 // Negation of an unknown bag of bits is still completely undefined.
6939 if (N1.isUndef())
6940 return getUNDEF(VT);
6941
6942 if (OpOpcode == ISD::FNEG) // --X -> X
6943 return N1.getOperand(i: 0);
6944 break;
6945 case ISD::FABS:
6946 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
6947 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
6948 break;
6949 case ISD::VSCALE:
6950 assert(VT == N1.getValueType() && "Unexpected VT!");
6951 break;
6952 case ISD::CTPOP:
6953 if (N1.getValueType().getScalarType() == MVT::i1)
6954 return N1;
6955 break;
6956 case ISD::CTLZ:
6957 case ISD::CTTZ:
6958 if (N1.getValueType().getScalarType() == MVT::i1)
6959 return getNOT(DL, Val: N1, VT: N1.getValueType());
6960 break;
6961 case ISD::CTLS:
6962 if (N1.getValueType().getScalarType() == MVT::i1)
6963 return getConstant(Val: 0, DL, VT);
6964 break;
6965 case ISD::VECREDUCE_ADD:
6966 if (N1.getValueType().getScalarType() == MVT::i1)
6967 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
6968 break;
6969 case ISD::VECREDUCE_SMIN:
6970 case ISD::VECREDUCE_UMAX:
6971 if (N1.getValueType().getScalarType() == MVT::i1)
6972 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
6973 break;
6974 case ISD::VECREDUCE_SMAX:
6975 case ISD::VECREDUCE_UMIN:
6976 if (N1.getValueType().getScalarType() == MVT::i1)
6977 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
6978 break;
6979 case ISD::SPLAT_VECTOR:
6980 assert(VT.isVector() && "Wrong return type!");
6981 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
6982 // that for now.
6983 assert((VT.getVectorElementType() == N1.getValueType() ||
6984 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
6985 (VT.getVectorElementType().isInteger() &&
6986 N1.getValueType().isInteger() &&
6987 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
6988 "Wrong operand type!");
6989 break;
6990 }
6991
6992 SDNode *N;
6993 SDVTList VTs = getVTList(VT);
6994 SDValue Ops[] = {N1};
6995 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
6996 FoldingSetNodeID ID;
6997 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
6998 void *IP = nullptr;
6999 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7000 E->intersectFlagsWith(Flags);
7001 return SDValue(E, 0);
7002 }
7003
7004 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7005 N->setFlags(Flags);
7006 createOperands(Node: N, Vals: Ops);
7007 CSEMap.InsertNode(N, InsertPos: IP);
7008 } else {
7009 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7010 createOperands(Node: N, Vals: Ops);
7011 }
7012
7013 InsertNode(N);
7014 SDValue V = SDValue(N, 0);
7015 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7016 return V;
7017}
7018
7019static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7020 const APInt &C2) {
7021 switch (Opcode) {
7022 case ISD::ADD: return C1 + C2;
7023 case ISD::SUB: return C1 - C2;
7024 case ISD::MUL: return C1 * C2;
7025 case ISD::AND: return C1 & C2;
7026 case ISD::OR: return C1 | C2;
7027 case ISD::XOR: return C1 ^ C2;
7028 case ISD::SHL: return C1 << C2;
7029 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7030 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7031 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7032 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7033 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7034 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7035 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7036 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7037 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7038 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7039 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7040 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7041 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7042 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7043 case ISD::UDIV:
7044 if (!C2.getBoolValue())
7045 break;
7046 return C1.udiv(RHS: C2);
7047 case ISD::UREM:
7048 if (!C2.getBoolValue())
7049 break;
7050 return C1.urem(RHS: C2);
7051 case ISD::SDIV:
7052 if (!C2.getBoolValue())
7053 break;
7054 return C1.sdiv(RHS: C2);
7055 case ISD::SREM:
7056 if (!C2.getBoolValue())
7057 break;
7058 return C1.srem(RHS: C2);
7059 case ISD::AVGFLOORS:
7060 return APIntOps::avgFloorS(C1, C2);
7061 case ISD::AVGFLOORU:
7062 return APIntOps::avgFloorU(C1, C2);
7063 case ISD::AVGCEILS:
7064 return APIntOps::avgCeilS(C1, C2);
7065 case ISD::AVGCEILU:
7066 return APIntOps::avgCeilU(C1, C2);
7067 case ISD::ABDS:
7068 return APIntOps::abds(A: C1, B: C2);
7069 case ISD::ABDU:
7070 return APIntOps::abdu(A: C1, B: C2);
7071 case ISD::MULHS:
7072 return APIntOps::mulhs(C1, C2);
7073 case ISD::MULHU:
7074 return APIntOps::mulhu(C1, C2);
7075 case ISD::CLMUL:
7076 return APIntOps::clmul(LHS: C1, RHS: C2);
7077 case ISD::CLMULR:
7078 return APIntOps::clmulr(LHS: C1, RHS: C2);
7079 case ISD::CLMULH:
7080 return APIntOps::clmulh(LHS: C1, RHS: C2);
7081 }
7082 return std::nullopt;
7083}
7084// Handle constant folding with UNDEF.
7085// TODO: Handle more cases.
7086static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7087 bool IsUndef1, const APInt &C2,
7088 bool IsUndef2) {
7089 if (!(IsUndef1 || IsUndef2))
7090 return FoldValue(Opcode, C1, C2);
7091
7092 // Fold and(x, undef) -> 0
7093 // Fold mul(x, undef) -> 0
7094 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7095 return APInt::getZero(numBits: C1.getBitWidth());
7096
7097 return std::nullopt;
7098}
7099
7100SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7101 const GlobalAddressSDNode *GA,
7102 const SDNode *N2) {
7103 if (GA->getOpcode() != ISD::GlobalAddress)
7104 return SDValue();
7105 if (!TLI->isOffsetFoldingLegal(GA))
7106 return SDValue();
7107 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7108 if (!C2)
7109 return SDValue();
7110 int64_t Offset = C2->getSExtValue();
7111 switch (Opcode) {
7112 case ISD::ADD:
7113 case ISD::PTRADD:
7114 break;
7115 case ISD::SUB: Offset = -uint64_t(Offset); break;
7116 default: return SDValue();
7117 }
7118 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7119 Offset: GA->getOffset() + uint64_t(Offset));
7120}
7121
7122bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7123 switch (Opcode) {
7124 case ISD::SDIV:
7125 case ISD::UDIV:
7126 case ISD::SREM:
7127 case ISD::UREM: {
7128 // If a divisor is zero/undef or any element of a divisor vector is
7129 // zero/undef, the whole op is undef.
7130 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7131 SDValue Divisor = Ops[1];
7132 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7133 return true;
7134
7135 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7136 llvm::any_of(Range: Divisor->op_values(),
7137 P: [](SDValue V) { return V.isUndef() ||
7138 isNullConstant(V); });
7139 // TODO: Handle signed overflow.
7140 }
7141 // TODO: Handle oversized shifts.
7142 default:
7143 return false;
7144 }
7145}
7146
7147SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7148 EVT VT, ArrayRef<SDValue> Ops,
7149 SDNodeFlags Flags) {
7150 // If the opcode is a target-specific ISD node, there's nothing we can
7151 // do here and the operand rules may not line up with the below, so
7152 // bail early.
7153 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7154 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7155 // foldCONCAT_VECTORS in getNode before this is called.
7156 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7157 return SDValue();
7158
7159 unsigned NumOps = Ops.size();
7160 if (NumOps == 0)
7161 return SDValue();
7162
7163 if (isUndef(Opcode, Ops))
7164 return getUNDEF(VT);
7165
7166 // Handle unary special cases.
7167 if (NumOps == 1) {
7168 SDValue N1 = Ops[0];
7169
7170 // Constant fold unary operations with an integer constant operand. Even
7171 // opaque constant will be folded, because the folding of unary operations
7172 // doesn't create new constants with different values. Nevertheless, the
7173 // opaque flag is preserved during folding to prevent future folding with
7174 // other constants.
7175 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7176 const APInt &Val = C->getAPIntValue();
7177 switch (Opcode) {
7178 case ISD::SIGN_EXTEND:
7179 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7180 isT: C->isTargetOpcode(), isO: C->isOpaque());
7181 case ISD::TRUNCATE:
7182 if (C->isOpaque())
7183 break;
7184 [[fallthrough]];
7185 case ISD::ZERO_EXTEND:
7186 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7187 isT: C->isTargetOpcode(), isO: C->isOpaque());
7188 case ISD::ANY_EXTEND:
7189 // Some targets like RISCV prefer to sign extend some types.
7190 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7191 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7192 isT: C->isTargetOpcode(), isO: C->isOpaque());
7193 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7194 isT: C->isTargetOpcode(), isO: C->isOpaque());
7195 case ISD::ABS:
7196 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7197 isO: C->isOpaque());
7198 case ISD::BITREVERSE:
7199 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7200 isO: C->isOpaque());
7201 case ISD::BSWAP:
7202 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7203 isO: C->isOpaque());
7204 case ISD::CTPOP:
7205 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7206 isO: C->isOpaque());
7207 case ISD::CTLZ:
7208 case ISD::CTLZ_ZERO_UNDEF:
7209 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7210 isO: C->isOpaque());
7211 case ISD::CTTZ:
7212 case ISD::CTTZ_ZERO_UNDEF:
7213 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7214 isO: C->isOpaque());
7215 case ISD::CTLS:
7216 // CTLS returns the number of extra sign bits so subtract one.
7217 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7218 isT: C->isTargetOpcode(), isO: C->isOpaque());
7219 case ISD::UINT_TO_FP:
7220 case ISD::SINT_TO_FP: {
7221 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7222 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7223 RM: APFloat::rmNearestTiesToEven);
7224 return getConstantFP(V: FPV, DL, VT);
7225 }
7226 case ISD::FP16_TO_FP:
7227 case ISD::BF16_TO_FP: {
7228 bool Ignored;
7229 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7230 : APFloat::BFloat(),
7231 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7232
7233 // This can return overflow, underflow, or inexact; we don't care.
7234 // FIXME need to be more flexible about rounding mode.
7235 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7236 losesInfo: &Ignored);
7237 return getConstantFP(V: FPV, DL, VT);
7238 }
7239 case ISD::STEP_VECTOR:
7240 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7241 return V;
7242 break;
7243 case ISD::BITCAST:
7244 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7245 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7246 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7247 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7248 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7249 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7250 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7251 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7252 break;
7253 }
7254 }
7255
7256 // Constant fold unary operations with a floating point constant operand.
7257 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7258 APFloat V = C->getValueAPF(); // make copy
7259 switch (Opcode) {
7260 case ISD::FNEG:
7261 V.changeSign();
7262 return getConstantFP(V, DL, VT);
7263 case ISD::FABS:
7264 V.clearSign();
7265 return getConstantFP(V, DL, VT);
7266 case ISD::FCEIL: {
7267 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7268 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7269 return getConstantFP(V, DL, VT);
7270 return SDValue();
7271 }
7272 case ISD::FTRUNC: {
7273 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7274 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7275 return getConstantFP(V, DL, VT);
7276 return SDValue();
7277 }
7278 case ISD::FFLOOR: {
7279 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7280 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7281 return getConstantFP(V, DL, VT);
7282 return SDValue();
7283 }
7284 case ISD::FP_EXTEND: {
7285 bool ignored;
7286 // This can return overflow, underflow, or inexact; we don't care.
7287 // FIXME need to be more flexible about rounding mode.
7288 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7289 losesInfo: &ignored);
7290 return getConstantFP(V, DL, VT);
7291 }
7292 case ISD::FP_TO_SINT:
7293 case ISD::FP_TO_UINT: {
7294 bool ignored;
7295 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7296 // FIXME need to be more flexible about rounding mode.
7297 APFloat::opStatus s =
7298 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7299 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7300 break;
7301 return getConstant(Val: IntVal, DL, VT);
7302 }
7303 case ISD::FP_TO_FP16:
7304 case ISD::FP_TO_BF16: {
7305 bool Ignored;
7306 // This can return overflow, underflow, or inexact; we don't care.
7307 // FIXME need to be more flexible about rounding mode.
7308 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7309 : APFloat::BFloat(),
7310 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7311 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7312 }
7313 case ISD::BITCAST:
7314 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7315 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7316 VT);
7317 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7318 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7319 VT);
7320 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7321 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7322 VT);
7323 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7324 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7325 break;
7326 }
7327 }
7328
7329 // Early-out if we failed to constant fold a bitcast.
7330 if (Opcode == ISD::BITCAST)
7331 return SDValue();
7332 }
7333
7334 // Handle binops special cases.
7335 if (NumOps == 2) {
7336 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7337 return CFP;
7338
7339 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7340 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7341 if (C1->isOpaque() || C2->isOpaque())
7342 return SDValue();
7343
7344 std::optional<APInt> FoldAttempt =
7345 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7346 if (!FoldAttempt)
7347 return SDValue();
7348
7349 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7350 assert((!Folded || !VT.isVector()) &&
7351 "Can't fold vectors ops with scalar operands");
7352 return Folded;
7353 }
7354 }
7355
7356 // fold (add Sym, c) -> Sym+c
7357 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7358 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7359 if (TLI->isCommutativeBinOp(Opcode))
7360 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7361 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7362
7363 // fold (sext_in_reg c1) -> c2
7364 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7365 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7366
7367 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7368 unsigned FromBits = EVT.getScalarSizeInBits();
7369 Val <<= Val.getBitWidth() - FromBits;
7370 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7371 return getConstant(Val, DL, VT: ConstantVT);
7372 };
7373
7374 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7375 const APInt &Val = C1->getAPIntValue();
7376 return SignExtendInReg(Val, VT);
7377 }
7378
7379 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7380 SmallVector<SDValue, 8> ScalarOps;
7381 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7382 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7383 SDValue Op = Ops[0].getOperand(i: I);
7384 if (Op.isUndef()) {
7385 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7386 continue;
7387 }
7388 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7389 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7390 }
7391 return getBuildVector(VT, DL, Ops: ScalarOps);
7392 }
7393
7394 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7395 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7396 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7397 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7398 Ops[0].getOperand(i: 0).getValueType()));
7399 }
7400 }
7401
7402 // Handle fshl/fshr special cases.
7403 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7404 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7405 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7406 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7407
7408 if (C1 && C2 && C3) {
7409 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7410 return SDValue();
7411 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7412 &V3 = C3->getAPIntValue();
7413
7414 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7415 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7416 return getConstant(Val: FoldedVal, DL, VT);
7417 }
7418 }
7419
7420 // Handle fma/fmad special cases.
7421 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7422 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7423 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7424 Ops[2].getValueType() == VT && "FMA types must match!");
7425 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
7426 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
7427 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
7428 if (C1 && C2 && C3) {
7429 APFloat V1 = C1->getValueAPF();
7430 const APFloat &V2 = C2->getValueAPF();
7431 const APFloat &V3 = C3->getValueAPF();
7432 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7433 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7434 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7435 } else
7436 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7437 return getConstantFP(V: V1, DL, VT);
7438 }
7439 }
7440
7441 // This is for vector folding only from here on.
7442 if (!VT.isVector())
7443 return SDValue();
7444
7445 ElementCount NumElts = VT.getVectorElementCount();
7446
7447 // See if we can fold through any bitcasted integer ops.
7448 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
7449 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7450 (Ops[0].getOpcode() == ISD::BITCAST ||
7451 Ops[1].getOpcode() == ISD::BITCAST)) {
7452 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
7453 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
7454 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
7455 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
7456 if (BV1 && BV2 && N1.getValueType().isInteger() &&
7457 N2.getValueType().isInteger()) {
7458 bool IsLE = getDataLayout().isLittleEndian();
7459 unsigned EltBits = VT.getScalarSizeInBits();
7460 SmallVector<APInt> RawBits1, RawBits2;
7461 BitVector UndefElts1, UndefElts2;
7462 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
7463 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
7464 SmallVector<APInt> RawBits;
7465 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
7466 std::optional<APInt> Fold = FoldValueWithUndef(
7467 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
7468 if (!Fold)
7469 break;
7470 RawBits.push_back(Elt: *Fold);
7471 }
7472 if (RawBits.size() == NumElts.getFixedValue()) {
7473 // We have constant folded, but we might need to cast this again back
7474 // to the original (possibly legalized) type.
7475 EVT BVVT, BVEltVT;
7476 if (N1.getValueType() == VT) {
7477 BVVT = N1.getValueType();
7478 BVEltVT = BV1->getOperand(Num: 0).getValueType();
7479 } else {
7480 BVVT = N2.getValueType();
7481 BVEltVT = BV2->getOperand(Num: 0).getValueType();
7482 }
7483 unsigned BVEltBits = BVEltVT.getSizeInBits();
7484 SmallVector<APInt> DstBits;
7485 BitVector DstUndefs;
7486 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
7487 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
7488 SrcUndefElements: BitVector(RawBits.size(), false));
7489 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
7490 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
7491 if (DstUndefs[I])
7492 continue;
7493 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
7494 }
7495 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
7496 }
7497 }
7498 }
7499 }
7500
7501 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
7502 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
7503 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
7504 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
7505 APInt RHSVal;
7506 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
7507 APInt NewStep = Opcode == ISD::MUL
7508 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
7509 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
7510 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
7511 }
7512 }
7513
7514 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
7515 return !Op.getValueType().isVector() ||
7516 Op.getValueType().getVectorElementCount() == NumElts;
7517 };
7518
7519 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
7520 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
7521 Op.getOpcode() == ISD::BUILD_VECTOR ||
7522 Op.getOpcode() == ISD::SPLAT_VECTOR;
7523 };
7524
7525 // All operands must be vector types with the same number of elements as
7526 // the result type and must be either UNDEF or a build/splat vector
7527 // or UNDEF scalars.
7528 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
7529 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
7530 return SDValue();
7531
7532 // If we are comparing vectors, then the result needs to be a i1 boolean that
7533 // is then extended back to the legal result type depending on how booleans
7534 // are represented.
7535 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
7536 ISD::NodeType ExtendCode =
7537 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
7538 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
7539 : ISD::SIGN_EXTEND;
7540
7541 // Find legal integer scalar type for constant promotion and
7542 // ensure that its scalar size is at least as large as source.
7543 EVT LegalSVT = VT.getScalarType();
7544 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
7545 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
7546 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
7547 return SDValue();
7548 }
7549
7550 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
7551 // only have one operand to check. For fixed-length vector types we may have
7552 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
7553 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
7554
7555 // Constant fold each scalar lane separately.
7556 SmallVector<SDValue, 4> ScalarResults;
7557 for (unsigned I = 0; I != NumVectorElts; I++) {
7558 SmallVector<SDValue, 4> ScalarOps;
7559 for (SDValue Op : Ops) {
7560 EVT InSVT = Op.getValueType().getScalarType();
7561 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
7562 Op.getOpcode() != ISD::SPLAT_VECTOR) {
7563 if (Op.isUndef())
7564 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
7565 else
7566 ScalarOps.push_back(Elt: Op);
7567 continue;
7568 }
7569
7570 SDValue ScalarOp =
7571 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
7572 EVT ScalarVT = ScalarOp.getValueType();
7573
7574 // Build vector (integer) scalar operands may need implicit
7575 // truncation - do this before constant folding.
7576 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
7577 // Don't create illegally-typed nodes unless they're constants or undef
7578 // - if we fail to constant fold we can't guarantee the (dead) nodes
7579 // we're creating will be cleaned up before being visited for
7580 // legalization.
7581 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
7582 !isa<ConstantSDNode>(Val: ScalarOp) &&
7583 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
7584 TargetLowering::TypeLegal)
7585 return SDValue();
7586 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
7587 }
7588
7589 ScalarOps.push_back(Elt: ScalarOp);
7590 }
7591
7592 // Constant fold the scalar operands.
7593 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
7594
7595 // Scalar folding only succeeded if the result is a constant or UNDEF.
7596 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
7597 ScalarResult.getOpcode() != ISD::ConstantFP)
7598 return SDValue();
7599
7600 // Legalize the (integer) scalar constant if necessary. We only do
7601 // this once we know the folding succeeded, since otherwise we would
7602 // get a node with illegal type which has a user.
7603 if (LegalSVT != SVT)
7604 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
7605
7606 ScalarResults.push_back(Elt: ScalarResult);
7607 }
7608
7609 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
7610 : getBuildVector(VT, DL, Ops: ScalarResults);
7611 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
7612 return V;
7613}
7614
7615SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
7616 EVT VT, ArrayRef<SDValue> Ops) {
7617 // TODO: Add support for unary/ternary fp opcodes.
7618 if (Ops.size() != 2)
7619 return SDValue();
7620
7621 // TODO: We don't do any constant folding for strict FP opcodes here, but we
7622 // should. That will require dealing with a potentially non-default
7623 // rounding mode, checking the "opStatus" return value from the APFloat
7624 // math calculations, and possibly other variations.
7625 SDValue N1 = Ops[0];
7626 SDValue N2 = Ops[1];
7627 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
7628 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
7629 if (N1CFP && N2CFP) {
7630 APFloat C1 = N1CFP->getValueAPF(); // make copy
7631 const APFloat &C2 = N2CFP->getValueAPF();
7632 switch (Opcode) {
7633 case ISD::FADD:
7634 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7635 return getConstantFP(V: C1, DL, VT);
7636 case ISD::FSUB:
7637 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7638 return getConstantFP(V: C1, DL, VT);
7639 case ISD::FMUL:
7640 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7641 return getConstantFP(V: C1, DL, VT);
7642 case ISD::FDIV:
7643 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7644 return getConstantFP(V: C1, DL, VT);
7645 case ISD::FREM:
7646 C1.mod(RHS: C2);
7647 return getConstantFP(V: C1, DL, VT);
7648 case ISD::FCOPYSIGN:
7649 C1.copySign(RHS: C2);
7650 return getConstantFP(V: C1, DL, VT);
7651 case ISD::FMINNUM:
7652 if (C1.isSignaling() || C2.isSignaling())
7653 return SDValue();
7654 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
7655 case ISD::FMAXNUM:
7656 if (C1.isSignaling() || C2.isSignaling())
7657 return SDValue();
7658 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
7659 case ISD::FMINIMUM:
7660 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
7661 case ISD::FMAXIMUM:
7662 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
7663 case ISD::FMINIMUMNUM:
7664 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
7665 case ISD::FMAXIMUMNUM:
7666 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
7667 default: break;
7668 }
7669 }
7670 if (N1CFP && Opcode == ISD::FP_ROUND) {
7671 APFloat C1 = N1CFP->getValueAPF(); // make copy
7672 bool Unused;
7673 // This can return overflow, underflow, or inexact; we don't care.
7674 // FIXME need to be more flexible about rounding mode.
7675 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7676 losesInfo: &Unused);
7677 return getConstantFP(V: C1, DL, VT);
7678 }
7679
7680 switch (Opcode) {
7681 case ISD::FSUB:
7682 // -0.0 - undef --> undef (consistent with "fneg undef")
7683 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
7684 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
7685 return getUNDEF(VT);
7686 [[fallthrough]];
7687
7688 case ISD::FADD:
7689 case ISD::FMUL:
7690 case ISD::FDIV:
7691 case ISD::FREM:
7692 // If both operands are undef, the result is undef. If 1 operand is undef,
7693 // the result is NaN. This should match the behavior of the IR optimizer.
7694 if (N1.isUndef() && N2.isUndef())
7695 return getUNDEF(VT);
7696 if (N1.isUndef() || N2.isUndef())
7697 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
7698 }
7699 return SDValue();
7700}
7701
7702SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
7703 const SDLoc &DL, EVT DstEltVT) {
7704 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
7705
7706 // If this is already the right type, we're done.
7707 if (SrcEltVT == DstEltVT)
7708 return SDValue(BV, 0);
7709
7710 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
7711 unsigned DstBitSize = DstEltVT.getSizeInBits();
7712
7713 // If this is a conversion of N elements of one type to N elements of another
7714 // type, convert each element. This handles FP<->INT cases.
7715 if (SrcBitSize == DstBitSize) {
7716 SmallVector<SDValue, 8> Ops;
7717 for (SDValue Op : BV->op_values()) {
7718 // If the vector element type is not legal, the BUILD_VECTOR operands
7719 // are promoted and implicitly truncated. Make that explicit here.
7720 if (Op.getValueType() != SrcEltVT)
7721 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
7722 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
7723 }
7724 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
7725 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
7726 return getBuildVector(VT, DL, Ops);
7727 }
7728
7729 // Otherwise, we're growing or shrinking the elements. To avoid having to
7730 // handle annoying details of growing/shrinking FP values, we convert them to
7731 // int first.
7732 if (SrcEltVT.isFloatingPoint()) {
7733 // Convert the input float vector to a int vector where the elements are the
7734 // same sizes.
7735 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
7736 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7737 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7738 DstEltVT);
7739 return SDValue();
7740 }
7741
7742 // Now we know the input is an integer vector. If the output is a FP type,
7743 // convert to integer first, then to FP of the right size.
7744 if (DstEltVT.isFloatingPoint()) {
7745 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
7746 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7747 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7748 DstEltVT);
7749 return SDValue();
7750 }
7751
7752 // Okay, we know the src/dst types are both integers of differing types.
7753 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
7754
7755 // Extract the constant raw bit data.
7756 BitVector UndefElements;
7757 SmallVector<APInt> RawBits;
7758 bool IsLE = getDataLayout().isLittleEndian();
7759 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
7760 return SDValue();
7761
7762 SmallVector<SDValue, 8> Ops;
7763 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
7764 if (UndefElements[I])
7765 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
7766 else
7767 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
7768 }
7769
7770 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
7771 return getBuildVector(VT, DL, Ops);
7772}
7773
7774SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
7775 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
7776
7777 // There's no need to assert on a byte-aligned pointer. All pointers are at
7778 // least byte aligned.
7779 if (A == Align(1))
7780 return Val;
7781
7782 SDVTList VTs = getVTList(VT: Val.getValueType());
7783 FoldingSetNodeID ID;
7784 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: VTs, OpList: {Val});
7785 ID.AddInteger(I: A.value());
7786
7787 void *IP = nullptr;
7788 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
7789 return SDValue(E, 0);
7790
7791 auto *N =
7792 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
7793 createOperands(Node: N, Vals: {Val});
7794
7795 CSEMap.InsertNode(N, InsertPos: IP);
7796 InsertNode(N);
7797
7798 SDValue V(N, 0);
7799 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7800 return V;
7801}
7802
7803SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7804 SDValue N1, SDValue N2) {
7805 SDNodeFlags Flags;
7806 if (Inserter)
7807 Flags = Inserter->getFlags();
7808 return getNode(Opcode, DL, VT, N1, N2, Flags);
7809}
7810
7811void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
7812 SDValue &N2) const {
7813 if (!TLI->isCommutativeBinOp(Opcode))
7814 return;
7815
7816 // Canonicalize:
7817 // binop(const, nonconst) -> binop(nonconst, const)
7818 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
7819 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
7820 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
7821 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
7822 if ((N1C && !N2C) || (N1CFP && !N2CFP))
7823 std::swap(a&: N1, b&: N2);
7824
7825 // Canonicalize:
7826 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
7827 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
7828 N2.getOpcode() == ISD::STEP_VECTOR)
7829 std::swap(a&: N1, b&: N2);
7830}
7831
7832SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7833 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
7834 assert(N1.getOpcode() != ISD::DELETED_NODE &&
7835 N2.getOpcode() != ISD::DELETED_NODE &&
7836 "Operand is DELETED_NODE!");
7837
7838 canonicalizeCommutativeBinop(Opcode, N1, N2);
7839
7840 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
7841 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
7842
7843 // Don't allow undefs in vector splats - we might be returning N2 when folding
7844 // to zero etc.
7845 ConstantSDNode *N2CV =
7846 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
7847
7848 switch (Opcode) {
7849 default: break;
7850 case ISD::TokenFactor:
7851 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
7852 N2.getValueType() == MVT::Other && "Invalid token factor!");
7853 // Fold trivial token factors.
7854 if (N1.getOpcode() == ISD::EntryToken) return N2;
7855 if (N2.getOpcode() == ISD::EntryToken) return N1;
7856 if (N1 == N2) return N1;
7857 break;
7858 case ISD::BUILD_VECTOR: {
7859 // Attempt to simplify BUILD_VECTOR.
7860 SDValue Ops[] = {N1, N2};
7861 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7862 return V;
7863 break;
7864 }
7865 case ISD::CONCAT_VECTORS: {
7866 SDValue Ops[] = {N1, N2};
7867 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
7868 return V;
7869 break;
7870 }
7871 case ISD::AND:
7872 assert(VT.isInteger() && "This operator does not apply to FP types!");
7873 assert(N1.getValueType() == N2.getValueType() &&
7874 N1.getValueType() == VT && "Binary operator types must match!");
7875 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
7876 // worth handling here.
7877 if (N2CV && N2CV->isZero())
7878 return N2;
7879 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
7880 return N1;
7881 break;
7882 case ISD::OR:
7883 case ISD::XOR:
7884 case ISD::ADD:
7885 case ISD::PTRADD:
7886 case ISD::SUB:
7887 assert(VT.isInteger() && "This operator does not apply to FP types!");
7888 assert(N1.getValueType() == N2.getValueType() &&
7889 N1.getValueType() == VT && "Binary operator types must match!");
7890 // The equal operand types requirement is unnecessarily strong for PTRADD.
7891 // However, the SelectionDAGBuilder does not generate PTRADDs with different
7892 // operand types, and we'd need to re-implement GEP's non-standard wrapping
7893 // logic everywhere where PTRADDs may be folded or combined to properly
7894 // support them. If/when we introduce pointer types to the SDAG, we will
7895 // need to relax this constraint.
7896
7897 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
7898 // it's worth handling here.
7899 if (N2CV && N2CV->isZero())
7900 return N1;
7901 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
7902 VT.getScalarType() == MVT::i1)
7903 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
7904 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
7905 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
7906 N2.getOpcode() == ISD::VSCALE) {
7907 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
7908 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
7909 return getVScale(DL, VT, MulImm: C1 + C2);
7910 }
7911 break;
7912 case ISD::MUL:
7913 assert(VT.isInteger() && "This operator does not apply to FP types!");
7914 assert(N1.getValueType() == N2.getValueType() &&
7915 N1.getValueType() == VT && "Binary operator types must match!");
7916 if (VT.getScalarType() == MVT::i1)
7917 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
7918 if (N2CV && N2CV->isZero())
7919 return N2;
7920 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7921 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
7922 const APInt &N2CImm = N2C->getAPIntValue();
7923 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
7924 }
7925 break;
7926 case ISD::UDIV:
7927 case ISD::UREM:
7928 case ISD::MULHU:
7929 case ISD::MULHS:
7930 case ISD::SDIV:
7931 case ISD::SREM:
7932 case ISD::SADDSAT:
7933 case ISD::SSUBSAT:
7934 case ISD::UADDSAT:
7935 case ISD::USUBSAT:
7936 assert(VT.isInteger() && "This operator does not apply to FP types!");
7937 assert(N1.getValueType() == N2.getValueType() &&
7938 N1.getValueType() == VT && "Binary operator types must match!");
7939 if (VT.getScalarType() == MVT::i1) {
7940 // fold (add_sat x, y) -> (or x, y) for bool types.
7941 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
7942 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
7943 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
7944 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
7945 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
7946 }
7947 break;
7948 case ISD::SCMP:
7949 case ISD::UCMP:
7950 assert(N1.getValueType() == N2.getValueType() &&
7951 "Types of operands of UCMP/SCMP must match");
7952 assert(N1.getValueType().isVector() == VT.isVector() &&
7953 "Operands and return type of must both be scalars or vectors");
7954 if (VT.isVector())
7955 assert(VT.getVectorElementCount() ==
7956 N1.getValueType().getVectorElementCount() &&
7957 "Result and operands must have the same number of elements");
7958 break;
7959 case ISD::AVGFLOORS:
7960 case ISD::AVGFLOORU:
7961 case ISD::AVGCEILS:
7962 case ISD::AVGCEILU:
7963 assert(VT.isInteger() && "This operator does not apply to FP types!");
7964 assert(N1.getValueType() == N2.getValueType() &&
7965 N1.getValueType() == VT && "Binary operator types must match!");
7966 break;
7967 case ISD::ABDS:
7968 case ISD::ABDU:
7969 assert(VT.isInteger() && "This operator does not apply to FP types!");
7970 assert(N1.getValueType() == N2.getValueType() &&
7971 N1.getValueType() == VT && "Binary operator types must match!");
7972 if (VT.getScalarType() == MVT::i1)
7973 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
7974 break;
7975 case ISD::SMIN:
7976 case ISD::UMAX:
7977 assert(VT.isInteger() && "This operator does not apply to FP types!");
7978 assert(N1.getValueType() == N2.getValueType() &&
7979 N1.getValueType() == VT && "Binary operator types must match!");
7980 if (VT.getScalarType() == MVT::i1)
7981 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
7982 break;
7983 case ISD::SMAX:
7984 case ISD::UMIN:
7985 assert(VT.isInteger() && "This operator does not apply to FP types!");
7986 assert(N1.getValueType() == N2.getValueType() &&
7987 N1.getValueType() == VT && "Binary operator types must match!");
7988 if (VT.getScalarType() == MVT::i1)
7989 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
7990 break;
7991 case ISD::FADD:
7992 case ISD::FSUB:
7993 case ISD::FMUL:
7994 case ISD::FDIV:
7995 case ISD::FREM:
7996 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7997 assert(N1.getValueType() == N2.getValueType() &&
7998 N1.getValueType() == VT && "Binary operator types must match!");
7999 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8000 return V;
8001 break;
8002 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8003 assert(N1.getValueType() == VT &&
8004 N1.getValueType().isFloatingPoint() &&
8005 N2.getValueType().isFloatingPoint() &&
8006 "Invalid FCOPYSIGN!");
8007 break;
8008 case ISD::SHL:
8009 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8010 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8011 const APInt &ShiftImm = N2C->getAPIntValue();
8012 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8013 }
8014 [[fallthrough]];
8015 case ISD::SRA:
8016 case ISD::SRL:
8017 if (SDValue V = simplifyShift(X: N1, Y: N2))
8018 return V;
8019 [[fallthrough]];
8020 case ISD::ROTL:
8021 case ISD::ROTR:
8022 case ISD::SSHLSAT:
8023 case ISD::USHLSAT:
8024 assert(VT == N1.getValueType() &&
8025 "Shift operators return type must be the same as their first arg");
8026 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8027 "Shifts only work on integers");
8028 assert((!VT.isVector() || VT == N2.getValueType()) &&
8029 "Vector shift amounts must be in the same as their first arg");
8030 // Verify that the shift amount VT is big enough to hold valid shift
8031 // amounts. This catches things like trying to shift an i1024 value by an
8032 // i8, which is easy to fall into in generic code that uses
8033 // TLI.getShiftAmount().
8034 assert(N2.getValueType().getScalarSizeInBits() >=
8035 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8036 "Invalid use of small shift amount with oversized value!");
8037
8038 // Always fold shifts of i1 values so the code generator doesn't need to
8039 // handle them. Since we know the size of the shift has to be less than the
8040 // size of the value, the shift/rotate count is guaranteed to be zero.
8041 if (VT == MVT::i1)
8042 return N1;
8043 if (N2CV && N2CV->isZero())
8044 return N1;
8045 break;
8046 case ISD::FP_ROUND:
8047 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8048 VT.bitsLE(N1.getValueType()) && N2C &&
8049 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8050 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8051 if (N1.getValueType() == VT) return N1; // noop conversion.
8052 break;
8053 case ISD::AssertNoFPClass: {
8054 assert(N1.getValueType().isFloatingPoint() &&
8055 "AssertNoFPClass is used for a non-floating type");
8056 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8057 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8058 assert(llvm::to_underlying(NoFPClass) <=
8059 BitmaskEnumDetail::Mask<FPClassTest>() &&
8060 "FPClassTest value too large");
8061 (void)NoFPClass;
8062 break;
8063 }
8064 case ISD::AssertSext:
8065 case ISD::AssertZext: {
8066 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8067 assert(VT == N1.getValueType() && "Not an inreg extend!");
8068 assert(VT.isInteger() && EVT.isInteger() &&
8069 "Cannot *_EXTEND_INREG FP types");
8070 assert(!EVT.isVector() &&
8071 "AssertSExt/AssertZExt type should be the vector element type "
8072 "rather than the vector type!");
8073 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8074 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8075 break;
8076 }
8077 case ISD::SIGN_EXTEND_INREG: {
8078 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8079 assert(VT == N1.getValueType() && "Not an inreg extend!");
8080 assert(VT.isInteger() && EVT.isInteger() &&
8081 "Cannot *_EXTEND_INREG FP types");
8082 assert(EVT.isVector() == VT.isVector() &&
8083 "SIGN_EXTEND_INREG type should be vector iff the operand "
8084 "type is vector!");
8085 assert((!EVT.isVector() ||
8086 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8087 "Vector element counts must match in SIGN_EXTEND_INREG");
8088 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8089 if (EVT == VT) return N1; // Not actually extending
8090 break;
8091 }
8092 case ISD::FP_TO_SINT_SAT:
8093 case ISD::FP_TO_UINT_SAT: {
8094 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8095 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8096 assert(N1.getValueType().isVector() == VT.isVector() &&
8097 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8098 "vector!");
8099 assert((!VT.isVector() || VT.getVectorElementCount() ==
8100 N1.getValueType().getVectorElementCount()) &&
8101 "Vector element counts must match in FP_TO_*INT_SAT");
8102 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8103 "Type to saturate to must be a scalar.");
8104 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8105 "Not extending!");
8106 break;
8107 }
8108 case ISD::EXTRACT_VECTOR_ELT:
8109 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8110 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8111 element type of the vector.");
8112
8113 // Extract from an undefined value or using an undefined index is undefined.
8114 if (N1.isUndef() || N2.isUndef())
8115 return getUNDEF(VT);
8116
8117 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
8118 // vectors. For scalable vectors we will provide appropriate support for
8119 // dealing with arbitrary indices.
8120 if (N2C && N1.getValueType().isFixedLengthVector() &&
8121 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8122 return getUNDEF(VT);
8123
8124 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8125 // expanding copies of large vectors from registers. This only works for
8126 // fixed length vectors, since we need to know the exact number of
8127 // elements.
8128 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8129 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8130 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8131 return getExtractVectorElt(DL, VT,
8132 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8133 Idx: N2C->getZExtValue() % Factor);
8134 }
8135
8136 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8137 // lowering is expanding large vector constants.
8138 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8139 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8140 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8141 N1.getValueType().isFixedLengthVector()) &&
8142 "BUILD_VECTOR used for scalable vectors");
8143 unsigned Index =
8144 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8145 SDValue Elt = N1.getOperand(i: Index);
8146
8147 if (VT != Elt.getValueType())
8148 // If the vector element type is not legal, the BUILD_VECTOR operands
8149 // are promoted and implicitly truncated, and the result implicitly
8150 // extended. Make that explicit here.
8151 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8152
8153 return Elt;
8154 }
8155
8156 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8157 // operations are lowered to scalars.
8158 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8159 // If the indices are the same, return the inserted element else
8160 // if the indices are known different, extract the element from
8161 // the original vector.
8162 SDValue N1Op2 = N1.getOperand(i: 2);
8163 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8164
8165 if (N1Op2C && N2C) {
8166 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8167 if (VT == N1.getOperand(i: 1).getValueType())
8168 return N1.getOperand(i: 1);
8169 if (VT.isFloatingPoint()) {
8170 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8171 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8172 }
8173 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8174 }
8175 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8176 }
8177 }
8178
8179 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8180 // when vector types are scalarized and v1iX is legal.
8181 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8182 // Here we are completely ignoring the extract element index (N2),
8183 // which is fine for fixed width vectors, since any index other than 0
8184 // is undefined anyway. However, this cannot be ignored for scalable
8185 // vectors - in theory we could support this, but we don't want to do this
8186 // without a profitability check.
8187 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8188 N1.getValueType().isFixedLengthVector() &&
8189 N1.getValueType().getVectorNumElements() == 1) {
8190 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8191 N2: N1.getOperand(i: 1));
8192 }
8193 break;
8194 case ISD::EXTRACT_ELEMENT:
8195 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8196 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8197 (N1.getValueType().isInteger() == VT.isInteger()) &&
8198 N1.getValueType() != VT &&
8199 "Wrong types for EXTRACT_ELEMENT!");
8200
8201 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8202 // 64-bit integers into 32-bit parts. Instead of building the extract of
8203 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8204 if (N1.getOpcode() == ISD::BUILD_PAIR)
8205 return N1.getOperand(i: N2C->getZExtValue());
8206
8207 // EXTRACT_ELEMENT of a constant int is also very common.
8208 if (N1C) {
8209 unsigned ElementSize = VT.getSizeInBits();
8210 unsigned Shift = ElementSize * N2C->getZExtValue();
8211 const APInt &Val = N1C->getAPIntValue();
8212 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8213 }
8214 break;
8215 case ISD::EXTRACT_SUBVECTOR: {
8216 EVT N1VT = N1.getValueType();
8217 assert(VT.isVector() && N1VT.isVector() &&
8218 "Extract subvector VTs must be vectors!");
8219 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8220 "Extract subvector VTs must have the same element type!");
8221 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8222 "Cannot extract a scalable vector from a fixed length vector!");
8223 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8224 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8225 "Extract subvector must be from larger vector to smaller vector!");
8226 assert(N2C && "Extract subvector index must be a constant");
8227 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8228 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8229 N1VT.getVectorMinNumElements()) &&
8230 "Extract subvector overflow!");
8231 assert(N2C->getAPIntValue().getBitWidth() ==
8232 TLI->getVectorIdxWidth(getDataLayout()) &&
8233 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8234 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8235 "Extract index is not a multiple of the output vector length");
8236
8237 // Trivial extraction.
8238 if (VT == N1VT)
8239 return N1;
8240
8241 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8242 if (N1.isUndef())
8243 return getUNDEF(VT);
8244
8245 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8246 // the concat have the same type as the extract.
8247 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8248 VT == N1.getOperand(i: 0).getValueType()) {
8249 unsigned Factor = VT.getVectorMinNumElements();
8250 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8251 }
8252
8253 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8254 // during shuffle legalization.
8255 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8256 VT == N1.getOperand(i: 1).getValueType())
8257 return N1.getOperand(i: 1);
8258 break;
8259 }
8260 }
8261
8262 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8263 switch (Opcode) {
8264 case ISD::XOR:
8265 case ISD::ADD:
8266 case ISD::PTRADD:
8267 case ISD::SUB:
8268 case ISD::SIGN_EXTEND_INREG:
8269 case ISD::UDIV:
8270 case ISD::SDIV:
8271 case ISD::UREM:
8272 case ISD::SREM:
8273 case ISD::MUL:
8274 case ISD::AND:
8275 case ISD::SSUBSAT:
8276 case ISD::USUBSAT:
8277 case ISD::UMIN:
8278 case ISD::OR:
8279 case ISD::SADDSAT:
8280 case ISD::UADDSAT:
8281 case ISD::UMAX:
8282 case ISD::SMAX:
8283 case ISD::SMIN:
8284 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8285 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8286 }
8287 }
8288
8289 // Canonicalize an UNDEF to the RHS, even over a constant.
8290 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8291 if (TLI->isCommutativeBinOp(Opcode)) {
8292 std::swap(a&: N1, b&: N2);
8293 } else {
8294 switch (Opcode) {
8295 case ISD::PTRADD:
8296 case ISD::SUB:
8297 // fold op(undef, non_undef_arg2) -> undef.
8298 return N1;
8299 case ISD::SIGN_EXTEND_INREG:
8300 case ISD::UDIV:
8301 case ISD::SDIV:
8302 case ISD::UREM:
8303 case ISD::SREM:
8304 case ISD::SSUBSAT:
8305 case ISD::USUBSAT:
8306 // fold op(undef, non_undef_arg2) -> 0.
8307 return getConstant(Val: 0, DL, VT);
8308 }
8309 }
8310 }
8311
8312 // Fold a bunch of operators when the RHS is undef.
8313 if (N2.getOpcode() == ISD::UNDEF) {
8314 switch (Opcode) {
8315 case ISD::XOR:
8316 if (N1.getOpcode() == ISD::UNDEF)
8317 // Handle undef ^ undef -> 0 special case. This is a common
8318 // idiom (misuse).
8319 return getConstant(Val: 0, DL, VT);
8320 [[fallthrough]];
8321 case ISD::ADD:
8322 case ISD::PTRADD:
8323 case ISD::SUB:
8324 // fold op(arg1, undef) -> undef.
8325 return N2;
8326 case ISD::UDIV:
8327 case ISD::SDIV:
8328 case ISD::UREM:
8329 case ISD::SREM:
8330 // fold op(arg1, undef) -> poison.
8331 return getPOISON(VT);
8332 case ISD::MUL:
8333 case ISD::AND:
8334 case ISD::SSUBSAT:
8335 case ISD::USUBSAT:
8336 case ISD::UMIN:
8337 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
8338 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
8339 case ISD::OR:
8340 case ISD::SADDSAT:
8341 case ISD::UADDSAT:
8342 case ISD::UMAX:
8343 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
8344 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
8345 case ISD::SMAX:
8346 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
8347 return N1.getOpcode() == ISD::UNDEF
8348 ? N2
8349 : getConstant(
8350 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
8351 VT);
8352 case ISD::SMIN:
8353 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
8354 return N1.getOpcode() == ISD::UNDEF
8355 ? N2
8356 : getConstant(
8357 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
8358 VT);
8359 }
8360 }
8361
8362 // Perform trivial constant folding.
8363 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
8364 return SV;
8365
8366 // Memoize this node if possible.
8367 SDNode *N;
8368 SDVTList VTs = getVTList(VT);
8369 SDValue Ops[] = {N1, N2};
8370 if (VT != MVT::Glue) {
8371 FoldingSetNodeID ID;
8372 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8373 void *IP = nullptr;
8374 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8375 E->intersectFlagsWith(Flags);
8376 return SDValue(E, 0);
8377 }
8378
8379 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8380 N->setFlags(Flags);
8381 createOperands(Node: N, Vals: Ops);
8382 CSEMap.InsertNode(N, InsertPos: IP);
8383 } else {
8384 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8385 createOperands(Node: N, Vals: Ops);
8386 }
8387
8388 InsertNode(N);
8389 SDValue V = SDValue(N, 0);
8390 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8391 return V;
8392}
8393
8394SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8395 SDValue N1, SDValue N2, SDValue N3) {
8396 SDNodeFlags Flags;
8397 if (Inserter)
8398 Flags = Inserter->getFlags();
8399 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
8400}
8401
8402SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8403 SDValue N1, SDValue N2, SDValue N3,
8404 const SDNodeFlags Flags) {
8405 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8406 N2.getOpcode() != ISD::DELETED_NODE &&
8407 N3.getOpcode() != ISD::DELETED_NODE &&
8408 "Operand is DELETED_NODE!");
8409 // Perform various simplifications.
8410 switch (Opcode) {
8411 case ISD::BUILD_VECTOR: {
8412 // Attempt to simplify BUILD_VECTOR.
8413 SDValue Ops[] = {N1, N2, N3};
8414 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8415 return V;
8416 break;
8417 }
8418 case ISD::CONCAT_VECTORS: {
8419 SDValue Ops[] = {N1, N2, N3};
8420 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8421 return V;
8422 break;
8423 }
8424 case ISD::SETCC: {
8425 assert(VT.isInteger() && "SETCC result type must be an integer!");
8426 assert(N1.getValueType() == N2.getValueType() &&
8427 "SETCC operands must have the same type!");
8428 assert(VT.isVector() == N1.getValueType().isVector() &&
8429 "SETCC type should be vector iff the operand type is vector!");
8430 assert((!VT.isVector() || VT.getVectorElementCount() ==
8431 N1.getValueType().getVectorElementCount()) &&
8432 "SETCC vector element counts must match!");
8433 // Use FoldSetCC to simplify SETCC's.
8434 if (SDValue V =
8435 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
8436 return V;
8437 break;
8438 }
8439 case ISD::SELECT:
8440 case ISD::VSELECT:
8441 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
8442 return V;
8443 break;
8444 case ISD::VECTOR_SHUFFLE:
8445 llvm_unreachable("should use getVectorShuffle constructor!");
8446 case ISD::VECTOR_SPLICE_LEFT:
8447 if (isNullConstant(V: N3))
8448 return N1;
8449 break;
8450 case ISD::VECTOR_SPLICE_RIGHT:
8451 if (isNullConstant(V: N3))
8452 return N2;
8453 break;
8454 case ISD::INSERT_VECTOR_ELT: {
8455 assert(VT.isVector() && VT == N1.getValueType() &&
8456 "INSERT_VECTOR_ELT vector type mismatch");
8457 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
8458 "INSERT_VECTOR_ELT scalar fp/int mismatch");
8459 assert((!VT.isFloatingPoint() ||
8460 VT.getVectorElementType() == N2.getValueType()) &&
8461 "INSERT_VECTOR_ELT fp scalar type mismatch");
8462 assert((!VT.isInteger() ||
8463 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
8464 "INSERT_VECTOR_ELT int scalar size mismatch");
8465
8466 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
8467 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
8468 // for scalable vectors where we will generate appropriate code to
8469 // deal with out-of-bounds cases correctly.
8470 if (N3C && VT.isFixedLengthVector() &&
8471 N3C->getZExtValue() >= VT.getVectorNumElements())
8472 return getUNDEF(VT);
8473
8474 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
8475 if (N3.isUndef())
8476 return getUNDEF(VT);
8477
8478 // If inserting poison, just use the input vector.
8479 if (N2.getOpcode() == ISD::POISON)
8480 return N1;
8481
8482 // Inserting undef into undef/poison is still undef.
8483 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8484 return getUNDEF(VT);
8485
8486 // If the inserted element is an UNDEF, just use the input vector.
8487 // But not if skipping the insert could make the result more poisonous.
8488 if (N2.isUndef()) {
8489 if (N3C && VT.isFixedLengthVector()) {
8490 APInt EltMask =
8491 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
8492 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8493 return N1;
8494 } else if (isGuaranteedNotToBePoison(Op: N1))
8495 return N1;
8496 }
8497 break;
8498 }
8499 case ISD::INSERT_SUBVECTOR: {
8500 // If inserting poison, just use the input vector,
8501 if (N2.getOpcode() == ISD::POISON)
8502 return N1;
8503
8504 // Inserting undef into undef/poison is still undef.
8505 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8506 return getUNDEF(VT);
8507
8508 EVT N2VT = N2.getValueType();
8509 assert(VT == N1.getValueType() &&
8510 "Dest and insert subvector source types must match!");
8511 assert(VT.isVector() && N2VT.isVector() &&
8512 "Insert subvector VTs must be vectors!");
8513 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
8514 "Insert subvector VTs must have the same element type!");
8515 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
8516 "Cannot insert a scalable vector into a fixed length vector!");
8517 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8518 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
8519 "Insert subvector must be from smaller vector to larger vector!");
8520 assert(isa<ConstantSDNode>(N3) &&
8521 "Insert subvector index must be constant");
8522 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8523 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
8524 VT.getVectorMinNumElements()) &&
8525 "Insert subvector overflow!");
8526 assert(N3->getAsAPIntVal().getBitWidth() ==
8527 TLI->getVectorIdxWidth(getDataLayout()) &&
8528 "Constant index for INSERT_SUBVECTOR has an invalid size");
8529
8530 // Trivial insertion.
8531 if (VT == N2VT)
8532 return N2;
8533
8534 // If this is an insert of an extracted vector into an undef/poison vector,
8535 // we can just use the input to the extract. But not if skipping the
8536 // extract+insert could make the result more poisonous.
8537 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8538 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
8539 if (N1.getOpcode() == ISD::POISON)
8540 return N2.getOperand(i: 0);
8541 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
8542 unsigned LoBit = N3->getAsZExtVal();
8543 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8544 APInt EltMask =
8545 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8546 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
8547 return N2.getOperand(i: 0);
8548 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
8549 return N2.getOperand(i: 0);
8550 }
8551
8552 // If the inserted subvector is UNDEF, just use the input vector.
8553 // But not if skipping the insert could make the result more poisonous.
8554 if (N2.isUndef()) {
8555 if (VT.isFixedLengthVector()) {
8556 unsigned LoBit = N3->getAsZExtVal();
8557 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8558 APInt EltMask =
8559 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8560 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8561 return N1;
8562 } else if (isGuaranteedNotToBePoison(Op: N1))
8563 return N1;
8564 }
8565 break;
8566 }
8567 case ISD::BITCAST:
8568 // Fold bit_convert nodes from a type to themselves.
8569 if (N1.getValueType() == VT)
8570 return N1;
8571 break;
8572 case ISD::VP_TRUNCATE:
8573 case ISD::VP_SIGN_EXTEND:
8574 case ISD::VP_ZERO_EXTEND:
8575 // Don't create noop casts.
8576 if (N1.getValueType() == VT)
8577 return N1;
8578 break;
8579 case ISD::VECTOR_COMPRESS: {
8580 [[maybe_unused]] EVT VecVT = N1.getValueType();
8581 [[maybe_unused]] EVT MaskVT = N2.getValueType();
8582 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
8583 assert(VT == VecVT && "Vector and result type don't match.");
8584 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
8585 "All inputs must be vectors.");
8586 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
8587 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
8588 "Vector and mask must have same number of elements.");
8589
8590 if (N1.isUndef() || N2.isUndef())
8591 return N3;
8592
8593 break;
8594 }
8595 case ISD::PARTIAL_REDUCE_UMLA:
8596 case ISD::PARTIAL_REDUCE_SMLA:
8597 case ISD::PARTIAL_REDUCE_SUMLA:
8598 case ISD::PARTIAL_REDUCE_FMLA: {
8599 [[maybe_unused]] EVT AccVT = N1.getValueType();
8600 [[maybe_unused]] EVT Input1VT = N2.getValueType();
8601 [[maybe_unused]] EVT Input2VT = N3.getValueType();
8602 assert(Input1VT.isVector() && Input1VT == Input2VT &&
8603 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8604 "node to have the same type!");
8605 assert(VT.isVector() && VT == AccVT &&
8606 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
8607 "the same type as its result!");
8608 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
8609 AccVT.getVectorElementCount()) &&
8610 "Expected the element count of the second and third operands of the "
8611 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
8612 "element count of the first operand and the result!");
8613 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
8614 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8615 "node to have an element type which is the same as or smaller than "
8616 "the element type of the first operand and result!");
8617 break;
8618 }
8619 }
8620
8621 // Perform trivial constant folding for arithmetic operators.
8622 switch (Opcode) {
8623 case ISD::FMA:
8624 case ISD::FMAD:
8625 case ISD::SETCC:
8626 case ISD::FSHL:
8627 case ISD::FSHR:
8628 if (SDValue SV =
8629 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
8630 return SV;
8631 break;
8632 }
8633
8634 // Memoize node if it doesn't produce a glue result.
8635 SDNode *N;
8636 SDVTList VTs = getVTList(VT);
8637 SDValue Ops[] = {N1, N2, N3};
8638 if (VT != MVT::Glue) {
8639 FoldingSetNodeID ID;
8640 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8641 void *IP = nullptr;
8642 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8643 E->intersectFlagsWith(Flags);
8644 return SDValue(E, 0);
8645 }
8646
8647 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8648 N->setFlags(Flags);
8649 createOperands(Node: N, Vals: Ops);
8650 CSEMap.InsertNode(N, InsertPos: IP);
8651 } else {
8652 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8653 createOperands(Node: N, Vals: Ops);
8654 }
8655
8656 InsertNode(N);
8657 SDValue V = SDValue(N, 0);
8658 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8659 return V;
8660}
8661
8662SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8663 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8664 const SDNodeFlags Flags) {
8665 SDValue Ops[] = { N1, N2, N3, N4 };
8666 return getNode(Opcode, DL, VT, Ops, Flags);
8667}
8668
8669SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8670 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
8671 SDNodeFlags Flags;
8672 if (Inserter)
8673 Flags = Inserter->getFlags();
8674 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
8675}
8676
8677SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8678 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8679 SDValue N5, const SDNodeFlags Flags) {
8680 SDValue Ops[] = { N1, N2, N3, N4, N5 };
8681 return getNode(Opcode, DL, VT, Ops, Flags);
8682}
8683
8684SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8685 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8686 SDValue N5) {
8687 SDNodeFlags Flags;
8688 if (Inserter)
8689 Flags = Inserter->getFlags();
8690 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
8691}
8692
8693/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
8694/// the incoming stack arguments to be loaded from the stack.
8695SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
8696 SmallVector<SDValue, 8> ArgChains;
8697
8698 // Include the original chain at the beginning of the list. When this is
8699 // used by target LowerCall hooks, this helps legalize find the
8700 // CALLSEQ_BEGIN node.
8701 ArgChains.push_back(Elt: Chain);
8702
8703 // Add a chain value for each stack argument.
8704 for (SDNode *U : getEntryNode().getNode()->users())
8705 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
8706 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
8707 if (FI->getIndex() < 0)
8708 ArgChains.push_back(Elt: SDValue(L, 1));
8709
8710 // Build a tokenfactor for all the chains.
8711 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
8712}
8713
8714/// getMemsetValue - Vectorized representation of the memset value
8715/// operand.
8716static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
8717 const SDLoc &dl) {
8718 assert(!Value.isUndef());
8719
8720 unsigned NumBits = VT.getScalarSizeInBits();
8721 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
8722 assert(C->getAPIntValue().getBitWidth() == 8);
8723 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
8724 if (VT.isInteger()) {
8725 bool IsOpaque = VT.getSizeInBits() > 64 ||
8726 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
8727 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
8728 }
8729 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
8730 }
8731
8732 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
8733 EVT IntVT = VT.getScalarType();
8734 if (!IntVT.isInteger())
8735 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
8736
8737 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
8738 if (NumBits > 8) {
8739 // Use a multiplication with 0x010101... to extend the input to the
8740 // required length.
8741 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
8742 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
8743 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
8744 }
8745
8746 if (VT != Value.getValueType() && !VT.isInteger())
8747 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
8748 if (VT != Value.getValueType())
8749 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
8750
8751 return Value;
8752}
8753
8754/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
8755/// used when a memcpy is turned into a memset when the source is a constant
8756/// string ptr.
8757static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
8758 const TargetLowering &TLI,
8759 const ConstantDataArraySlice &Slice) {
8760 // Handle vector with all elements zero.
8761 if (Slice.Array == nullptr) {
8762 if (VT.isInteger())
8763 return DAG.getConstant(Val: 0, DL: dl, VT);
8764 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
8765 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
8766 }
8767
8768 assert(!VT.isVector() && "Can't handle vector type here!");
8769 unsigned NumVTBits = VT.getSizeInBits();
8770 unsigned NumVTBytes = NumVTBits / 8;
8771 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
8772
8773 APInt Val(NumVTBits, 0);
8774 if (DAG.getDataLayout().isLittleEndian()) {
8775 for (unsigned i = 0; i != NumBytes; ++i)
8776 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
8777 } else {
8778 for (unsigned i = 0; i != NumBytes; ++i)
8779 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
8780 }
8781
8782 // If the "cost" of materializing the integer immediate is less than the cost
8783 // of a load, then it is cost effective to turn the load into the immediate.
8784 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
8785 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
8786 return DAG.getConstant(Val, DL: dl, VT);
8787 return SDValue();
8788}
8789
8790SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
8791 const SDLoc &DL,
8792 const SDNodeFlags Flags) {
8793 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
8794 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
8795}
8796
8797SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
8798 const SDLoc &DL,
8799 const SDNodeFlags Flags) {
8800 assert(Offset.getValueType().isInteger());
8801 EVT BasePtrVT = Ptr.getValueType();
8802 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
8803 PtrVT: BasePtrVT))
8804 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
8805 // InBounds only applies to PTRADD, don't set it if we generate ADD.
8806 SDNodeFlags AddFlags = Flags;
8807 AddFlags.setInBounds(false);
8808 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
8809}
8810
8811/// Returns true if memcpy source is constant data.
8812static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
8813 uint64_t SrcDelta = 0;
8814 GlobalAddressSDNode *G = nullptr;
8815 if (Src.getOpcode() == ISD::GlobalAddress)
8816 G = cast<GlobalAddressSDNode>(Val&: Src);
8817 else if (Src->isAnyAdd() &&
8818 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
8819 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
8820 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
8821 SrcDelta = Src.getConstantOperandVal(i: 1);
8822 }
8823 if (!G)
8824 return false;
8825
8826 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
8827 Offset: SrcDelta + G->getOffset());
8828}
8829
8830static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
8831 SelectionDAG &DAG) {
8832 // On Darwin, -Os means optimize for size without hurting performance, so
8833 // only really optimize for size when -Oz (MinSize) is used.
8834 if (MF.getTarget().getTargetTriple().isOSDarwin())
8835 return MF.getFunction().hasMinSize();
8836 return DAG.shouldOptForSize();
8837}
8838
8839static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
8840 SmallVector<SDValue, 32> &OutChains, unsigned From,
8841 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
8842 SmallVector<SDValue, 16> &OutStoreChains) {
8843 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
8844 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
8845 SmallVector<SDValue, 16> GluedLoadChains;
8846 for (unsigned i = From; i < To; ++i) {
8847 OutChains.push_back(Elt: OutLoadChains[i]);
8848 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
8849 }
8850
8851 // Chain for all loads.
8852 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
8853 Ops: GluedLoadChains);
8854
8855 for (unsigned i = From; i < To; ++i) {
8856 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
8857 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
8858 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
8859 MMO: ST->getMemOperand());
8860 OutChains.push_back(Elt: NewStore);
8861 }
8862}
8863
8864static SDValue getMemcpyLoadsAndStores(
8865 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
8866 uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline,
8867 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
8868 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
8869 // Turn a memcpy of undef to nop.
8870 // FIXME: We need to honor volatile even is Src is undef.
8871 if (Src.isUndef())
8872 return Chain;
8873
8874 // Expand memcpy to a series of load and store ops if the size operand falls
8875 // below a certain threshold.
8876 // TODO: In the AlwaysInline case, if the size is big then generate a loop
8877 // rather than maybe a humongous number of loads and stores.
8878 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8879 const DataLayout &DL = DAG.getDataLayout();
8880 LLVMContext &C = *DAG.getContext();
8881 std::vector<EVT> MemOps;
8882 bool DstAlignCanChange = false;
8883 MachineFunction &MF = DAG.getMachineFunction();
8884 MachineFrameInfo &MFI = MF.getFrameInfo();
8885 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8886 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
8887 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
8888 DstAlignCanChange = true;
8889 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
8890 if (!SrcAlign || Alignment > *SrcAlign)
8891 SrcAlign = Alignment;
8892 assert(SrcAlign && "SrcAlign must be set");
8893 ConstantDataArraySlice Slice;
8894 // If marked as volatile, perform a copy even when marked as constant.
8895 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
8896 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
8897 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
8898 const MemOp Op = isZeroConstant
8899 ? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment,
8900 /*IsZeroMemset*/ true, IsVolatile: isVol)
8901 : MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment,
8902 SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
8903 if (!TLI.findOptimalMemOpLowering(
8904 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
8905 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
8906 return SDValue();
8907
8908 if (DstAlignCanChange) {
8909 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
8910 Align NewAlign = DL.getABITypeAlign(Ty);
8911
8912 // Don't promote to an alignment that would require dynamic stack
8913 // realignment which may conflict with optimizations such as tail call
8914 // optimization.
8915 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
8916 if (!TRI->hasStackRealignment(MF))
8917 if (MaybeAlign StackAlign = DL.getStackAlignment())
8918 NewAlign = std::min(a: NewAlign, b: *StackAlign);
8919
8920 if (NewAlign > Alignment) {
8921 // Give the stack frame object a larger alignment if needed.
8922 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
8923 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
8924 Alignment = NewAlign;
8925 }
8926 }
8927
8928 // Prepare AAInfo for loads/stores after lowering this memcpy.
8929 AAMDNodes NewAAInfo = AAInfo;
8930 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8931
8932 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
8933 bool isConstant =
8934 BatchAA && SrcVal &&
8935 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
8936
8937 MachineMemOperand::Flags MMOFlags =
8938 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
8939 SmallVector<SDValue, 16> OutLoadChains;
8940 SmallVector<SDValue, 16> OutStoreChains;
8941 SmallVector<SDValue, 32> OutChains;
8942 unsigned NumMemOps = MemOps.size();
8943 uint64_t SrcOff = 0, DstOff = 0;
8944 for (unsigned i = 0; i != NumMemOps; ++i) {
8945 EVT VT = MemOps[i];
8946 unsigned VTSize = VT.getSizeInBits() / 8;
8947 SDValue Value, Store;
8948
8949 if (VTSize > Size) {
8950 // Issuing an unaligned load / store pair that overlaps with the previous
8951 // pair. Adjust the offset accordingly.
8952 assert(i == NumMemOps-1 && i != 0);
8953 SrcOff -= VTSize - Size;
8954 DstOff -= VTSize - Size;
8955 }
8956
8957 if (CopyFromConstant &&
8958 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
8959 // It's unlikely a store of a vector immediate can be done in a single
8960 // instruction. It would require a load from a constantpool first.
8961 // We only handle zero vectors here.
8962 // FIXME: Handle other cases where store of vector immediate is done in
8963 // a single instruction.
8964 ConstantDataArraySlice SubSlice;
8965 if (SrcOff < Slice.Length) {
8966 SubSlice = Slice;
8967 SubSlice.move(Delta: SrcOff);
8968 } else {
8969 // This is an out-of-bounds access and hence UB. Pretend we read zero.
8970 SubSlice.Array = nullptr;
8971 SubSlice.Offset = 0;
8972 SubSlice.Length = VTSize;
8973 }
8974 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
8975 if (Value.getNode()) {
8976 Store = DAG.getStore(
8977 Chain, dl, Val: Value,
8978 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
8979 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
8980 OutChains.push_back(Elt: Store);
8981 }
8982 }
8983
8984 if (!Store.getNode()) {
8985 // The type might not be legal for the target. This should only happen
8986 // if the type is smaller than a legal type, as on PPC, so the right
8987 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
8988 // to Load/Store if NVT==VT.
8989 // FIXME does the case above also need this?
8990 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
8991 assert(NVT.bitsGE(VT));
8992
8993 bool isDereferenceable =
8994 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
8995 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
8996 if (isDereferenceable)
8997 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
8998 if (isConstant)
8999 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9000
9001 Value = DAG.getExtLoad(
9002 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9003 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9004 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9005 Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9006 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9007
9008 Store = DAG.getTruncStore(
9009 Chain, dl, Val: Value,
9010 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9011 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo);
9012 OutStoreChains.push_back(Elt: Store);
9013 }
9014 SrcOff += VTSize;
9015 DstOff += VTSize;
9016 Size -= VTSize;
9017 }
9018
9019 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9020 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9021 unsigned NumLdStInMemcpy = OutStoreChains.size();
9022
9023 if (NumLdStInMemcpy) {
9024 // It may be that memcpy might be converted to memset if it's memcpy
9025 // of constants. In such a case, we won't have loads and stores, but
9026 // just stores. In the absence of loads, there is nothing to gang up.
9027 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9028 // If target does not care, just leave as it.
9029 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9030 OutChains.push_back(Elt: OutLoadChains[i]);
9031 OutChains.push_back(Elt: OutStoreChains[i]);
9032 }
9033 } else {
9034 // Ld/St less than/equal limit set by target.
9035 if (NumLdStInMemcpy <= GluedLdStLimit) {
9036 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9037 To: NumLdStInMemcpy, OutLoadChains,
9038 OutStoreChains);
9039 } else {
9040 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9041 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9042 unsigned GlueIter = 0;
9043
9044 // Residual ld/st.
9045 if (RemainingLdStInMemcpy) {
9046 chainLoadsAndStoresForMemcpy(
9047 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9048 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9049 }
9050
9051 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9052 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9053 GlueIter - GluedLdStLimit;
9054 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9055 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9056 OutLoadChains, OutStoreChains);
9057 GlueIter += GluedLdStLimit;
9058 }
9059 }
9060 }
9061 }
9062 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9063}
9064
9065static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
9066 SDValue Chain, SDValue Dst, SDValue Src,
9067 uint64_t Size, Align Alignment,
9068 bool isVol, bool AlwaysInline,
9069 MachinePointerInfo DstPtrInfo,
9070 MachinePointerInfo SrcPtrInfo,
9071 const AAMDNodes &AAInfo) {
9072 // Turn a memmove of undef to nop.
9073 // FIXME: We need to honor volatile even is Src is undef.
9074 if (Src.isUndef())
9075 return Chain;
9076
9077 // Expand memmove to a series of load and store ops if the size operand falls
9078 // below a certain threshold.
9079 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9080 const DataLayout &DL = DAG.getDataLayout();
9081 LLVMContext &C = *DAG.getContext();
9082 std::vector<EVT> MemOps;
9083 bool DstAlignCanChange = false;
9084 MachineFunction &MF = DAG.getMachineFunction();
9085 MachineFrameInfo &MFI = MF.getFrameInfo();
9086 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9087 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9088 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9089 DstAlignCanChange = true;
9090 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
9091 if (!SrcAlign || Alignment > *SrcAlign)
9092 SrcAlign = Alignment;
9093 assert(SrcAlign && "SrcAlign must be set");
9094 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9095 if (!TLI.findOptimalMemOpLowering(
9096 Context&: C, MemOps, Limit,
9097 Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign,
9098 /*IsVolatile*/ true),
9099 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9100 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9101 return SDValue();
9102
9103 if (DstAlignCanChange) {
9104 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9105 Align NewAlign = DL.getABITypeAlign(Ty);
9106
9107 // Don't promote to an alignment that would require dynamic stack
9108 // realignment which may conflict with optimizations such as tail call
9109 // optimization.
9110 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9111 if (!TRI->hasStackRealignment(MF))
9112 if (MaybeAlign StackAlign = DL.getStackAlignment())
9113 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9114
9115 if (NewAlign > Alignment) {
9116 // Give the stack frame object a larger alignment if needed.
9117 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9118 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9119 Alignment = NewAlign;
9120 }
9121 }
9122
9123 // Prepare AAInfo for loads/stores after lowering this memmove.
9124 AAMDNodes NewAAInfo = AAInfo;
9125 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9126
9127 MachineMemOperand::Flags MMOFlags =
9128 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9129 uint64_t SrcOff = 0, DstOff = 0;
9130 SmallVector<SDValue, 8> LoadValues;
9131 SmallVector<SDValue, 8> LoadChains;
9132 SmallVector<SDValue, 8> OutChains;
9133 unsigned NumMemOps = MemOps.size();
9134 for (unsigned i = 0; i < NumMemOps; i++) {
9135 EVT VT = MemOps[i];
9136 unsigned VTSize = VT.getSizeInBits() / 8;
9137 SDValue Value;
9138
9139 bool isDereferenceable =
9140 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9141 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9142 if (isDereferenceable)
9143 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9144
9145 Value = DAG.getLoad(
9146 VT, dl, Chain,
9147 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9148 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: *SrcAlign, MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9149 LoadValues.push_back(Elt: Value);
9150 LoadChains.push_back(Elt: Value.getValue(R: 1));
9151 SrcOff += VTSize;
9152 }
9153 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
9154 OutChains.clear();
9155 for (unsigned i = 0; i < NumMemOps; i++) {
9156 EVT VT = MemOps[i];
9157 unsigned VTSize = VT.getSizeInBits() / 8;
9158 SDValue Store;
9159
9160 Store = DAG.getStore(
9161 Chain, dl, Val: LoadValues[i],
9162 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9163 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
9164 OutChains.push_back(Elt: Store);
9165 DstOff += VTSize;
9166 }
9167
9168 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9169}
9170
9171/// Lower the call to 'memset' intrinsic function into a series of store
9172/// operations.
9173///
9174/// \param DAG Selection DAG where lowered code is placed.
9175/// \param dl Link to corresponding IR location.
9176/// \param Chain Control flow dependency.
9177/// \param Dst Pointer to destination memory location.
9178/// \param Src Value of byte to write into the memory.
9179/// \param Size Number of bytes to write.
9180/// \param Alignment Alignment of the destination in bytes.
9181/// \param isVol True if destination is volatile.
9182/// \param AlwaysInline Makes sure no function call is generated.
9183/// \param DstPtrInfo IR information on the memory pointer.
9184/// \returns New head in the control flow, if lowering was successful, empty
9185/// SDValue otherwise.
9186///
9187/// The function tries to replace 'llvm.memset' intrinsic with several store
9188/// operations and value calculation code. This is usually profitable for small
9189/// memory size or when the semantic requires inlining.
9190static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9191 SDValue Chain, SDValue Dst, SDValue Src,
9192 uint64_t Size, Align Alignment, bool isVol,
9193 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9194 const AAMDNodes &AAInfo) {
9195 // Turn a memset of undef to nop.
9196 // FIXME: We need to honor volatile even is Src is undef.
9197 if (Src.isUndef())
9198 return Chain;
9199
9200 // Expand memset to a series of load/store ops if the size operand
9201 // falls below a certain threshold.
9202 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9203 std::vector<EVT> MemOps;
9204 bool DstAlignCanChange = false;
9205 LLVMContext &C = *DAG.getContext();
9206 MachineFunction &MF = DAG.getMachineFunction();
9207 MachineFrameInfo &MFI = MF.getFrameInfo();
9208 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9209 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9210 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9211 DstAlignCanChange = true;
9212 bool IsZeroVal = isNullConstant(V: Src);
9213 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9214
9215 EVT LargestVT;
9216 if (!TLI.findOptimalMemOpLowering(
9217 Context&: C, MemOps, Limit,
9218 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9219 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9220 LargestVT: &LargestVT))
9221 return SDValue();
9222
9223 if (DstAlignCanChange) {
9224 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9225 const DataLayout &DL = DAG.getDataLayout();
9226 Align NewAlign = DL.getABITypeAlign(Ty);
9227
9228 // Don't promote to an alignment that would require dynamic stack
9229 // realignment which may conflict with optimizations such as tail call
9230 // optimization.
9231 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9232 if (!TRI->hasStackRealignment(MF))
9233 if (MaybeAlign StackAlign = DL.getStackAlignment())
9234 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9235
9236 if (NewAlign > Alignment) {
9237 // Give the stack frame object a larger alignment if needed.
9238 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9239 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9240 Alignment = NewAlign;
9241 }
9242 }
9243
9244 SmallVector<SDValue, 8> OutChains;
9245 uint64_t DstOff = 0;
9246 unsigned NumMemOps = MemOps.size();
9247
9248 // Find the largest store and generate the bit pattern for it.
9249 // If target didn't set LargestVT, compute it from MemOps.
9250 if (!LargestVT.isSimple()) {
9251 LargestVT = MemOps[0];
9252 for (unsigned i = 1; i < NumMemOps; i++)
9253 if (MemOps[i].bitsGT(VT: LargestVT))
9254 LargestVT = MemOps[i];
9255 }
9256 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9257
9258 // Prepare AAInfo for loads/stores after lowering this memset.
9259 AAMDNodes NewAAInfo = AAInfo;
9260 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9261
9262 for (unsigned i = 0; i < NumMemOps; i++) {
9263 EVT VT = MemOps[i];
9264 unsigned VTSize = VT.getSizeInBits() / 8;
9265 // The target should specify store types that exactly cover the memset size
9266 // (with the last store potentially being oversized for overlapping stores).
9267 assert(Size > 0 && "Target specified more stores than needed in "
9268 "findOptimalMemOpLowering");
9269 if (VTSize > Size) {
9270 // Issuing an unaligned load / store pair that overlaps with the previous
9271 // pair. Adjust the offset accordingly.
9272 assert(i == NumMemOps-1 && i != 0);
9273 DstOff -= VTSize - Size;
9274 }
9275
9276 // If this store is smaller than the largest store see whether we can get
9277 // the smaller value for free with a truncate or extract vector element and
9278 // then store.
9279 SDValue Value = MemSetValue;
9280 if (VT.bitsLT(VT: LargestVT)) {
9281 unsigned Index;
9282 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9283 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
9284 if (!LargestVT.isVector() && !VT.isVector() &&
9285 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
9286 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
9287 else if (LargestVT.isVector() && !VT.isVector() &&
9288 TLI.shallExtractConstSplatVectorElementToStore(
9289 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
9290 ElemSizeInBits: VT.getSizeInBits(), Index) &&
9291 TLI.isTypeLegal(VT: SVT) &&
9292 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
9293 // Target which can combine store(extractelement VectorTy, Idx) can get
9294 // the smaller value for free.
9295 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
9296 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
9297 } else
9298 Value = getMemsetValue(Value: Src, VT, DAG, dl);
9299 }
9300 assert(Value.getValueType() == VT && "Value with wrong type.");
9301 SDValue Store = DAG.getStore(
9302 Chain, dl, Val: Value,
9303 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9304 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
9305 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
9306 AAInfo: NewAAInfo);
9307 OutChains.push_back(Elt: Store);
9308 DstOff += VT.getSizeInBits() / 8;
9309 // For oversized overlapping stores, only subtract the remaining bytes.
9310 // For normal stores, subtract the full store size.
9311 if (VTSize > Size) {
9312 Size = 0;
9313 } else {
9314 Size -= VTSize;
9315 }
9316 }
9317
9318 // After processing all stores, Size should be exactly 0. Any remaining bytes
9319 // indicate a bug in the target's findOptimalMemOpLowering implementation.
9320 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
9321 "stores that exactly cover the memset size");
9322
9323 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9324}
9325
9326static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
9327 unsigned AS) {
9328 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
9329 // pointer operands can be losslessly bitcasted to pointers of address space 0
9330 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
9331 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
9332 Twine(AS));
9333 }
9334}
9335
9336static bool isInTailCallPositionWrapper(const CallInst *CI,
9337 const SelectionDAG *SelDAG,
9338 bool AllowReturnsFirstArg) {
9339 if (!CI || !CI->isTailCall())
9340 return false;
9341 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
9342 // helper symbol we lower to.
9343 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
9344 ReturnsFirstArg: AllowReturnsFirstArg &&
9345 funcReturnsFirstArgOfCall(CI: *CI));
9346}
9347
9348static std::pair<SDValue, SDValue>
9349getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
9350 TargetLowering::ArgListTy &&Args,
9351 const CallInst *CI, RTLIB::Libcall Call,
9352 SelectionDAG *DAG, const TargetLowering *TLI) {
9353 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
9354
9355 if (LCImpl == RTLIB::Unsupported)
9356 return {};
9357
9358 TargetLowering::CallLoweringInfo CLI(*DAG);
9359 bool IsTailCall =
9360 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true);
9361 SDValue Callee =
9362 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
9363
9364 CLI.setDebugLoc(dl)
9365 .setChain(Chain)
9366 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9367 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
9368 .setTailCall(IsTailCall);
9369
9370 return TLI->LowerCallTo(CLI);
9371}
9372
9373std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
9374 const SDLoc &dl, SDValue S1,
9375 SDValue S2,
9376 const CallInst *CI) {
9377 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9378 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9379 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9380 Call: RTLIB::STRCMP, DAG: this, TLI);
9381}
9382
9383std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
9384 const SDLoc &dl, SDValue S1,
9385 SDValue S2,
9386 const CallInst *CI) {
9387 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9388 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9389 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9390 Call: RTLIB::STRSTR, DAG: this, TLI);
9391}
9392
9393std::pair<SDValue, SDValue>
9394SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
9395 SDValue Mem1, SDValue Size, const CallInst *CI) {
9396 RTLIB::LibcallImpl MemcmpImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMCMP);
9397 if (MemcmpImpl == RTLIB::Unsupported)
9398 return {};
9399
9400 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9401 TargetLowering::ArgListTy Args = {
9402 {Mem0, PT},
9403 {Mem1, PT},
9404 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9405
9406 TargetLowering::CallLoweringInfo CLI(*this);
9407 bool IsTailCall =
9408 isInTailCallPositionWrapper(CI, SelDAG: this, /*AllowReturnsFirstArg*/ true);
9409
9410 CLI.setDebugLoc(dl)
9411 .setChain(Chain)
9412 .setLibCallee(
9413 CC: Libcalls->getLibcallImplCallingConv(Call: MemcmpImpl),
9414 ResultType: Type::getInt32Ty(C&: *getContext()),
9415 Target: getExternalSymbol(Libcall: MemcmpImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9416 ArgsList: std::move(Args))
9417 .setTailCall(IsTailCall);
9418
9419 return TLI->LowerCallTo(CLI);
9420}
9421
9422std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
9423 const SDLoc &dl,
9424 SDValue Dst, SDValue Src,
9425 const CallInst *CI) {
9426 RTLIB::LibcallImpl LCImpl = Libcalls->getLibcallImpl(Call: RTLIB::STRCPY);
9427 if (LCImpl == RTLIB::Unsupported)
9428 return {};
9429
9430 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9431 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
9432
9433 TargetLowering::CallLoweringInfo CLI(*this);
9434 bool IsTailCall =
9435 isInTailCallPositionWrapper(CI, SelDAG: this, /*AllowReturnsFirstArg=*/true);
9436
9437 CLI.setDebugLoc(dl)
9438 .setChain(Chain)
9439 .setLibCallee(
9440 CC: Libcalls->getLibcallImplCallingConv(Call: LCImpl), ResultType: CI->getType(),
9441 Target: getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9442 ArgsList: std::move(Args))
9443 .setTailCall(IsTailCall);
9444
9445 return TLI->LowerCallTo(CLI);
9446}
9447
9448std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
9449 const SDLoc &dl,
9450 SDValue Src,
9451 const CallInst *CI) {
9452 RTLIB::LibcallImpl StrlenImpl = Libcalls->getLibcallImpl(Call: RTLIB::STRLEN);
9453 if (StrlenImpl == RTLIB::Unsupported)
9454 return {};
9455
9456 // Emit a library call.
9457 TargetLowering::ArgListTy Args = {
9458 {Src, PointerType::getUnqual(C&: *getContext())}};
9459
9460 TargetLowering::CallLoweringInfo CLI(*this);
9461 bool IsTailCall =
9462 isInTailCallPositionWrapper(CI, SelDAG: this, /*AllowReturnsFirstArg*/ true);
9463
9464 CLI.setDebugLoc(dl)
9465 .setChain(Chain)
9466 .setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: StrlenImpl),
9467 ResultType: CI->getType(),
9468 Target: getExternalSymbol(
9469 Libcall: StrlenImpl, VT: TLI->getProgramPointerTy(DL: getDataLayout())),
9470 ArgsList: std::move(Args))
9471 .setTailCall(IsTailCall);
9472
9473 return TLI->LowerCallTo(CLI);
9474}
9475
9476SDValue SelectionDAG::getMemcpy(
9477 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
9478 Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
9479 std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
9480 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9481 BatchAAResults *BatchAA) {
9482 // Check to see if we should lower the memcpy to loads and stores first.
9483 // For cases within the target-specified limits, this is the best choice.
9484 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9485 if (ConstantSize) {
9486 // Memcpy with size zero? Just return the original chain.
9487 if (ConstantSize->isZero())
9488 return Chain;
9489
9490 SDValue Result = getMemcpyLoadsAndStores(
9491 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9492 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9493 if (Result.getNode())
9494 return Result;
9495 }
9496
9497 // Then check to see if we should lower the memcpy with target-specific
9498 // code. If the target chooses to do this, this is the next best.
9499 if (TSI) {
9500 SDValue Result = TSI->EmitTargetCodeForMemcpy(
9501 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
9502 DstPtrInfo, SrcPtrInfo);
9503 if (Result.getNode())
9504 return Result;
9505 }
9506
9507 // If we really need inline code and the target declined to provide it,
9508 // use a (potentially long) sequence of loads and stores.
9509 if (AlwaysInline) {
9510 assert(ConstantSize && "AlwaysInline requires a constant size!");
9511 return getMemcpyLoadsAndStores(
9512 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9513 isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9514 }
9515
9516 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9517 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9518
9519 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
9520 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
9521 // respect volatile, so they may do things like read or write memory
9522 // beyond the given memory regions. But fixing this isn't easy, and most
9523 // people don't care.
9524
9525 // Emit a library call.
9526 TargetLowering::ArgListTy Args;
9527 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9528 Args.emplace_back(args&: Dst, args&: PtrTy);
9529 Args.emplace_back(args&: Src, args&: PtrTy);
9530 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9531 // FIXME: pass in SDLoc
9532 TargetLowering::CallLoweringInfo CLI(*this);
9533 bool IsTailCall = false;
9534 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
9535
9536 if (OverrideTailCall.has_value()) {
9537 IsTailCall = *OverrideTailCall;
9538 } else {
9539 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
9540 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
9541 }
9542
9543 CLI.setDebugLoc(dl)
9544 .setChain(Chain)
9545 .setLibCallee(
9546 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
9547 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9548 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9549 ArgsList: std::move(Args))
9550 .setDiscardResult()
9551 .setTailCall(IsTailCall);
9552
9553 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9554 return CallResult.second;
9555}
9556
9557SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
9558 SDValue Dst, SDValue Src, SDValue Size,
9559 Type *SizeTy, unsigned ElemSz,
9560 bool isTailCall,
9561 MachinePointerInfo DstPtrInfo,
9562 MachinePointerInfo SrcPtrInfo) {
9563 // Emit a library call.
9564 TargetLowering::ArgListTy Args;
9565 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
9566 Args.emplace_back(args&: Dst, args&: ArgTy);
9567 Args.emplace_back(args&: Src, args&: ArgTy);
9568 Args.emplace_back(args&: Size, args&: SizeTy);
9569
9570 RTLIB::Libcall LibraryCall =
9571 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9572 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9573 if (LibcallImpl == RTLIB::Unsupported)
9574 report_fatal_error(reason: "Unsupported element size");
9575
9576 TargetLowering::CallLoweringInfo CLI(*this);
9577 CLI.setDebugLoc(dl)
9578 .setChain(Chain)
9579 .setLibCallee(
9580 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9581 ResultType: Type::getVoidTy(C&: *getContext()),
9582 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9583 ArgsList: std::move(Args))
9584 .setDiscardResult()
9585 .setTailCall(isTailCall);
9586
9587 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9588 return CallResult.second;
9589}
9590
9591SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
9592 SDValue Src, SDValue Size, Align Alignment,
9593 bool isVol, const CallInst *CI,
9594 std::optional<bool> OverrideTailCall,
9595 MachinePointerInfo DstPtrInfo,
9596 MachinePointerInfo SrcPtrInfo,
9597 const AAMDNodes &AAInfo,
9598 BatchAAResults *BatchAA) {
9599 // Check to see if we should lower the memmove to loads and stores first.
9600 // For cases within the target-specified limits, this is the best choice.
9601 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9602 if (ConstantSize) {
9603 // Memmove with size zero? Just return the original chain.
9604 if (ConstantSize->isZero())
9605 return Chain;
9606
9607 SDValue Result = getMemmoveLoadsAndStores(
9608 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9609 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
9610 if (Result.getNode())
9611 return Result;
9612 }
9613
9614 // Then check to see if we should lower the memmove with target-specific
9615 // code. If the target chooses to do this, this is the next best.
9616 if (TSI) {
9617 SDValue Result =
9618 TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size,
9619 Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo);
9620 if (Result.getNode())
9621 return Result;
9622 }
9623
9624 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9625 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9626
9627 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
9628 // not be safe. See memcpy above for more details.
9629
9630 // Emit a library call.
9631 TargetLowering::ArgListTy Args;
9632 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9633 Args.emplace_back(args&: Dst, args&: PtrTy);
9634 Args.emplace_back(args&: Src, args&: PtrTy);
9635 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9636 // FIXME: pass in SDLoc
9637 TargetLowering::CallLoweringInfo CLI(*this);
9638
9639 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
9640
9641 bool IsTailCall = false;
9642 if (OverrideTailCall.has_value()) {
9643 IsTailCall = *OverrideTailCall;
9644 } else {
9645 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
9646 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
9647 }
9648
9649 CLI.setDebugLoc(dl)
9650 .setChain(Chain)
9651 .setLibCallee(
9652 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
9653 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9654 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9655 ArgsList: std::move(Args))
9656 .setDiscardResult()
9657 .setTailCall(IsTailCall);
9658
9659 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9660 return CallResult.second;
9661}
9662
9663SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
9664 SDValue Dst, SDValue Src, SDValue Size,
9665 Type *SizeTy, unsigned ElemSz,
9666 bool isTailCall,
9667 MachinePointerInfo DstPtrInfo,
9668 MachinePointerInfo SrcPtrInfo) {
9669 // Emit a library call.
9670 TargetLowering::ArgListTy Args;
9671 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
9672 Args.emplace_back(args&: Dst, args&: IntPtrTy);
9673 Args.emplace_back(args&: Src, args&: IntPtrTy);
9674 Args.emplace_back(args&: Size, args&: SizeTy);
9675
9676 RTLIB::Libcall LibraryCall =
9677 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9678 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9679 if (LibcallImpl == RTLIB::Unsupported)
9680 report_fatal_error(reason: "Unsupported element size");
9681
9682 TargetLowering::CallLoweringInfo CLI(*this);
9683 CLI.setDebugLoc(dl)
9684 .setChain(Chain)
9685 .setLibCallee(
9686 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9687 ResultType: Type::getVoidTy(C&: *getContext()),
9688 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9689 ArgsList: std::move(Args))
9690 .setDiscardResult()
9691 .setTailCall(isTailCall);
9692
9693 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9694 return CallResult.second;
9695}
9696
9697SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
9698 SDValue Src, SDValue Size, Align Alignment,
9699 bool isVol, bool AlwaysInline,
9700 const CallInst *CI,
9701 MachinePointerInfo DstPtrInfo,
9702 const AAMDNodes &AAInfo) {
9703 // Check to see if we should lower the memset to stores first.
9704 // For cases within the target-specified limits, this is the best choice.
9705 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9706 if (ConstantSize) {
9707 // Memset with size zero? Just return the original chain.
9708 if (ConstantSize->isZero())
9709 return Chain;
9710
9711 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9712 Size: ConstantSize->getZExtValue(), Alignment,
9713 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
9714
9715 if (Result.getNode())
9716 return Result;
9717 }
9718
9719 // Then check to see if we should lower the memset with target-specific
9720 // code. If the target chooses to do this, this is the next best.
9721 if (TSI) {
9722 SDValue Result = TSI->EmitTargetCodeForMemset(
9723 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
9724 if (Result.getNode())
9725 return Result;
9726 }
9727
9728 // If we really need inline code and the target declined to provide it,
9729 // use a (potentially long) sequence of loads and stores.
9730 if (AlwaysInline) {
9731 assert(ConstantSize && "AlwaysInline requires a constant size!");
9732 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9733 Size: ConstantSize->getZExtValue(), Alignment,
9734 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
9735 assert(Result &&
9736 "getMemsetStores must return a valid sequence when AlwaysInline");
9737 return Result;
9738 }
9739
9740 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9741
9742 // Emit a library call.
9743 auto &Ctx = *getContext();
9744 const auto& DL = getDataLayout();
9745
9746 TargetLowering::CallLoweringInfo CLI(*this);
9747 // FIXME: pass in SDLoc
9748 CLI.setDebugLoc(dl).setChain(Chain);
9749
9750 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
9751 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
9752
9753 // If zeroing out and bzero is present, use it.
9754 if (UseBZero) {
9755 TargetLowering::ArgListTy Args;
9756 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
9757 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
9758 CLI.setLibCallee(
9759 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
9760 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
9761 } else {
9762 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
9763
9764 TargetLowering::ArgListTy Args;
9765 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
9766 Args.emplace_back(args&: Src, args: Src.getValueType().getTypeForEVT(Context&: Ctx));
9767 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
9768 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
9769 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
9770 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
9771 ArgsList: std::move(Args));
9772 }
9773
9774 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
9775 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
9776
9777 // If we're going to use bzero, make sure not to tail call unless the
9778 // subsequent return doesn't need a value, as bzero doesn't return the first
9779 // arg unlike memset.
9780 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
9781 bool IsTailCall =
9782 CI && CI->isTailCall() &&
9783 isInTailCallPosition(Call: *CI, TM: getTarget(), ReturnsFirstArg: ReturnsFirstArg && LowersToMemset);
9784 CLI.setDiscardResult().setTailCall(IsTailCall);
9785
9786 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9787 return CallResult.second;
9788}
9789
9790SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
9791 SDValue Dst, SDValue Value, SDValue Size,
9792 Type *SizeTy, unsigned ElemSz,
9793 bool isTailCall,
9794 MachinePointerInfo DstPtrInfo) {
9795 // Emit a library call.
9796 TargetLowering::ArgListTy Args;
9797 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
9798 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
9799 Args.emplace_back(args&: Size, args&: SizeTy);
9800
9801 RTLIB::Libcall LibraryCall =
9802 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9803 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9804 if (LibcallImpl == RTLIB::Unsupported)
9805 report_fatal_error(reason: "Unsupported element size");
9806
9807 TargetLowering::CallLoweringInfo CLI(*this);
9808 CLI.setDebugLoc(dl)
9809 .setChain(Chain)
9810 .setLibCallee(
9811 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9812 ResultType: Type::getVoidTy(C&: *getContext()),
9813 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9814 ArgsList: std::move(Args))
9815 .setDiscardResult()
9816 .setTailCall(isTailCall);
9817
9818 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9819 return CallResult.second;
9820}
9821
9822SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9823 SDVTList VTList, ArrayRef<SDValue> Ops,
9824 MachineMemOperand *MMO,
9825 ISD::LoadExtType ExtType) {
9826 FoldingSetNodeID ID;
9827 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
9828 ID.AddInteger(I: MemVT.getRawBits());
9829 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
9830 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
9831 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9832 ID.AddInteger(I: MMO->getFlags());
9833 void* IP = nullptr;
9834 if (auto *E = cast_or_null<AtomicSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
9835 E->refineAlignment(NewMMO: MMO);
9836 E->refineRanges(NewMMO: MMO);
9837 return SDValue(E, 0);
9838 }
9839
9840 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
9841 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
9842 createOperands(Node: N, Vals: Ops);
9843
9844 CSEMap.InsertNode(N, InsertPos: IP);
9845 InsertNode(N);
9846 SDValue V(N, 0);
9847 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9848 return V;
9849}
9850
9851SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
9852 EVT MemVT, SDVTList VTs, SDValue Chain,
9853 SDValue Ptr, SDValue Cmp, SDValue Swp,
9854 MachineMemOperand *MMO) {
9855 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
9856 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
9857 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
9858
9859 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
9860 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9861}
9862
9863SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9864 SDValue Chain, SDValue Ptr, SDValue Val,
9865 MachineMemOperand *MMO) {
9866 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
9867 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
9868 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
9869 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
9870 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
9871 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
9872 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
9873 Opcode == ISD::ATOMIC_LOAD_FMIN ||
9874 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
9875 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
9876 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
9877 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
9878 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
9879 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
9880 Opcode == ISD::ATOMIC_STORE) &&
9881 "Invalid Atomic Op");
9882
9883 EVT VT = Val.getValueType();
9884
9885 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
9886 getVTList(VT1: VT, VT2: MVT::Other);
9887 SDValue Ops[] = {Chain, Ptr, Val};
9888 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9889}
9890
9891SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
9892 EVT MemVT, EVT VT, SDValue Chain,
9893 SDValue Ptr, MachineMemOperand *MMO) {
9894 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
9895 SDValue Ops[] = {Chain, Ptr};
9896 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
9897}
9898
9899/// getMergeValues - Create a MERGE_VALUES node from the given operands.
9900SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
9901 if (Ops.size() == 1)
9902 return Ops[0];
9903
9904 SmallVector<EVT, 4> VTs;
9905 VTs.reserve(N: Ops.size());
9906 for (const SDValue &Op : Ops)
9907 VTs.push_back(Elt: Op.getValueType());
9908 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
9909}
9910
9911SDValue SelectionDAG::getMemIntrinsicNode(
9912 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
9913 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
9914 MachineMemOperand::Flags Flags, LocationSize Size,
9915 const AAMDNodes &AAInfo) {
9916 if (Size.hasValue() && !Size.getValue())
9917 Size = LocationSize::precise(Value: MemVT.getStoreSize());
9918
9919 MachineFunction &MF = getMachineFunction();
9920 MachineMemOperand *MMO =
9921 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, AAInfo);
9922
9923 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
9924}
9925
9926SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
9927 SDVTList VTList,
9928 ArrayRef<SDValue> Ops, EVT MemVT,
9929 MachineMemOperand *MMO) {
9930 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
9931}
9932
9933SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
9934 SDVTList VTList,
9935 ArrayRef<SDValue> Ops, EVT MemVT,
9936 ArrayRef<MachineMemOperand *> MMOs) {
9937 assert(!MMOs.empty() && "Must have at least one MMO");
9938 assert(
9939 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
9940 Opcode == ISD::PREFETCH ||
9941 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
9942 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
9943 "Opcode is not a memory-accessing opcode!");
9944
9945 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
9946 if (MMOs.size() == 1) {
9947 MemRefs = MMOs[0];
9948 } else {
9949 // Allocate: [size_t count][MMO*][MMO*]...
9950 size_t AllocSize =
9951 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
9952 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
9953 size_t *CountPtr = static_cast<size_t *>(Buffer);
9954 *CountPtr = MMOs.size();
9955 MachineMemOperand **Array =
9956 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
9957 llvm::copy(Range&: MMOs, Out: Array);
9958 MemRefs = Array;
9959 }
9960
9961 // Memoize the node unless it returns a glue result.
9962 MemIntrinsicSDNode *N;
9963 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9964 FoldingSetNodeID ID;
9965 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
9966 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
9967 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
9968 ID.AddInteger(I: MemVT.getRawBits());
9969 for (const MachineMemOperand *MMO : MMOs) {
9970 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9971 ID.AddInteger(I: MMO->getFlags());
9972 }
9973 void *IP = nullptr;
9974 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9975 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
9976 return SDValue(E, 0);
9977 }
9978
9979 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9980 Args&: VTList, Args&: MemVT, Args&: MemRefs);
9981 createOperands(Node: N, Vals: Ops);
9982 CSEMap.InsertNode(N, InsertPos: IP);
9983 } else {
9984 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9985 Args&: VTList, Args&: MemVT, Args&: MemRefs);
9986 createOperands(Node: N, Vals: Ops);
9987 }
9988 InsertNode(N);
9989 SDValue V(N, 0);
9990 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9991 return V;
9992}
9993
9994SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
9995 SDValue Chain, int FrameIndex) {
9996 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
9997 const auto VTs = getVTList(VT: MVT::Other);
9998 SDValue Ops[2] = {
9999 Chain,
10000 getFrameIndex(FI: FrameIndex,
10001 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10002 isTarget: true)};
10003
10004 FoldingSetNodeID ID;
10005 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10006 ID.AddInteger(I: FrameIndex);
10007 void *IP = nullptr;
10008 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10009 return SDValue(E, 0);
10010
10011 LifetimeSDNode *N =
10012 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10013 createOperands(Node: N, Vals: Ops);
10014 CSEMap.InsertNode(N, InsertPos: IP);
10015 InsertNode(N);
10016 SDValue V(N, 0);
10017 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10018 return V;
10019}
10020
10021SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10022 uint64_t Guid, uint64_t Index,
10023 uint32_t Attr) {
10024 const unsigned Opcode = ISD::PSEUDO_PROBE;
10025 const auto VTs = getVTList(VT: MVT::Other);
10026 SDValue Ops[] = {Chain};
10027 FoldingSetNodeID ID;
10028 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10029 ID.AddInteger(I: Guid);
10030 ID.AddInteger(I: Index);
10031 void *IP = nullptr;
10032 if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
10033 return SDValue(E, 0);
10034
10035 auto *N = newSDNode<PseudoProbeSDNode>(
10036 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10037 createOperands(Node: N, Vals: Ops);
10038 CSEMap.InsertNode(N, InsertPos: IP);
10039 InsertNode(N);
10040 SDValue V(N, 0);
10041 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10042 return V;
10043}
10044
10045/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10046/// MachinePointerInfo record from it. This is particularly useful because the
10047/// code generator has many cases where it doesn't bother passing in a
10048/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10049static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10050 SelectionDAG &DAG, SDValue Ptr,
10051 int64_t Offset = 0) {
10052 // If this is FI+Offset, we can model it.
10053 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10054 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10055 FI: FI->getIndex(), Offset);
10056
10057 // If this is (FI+Offset1)+Offset2, we can model it.
10058 if (Ptr.getOpcode() != ISD::ADD ||
10059 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10060 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10061 return Info;
10062
10063 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10064 return MachinePointerInfo::getFixedStack(
10065 MF&: DAG.getMachineFunction(), FI,
10066 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10067}
10068
10069/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10070/// MachinePointerInfo record from it. This is particularly useful because the
10071/// code generator has many cases where it doesn't bother passing in a
10072/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10073static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10074 SelectionDAG &DAG, SDValue Ptr,
10075 SDValue OffsetOp) {
10076 // If the 'Offset' value isn't a constant, we can't handle this.
10077 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10078 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10079 if (OffsetOp.isUndef())
10080 return InferPointerInfo(Info, DAG, Ptr);
10081 return Info;
10082}
10083
10084SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10085 EVT VT, const SDLoc &dl, SDValue Chain,
10086 SDValue Ptr, SDValue Offset,
10087 MachinePointerInfo PtrInfo, EVT MemVT,
10088 Align Alignment,
10089 MachineMemOperand::Flags MMOFlags,
10090 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10091 assert(Chain.getValueType() == MVT::Other &&
10092 "Invalid chain type");
10093
10094 MMOFlags |= MachineMemOperand::MOLoad;
10095 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10096 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10097 // clients.
10098 if (PtrInfo.V.isNull())
10099 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10100
10101 TypeSize Size = MemVT.getStoreSize();
10102 MachineFunction &MF = getMachineFunction();
10103 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10104 BaseAlignment: Alignment, AAInfo, Ranges);
10105 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10106}
10107
10108SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10109 EVT VT, const SDLoc &dl, SDValue Chain,
10110 SDValue Ptr, SDValue Offset, EVT MemVT,
10111 MachineMemOperand *MMO) {
10112 if (VT == MemVT) {
10113 ExtType = ISD::NON_EXTLOAD;
10114 } else if (ExtType == ISD::NON_EXTLOAD) {
10115 assert(VT == MemVT && "Non-extending load from different memory type!");
10116 } else {
10117 // Extending load.
10118 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10119 "Should only be an extending load, not truncating!");
10120 assert(VT.isInteger() == MemVT.isInteger() &&
10121 "Cannot convert from FP to Int or Int -> FP!");
10122 assert(VT.isVector() == MemVT.isVector() &&
10123 "Cannot use an ext load to convert to or from a vector!");
10124 assert((!VT.isVector() ||
10125 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10126 "Cannot use an ext load to change the number of vector elements!");
10127 }
10128
10129 assert((!MMO->getRanges() ||
10130 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10131 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10132 MemVT.isInteger())) &&
10133 "Range metadata and load type must match!");
10134
10135 bool Indexed = AM != ISD::UNINDEXED;
10136 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10137
10138 SDVTList VTs = Indexed ?
10139 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10140 SDValue Ops[] = { Chain, Ptr, Offset };
10141 FoldingSetNodeID ID;
10142 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
10143 ID.AddInteger(I: MemVT.getRawBits());
10144 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10145 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10146 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10147 ID.AddInteger(I: MMO->getFlags());
10148 void *IP = nullptr;
10149 if (auto *E = cast_or_null<LoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10150 E->refineAlignment(NewMMO: MMO);
10151 E->refineRanges(NewMMO: MMO);
10152 return SDValue(E, 0);
10153 }
10154 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10155 Args&: ExtType, Args&: MemVT, Args&: MMO);
10156 createOperands(Node: N, Vals: Ops);
10157
10158 CSEMap.InsertNode(N, InsertPos: IP);
10159 InsertNode(N);
10160 SDValue V(N, 0);
10161 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10162 return V;
10163}
10164
10165SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10166 SDValue Ptr, MachinePointerInfo PtrInfo,
10167 MaybeAlign Alignment,
10168 MachineMemOperand::Flags MMOFlags,
10169 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10170 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10171 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10172 PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
10173}
10174
10175SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10176 SDValue Ptr, MachineMemOperand *MMO) {
10177 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10178 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10179 MemVT: VT, MMO);
10180}
10181
10182SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10183 EVT VT, SDValue Chain, SDValue Ptr,
10184 MachinePointerInfo PtrInfo, EVT MemVT,
10185 MaybeAlign Alignment,
10186 MachineMemOperand::Flags MMOFlags,
10187 const AAMDNodes &AAInfo) {
10188 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10189 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10190 MemVT, Alignment, MMOFlags, AAInfo);
10191}
10192
10193SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10194 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10195 MachineMemOperand *MMO) {
10196 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10197 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10198 MemVT, MMO);
10199}
10200
10201SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10202 SDValue Base, SDValue Offset,
10203 ISD::MemIndexedMode AM) {
10204 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10205 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10206 // Don't propagate the invariant or dereferenceable flags.
10207 auto MMOFlags =
10208 LD->getMemOperand()->getFlags() &
10209 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10210 return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10211 Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
10212 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
10213}
10214
10215SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10216 SDValue Ptr, MachinePointerInfo PtrInfo,
10217 Align Alignment,
10218 MachineMemOperand::Flags MMOFlags,
10219 const AAMDNodes &AAInfo) {
10220 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10221
10222 MMOFlags |= MachineMemOperand::MOStore;
10223 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10224
10225 if (PtrInfo.V.isNull())
10226 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10227
10228 MachineFunction &MF = getMachineFunction();
10229 TypeSize Size = Val.getValueType().getStoreSize();
10230 MachineMemOperand *MMO =
10231 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, AAInfo);
10232 return getStore(Chain, dl, Val, Ptr, MMO);
10233}
10234
10235SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10236 SDValue Ptr, MachineMemOperand *MMO) {
10237 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10238 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10239 AM: ISD::UNINDEXED);
10240}
10241
10242SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10243 SDValue Ptr, SDValue Offset, EVT SVT,
10244 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
10245 bool IsTruncating) {
10246 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10247 EVT VT = Val.getValueType();
10248 if (VT == SVT) {
10249 IsTruncating = false;
10250 } else if (!IsTruncating) {
10251 assert(VT == SVT && "No-truncating store from different memory type!");
10252 } else {
10253 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10254 "Should only be a truncating store, not extending!");
10255 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10256 assert(VT.isVector() == SVT.isVector() &&
10257 "Cannot use trunc store to convert to or from a vector!");
10258 assert((!VT.isVector() ||
10259 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10260 "Cannot use trunc store to change the number of vector elements!");
10261 }
10262
10263 bool Indexed = AM != ISD::UNINDEXED;
10264 assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");
10265 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10266 : getVTList(VT: MVT::Other);
10267 SDValue Ops[] = {Chain, Val, Ptr, Offset};
10268 FoldingSetNodeID ID;
10269 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
10270 ID.AddInteger(I: SVT.getRawBits());
10271 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
10272 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
10273 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10274 ID.AddInteger(I: MMO->getFlags());
10275 void *IP = nullptr;
10276 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10277 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10278 return SDValue(E, 0);
10279 }
10280 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10281 Args&: IsTruncating, Args&: SVT, Args&: MMO);
10282 createOperands(Node: N, Vals: Ops);
10283
10284 CSEMap.InsertNode(N, InsertPos: IP);
10285 InsertNode(N);
10286 SDValue V(N, 0);
10287 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10288 return V;
10289}
10290
10291SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10292 SDValue Ptr, MachinePointerInfo PtrInfo,
10293 EVT SVT, Align Alignment,
10294 MachineMemOperand::Flags MMOFlags,
10295 const AAMDNodes &AAInfo) {
10296 assert(Chain.getValueType() == MVT::Other &&
10297 "Invalid chain type");
10298
10299 MMOFlags |= MachineMemOperand::MOStore;
10300 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10301
10302 if (PtrInfo.V.isNull())
10303 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10304
10305 MachineFunction &MF = getMachineFunction();
10306 MachineMemOperand *MMO = MF.getMachineMemOperand(
10307 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10308 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
10309}
10310
10311SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10312 SDValue Ptr, EVT SVT,
10313 MachineMemOperand *MMO) {
10314 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10315 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
10316}
10317
10318SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
10319 SDValue Base, SDValue Offset,
10320 ISD::MemIndexedMode AM) {
10321 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
10322 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
10323 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
10324 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
10325 IsTruncating: ST->isTruncatingStore());
10326}
10327
10328SDValue SelectionDAG::getLoadVP(
10329 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
10330 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
10331 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
10332 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
10333 const MDNode *Ranges, bool IsExpanding) {
10334 MMOFlags |= MachineMemOperand::MOLoad;
10335 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10336 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10337 // clients.
10338 if (PtrInfo.V.isNull())
10339 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10340
10341 TypeSize Size = MemVT.getStoreSize();
10342 MachineFunction &MF = getMachineFunction();
10343 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10344 BaseAlignment: Alignment, AAInfo, Ranges);
10345 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
10346 MMO, IsExpanding);
10347}
10348
10349SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
10350 ISD::LoadExtType ExtType, EVT VT,
10351 const SDLoc &dl, SDValue Chain, SDValue Ptr,
10352 SDValue Offset, SDValue Mask, SDValue EVL,
10353 EVT MemVT, MachineMemOperand *MMO,
10354 bool IsExpanding) {
10355 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10356 assert(Mask.getValueType().getVectorElementCount() ==
10357 VT.getVectorElementCount() &&
10358 "Vector width mismatch between mask and data");
10359
10360 bool Indexed = AM != ISD::UNINDEXED;
10361 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10362
10363 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10364 : getVTList(VT1: VT, VT2: MVT::Other);
10365 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
10366 FoldingSetNodeID ID;
10367 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
10368 ID.AddInteger(I: MemVT.getRawBits());
10369 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
10370 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10371 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10372 ID.AddInteger(I: MMO->getFlags());
10373 void *IP = nullptr;
10374 if (auto *E = cast_or_null<VPLoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10375 E->refineAlignment(NewMMO: MMO);
10376 E->refineRanges(NewMMO: MMO);
10377 return SDValue(E, 0);
10378 }
10379 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10380 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10381 createOperands(Node: N, Vals: Ops);
10382
10383 CSEMap.InsertNode(N, InsertPos: IP);
10384 InsertNode(N);
10385 SDValue V(N, 0);
10386 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10387 return V;
10388}
10389
10390SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10391 SDValue Ptr, SDValue Mask, SDValue EVL,
10392 MachinePointerInfo PtrInfo,
10393 MaybeAlign Alignment,
10394 MachineMemOperand::Flags MMOFlags,
10395 const AAMDNodes &AAInfo, const MDNode *Ranges,
10396 bool IsExpanding) {
10397 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10398 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10399 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
10400 IsExpanding);
10401}
10402
10403SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10404 SDValue Ptr, SDValue Mask, SDValue EVL,
10405 MachineMemOperand *MMO, bool IsExpanding) {
10406 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10407 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10408 Mask, EVL, MemVT: VT, MMO, IsExpanding);
10409}
10410
10411SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10412 EVT VT, SDValue Chain, SDValue Ptr,
10413 SDValue Mask, SDValue EVL,
10414 MachinePointerInfo PtrInfo, EVT MemVT,
10415 MaybeAlign Alignment,
10416 MachineMemOperand::Flags MMOFlags,
10417 const AAMDNodes &AAInfo, bool IsExpanding) {
10418 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10419 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10420 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
10421 IsExpanding);
10422}
10423
10424SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10425 EVT VT, SDValue Chain, SDValue Ptr,
10426 SDValue Mask, SDValue EVL, EVT MemVT,
10427 MachineMemOperand *MMO, bool IsExpanding) {
10428 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10429 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10430 EVL, MemVT, MMO, IsExpanding);
10431}
10432
10433SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
10434 SDValue Base, SDValue Offset,
10435 ISD::MemIndexedMode AM) {
10436 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
10437 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10438 // Don't propagate the invariant or dereferenceable flags.
10439 auto MMOFlags =
10440 LD->getMemOperand()->getFlags() &
10441 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10442 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10443 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
10444 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
10445 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
10446 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
10447}
10448
10449SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
10450 SDValue Ptr, SDValue Offset, SDValue Mask,
10451 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
10452 ISD::MemIndexedMode AM, bool IsTruncating,
10453 bool IsCompressing) {
10454 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10455 assert(Mask.getValueType().getVectorElementCount() ==
10456 Val.getValueType().getVectorElementCount() &&
10457 "Vector width mismatch between mask and data");
10458
10459 bool Indexed = AM != ISD::UNINDEXED;
10460 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10461 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10462 : getVTList(VT: MVT::Other);
10463 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
10464 FoldingSetNodeID ID;
10465 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10466 ID.AddInteger(I: MemVT.getRawBits());
10467 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10468 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10469 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10470 ID.AddInteger(I: MMO->getFlags());
10471 void *IP = nullptr;
10472 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10473 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10474 return SDValue(E, 0);
10475 }
10476 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10477 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10478 createOperands(Node: N, Vals: Ops);
10479
10480 CSEMap.InsertNode(N, InsertPos: IP);
10481 InsertNode(N);
10482 SDValue V(N, 0);
10483 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10484 return V;
10485}
10486
10487SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10488 SDValue Val, SDValue Ptr, SDValue Mask,
10489 SDValue EVL, MachinePointerInfo PtrInfo,
10490 EVT SVT, Align Alignment,
10491 MachineMemOperand::Flags MMOFlags,
10492 const AAMDNodes &AAInfo,
10493 bool IsCompressing) {
10494 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10495
10496 MMOFlags |= MachineMemOperand::MOStore;
10497 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10498
10499 if (PtrInfo.V.isNull())
10500 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10501
10502 MachineFunction &MF = getMachineFunction();
10503 MachineMemOperand *MMO = MF.getMachineMemOperand(
10504 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10505 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
10506 IsCompressing);
10507}
10508
10509SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10510 SDValue Val, SDValue Ptr, SDValue Mask,
10511 SDValue EVL, EVT SVT,
10512 MachineMemOperand *MMO,
10513 bool IsCompressing) {
10514 EVT VT = Val.getValueType();
10515
10516 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10517 if (VT == SVT)
10518 return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
10519 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10520 /*IsTruncating*/ false, IsCompressing);
10521
10522 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10523 "Should only be a truncating store, not extending!");
10524 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10525 assert(VT.isVector() == SVT.isVector() &&
10526 "Cannot use trunc store to convert to or from a vector!");
10527 assert((!VT.isVector() ||
10528 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10529 "Cannot use trunc store to change the number of vector elements!");
10530
10531 SDVTList VTs = getVTList(VT: MVT::Other);
10532 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10533 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
10534 FoldingSetNodeID ID;
10535 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10536 ID.AddInteger(I: SVT.getRawBits());
10537 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10538 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10539 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10540 ID.AddInteger(I: MMO->getFlags());
10541 void *IP = nullptr;
10542 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10543 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10544 return SDValue(E, 0);
10545 }
10546 auto *N =
10547 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10548 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
10549 createOperands(Node: N, Vals: Ops);
10550
10551 CSEMap.InsertNode(N, InsertPos: IP);
10552 InsertNode(N);
10553 SDValue V(N, 0);
10554 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10555 return V;
10556}
10557
10558SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
10559 SDValue Base, SDValue Offset,
10560 ISD::MemIndexedMode AM) {
10561 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
10562 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
10563 SDVTList VTs = getVTList(VT1: Base.getValueType(), VT2: MVT::Other);
10564 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
10565 Offset, ST->getMask(), ST->getVectorLength()};
10566 FoldingSetNodeID ID;
10567 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10568 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
10569 ID.AddInteger(I: ST->getRawSubclassData());
10570 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
10571 ID.AddInteger(I: ST->getMemOperand()->getFlags());
10572 void *IP = nullptr;
10573 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10574 return SDValue(E, 0);
10575
10576 auto *N = newSDNode<VPStoreSDNode>(
10577 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
10578 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
10579 createOperands(Node: N, Vals: Ops);
10580
10581 CSEMap.InsertNode(N, InsertPos: IP);
10582 InsertNode(N);
10583 SDValue V(N, 0);
10584 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10585 return V;
10586}
10587
10588SDValue SelectionDAG::getStridedLoadVP(
10589 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
10590 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
10591 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
10592 bool Indexed = AM != ISD::UNINDEXED;
10593 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10594
10595 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
10596 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10597 : getVTList(VT1: VT, VT2: MVT::Other);
10598 FoldingSetNodeID ID;
10599 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
10600 ID.AddInteger(I: VT.getRawBits());
10601 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
10602 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10603 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10604
10605 void *IP = nullptr;
10606 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10607 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10608 return SDValue(E, 0);
10609 }
10610
10611 auto *N =
10612 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
10613 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10614 createOperands(Node: N, Vals: Ops);
10615 CSEMap.InsertNode(N, InsertPos: IP);
10616 InsertNode(N);
10617 SDValue V(N, 0);
10618 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10619 return V;
10620}
10621
10622SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
10623 SDValue Ptr, SDValue Stride,
10624 SDValue Mask, SDValue EVL,
10625 MachineMemOperand *MMO,
10626 bool IsExpanding) {
10627 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10628 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
10629 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
10630}
10631
10632SDValue SelectionDAG::getExtStridedLoadVP(
10633 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
10634 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
10635 MachineMemOperand *MMO, bool IsExpanding) {
10636 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10637 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
10638 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
10639}
10640
10641SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
10642 SDValue Val, SDValue Ptr,
10643 SDValue Offset, SDValue Stride,
10644 SDValue Mask, SDValue EVL, EVT MemVT,
10645 MachineMemOperand *MMO,
10646 ISD::MemIndexedMode AM,
10647 bool IsTruncating, bool IsCompressing) {
10648 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10649 bool Indexed = AM != ISD::UNINDEXED;
10650 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10651 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10652 : getVTList(VT: MVT::Other);
10653 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
10654 FoldingSetNodeID ID;
10655 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10656 ID.AddInteger(I: MemVT.getRawBits());
10657 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10658 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10659 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10660 void *IP = nullptr;
10661 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10662 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10663 return SDValue(E, 0);
10664 }
10665 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10666 Args&: VTs, Args&: AM, Args&: IsTruncating,
10667 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10668 createOperands(Node: N, Vals: Ops);
10669
10670 CSEMap.InsertNode(N, InsertPos: IP);
10671 InsertNode(N);
10672 SDValue V(N, 0);
10673 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10674 return V;
10675}
10676
10677SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
10678 SDValue Val, SDValue Ptr,
10679 SDValue Stride, SDValue Mask,
10680 SDValue EVL, EVT SVT,
10681 MachineMemOperand *MMO,
10682 bool IsCompressing) {
10683 EVT VT = Val.getValueType();
10684
10685 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10686 if (VT == SVT)
10687 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
10688 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10689 /*IsTruncating*/ false, IsCompressing);
10690
10691 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10692 "Should only be a truncating store, not extending!");
10693 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10694 assert(VT.isVector() == SVT.isVector() &&
10695 "Cannot use trunc store to convert to or from a vector!");
10696 assert((!VT.isVector() ||
10697 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10698 "Cannot use trunc store to change the number of vector elements!");
10699
10700 SDVTList VTs = getVTList(VT: MVT::Other);
10701 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10702 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
10703 FoldingSetNodeID ID;
10704 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10705 ID.AddInteger(I: SVT.getRawBits());
10706 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10707 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10708 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10709 void *IP = nullptr;
10710 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10711 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10712 return SDValue(E, 0);
10713 }
10714 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10715 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
10716 Args&: IsCompressing, Args&: SVT, Args&: MMO);
10717 createOperands(Node: N, Vals: Ops);
10718
10719 CSEMap.InsertNode(N, InsertPos: IP);
10720 InsertNode(N);
10721 SDValue V(N, 0);
10722 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10723 return V;
10724}
10725
10726SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10727 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
10728 ISD::MemIndexType IndexType) {
10729 assert(Ops.size() == 6 && "Incompatible number of operands");
10730
10731 FoldingSetNodeID ID;
10732 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
10733 ID.AddInteger(I: VT.getRawBits());
10734 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
10735 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10736 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10737 ID.AddInteger(I: MMO->getFlags());
10738 void *IP = nullptr;
10739 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10740 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10741 return SDValue(E, 0);
10742 }
10743
10744 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10745 Args&: VT, Args&: MMO, Args&: IndexType);
10746 createOperands(Node: N, Vals: Ops);
10747
10748 assert(N->getMask().getValueType().getVectorElementCount() ==
10749 N->getValueType(0).getVectorElementCount() &&
10750 "Vector width mismatch between mask and data");
10751 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10752 N->getValueType(0).getVectorElementCount().isScalable() &&
10753 "Scalable flags of index and data do not match");
10754 assert(ElementCount::isKnownGE(
10755 N->getIndex().getValueType().getVectorElementCount(),
10756 N->getValueType(0).getVectorElementCount()) &&
10757 "Vector width mismatch between index and data");
10758 assert(isa<ConstantSDNode>(N->getScale()) &&
10759 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10760 "Scale should be a constant power of 2");
10761
10762 CSEMap.InsertNode(N, InsertPos: IP);
10763 InsertNode(N);
10764 SDValue V(N, 0);
10765 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10766 return V;
10767}
10768
10769SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10770 ArrayRef<SDValue> Ops,
10771 MachineMemOperand *MMO,
10772 ISD::MemIndexType IndexType) {
10773 assert(Ops.size() == 7 && "Incompatible number of operands");
10774
10775 FoldingSetNodeID ID;
10776 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
10777 ID.AddInteger(I: VT.getRawBits());
10778 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
10779 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10780 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10781 ID.AddInteger(I: MMO->getFlags());
10782 void *IP = nullptr;
10783 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10784 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10785 return SDValue(E, 0);
10786 }
10787 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10788 Args&: VT, Args&: MMO, Args&: IndexType);
10789 createOperands(Node: N, Vals: Ops);
10790
10791 assert(N->getMask().getValueType().getVectorElementCount() ==
10792 N->getValue().getValueType().getVectorElementCount() &&
10793 "Vector width mismatch between mask and data");
10794 assert(
10795 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10796 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10797 "Scalable flags of index and data do not match");
10798 assert(ElementCount::isKnownGE(
10799 N->getIndex().getValueType().getVectorElementCount(),
10800 N->getValue().getValueType().getVectorElementCount()) &&
10801 "Vector width mismatch between index and data");
10802 assert(isa<ConstantSDNode>(N->getScale()) &&
10803 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10804 "Scale should be a constant power of 2");
10805
10806 CSEMap.InsertNode(N, InsertPos: IP);
10807 InsertNode(N);
10808 SDValue V(N, 0);
10809 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10810 return V;
10811}
10812
10813SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10814 SDValue Base, SDValue Offset, SDValue Mask,
10815 SDValue PassThru, EVT MemVT,
10816 MachineMemOperand *MMO,
10817 ISD::MemIndexedMode AM,
10818 ISD::LoadExtType ExtTy, bool isExpanding) {
10819 bool Indexed = AM != ISD::UNINDEXED;
10820 assert((Indexed || Offset.isUndef()) &&
10821 "Unindexed masked load with an offset!");
10822 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
10823 : getVTList(VT1: VT, VT2: MVT::Other);
10824 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
10825 FoldingSetNodeID ID;
10826 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
10827 ID.AddInteger(I: MemVT.getRawBits());
10828 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
10829 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
10830 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10831 ID.AddInteger(I: MMO->getFlags());
10832 void *IP = nullptr;
10833 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10834 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10835 return SDValue(E, 0);
10836 }
10837 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10838 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
10839 createOperands(Node: N, Vals: Ops);
10840
10841 CSEMap.InsertNode(N, InsertPos: IP);
10842 InsertNode(N);
10843 SDValue V(N, 0);
10844 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10845 return V;
10846}
10847
10848SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
10849 SDValue Base, SDValue Offset,
10850 ISD::MemIndexedMode AM) {
10851 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
10852 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
10853 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
10854 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
10855 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
10856 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
10857}
10858
10859SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
10860 SDValue Val, SDValue Base, SDValue Offset,
10861 SDValue Mask, EVT MemVT,
10862 MachineMemOperand *MMO,
10863 ISD::MemIndexedMode AM, bool IsTruncating,
10864 bool IsCompressing) {
10865 assert(Chain.getValueType() == MVT::Other &&
10866 "Invalid chain type");
10867 bool Indexed = AM != ISD::UNINDEXED;
10868 assert((Indexed || Offset.isUndef()) &&
10869 "Unindexed masked store with an offset!");
10870 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
10871 : getVTList(VT: MVT::Other);
10872 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
10873 FoldingSetNodeID ID;
10874 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
10875 ID.AddInteger(I: MemVT.getRawBits());
10876 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
10877 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10878 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10879 ID.AddInteger(I: MMO->getFlags());
10880 void *IP = nullptr;
10881 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10882 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10883 return SDValue(E, 0);
10884 }
10885 auto *N =
10886 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10887 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10888 createOperands(Node: N, Vals: Ops);
10889
10890 CSEMap.InsertNode(N, InsertPos: IP);
10891 InsertNode(N);
10892 SDValue V(N, 0);
10893 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10894 return V;
10895}
10896
10897SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
10898 SDValue Base, SDValue Offset,
10899 ISD::MemIndexedMode AM) {
10900 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
10901 assert(ST->getOffset().isUndef() &&
10902 "Masked store is already a indexed store!");
10903 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
10904 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
10905 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
10906}
10907
10908SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10909 ArrayRef<SDValue> Ops,
10910 MachineMemOperand *MMO,
10911 ISD::MemIndexType IndexType,
10912 ISD::LoadExtType ExtTy) {
10913 assert(Ops.size() == 6 && "Incompatible number of operands");
10914
10915 FoldingSetNodeID ID;
10916 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
10917 ID.AddInteger(I: MemVT.getRawBits());
10918 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
10919 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
10920 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10921 ID.AddInteger(I: MMO->getFlags());
10922 void *IP = nullptr;
10923 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10924 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10925 return SDValue(E, 0);
10926 }
10927
10928 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10929 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
10930 createOperands(Node: N, Vals: Ops);
10931
10932 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
10933 "Incompatible type of the PassThru value in MaskedGatherSDNode");
10934 assert(N->getMask().getValueType().getVectorElementCount() ==
10935 N->getValueType(0).getVectorElementCount() &&
10936 "Vector width mismatch between mask and data");
10937 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10938 N->getValueType(0).getVectorElementCount().isScalable() &&
10939 "Scalable flags of index and data do not match");
10940 assert(ElementCount::isKnownGE(
10941 N->getIndex().getValueType().getVectorElementCount(),
10942 N->getValueType(0).getVectorElementCount()) &&
10943 "Vector width mismatch between index and data");
10944 assert(isa<ConstantSDNode>(N->getScale()) &&
10945 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10946 "Scale should be a constant power of 2");
10947
10948 CSEMap.InsertNode(N, InsertPos: IP);
10949 InsertNode(N);
10950 SDValue V(N, 0);
10951 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10952 return V;
10953}
10954
10955SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10956 ArrayRef<SDValue> Ops,
10957 MachineMemOperand *MMO,
10958 ISD::MemIndexType IndexType,
10959 bool IsTrunc) {
10960 assert(Ops.size() == 6 && "Incompatible number of operands");
10961
10962 FoldingSetNodeID ID;
10963 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
10964 ID.AddInteger(I: MemVT.getRawBits());
10965 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
10966 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
10967 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10968 ID.AddInteger(I: MMO->getFlags());
10969 void *IP = nullptr;
10970 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10971 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10972 return SDValue(E, 0);
10973 }
10974
10975 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10976 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
10977 createOperands(Node: N, Vals: Ops);
10978
10979 assert(N->getMask().getValueType().getVectorElementCount() ==
10980 N->getValue().getValueType().getVectorElementCount() &&
10981 "Vector width mismatch between mask and data");
10982 assert(
10983 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10984 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10985 "Scalable flags of index and data do not match");
10986 assert(ElementCount::isKnownGE(
10987 N->getIndex().getValueType().getVectorElementCount(),
10988 N->getValue().getValueType().getVectorElementCount()) &&
10989 "Vector width mismatch between index and data");
10990 assert(isa<ConstantSDNode>(N->getScale()) &&
10991 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10992 "Scale should be a constant power of 2");
10993
10994 CSEMap.InsertNode(N, InsertPos: IP);
10995 InsertNode(N);
10996 SDValue V(N, 0);
10997 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10998 return V;
10999}
11000
11001SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11002 const SDLoc &dl, ArrayRef<SDValue> Ops,
11003 MachineMemOperand *MMO,
11004 ISD::MemIndexType IndexType) {
11005 assert(Ops.size() == 7 && "Incompatible number of operands");
11006
11007 FoldingSetNodeID ID;
11008 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTList: VTs, OpList: Ops);
11009 ID.AddInteger(I: MemVT.getRawBits());
11010 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11011 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11012 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11013 ID.AddInteger(I: MMO->getFlags());
11014 void *IP = nullptr;
11015 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11016 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11017 return SDValue(E, 0);
11018 }
11019
11020 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11021 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11022 createOperands(Node: N, Vals: Ops);
11023
11024 assert(N->getMask().getValueType().getVectorElementCount() ==
11025 N->getIndex().getValueType().getVectorElementCount() &&
11026 "Vector width mismatch between mask and data");
11027 assert(isa<ConstantSDNode>(N->getScale()) &&
11028 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11029 "Scale should be a constant power of 2");
11030 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11031
11032 CSEMap.InsertNode(N, InsertPos: IP);
11033 InsertNode(N);
11034 SDValue V(N, 0);
11035 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11036 return V;
11037}
11038
11039SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11040 SDValue Ptr, SDValue Mask, SDValue EVL,
11041 MachineMemOperand *MMO) {
11042 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11043 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11044 FoldingSetNodeID ID;
11045 AddNodeIDNode(ID, OpC: ISD::VP_LOAD_FF, VTList: VTs, OpList: Ops);
11046 ID.AddInteger(I: VT.getRawBits());
11047 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11048 Args&: VTs, Args&: VT, Args&: MMO));
11049 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11050 ID.AddInteger(I: MMO->getFlags());
11051 void *IP = nullptr;
11052 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11053 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11054 return SDValue(E, 0);
11055 }
11056 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11057 Args&: VT, Args&: MMO);
11058 createOperands(Node: N, Vals: Ops);
11059
11060 CSEMap.InsertNode(N, InsertPos: IP);
11061 InsertNode(N);
11062 SDValue V(N, 0);
11063 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11064 return V;
11065}
11066
11067SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11068 EVT MemVT, MachineMemOperand *MMO) {
11069 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11070 SDVTList VTs = getVTList(VT: MVT::Other);
11071 SDValue Ops[] = {Chain, Ptr};
11072 FoldingSetNodeID ID;
11073 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
11074 ID.AddInteger(I: MemVT.getRawBits());
11075 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11076 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11077 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11078 ID.AddInteger(I: MMO->getFlags());
11079 void *IP = nullptr;
11080 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11081 return SDValue(E, 0);
11082
11083 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11084 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11085 createOperands(Node: N, Vals: Ops);
11086
11087 CSEMap.InsertNode(N, InsertPos: IP);
11088 InsertNode(N);
11089 SDValue V(N, 0);
11090 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11091 return V;
11092}
11093
11094SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11095 EVT MemVT, MachineMemOperand *MMO) {
11096 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11097 SDVTList VTs = getVTList(VT: MVT::Other);
11098 SDValue Ops[] = {Chain, Ptr};
11099 FoldingSetNodeID ID;
11100 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
11101 ID.AddInteger(I: MemVT.getRawBits());
11102 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11103 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11104 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11105 ID.AddInteger(I: MMO->getFlags());
11106 void *IP = nullptr;
11107 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11108 return SDValue(E, 0);
11109
11110 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11111 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11112 createOperands(Node: N, Vals: Ops);
11113
11114 CSEMap.InsertNode(N, InsertPos: IP);
11115 InsertNode(N);
11116 SDValue V(N, 0);
11117 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11118 return V;
11119}
11120
11121SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11122 // select undef, T, F --> T (if T is a constant), otherwise F
11123 // select, ?, undef, F --> F
11124 // select, ?, T, undef --> T
11125 if (Cond.isUndef())
11126 return isConstantValueOfAnyType(N: T) ? T : F;
11127 if (T.isUndef())
11128 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11129 if (F.isUndef())
11130 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11131
11132 // select true, T, F --> T
11133 // select false, T, F --> F
11134 if (auto C = isBoolConstant(N: Cond))
11135 return *C ? T : F;
11136
11137 // select ?, T, T --> T
11138 if (T == F)
11139 return T;
11140
11141 return SDValue();
11142}
11143
11144SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11145 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11146 if (X.isUndef())
11147 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11148 // shift X, undef --> undef (because it may shift by the bitwidth)
11149 if (Y.isUndef())
11150 return getUNDEF(VT: X.getValueType());
11151
11152 // shift 0, Y --> 0
11153 // shift X, 0 --> X
11154 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11155 return X;
11156
11157 // shift X, C >= bitwidth(X) --> undef
11158 // All vector elements must be too big (or undef) to avoid partial undefs.
11159 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11160 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11161 };
11162 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11163 return getUNDEF(VT: X.getValueType());
11164
11165 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11166 if (X.getValueType().getScalarType() == MVT::i1)
11167 return X;
11168
11169 return SDValue();
11170}
11171
11172SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11173 SDNodeFlags Flags) {
11174 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11175 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11176 // operation is poison. That result can be relaxed to undef.
11177 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11178 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11179 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11180 (YC && YC->getValueAPF().isNaN());
11181 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11182 (YC && YC->getValueAPF().isInfinity());
11183
11184 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11185 return getUNDEF(VT: X.getValueType());
11186
11187 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11188 return getUNDEF(VT: X.getValueType());
11189
11190 if (!YC)
11191 return SDValue();
11192
11193 // X + -0.0 --> X
11194 if (Opcode == ISD::FADD)
11195 if (YC->getValueAPF().isNegZero())
11196 return X;
11197
11198 // X - +0.0 --> X
11199 if (Opcode == ISD::FSUB)
11200 if (YC->getValueAPF().isPosZero())
11201 return X;
11202
11203 // X * 1.0 --> X
11204 // X / 1.0 --> X
11205 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11206 if (YC->getValueAPF().isExactlyValue(V: 1.0))
11207 return X;
11208
11209 // X * 0.0 --> 0.0
11210 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11211 if (YC->getValueAPF().isZero())
11212 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11213
11214 return SDValue();
11215}
11216
11217SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11218 SDValue Ptr, SDValue SV, unsigned Align) {
11219 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11220 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11221}
11222
11223SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11224 ArrayRef<SDUse> Ops) {
11225 switch (Ops.size()) {
11226 case 0: return getNode(Opcode, DL, VT);
11227 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11228 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11229 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11230 default: break;
11231 }
11232
11233 // Copy from an SDUse array into an SDValue array for use with
11234 // the regular getNode logic.
11235 SmallVector<SDValue, 8> NewOps(Ops);
11236 return getNode(Opcode, DL, VT, Ops: NewOps);
11237}
11238
11239SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11240 ArrayRef<SDValue> Ops) {
11241 SDNodeFlags Flags;
11242 if (Inserter)
11243 Flags = Inserter->getFlags();
11244 return getNode(Opcode, DL, VT, Ops, Flags);
11245}
11246
11247SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11248 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11249 unsigned NumOps = Ops.size();
11250 switch (NumOps) {
11251 case 0: return getNode(Opcode, DL, VT);
11252 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11253 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11254 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11255 default: break;
11256 }
11257
11258#ifndef NDEBUG
11259 for (const auto &Op : Ops)
11260 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11261 "Operand is DELETED_NODE!");
11262#endif
11263
11264 switch (Opcode) {
11265 default: break;
11266 case ISD::BUILD_VECTOR:
11267 // Attempt to simplify BUILD_VECTOR.
11268 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11269 return V;
11270 break;
11271 case ISD::CONCAT_VECTORS:
11272 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11273 return V;
11274 break;
11275 case ISD::SELECT_CC:
11276 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11277 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11278 "LHS and RHS of condition must have same type!");
11279 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11280 "True and False arms of SelectCC must have same type!");
11281 assert(Ops[2].getValueType() == VT &&
11282 "select_cc node must be of same type as true and false value!");
11283 assert((!Ops[0].getValueType().isVector() ||
11284 Ops[0].getValueType().getVectorElementCount() ==
11285 VT.getVectorElementCount()) &&
11286 "Expected select_cc with vector result to have the same sized "
11287 "comparison type!");
11288 break;
11289 case ISD::BR_CC:
11290 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11291 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11292 "LHS/RHS of comparison should match types!");
11293 break;
11294 case ISD::VP_ADD:
11295 case ISD::VP_SUB:
11296 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
11297 if (VT.getScalarType() == MVT::i1)
11298 Opcode = ISD::VP_XOR;
11299 break;
11300 case ISD::VP_MUL:
11301 // If it is VP_MUL mask operation then turn it to VP_AND
11302 if (VT.getScalarType() == MVT::i1)
11303 Opcode = ISD::VP_AND;
11304 break;
11305 case ISD::VP_REDUCE_MUL:
11306 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11307 if (VT == MVT::i1)
11308 Opcode = ISD::VP_REDUCE_AND;
11309 break;
11310 case ISD::VP_REDUCE_ADD:
11311 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11312 if (VT == MVT::i1)
11313 Opcode = ISD::VP_REDUCE_XOR;
11314 break;
11315 case ISD::VP_REDUCE_SMAX:
11316 case ISD::VP_REDUCE_UMIN:
11317 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11318 // VP_REDUCE_AND.
11319 if (VT == MVT::i1)
11320 Opcode = ISD::VP_REDUCE_AND;
11321 break;
11322 case ISD::VP_REDUCE_SMIN:
11323 case ISD::VP_REDUCE_UMAX:
11324 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11325 // VP_REDUCE_OR.
11326 if (VT == MVT::i1)
11327 Opcode = ISD::VP_REDUCE_OR;
11328 break;
11329 }
11330
11331 // Memoize nodes.
11332 SDNode *N;
11333 SDVTList VTs = getVTList(VT);
11334
11335 if (VT != MVT::Glue) {
11336 FoldingSetNodeID ID;
11337 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
11338 void *IP = nullptr;
11339
11340 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11341 E->intersectFlagsWith(Flags);
11342 return SDValue(E, 0);
11343 }
11344
11345 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11346 createOperands(Node: N, Vals: Ops);
11347
11348 CSEMap.InsertNode(N, InsertPos: IP);
11349 } else {
11350 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11351 createOperands(Node: N, Vals: Ops);
11352 }
11353
11354 N->setFlags(Flags);
11355 InsertNode(N);
11356 SDValue V(N, 0);
11357 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11358 return V;
11359}
11360
11361SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11362 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
11363 SDNodeFlags Flags;
11364 if (Inserter)
11365 Flags = Inserter->getFlags();
11366 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11367}
11368
11369SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11370 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
11371 const SDNodeFlags Flags) {
11372 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11373}
11374
11375SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11376 ArrayRef<SDValue> Ops) {
11377 SDNodeFlags Flags;
11378 if (Inserter)
11379 Flags = Inserter->getFlags();
11380 return getNode(Opcode, DL, VTList, Ops, Flags);
11381}
11382
11383SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11384 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11385 if (VTList.NumVTs == 1)
11386 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
11387
11388#ifndef NDEBUG
11389 for (const auto &Op : Ops)
11390 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11391 "Operand is DELETED_NODE!");
11392#endif
11393
11394 switch (Opcode) {
11395 case ISD::SADDO:
11396 case ISD::UADDO:
11397 case ISD::SSUBO:
11398 case ISD::USUBO: {
11399 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11400 "Invalid add/sub overflow op!");
11401 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11402 Ops[0].getValueType() == Ops[1].getValueType() &&
11403 Ops[0].getValueType() == VTList.VTs[0] &&
11404 "Binary operator types must match!");
11405 SDValue N1 = Ops[0], N2 = Ops[1];
11406 canonicalizeCommutativeBinop(Opcode, N1, N2);
11407
11408 // (X +- 0) -> X with zero-overflow.
11409 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
11410 /*AllowTruncation*/ true);
11411 if (N2CV && N2CV->isZero()) {
11412 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
11413 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
11414 }
11415
11416 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
11417 VTList.VTs[1].getScalarType() == MVT::i1) {
11418 SDValue F1 = getFreeze(V: N1);
11419 SDValue F2 = getFreeze(V: N2);
11420 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
11421 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
11422 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11423 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11424 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
11425 Flags);
11426 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
11427 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
11428 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
11429 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11430 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11431 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
11432 Flags);
11433 }
11434 }
11435 break;
11436 }
11437 case ISD::SADDO_CARRY:
11438 case ISD::UADDO_CARRY:
11439 case ISD::SSUBO_CARRY:
11440 case ISD::USUBO_CARRY:
11441 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
11442 "Invalid add/sub overflow op!");
11443 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11444 Ops[0].getValueType() == Ops[1].getValueType() &&
11445 Ops[0].getValueType() == VTList.VTs[0] &&
11446 Ops[2].getValueType() == VTList.VTs[1] &&
11447 "Binary operator types must match!");
11448 break;
11449 case ISD::SMUL_LOHI:
11450 case ISD::UMUL_LOHI: {
11451 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
11452 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
11453 VTList.VTs[0] == Ops[0].getValueType() &&
11454 VTList.VTs[0] == Ops[1].getValueType() &&
11455 "Binary operator types must match!");
11456 // Constant fold.
11457 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
11458 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
11459 if (LHS && RHS) {
11460 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
11461 unsigned OutWidth = Width * 2;
11462 APInt Val = LHS->getAPIntValue();
11463 APInt Mul = RHS->getAPIntValue();
11464 if (Opcode == ISD::SMUL_LOHI) {
11465 Val = Val.sext(width: OutWidth);
11466 Mul = Mul.sext(width: OutWidth);
11467 } else {
11468 Val = Val.zext(width: OutWidth);
11469 Mul = Mul.zext(width: OutWidth);
11470 }
11471 Val *= Mul;
11472
11473 SDValue Hi =
11474 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
11475 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
11476 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
11477 }
11478 break;
11479 }
11480 case ISD::FFREXP: {
11481 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
11482 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
11483 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
11484
11485 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
11486 int FrexpExp;
11487 APFloat FrexpMant =
11488 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
11489 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
11490 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
11491 DL, VT: VTList.VTs[1]);
11492 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
11493 }
11494
11495 break;
11496 }
11497 case ISD::STRICT_FP_EXTEND:
11498 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11499 "Invalid STRICT_FP_EXTEND!");
11500 assert(VTList.VTs[0].isFloatingPoint() &&
11501 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
11502 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11503 "STRICT_FP_EXTEND result type should be vector iff the operand "
11504 "type is vector!");
11505 assert((!VTList.VTs[0].isVector() ||
11506 VTList.VTs[0].getVectorElementCount() ==
11507 Ops[1].getValueType().getVectorElementCount()) &&
11508 "Vector element count mismatch!");
11509 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
11510 "Invalid fpext node, dst <= src!");
11511 break;
11512 case ISD::STRICT_FP_ROUND:
11513 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
11514 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11515 "STRICT_FP_ROUND result type should be vector iff the operand "
11516 "type is vector!");
11517 assert((!VTList.VTs[0].isVector() ||
11518 VTList.VTs[0].getVectorElementCount() ==
11519 Ops[1].getValueType().getVectorElementCount()) &&
11520 "Vector element count mismatch!");
11521 assert(VTList.VTs[0].isFloatingPoint() &&
11522 Ops[1].getValueType().isFloatingPoint() &&
11523 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
11524 Ops[2].getOpcode() == ISD::TargetConstant &&
11525 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
11526 "Invalid STRICT_FP_ROUND!");
11527 break;
11528 }
11529
11530 // Memoize the node unless it returns a glue result.
11531 SDNode *N;
11532 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
11533 FoldingSetNodeID ID;
11534 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
11535 void *IP = nullptr;
11536 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11537 E->intersectFlagsWith(Flags);
11538 return SDValue(E, 0);
11539 }
11540
11541 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11542 createOperands(Node: N, Vals: Ops);
11543 CSEMap.InsertNode(N, InsertPos: IP);
11544 } else {
11545 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11546 createOperands(Node: N, Vals: Ops);
11547 }
11548
11549 N->setFlags(Flags);
11550 InsertNode(N);
11551 SDValue V(N, 0);
11552 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11553 return V;
11554}
11555
11556SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11557 SDVTList VTList) {
11558 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
11559}
11560
11561SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11562 SDValue N1) {
11563 SDValue Ops[] = { N1 };
11564 return getNode(Opcode, DL, VTList, Ops);
11565}
11566
11567SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11568 SDValue N1, SDValue N2) {
11569 SDValue Ops[] = { N1, N2 };
11570 return getNode(Opcode, DL, VTList, Ops);
11571}
11572
11573SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11574 SDValue N1, SDValue N2, SDValue N3) {
11575 SDValue Ops[] = { N1, N2, N3 };
11576 return getNode(Opcode, DL, VTList, Ops);
11577}
11578
11579SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11580 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
11581 SDValue Ops[] = { N1, N2, N3, N4 };
11582 return getNode(Opcode, DL, VTList, Ops);
11583}
11584
11585SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11586 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
11587 SDValue N5) {
11588 SDValue Ops[] = { N1, N2, N3, N4, N5 };
11589 return getNode(Opcode, DL, VTList, Ops);
11590}
11591
11592SDVTList SelectionDAG::getVTList(EVT VT) {
11593 if (!VT.isExtended())
11594 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
11595
11596 return makeVTList(VTs: &(*EVTs.insert(x: VT).first), NumVTs: 1);
11597}
11598
11599SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
11600 FoldingSetNodeID ID;
11601 ID.AddInteger(I: 2U);
11602 ID.AddInteger(I: VT1.getRawBits());
11603 ID.AddInteger(I: VT2.getRawBits());
11604
11605 void *IP = nullptr;
11606 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11607 if (!Result) {
11608 EVT *Array = Allocator.Allocate<EVT>(Num: 2);
11609 Array[0] = VT1;
11610 Array[1] = VT2;
11611 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
11612 VTListMap.InsertNode(N: Result, InsertPos: IP);
11613 }
11614 return Result->getSDVTList();
11615}
11616
11617SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
11618 FoldingSetNodeID ID;
11619 ID.AddInteger(I: 3U);
11620 ID.AddInteger(I: VT1.getRawBits());
11621 ID.AddInteger(I: VT2.getRawBits());
11622 ID.AddInteger(I: VT3.getRawBits());
11623
11624 void *IP = nullptr;
11625 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11626 if (!Result) {
11627 EVT *Array = Allocator.Allocate<EVT>(Num: 3);
11628 Array[0] = VT1;
11629 Array[1] = VT2;
11630 Array[2] = VT3;
11631 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
11632 VTListMap.InsertNode(N: Result, InsertPos: IP);
11633 }
11634 return Result->getSDVTList();
11635}
11636
11637SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
11638 FoldingSetNodeID ID;
11639 ID.AddInteger(I: 4U);
11640 ID.AddInteger(I: VT1.getRawBits());
11641 ID.AddInteger(I: VT2.getRawBits());
11642 ID.AddInteger(I: VT3.getRawBits());
11643 ID.AddInteger(I: VT4.getRawBits());
11644
11645 void *IP = nullptr;
11646 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11647 if (!Result) {
11648 EVT *Array = Allocator.Allocate<EVT>(Num: 4);
11649 Array[0] = VT1;
11650 Array[1] = VT2;
11651 Array[2] = VT3;
11652 Array[3] = VT4;
11653 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
11654 VTListMap.InsertNode(N: Result, InsertPos: IP);
11655 }
11656 return Result->getSDVTList();
11657}
11658
11659SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
11660 unsigned NumVTs = VTs.size();
11661 FoldingSetNodeID ID;
11662 ID.AddInteger(I: NumVTs);
11663 for (unsigned index = 0; index < NumVTs; index++) {
11664 ID.AddInteger(I: VTs[index].getRawBits());
11665 }
11666
11667 void *IP = nullptr;
11668 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11669 if (!Result) {
11670 EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
11671 llvm::copy(Range&: VTs, Out: Array);
11672 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
11673 VTListMap.InsertNode(N: Result, InsertPos: IP);
11674 }
11675 return Result->getSDVTList();
11676}
11677
11678
11679/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
11680/// specified operands. If the resultant node already exists in the DAG,
11681/// this does not modify the specified node, instead it returns the node that
11682/// already exists. If the resultant node does not exist in the DAG, the
11683/// input node is returned. As a degenerate case, if you specify the same
11684/// input operands as the node already has, the input node is returned.
11685SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
11686 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
11687
11688 // Check to see if there is no change.
11689 if (Op == N->getOperand(Num: 0)) return N;
11690
11691 // See if the modified node already exists.
11692 void *InsertPos = nullptr;
11693 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
11694 return Existing;
11695
11696 // Nope it doesn't. Remove the node from its current place in the maps.
11697 if (InsertPos)
11698 if (!RemoveNodeFromCSEMaps(N))
11699 InsertPos = nullptr;
11700
11701 // Now we update the operands.
11702 N->OperandList[0].set(Op);
11703
11704 updateDivergence(N);
11705 // If this gets put into a CSE map, add it.
11706 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11707 return N;
11708}
11709
11710SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
11711 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
11712
11713 // Check to see if there is no change.
11714 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
11715 return N; // No operands changed, just return the input node.
11716
11717 // See if the modified node already exists.
11718 void *InsertPos = nullptr;
11719 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
11720 return Existing;
11721
11722 // Nope it doesn't. Remove the node from its current place in the maps.
11723 if (InsertPos)
11724 if (!RemoveNodeFromCSEMaps(N))
11725 InsertPos = nullptr;
11726
11727 // Now we update the operands.
11728 if (N->OperandList[0] != Op1)
11729 N->OperandList[0].set(Op1);
11730 if (N->OperandList[1] != Op2)
11731 N->OperandList[1].set(Op2);
11732
11733 updateDivergence(N);
11734 // If this gets put into a CSE map, add it.
11735 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11736 return N;
11737}
11738
11739SDNode *SelectionDAG::
11740UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
11741 SDValue Ops[] = { Op1, Op2, Op3 };
11742 return UpdateNodeOperands(N, Ops);
11743}
11744
11745SDNode *SelectionDAG::
11746UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11747 SDValue Op3, SDValue Op4) {
11748 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
11749 return UpdateNodeOperands(N, Ops);
11750}
11751
11752SDNode *SelectionDAG::
11753UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11754 SDValue Op3, SDValue Op4, SDValue Op5) {
11755 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
11756 return UpdateNodeOperands(N, Ops);
11757}
11758
11759SDNode *SelectionDAG::
11760UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
11761 unsigned NumOps = Ops.size();
11762 assert(N->getNumOperands() == NumOps &&
11763 "Update with wrong number of operands");
11764
11765 // If no operands changed just return the input node.
11766 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
11767 return N;
11768
11769 // See if the modified node already exists.
11770 void *InsertPos = nullptr;
11771 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
11772 return Existing;
11773
11774 // Nope it doesn't. Remove the node from its current place in the maps.
11775 if (InsertPos)
11776 if (!RemoveNodeFromCSEMaps(N))
11777 InsertPos = nullptr;
11778
11779 // Now we update the operands.
11780 for (unsigned i = 0; i != NumOps; ++i)
11781 if (N->OperandList[i] != Ops[i])
11782 N->OperandList[i].set(Ops[i]);
11783
11784 updateDivergence(N);
11785 // If this gets put into a CSE map, add it.
11786 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11787 return N;
11788}
11789
11790/// DropOperands - Release the operands and set this node to have
11791/// zero operands.
11792void SDNode::DropOperands() {
11793 // Unlike the code in MorphNodeTo that does this, we don't need to
11794 // watch for dead nodes here.
11795 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
11796 SDUse &Use = *I++;
11797 Use.set(SDValue());
11798 }
11799}
11800
11801void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
11802 ArrayRef<MachineMemOperand *> NewMemRefs) {
11803 if (NewMemRefs.empty()) {
11804 N->clearMemRefs();
11805 return;
11806 }
11807
11808 // Check if we can avoid allocating by storing a single reference directly.
11809 if (NewMemRefs.size() == 1) {
11810 N->MemRefs = NewMemRefs[0];
11811 N->NumMemRefs = 1;
11812 return;
11813 }
11814
11815 MachineMemOperand **MemRefsBuffer =
11816 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
11817 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
11818 N->MemRefs = MemRefsBuffer;
11819 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
11820}
11821
11822/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
11823/// machine opcode.
11824///
11825SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11826 EVT VT) {
11827 SDVTList VTs = getVTList(VT);
11828 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11829}
11830
11831SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11832 EVT VT, SDValue Op1) {
11833 SDVTList VTs = getVTList(VT);
11834 SDValue Ops[] = { Op1 };
11835 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11836}
11837
11838SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11839 EVT VT, SDValue Op1,
11840 SDValue Op2) {
11841 SDVTList VTs = getVTList(VT);
11842 SDValue Ops[] = { Op1, Op2 };
11843 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11844}
11845
11846SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11847 EVT VT, SDValue Op1,
11848 SDValue Op2, SDValue Op3) {
11849 SDVTList VTs = getVTList(VT);
11850 SDValue Ops[] = { Op1, Op2, Op3 };
11851 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11852}
11853
11854SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11855 EVT VT, ArrayRef<SDValue> Ops) {
11856 SDVTList VTs = getVTList(VT);
11857 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11858}
11859
11860SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11861 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
11862 SDVTList VTs = getVTList(VT1, VT2);
11863 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11864}
11865
11866SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11867 EVT VT1, EVT VT2) {
11868 SDVTList VTs = getVTList(VT1, VT2);
11869 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11870}
11871
11872SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11873 EVT VT1, EVT VT2, EVT VT3,
11874 ArrayRef<SDValue> Ops) {
11875 SDVTList VTs = getVTList(VT1, VT2, VT3);
11876 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11877}
11878
11879SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11880 EVT VT1, EVT VT2,
11881 SDValue Op1, SDValue Op2) {
11882 SDVTList VTs = getVTList(VT1, VT2);
11883 SDValue Ops[] = { Op1, Op2 };
11884 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11885}
11886
11887SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11888 SDVTList VTs,ArrayRef<SDValue> Ops) {
11889 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
11890 // Reset the NodeID to -1.
11891 New->setNodeId(-1);
11892 if (New != N) {
11893 ReplaceAllUsesWith(From: N, To: New);
11894 RemoveDeadNode(N);
11895 }
11896 return New;
11897}
11898
11899/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
11900/// the line number information on the merged node since it is not possible to
11901/// preserve the information that operation is associated with multiple lines.
11902/// This will make the debugger working better at -O0, were there is a higher
11903/// probability having other instructions associated with that line.
11904///
11905/// For IROrder, we keep the smaller of the two
11906SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
11907 DebugLoc NLoc = N->getDebugLoc();
11908 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
11909 N->setDebugLoc(DebugLoc());
11910 }
11911 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
11912 N->setIROrder(Order);
11913 return N;
11914}
11915
11916/// MorphNodeTo - This *mutates* the specified node to have the specified
11917/// return type, opcode, and operands.
11918///
11919/// Note that MorphNodeTo returns the resultant node. If there is already a
11920/// node of the specified opcode and operands, it returns that node instead of
11921/// the current one. Note that the SDLoc need not be the same.
11922///
11923/// Using MorphNodeTo is faster than creating a new node and swapping it in
11924/// with ReplaceAllUsesWith both because it often avoids allocating a new
11925/// node, and because it doesn't require CSE recalculation for any of
11926/// the node's users.
11927///
11928/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
11929/// As a consequence it isn't appropriate to use from within the DAG combiner or
11930/// the legalizer which maintain worklists that would need to be updated when
11931/// deleting things.
11932SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
11933 SDVTList VTs, ArrayRef<SDValue> Ops) {
11934 // If an identical node already exists, use it.
11935 void *IP = nullptr;
11936 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
11937 FoldingSetNodeID ID;
11938 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
11939 if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
11940 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
11941 }
11942
11943 if (!RemoveNodeFromCSEMaps(N))
11944 IP = nullptr;
11945
11946 // Start the morphing.
11947 N->NodeType = Opc;
11948 N->ValueList = VTs.VTs;
11949 N->NumValues = VTs.NumVTs;
11950
11951 // Clear the operands list, updating used nodes to remove this from their
11952 // use list. Keep track of any operands that become dead as a result.
11953 SmallPtrSet<SDNode*, 16> DeadNodeSet;
11954 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
11955 SDUse &Use = *I++;
11956 SDNode *Used = Use.getNode();
11957 Use.set(SDValue());
11958 if (Used->use_empty())
11959 DeadNodeSet.insert(Ptr: Used);
11960 }
11961
11962 // For MachineNode, initialize the memory references information.
11963 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
11964 MN->clearMemRefs();
11965
11966 // Swap for an appropriately sized array from the recycler.
11967 removeOperands(Node: N);
11968 createOperands(Node: N, Vals: Ops);
11969
11970 // Delete any nodes that are still dead after adding the uses for the
11971 // new operands.
11972 if (!DeadNodeSet.empty()) {
11973 SmallVector<SDNode *, 16> DeadNodes;
11974 for (SDNode *N : DeadNodeSet)
11975 if (N->use_empty())
11976 DeadNodes.push_back(Elt: N);
11977 RemoveDeadNodes(DeadNodes);
11978 }
11979
11980 if (IP)
11981 CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
11982 return N;
11983}
11984
11985SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
11986 unsigned OrigOpc = Node->getOpcode();
11987 unsigned NewOpc;
11988 switch (OrigOpc) {
11989 default:
11990 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
11991#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11992 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
11993#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11994 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
11995#include "llvm/IR/ConstrainedOps.def"
11996 }
11997
11998 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
11999
12000 // We're taking this node out of the chain, so we need to re-link things.
12001 SDValue InputChain = Node->getOperand(Num: 0);
12002 SDValue OutputChain = SDValue(Node, 1);
12003 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12004
12005 SmallVector<SDValue, 3> Ops;
12006 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12007 Ops.push_back(Elt: Node->getOperand(Num: i));
12008
12009 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12010 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12011
12012 // MorphNodeTo can operate in two ways: if an existing node with the
12013 // specified operands exists, it can just return it. Otherwise, it
12014 // updates the node in place to have the requested operands.
12015 if (Res == Node) {
12016 // If we updated the node in place, reset the node ID. To the isel,
12017 // this should be just like a newly allocated machine node.
12018 Res->setNodeId(-1);
12019 } else {
12020 ReplaceAllUsesWith(From: Node, To: Res);
12021 RemoveDeadNode(N: Node);
12022 }
12023
12024 return Res;
12025}
12026
12027/// getMachineNode - These are used for target selectors to create a new node
12028/// with specified return type(s), MachineInstr opcode, and operands.
12029///
12030/// Note that getMachineNode returns the resultant node. If there is already a
12031/// node of the specified opcode and operands, it returns that node instead of
12032/// the current one.
12033MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12034 EVT VT) {
12035 SDVTList VTs = getVTList(VT);
12036 return getMachineNode(Opcode, dl, VTs, Ops: {});
12037}
12038
12039MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12040 EVT VT, SDValue Op1) {
12041 SDVTList VTs = getVTList(VT);
12042 SDValue Ops[] = { Op1 };
12043 return getMachineNode(Opcode, dl, VTs, Ops);
12044}
12045
12046MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12047 EVT VT, SDValue Op1, SDValue Op2) {
12048 SDVTList VTs = getVTList(VT);
12049 SDValue Ops[] = { Op1, Op2 };
12050 return getMachineNode(Opcode, dl, VTs, Ops);
12051}
12052
12053MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12054 EVT VT, SDValue Op1, SDValue Op2,
12055 SDValue Op3) {
12056 SDVTList VTs = getVTList(VT);
12057 SDValue Ops[] = { Op1, Op2, Op3 };
12058 return getMachineNode(Opcode, dl, VTs, Ops);
12059}
12060
12061MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12062 EVT VT, ArrayRef<SDValue> Ops) {
12063 SDVTList VTs = getVTList(VT);
12064 return getMachineNode(Opcode, dl, VTs, Ops);
12065}
12066
12067MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12068 EVT VT1, EVT VT2, SDValue Op1,
12069 SDValue Op2) {
12070 SDVTList VTs = getVTList(VT1, VT2);
12071 SDValue Ops[] = { Op1, Op2 };
12072 return getMachineNode(Opcode, dl, VTs, Ops);
12073}
12074
12075MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12076 EVT VT1, EVT VT2, SDValue Op1,
12077 SDValue Op2, SDValue Op3) {
12078 SDVTList VTs = getVTList(VT1, VT2);
12079 SDValue Ops[] = { Op1, Op2, Op3 };
12080 return getMachineNode(Opcode, dl, VTs, Ops);
12081}
12082
12083MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12084 EVT VT1, EVT VT2,
12085 ArrayRef<SDValue> Ops) {
12086 SDVTList VTs = getVTList(VT1, VT2);
12087 return getMachineNode(Opcode, dl, VTs, Ops);
12088}
12089
12090MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12091 EVT VT1, EVT VT2, EVT VT3,
12092 SDValue Op1, SDValue Op2) {
12093 SDVTList VTs = getVTList(VT1, VT2, VT3);
12094 SDValue Ops[] = { Op1, Op2 };
12095 return getMachineNode(Opcode, dl, VTs, Ops);
12096}
12097
12098MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12099 EVT VT1, EVT VT2, EVT VT3,
12100 SDValue Op1, SDValue Op2,
12101 SDValue Op3) {
12102 SDVTList VTs = getVTList(VT1, VT2, VT3);
12103 SDValue Ops[] = { Op1, Op2, Op3 };
12104 return getMachineNode(Opcode, dl, VTs, Ops);
12105}
12106
12107MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12108 EVT VT1, EVT VT2, EVT VT3,
12109 ArrayRef<SDValue> Ops) {
12110 SDVTList VTs = getVTList(VT1, VT2, VT3);
12111 return getMachineNode(Opcode, dl, VTs, Ops);
12112}
12113
12114MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12115 ArrayRef<EVT> ResultTys,
12116 ArrayRef<SDValue> Ops) {
12117 SDVTList VTs = getVTList(VTs: ResultTys);
12118 return getMachineNode(Opcode, dl, VTs, Ops);
12119}
12120
12121MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12122 SDVTList VTs,
12123 ArrayRef<SDValue> Ops) {
12124 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12125 MachineSDNode *N;
12126 void *IP = nullptr;
12127
12128 if (DoCSE) {
12129 FoldingSetNodeID ID;
12130 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
12131 IP = nullptr;
12132 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
12133 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12134 }
12135 }
12136
12137 // Allocate a new MachineSDNode.
12138 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12139 createOperands(Node: N, Vals: Ops);
12140
12141 if (DoCSE)
12142 CSEMap.InsertNode(N, InsertPos: IP);
12143
12144 InsertNode(N);
12145 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12146 return N;
12147}
12148
12149/// getTargetExtractSubreg - A convenience function for creating
12150/// TargetOpcode::EXTRACT_SUBREG nodes.
12151SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12152 SDValue Operand) {
12153 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12154 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12155 VT, Op1: Operand, Op2: SRIdxVal);
12156 return SDValue(Subreg, 0);
12157}
12158
12159/// getTargetInsertSubreg - A convenience function for creating
12160/// TargetOpcode::INSERT_SUBREG nodes.
12161SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12162 SDValue Operand, SDValue Subreg) {
12163 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12164 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12165 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12166 return SDValue(Result, 0);
12167}
12168
12169/// getNodeIfExists - Get the specified node if it's already available, or
12170/// else return NULL.
12171SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12172 ArrayRef<SDValue> Ops,
12173 bool AllowCommute) {
12174 SDNodeFlags Flags;
12175 if (Inserter)
12176 Flags = Inserter->getFlags();
12177 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12178}
12179
12180SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12181 ArrayRef<SDValue> Ops,
12182 const SDNodeFlags Flags,
12183 bool AllowCommute) {
12184 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12185 return nullptr;
12186
12187 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12188 FoldingSetNodeID ID;
12189 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: LookupOps);
12190 void *IP = nullptr;
12191 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) {
12192 E->intersectFlagsWith(Flags);
12193 return E;
12194 }
12195 return nullptr;
12196 };
12197
12198 if (SDNode *Existing = Lookup(Ops))
12199 return Existing;
12200
12201 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12202 return Lookup({Ops[1], Ops[0]});
12203
12204 return nullptr;
12205}
12206
12207/// doesNodeExist - Check if a node exists without modifying its flags.
12208bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12209 ArrayRef<SDValue> Ops) {
12210 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12211 FoldingSetNodeID ID;
12212 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12213 void *IP = nullptr;
12214 if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
12215 return true;
12216 }
12217 return false;
12218}
12219
12220/// getDbgValue - Creates a SDDbgValue node.
12221///
12222/// SDNode
12223SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12224 SDNode *N, unsigned R, bool IsIndirect,
12225 const DebugLoc &DL, unsigned O) {
12226 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12227 "Expected inlined-at fields to agree");
12228 return new (DbgInfo->getAlloc())
12229 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12230 {}, IsIndirect, DL, O,
12231 /*IsVariadic=*/false);
12232}
12233
12234/// Constant
12235SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12236 DIExpression *Expr,
12237 const Value *C,
12238 const DebugLoc &DL, unsigned O) {
12239 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12240 "Expected inlined-at fields to agree");
12241 return new (DbgInfo->getAlloc())
12242 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12243 /*IsIndirect=*/false, DL, O,
12244 /*IsVariadic=*/false);
12245}
12246
12247/// FrameIndex
12248SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12249 DIExpression *Expr, unsigned FI,
12250 bool IsIndirect,
12251 const DebugLoc &DL,
12252 unsigned O) {
12253 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12254 "Expected inlined-at fields to agree");
12255 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12256}
12257
12258/// FrameIndex with dependencies
12259SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12260 DIExpression *Expr, unsigned FI,
12261 ArrayRef<SDNode *> Dependencies,
12262 bool IsIndirect,
12263 const DebugLoc &DL,
12264 unsigned O) {
12265 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12266 "Expected inlined-at fields to agree");
12267 return new (DbgInfo->getAlloc())
12268 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12269 Dependencies, IsIndirect, DL, O,
12270 /*IsVariadic=*/false);
12271}
12272
12273/// VReg
12274SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12275 Register VReg, bool IsIndirect,
12276 const DebugLoc &DL, unsigned O) {
12277 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12278 "Expected inlined-at fields to agree");
12279 return new (DbgInfo->getAlloc())
12280 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12281 {}, IsIndirect, DL, O,
12282 /*IsVariadic=*/false);
12283}
12284
12285SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12286 ArrayRef<SDDbgOperand> Locs,
12287 ArrayRef<SDNode *> Dependencies,
12288 bool IsIndirect, const DebugLoc &DL,
12289 unsigned O, bool IsVariadic) {
12290 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12291 "Expected inlined-at fields to agree");
12292 return new (DbgInfo->getAlloc())
12293 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12294 DL, O, IsVariadic);
12295}
12296
12297void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12298 unsigned OffsetInBits, unsigned SizeInBits,
12299 bool InvalidateDbg) {
12300 SDNode *FromNode = From.getNode();
12301 SDNode *ToNode = To.getNode();
12302 assert(FromNode && ToNode && "Can't modify dbg values");
12303
12304 // PR35338
12305 // TODO: assert(From != To && "Redundant dbg value transfer");
12306 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12307 if (From == To || FromNode == ToNode)
12308 return;
12309
12310 if (!FromNode->getHasDebugValue())
12311 return;
12312
12313 SDDbgOperand FromLocOp =
12314 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12315 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12316
12317 SmallVector<SDDbgValue *, 2> ClonedDVs;
12318 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12319 if (Dbg->isInvalidated())
12320 continue;
12321
12322 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12323
12324 // Create a new location ops vector that is equal to the old vector, but
12325 // with each instance of FromLocOp replaced with ToLocOp.
12326 bool Changed = false;
12327 auto NewLocOps = Dbg->copyLocationOps();
12328 std::replace_if(
12329 first: NewLocOps.begin(), last: NewLocOps.end(),
12330 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12331 bool Match = Op == FromLocOp;
12332 Changed |= Match;
12333 return Match;
12334 },
12335 new_value: ToLocOp);
12336 // Ignore this SDDbgValue if we didn't find a matching location.
12337 if (!Changed)
12338 continue;
12339
12340 DIVariable *Var = Dbg->getVariable();
12341 auto *Expr = Dbg->getExpression();
12342 // If a fragment is requested, update the expression.
12343 if (SizeInBits) {
12344 // When splitting a larger (e.g., sign-extended) value whose
12345 // lower bits are described with an SDDbgValue, do not attempt
12346 // to transfer the SDDbgValue to the upper bits.
12347 if (auto FI = Expr->getFragmentInfo())
12348 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12349 continue;
12350 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12351 SizeInBits);
12352 if (!Fragment)
12353 continue;
12354 Expr = *Fragment;
12355 }
12356
12357 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12358 // Clone the SDDbgValue and move it to To.
12359 SDDbgValue *Clone = getDbgValueList(
12360 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12361 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12362 IsVariadic: Dbg->isVariadic());
12363 ClonedDVs.push_back(Elt: Clone);
12364
12365 if (InvalidateDbg) {
12366 // Invalidate value and indicate the SDDbgValue should not be emitted.
12367 Dbg->setIsInvalidated();
12368 Dbg->setIsEmitted();
12369 }
12370 }
12371
12372 for (SDDbgValue *Dbg : ClonedDVs) {
12373 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12374 "Transferred DbgValues should depend on the new SDNode");
12375 AddDbgValue(DB: Dbg, isParameter: false);
12376 }
12377}
12378
12379void SelectionDAG::salvageDebugInfo(SDNode &N) {
12380 if (!N.getHasDebugValue())
12381 return;
12382
12383 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
12384 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
12385 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
12386 return SDDbgOperand::fromNode(Node, ResNo);
12387 };
12388
12389 SmallVector<SDDbgValue *, 2> ClonedDVs;
12390 for (auto *DV : GetDbgValues(SD: &N)) {
12391 if (DV->isInvalidated())
12392 continue;
12393 switch (N.getOpcode()) {
12394 default:
12395 break;
12396 case ISD::ADD: {
12397 SDValue N0 = N.getOperand(Num: 0);
12398 SDValue N1 = N.getOperand(Num: 1);
12399 if (!isa<ConstantSDNode>(Val: N0)) {
12400 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
12401 uint64_t Offset;
12402 if (RHSConstant)
12403 Offset = N.getConstantOperandVal(Num: 1);
12404 // We are not allowed to turn indirect debug values variadic, so
12405 // don't salvage those.
12406 if (!RHSConstant && DV->isIndirect())
12407 continue;
12408
12409 // Rewrite an ADD constant node into a DIExpression. Since we are
12410 // performing arithmetic to compute the variable's *value* in the
12411 // DIExpression, we need to mark the expression with a
12412 // DW_OP_stack_value.
12413 auto *DIExpr = DV->getExpression();
12414 auto NewLocOps = DV->copyLocationOps();
12415 bool Changed = false;
12416 size_t OrigLocOpsSize = NewLocOps.size();
12417 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
12418 // We're not given a ResNo to compare against because the whole
12419 // node is going away. We know that any ISD::ADD only has one
12420 // result, so we can assume any node match is using the result.
12421 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12422 NewLocOps[i].getSDNode() != &N)
12423 continue;
12424 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12425 if (RHSConstant) {
12426 SmallVector<uint64_t, 3> ExprOps;
12427 DIExpression::appendOffset(Ops&: ExprOps, Offset);
12428 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12429 } else {
12430 // Convert to a variadic expression (if not already).
12431 // convertToVariadicExpression() returns a const pointer, so we use
12432 // a temporary const variable here.
12433 const auto *TmpDIExpr =
12434 DIExpression::convertToVariadicExpression(Expr: DIExpr);
12435 SmallVector<uint64_t, 3> ExprOps;
12436 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
12437 ExprOps.push_back(Elt: NewLocOps.size());
12438 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
12439 SDDbgOperand RHS =
12440 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
12441 NewLocOps.push_back(Elt: RHS);
12442 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12443 }
12444 Changed = true;
12445 }
12446 (void)Changed;
12447 assert(Changed && "Salvage target doesn't use N");
12448
12449 bool IsVariadic =
12450 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
12451
12452 auto AdditionalDependencies = DV->getAdditionalDependencies();
12453 SDDbgValue *Clone = getDbgValueList(
12454 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
12455 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
12456 ClonedDVs.push_back(Elt: Clone);
12457 DV->setIsInvalidated();
12458 DV->setIsEmitted();
12459 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
12460 N0.getNode()->dumprFull(this);
12461 dbgs() << " into " << *DIExpr << '\n');
12462 }
12463 break;
12464 }
12465 case ISD::TRUNCATE: {
12466 SDValue N0 = N.getOperand(Num: 0);
12467 TypeSize FromSize = N0.getValueSizeInBits();
12468 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
12469
12470 DIExpression *DbgExpression = DV->getExpression();
12471 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
12472 auto NewLocOps = DV->copyLocationOps();
12473 bool Changed = false;
12474 for (size_t i = 0; i < NewLocOps.size(); ++i) {
12475 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12476 NewLocOps[i].getSDNode() != &N)
12477 continue;
12478
12479 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12480 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
12481 Changed = true;
12482 }
12483 assert(Changed && "Salvage target doesn't use N");
12484 (void)Changed;
12485
12486 SDDbgValue *Clone =
12487 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
12488 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
12489 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
12490
12491 ClonedDVs.push_back(Elt: Clone);
12492 DV->setIsInvalidated();
12493 DV->setIsEmitted();
12494 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
12495 dbgs() << " into " << *DbgExpression << '\n');
12496 break;
12497 }
12498 }
12499 }
12500
12501 for (SDDbgValue *Dbg : ClonedDVs) {
12502 assert((!Dbg->getSDNodes().empty() ||
12503 llvm::any_of(Dbg->getLocationOps(),
12504 [&](const SDDbgOperand &Op) {
12505 return Op.getKind() == SDDbgOperand::FRAMEIX;
12506 })) &&
12507 "Salvaged DbgValue should depend on a new SDNode");
12508 AddDbgValue(DB: Dbg, isParameter: false);
12509 }
12510}
12511
12512/// Creates a SDDbgLabel node.
12513SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
12514 const DebugLoc &DL, unsigned O) {
12515 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
12516 "Expected inlined-at fields to agree");
12517 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
12518}
12519
12520namespace {
12521
12522/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
12523/// pointed to by a use iterator is deleted, increment the use iterator
12524/// so that it doesn't dangle.
12525///
12526class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
12527 SDNode::use_iterator &UI;
12528 SDNode::use_iterator &UE;
12529
12530 void NodeDeleted(SDNode *N, SDNode *E) override {
12531 // Increment the iterator as needed.
12532 while (UI != UE && N == UI->getUser())
12533 ++UI;
12534 }
12535
12536public:
12537 RAUWUpdateListener(SelectionDAG &d,
12538 SDNode::use_iterator &ui,
12539 SDNode::use_iterator &ue)
12540 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
12541};
12542
12543} // end anonymous namespace
12544
12545/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12546/// This can cause recursive merging of nodes in the DAG.
12547///
12548/// This version assumes From has a single result value.
12549///
12550void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
12551 SDNode *From = FromN.getNode();
12552 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
12553 "Cannot replace with this method!");
12554 assert(From != To.getNode() && "Cannot replace uses of with self");
12555
12556 // Preserve Debug Values
12557 transferDbgValues(From: FromN, To);
12558 // Preserve extra info.
12559 copyExtraInfo(From, To: To.getNode());
12560
12561 // Iterate over all the existing uses of From. New uses will be added
12562 // to the beginning of the use list, which we avoid visiting.
12563 // This specifically avoids visiting uses of From that arise while the
12564 // replacement is happening, because any such uses would be the result
12565 // of CSE: If an existing node looks like From after one of its operands
12566 // is replaced by To, we don't want to replace of all its users with To
12567 // too. See PR3018 for more info.
12568 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12569 RAUWUpdateListener Listener(*this, UI, UE);
12570 while (UI != UE) {
12571 SDNode *User = UI->getUser();
12572
12573 // This node is about to morph, remove its old self from the CSE maps.
12574 RemoveNodeFromCSEMaps(N: User);
12575
12576 // A user can appear in a use list multiple times, and when this
12577 // happens the uses are usually next to each other in the list.
12578 // To help reduce the number of CSE recomputations, process all
12579 // the uses of this user that we can find this way.
12580 do {
12581 SDUse &Use = *UI;
12582 ++UI;
12583 Use.set(To);
12584 if (To->isDivergent() != From->isDivergent())
12585 updateDivergence(N: User);
12586 } while (UI != UE && UI->getUser() == User);
12587 // Now that we have modified User, add it back to the CSE maps. If it
12588 // already exists there, recursively merge the results together.
12589 AddModifiedNodeToCSEMaps(N: User);
12590 }
12591
12592 // If we just RAUW'd the root, take note.
12593 if (FromN == getRoot())
12594 setRoot(To);
12595}
12596
12597/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12598/// This can cause recursive merging of nodes in the DAG.
12599///
12600/// This version assumes that for each value of From, there is a
12601/// corresponding value in To in the same position with the same type.
12602///
12603void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
12604#ifndef NDEBUG
12605 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12606 assert((!From->hasAnyUseOfValue(i) ||
12607 From->getValueType(i) == To->getValueType(i)) &&
12608 "Cannot use this version of ReplaceAllUsesWith!");
12609#endif
12610
12611 // Handle the trivial case.
12612 if (From == To)
12613 return;
12614
12615 // Preserve Debug Info. Only do this if there's a use.
12616 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12617 if (From->hasAnyUseOfValue(Value: i)) {
12618 assert((i < To->getNumValues()) && "Invalid To location");
12619 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
12620 }
12621 // Preserve extra info.
12622 copyExtraInfo(From, To);
12623
12624 // Iterate over just the existing users of From. See the comments in
12625 // the ReplaceAllUsesWith above.
12626 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12627 RAUWUpdateListener Listener(*this, UI, UE);
12628 while (UI != UE) {
12629 SDNode *User = UI->getUser();
12630
12631 // This node is about to morph, remove its old self from the CSE maps.
12632 RemoveNodeFromCSEMaps(N: User);
12633
12634 // A user can appear in a use list multiple times, and when this
12635 // happens the uses are usually next to each other in the list.
12636 // To help reduce the number of CSE recomputations, process all
12637 // the uses of this user that we can find this way.
12638 do {
12639 SDUse &Use = *UI;
12640 ++UI;
12641 Use.setNode(To);
12642 if (To->isDivergent() != From->isDivergent())
12643 updateDivergence(N: User);
12644 } while (UI != UE && UI->getUser() == User);
12645
12646 // Now that we have modified User, add it back to the CSE maps. If it
12647 // already exists there, recursively merge the results together.
12648 AddModifiedNodeToCSEMaps(N: User);
12649 }
12650
12651 // If we just RAUW'd the root, take note.
12652 if (From == getRoot().getNode())
12653 setRoot(SDValue(To, getRoot().getResNo()));
12654}
12655
12656/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12657/// This can cause recursive merging of nodes in the DAG.
12658///
12659/// This version can replace From with any result values. To must match the
12660/// number and types of values returned by From.
12661void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
12662 if (From->getNumValues() == 1) // Handle the simple case efficiently.
12663 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
12664
12665 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
12666 // Preserve Debug Info.
12667 transferDbgValues(From: SDValue(From, i), To: To[i]);
12668 // Preserve extra info.
12669 copyExtraInfo(From, To: To[i].getNode());
12670 }
12671
12672 // Iterate over just the existing users of From. See the comments in
12673 // the ReplaceAllUsesWith above.
12674 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12675 RAUWUpdateListener Listener(*this, UI, UE);
12676 while (UI != UE) {
12677 SDNode *User = UI->getUser();
12678
12679 // This node is about to morph, remove its old self from the CSE maps.
12680 RemoveNodeFromCSEMaps(N: User);
12681
12682 // A user can appear in a use list multiple times, and when this happens the
12683 // uses are usually next to each other in the list. To help reduce the
12684 // number of CSE and divergence recomputations, process all the uses of this
12685 // user that we can find this way.
12686 bool To_IsDivergent = false;
12687 do {
12688 SDUse &Use = *UI;
12689 const SDValue &ToOp = To[Use.getResNo()];
12690 ++UI;
12691 Use.set(ToOp);
12692 if (ToOp.getValueType() != MVT::Other)
12693 To_IsDivergent |= ToOp->isDivergent();
12694 } while (UI != UE && UI->getUser() == User);
12695
12696 if (To_IsDivergent != From->isDivergent())
12697 updateDivergence(N: User);
12698
12699 // Now that we have modified User, add it back to the CSE maps. If it
12700 // already exists there, recursively merge the results together.
12701 AddModifiedNodeToCSEMaps(N: User);
12702 }
12703
12704 // If we just RAUW'd the root, take note.
12705 if (From == getRoot().getNode())
12706 setRoot(SDValue(To[getRoot().getResNo()]));
12707}
12708
12709/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
12710/// uses of other values produced by From.getNode() alone. The Deleted
12711/// vector is handled the same way as for ReplaceAllUsesWith.
12712void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
12713 // Handle the really simple, really trivial case efficiently.
12714 if (From == To) return;
12715
12716 // Handle the simple, trivial, case efficiently.
12717 if (From.getNode()->getNumValues() == 1) {
12718 ReplaceAllUsesWith(FromN: From, To);
12719 return;
12720 }
12721
12722 // Preserve Debug Info.
12723 transferDbgValues(From, To);
12724 copyExtraInfo(From: From.getNode(), To: To.getNode());
12725
12726 // Iterate over just the existing users of From. See the comments in
12727 // the ReplaceAllUsesWith above.
12728 SDNode::use_iterator UI = From.getNode()->use_begin(),
12729 UE = From.getNode()->use_end();
12730 RAUWUpdateListener Listener(*this, UI, UE);
12731 while (UI != UE) {
12732 SDNode *User = UI->getUser();
12733 bool UserRemovedFromCSEMaps = false;
12734
12735 // A user can appear in a use list multiple times, and when this
12736 // happens the uses are usually next to each other in the list.
12737 // To help reduce the number of CSE recomputations, process all
12738 // the uses of this user that we can find this way.
12739 do {
12740 SDUse &Use = *UI;
12741
12742 // Skip uses of different values from the same node.
12743 if (Use.getResNo() != From.getResNo()) {
12744 ++UI;
12745 continue;
12746 }
12747
12748 // If this node hasn't been modified yet, it's still in the CSE maps,
12749 // so remove its old self from the CSE maps.
12750 if (!UserRemovedFromCSEMaps) {
12751 RemoveNodeFromCSEMaps(N: User);
12752 UserRemovedFromCSEMaps = true;
12753 }
12754
12755 ++UI;
12756 Use.set(To);
12757 if (To->isDivergent() != From->isDivergent())
12758 updateDivergence(N: User);
12759 } while (UI != UE && UI->getUser() == User);
12760 // We are iterating over all uses of the From node, so if a use
12761 // doesn't use the specific value, no changes are made.
12762 if (!UserRemovedFromCSEMaps)
12763 continue;
12764
12765 // Now that we have modified User, add it back to the CSE maps. If it
12766 // already exists there, recursively merge the results together.
12767 AddModifiedNodeToCSEMaps(N: User);
12768 }
12769
12770 // If we just RAUW'd the root, take note.
12771 if (From == getRoot())
12772 setRoot(To);
12773}
12774
12775namespace {
12776
12777/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
12778/// to record information about a use.
12779struct UseMemo {
12780 SDNode *User;
12781 unsigned Index;
12782 SDUse *Use;
12783};
12784
12785/// operator< - Sort Memos by User.
12786bool operator<(const UseMemo &L, const UseMemo &R) {
12787 return (intptr_t)L.User < (intptr_t)R.User;
12788}
12789
12790/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
12791/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
12792/// the node already has been taken care of recursively.
12793class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
12794 SmallVectorImpl<UseMemo> &Uses;
12795
12796 void NodeDeleted(SDNode *N, SDNode *E) override {
12797 for (UseMemo &Memo : Uses)
12798 if (Memo.User == N)
12799 Memo.User = nullptr;
12800 }
12801
12802public:
12803 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
12804 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
12805};
12806
12807} // end anonymous namespace
12808
12809/// Return true if a glue output should propagate divergence information.
12810static bool gluePropagatesDivergence(const SDNode *Node) {
12811 switch (Node->getOpcode()) {
12812 case ISD::CopyFromReg:
12813 case ISD::CopyToReg:
12814 return false;
12815 default:
12816 return true;
12817 }
12818
12819 llvm_unreachable("covered opcode switch");
12820}
12821
12822bool SelectionDAG::calculateDivergence(SDNode *N) {
12823 if (TLI->isSDNodeAlwaysUniform(N)) {
12824 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
12825 "Conflicting divergence information!");
12826 return false;
12827 }
12828 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
12829 return true;
12830 for (const auto &Op : N->ops()) {
12831 EVT VT = Op.getValueType();
12832
12833 // Skip Chain. It does not carry divergence.
12834 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
12835 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
12836 return true;
12837 }
12838 return false;
12839}
12840
12841void SelectionDAG::updateDivergence(SDNode *N) {
12842 SmallVector<SDNode *, 16> Worklist(1, N);
12843 do {
12844 N = Worklist.pop_back_val();
12845 bool IsDivergent = calculateDivergence(N);
12846 if (N->SDNodeBits.IsDivergent != IsDivergent) {
12847 N->SDNodeBits.IsDivergent = IsDivergent;
12848 llvm::append_range(C&: Worklist, R: N->users());
12849 }
12850 } while (!Worklist.empty());
12851}
12852
12853void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
12854 DenseMap<SDNode *, unsigned> Degree;
12855 Order.reserve(n: AllNodes.size());
12856 for (auto &N : allnodes()) {
12857 unsigned NOps = N.getNumOperands();
12858 Degree[&N] = NOps;
12859 if (0 == NOps)
12860 Order.push_back(x: &N);
12861 }
12862 for (size_t I = 0; I != Order.size(); ++I) {
12863 SDNode *N = Order[I];
12864 for (auto *U : N->users()) {
12865 unsigned &UnsortedOps = Degree[U];
12866 if (0 == --UnsortedOps)
12867 Order.push_back(x: U);
12868 }
12869 }
12870}
12871
12872#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
12873void SelectionDAG::VerifyDAGDivergence() {
12874 std::vector<SDNode *> TopoOrder;
12875 CreateTopologicalOrder(TopoOrder);
12876 for (auto *N : TopoOrder) {
12877 assert(calculateDivergence(N) == N->isDivergent() &&
12878 "Divergence bit inconsistency detected");
12879 }
12880}
12881#endif
12882
12883/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
12884/// uses of other values produced by From.getNode() alone. The same value
12885/// may appear in both the From and To list. The Deleted vector is
12886/// handled the same way as for ReplaceAllUsesWith.
12887void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
12888 const SDValue *To,
12889 unsigned Num){
12890 // Handle the simple, trivial case efficiently.
12891 if (Num == 1)
12892 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
12893
12894 transferDbgValues(From: *From, To: *To);
12895 copyExtraInfo(From: From->getNode(), To: To->getNode());
12896
12897 // Read up all the uses and make records of them. This helps
12898 // processing new uses that are introduced during the
12899 // replacement process.
12900 SmallVector<UseMemo, 4> Uses;
12901 for (unsigned i = 0; i != Num; ++i) {
12902 unsigned FromResNo = From[i].getResNo();
12903 SDNode *FromNode = From[i].getNode();
12904 for (SDUse &Use : FromNode->uses()) {
12905 if (Use.getResNo() == FromResNo) {
12906 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
12907 Uses.push_back(Elt: Memo);
12908 }
12909 }
12910 }
12911
12912 // Sort the uses, so that all the uses from a given User are together.
12913 llvm::sort(C&: Uses);
12914 RAUOVWUpdateListener Listener(*this, Uses);
12915
12916 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
12917 UseIndex != UseIndexEnd; ) {
12918 // We know that this user uses some value of From. If it is the right
12919 // value, update it.
12920 SDNode *User = Uses[UseIndex].User;
12921 // If the node has been deleted by recursive CSE updates when updating
12922 // another node, then just skip this entry.
12923 if (User == nullptr) {
12924 ++UseIndex;
12925 continue;
12926 }
12927
12928 // This node is about to morph, remove its old self from the CSE maps.
12929 RemoveNodeFromCSEMaps(N: User);
12930
12931 // The Uses array is sorted, so all the uses for a given User
12932 // are next to each other in the list.
12933 // To help reduce the number of CSE recomputations, process all
12934 // the uses of this user that we can find this way.
12935 do {
12936 unsigned i = Uses[UseIndex].Index;
12937 SDUse &Use = *Uses[UseIndex].Use;
12938 ++UseIndex;
12939
12940 Use.set(To[i]);
12941 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
12942
12943 // Now that we have modified User, add it back to the CSE maps. If it
12944 // already exists there, recursively merge the results together.
12945 AddModifiedNodeToCSEMaps(N: User);
12946 }
12947}
12948
12949/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
12950/// based on their topological order. It returns the maximum id and a vector
12951/// of the SDNodes* in assigned order by reference.
12952unsigned SelectionDAG::AssignTopologicalOrder() {
12953 unsigned DAGSize = 0;
12954
12955 // SortedPos tracks the progress of the algorithm. Nodes before it are
12956 // sorted, nodes after it are unsorted. When the algorithm completes
12957 // it is at the end of the list.
12958 allnodes_iterator SortedPos = allnodes_begin();
12959
12960 // Visit all the nodes. Move nodes with no operands to the front of
12961 // the list immediately. Annotate nodes that do have operands with their
12962 // operand count. Before we do this, the Node Id fields of the nodes
12963 // may contain arbitrary values. After, the Node Id fields for nodes
12964 // before SortedPos will contain the topological sort index, and the
12965 // Node Id fields for nodes At SortedPos and after will contain the
12966 // count of outstanding operands.
12967 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
12968 checkForCycles(N: &N, DAG: this);
12969 unsigned Degree = N.getNumOperands();
12970 if (Degree == 0) {
12971 // A node with no uses, add it to the result array immediately.
12972 N.setNodeId(DAGSize++);
12973 allnodes_iterator Q(&N);
12974 if (Q != SortedPos)
12975 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
12976 assert(SortedPos != AllNodes.end() && "Overran node list");
12977 ++SortedPos;
12978 } else {
12979 // Temporarily use the Node Id as scratch space for the degree count.
12980 N.setNodeId(Degree);
12981 }
12982 }
12983
12984 // Visit all the nodes. As we iterate, move nodes into sorted order,
12985 // such that by the time the end is reached all nodes will be sorted.
12986 for (SDNode &Node : allnodes()) {
12987 SDNode *N = &Node;
12988 checkForCycles(N, DAG: this);
12989 // N is in sorted position, so all its uses have one less operand
12990 // that needs to be sorted.
12991 for (SDNode *P : N->users()) {
12992 unsigned Degree = P->getNodeId();
12993 assert(Degree != 0 && "Invalid node degree");
12994 --Degree;
12995 if (Degree == 0) {
12996 // All of P's operands are sorted, so P may sorted now.
12997 P->setNodeId(DAGSize++);
12998 if (P->getIterator() != SortedPos)
12999 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13000 assert(SortedPos != AllNodes.end() && "Overran node list");
13001 ++SortedPos;
13002 } else {
13003 // Update P's outstanding operand count.
13004 P->setNodeId(Degree);
13005 }
13006 }
13007 if (Node.getIterator() == SortedPos) {
13008#ifndef NDEBUG
13009 allnodes_iterator I(N);
13010 SDNode *S = &*++I;
13011 dbgs() << "Overran sorted position:\n";
13012 S->dumprFull(this); dbgs() << "\n";
13013 dbgs() << "Checking if this is due to cycles\n";
13014 checkForCycles(this, true);
13015#endif
13016 llvm_unreachable(nullptr);
13017 }
13018 }
13019
13020 assert(SortedPos == AllNodes.end() &&
13021 "Topological sort incomplete!");
13022 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13023 "First node in topological sort is not the entry token!");
13024 assert(AllNodes.front().getNodeId() == 0 &&
13025 "First node in topological sort has non-zero id!");
13026 assert(AllNodes.front().getNumOperands() == 0 &&
13027 "First node in topological sort has operands!");
13028 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13029 "Last node in topologic sort has unexpected id!");
13030 assert(AllNodes.back().use_empty() &&
13031 "Last node in topologic sort has users!");
13032 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13033 return DAGSize;
13034}
13035
13036void SelectionDAG::getTopologicallyOrderedNodes(
13037 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13038 SortedNodes.clear();
13039 // Node -> remaining number of outstanding operands.
13040 DenseMap<const SDNode *, unsigned> RemainingOperands;
13041
13042 // Put nodes without any operands into SortedNodes first.
13043 for (const SDNode &N : allnodes()) {
13044 checkForCycles(N: &N, DAG: this);
13045 unsigned NumOperands = N.getNumOperands();
13046 if (NumOperands == 0)
13047 SortedNodes.push_back(Elt: &N);
13048 else
13049 // Record their total number of outstanding operands.
13050 RemainingOperands[&N] = NumOperands;
13051 }
13052
13053 // A node is pushed into SortedNodes when all of its operands (predecessors in
13054 // the graph) are also in SortedNodes.
13055 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13056 const SDNode *N = SortedNodes[i];
13057 for (const SDNode *U : N->users()) {
13058 // HandleSDNode is never part of a DAG and therefore has no entry in
13059 // RemainingOperands.
13060 if (U->getOpcode() == ISD::HANDLENODE)
13061 continue;
13062 unsigned &NumRemOperands = RemainingOperands[U];
13063 assert(NumRemOperands && "Invalid number of remaining operands");
13064 --NumRemOperands;
13065 if (!NumRemOperands)
13066 SortedNodes.push_back(Elt: U);
13067 }
13068 }
13069
13070 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13071 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13072 "First node in topological sort is not the entry token");
13073 assert(SortedNodes.front()->getNumOperands() == 0 &&
13074 "First node in topological sort has operands");
13075}
13076
13077/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13078/// value is produced by SD.
13079void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13080 for (SDNode *SD : DB->getSDNodes()) {
13081 if (!SD)
13082 continue;
13083 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13084 SD->setHasDebugValue(true);
13085 }
13086 DbgInfo->add(V: DB, isParameter);
13087}
13088
13089void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13090
13091SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13092 SDValue NewMemOpChain) {
13093 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13094 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13095 // The new memory operation must have the same position as the old load in
13096 // terms of memory dependency. Create a TokenFactor for the old load and new
13097 // memory operation and update uses of the old load's output chain to use that
13098 // TokenFactor.
13099 if (OldChain == NewMemOpChain || OldChain.use_empty())
13100 return NewMemOpChain;
13101
13102 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13103 N1: OldChain, N2: NewMemOpChain);
13104 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13105 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13106 return TokenFactor;
13107}
13108
13109SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13110 SDValue NewMemOp) {
13111 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13112 SDValue OldChain = SDValue(OldLoad, 1);
13113 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13114 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13115}
13116
13117SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13118 Function **OutFunction) {
13119 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13120
13121 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13122 auto *Module = MF->getFunction().getParent();
13123 auto *Function = Module->getFunction(Name: Symbol);
13124
13125 if (OutFunction != nullptr)
13126 *OutFunction = Function;
13127
13128 if (Function != nullptr) {
13129 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13130 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13131 }
13132
13133 std::string ErrorStr;
13134 raw_string_ostream ErrorFormatter(ErrorStr);
13135 ErrorFormatter << "Undefined external symbol ";
13136 ErrorFormatter << '"' << Symbol << '"';
13137 report_fatal_error(reason: Twine(ErrorStr));
13138}
13139
13140//===----------------------------------------------------------------------===//
13141// SDNode Class
13142//===----------------------------------------------------------------------===//
13143
13144bool llvm::isNullConstant(SDValue V) {
13145 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13146 return Const != nullptr && Const->isZero();
13147}
13148
13149bool llvm::isNullConstantOrUndef(SDValue V) {
13150 return V.isUndef() || isNullConstant(V);
13151}
13152
13153bool llvm::isNullFPConstant(SDValue V) {
13154 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13155 return Const != nullptr && Const->isZero() && !Const->isNegative();
13156}
13157
13158bool llvm::isAllOnesConstant(SDValue V) {
13159 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13160 return Const != nullptr && Const->isAllOnes();
13161}
13162
13163bool llvm::isOneConstant(SDValue V) {
13164 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13165 return Const != nullptr && Const->isOne();
13166}
13167
13168bool llvm::isMinSignedConstant(SDValue V) {
13169 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13170 return Const != nullptr && Const->isMinSignedValue();
13171}
13172
13173bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
13174 unsigned OperandNo) {
13175 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13176 // TODO: Target-specific opcodes could be added.
13177 if (auto *ConstV = isConstOrConstSplat(N: V, /*AllowUndefs*/ false,
13178 /*AllowTruncation*/ true)) {
13179 APInt Const = ConstV->getAPIntValue().trunc(width: V.getScalarValueSizeInBits());
13180 switch (Opcode) {
13181 case ISD::ADD:
13182 case ISD::OR:
13183 case ISD::XOR:
13184 case ISD::UMAX:
13185 return Const.isZero();
13186 case ISD::MUL:
13187 return Const.isOne();
13188 case ISD::AND:
13189 case ISD::UMIN:
13190 return Const.isAllOnes();
13191 case ISD::SMAX:
13192 return Const.isMinSignedValue();
13193 case ISD::SMIN:
13194 return Const.isMaxSignedValue();
13195 case ISD::SUB:
13196 case ISD::SHL:
13197 case ISD::SRA:
13198 case ISD::SRL:
13199 return OperandNo == 1 && Const.isZero();
13200 case ISD::UDIV:
13201 case ISD::SDIV:
13202 return OperandNo == 1 && Const.isOne();
13203 }
13204 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) {
13205 switch (Opcode) {
13206 case ISD::FADD:
13207 return ConstFP->isZero() &&
13208 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13209 case ISD::FSUB:
13210 return OperandNo == 1 && ConstFP->isZero() &&
13211 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13212 case ISD::FMUL:
13213 return ConstFP->isExactlyValue(V: 1.0);
13214 case ISD::FDIV:
13215 return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0);
13216 case ISD::FMINNUM:
13217 case ISD::FMAXNUM: {
13218 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
13219 EVT VT = V.getValueType();
13220 const fltSemantics &Semantics = VT.getFltSemantics();
13221 APFloat NeutralAF = !Flags.hasNoNaNs()
13222 ? APFloat::getQNaN(Sem: Semantics)
13223 : !Flags.hasNoInfs()
13224 ? APFloat::getInf(Sem: Semantics)
13225 : APFloat::getLargest(Sem: Semantics);
13226 if (Opcode == ISD::FMAXNUM)
13227 NeutralAF.changeSign();
13228
13229 return ConstFP->isExactlyValue(V: NeutralAF);
13230 }
13231 }
13232 }
13233 return false;
13234}
13235
13236SDValue llvm::peekThroughBitcasts(SDValue V) {
13237 while (V.getOpcode() == ISD::BITCAST)
13238 V = V.getOperand(i: 0);
13239 return V;
13240}
13241
13242SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13243 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13244 V = V.getOperand(i: 0);
13245 return V;
13246}
13247
13248SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13249 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13250 V = V.getOperand(i: 0);
13251 return V;
13252}
13253
13254SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13255 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13256 SDValue InVec = V.getOperand(i: 0);
13257 SDValue EltNo = V.getOperand(i: 2);
13258 EVT VT = InVec.getValueType();
13259 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13260 if (IndexC && VT.isFixedLengthVector() &&
13261 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13262 !DemandedElts[IndexC->getZExtValue()]) {
13263 V = InVec;
13264 continue;
13265 }
13266 break;
13267 }
13268 return V;
13269}
13270
13271SDValue llvm::peekThroughTruncates(SDValue V) {
13272 while (V.getOpcode() == ISD::TRUNCATE)
13273 V = V.getOperand(i: 0);
13274 return V;
13275}
13276
13277bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13278 if (V.getOpcode() != ISD::XOR)
13279 return false;
13280 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13281 unsigned NumBits = V.getScalarValueSizeInBits();
13282 ConstantSDNode *C =
13283 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13284 return C && (C->getAPIntValue().countr_one() >= NumBits);
13285}
13286
13287ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13288 bool AllowTruncation) {
13289 EVT VT = N.getValueType();
13290 APInt DemandedElts = VT.isFixedLengthVector()
13291 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
13292 : APInt(1, 1);
13293 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13294}
13295
13296ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13297 bool AllowUndefs,
13298 bool AllowTruncation) {
13299 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13300 return CN;
13301
13302 // SplatVectors can truncate their operands. Ignore that case here unless
13303 // AllowTruncation is set.
13304 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13305 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13306 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13307 EVT CVT = CN->getValueType(ResNo: 0);
13308 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13309 if (AllowTruncation || CVT == VecEltVT)
13310 return CN;
13311 }
13312 }
13313
13314 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13315 BitVector UndefElements;
13316 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13317
13318 // BuildVectors can truncate their operands. Ignore that case here unless
13319 // AllowTruncation is set.
13320 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13321 if (CN && (UndefElements.none() || AllowUndefs)) {
13322 EVT CVT = CN->getValueType(ResNo: 0);
13323 EVT NSVT = N.getValueType().getScalarType();
13324 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13325 if (AllowTruncation || (CVT == NSVT))
13326 return CN;
13327 }
13328 }
13329
13330 return nullptr;
13331}
13332
13333ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13334 EVT VT = N.getValueType();
13335 APInt DemandedElts = VT.isFixedLengthVector()
13336 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
13337 : APInt(1, 1);
13338 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13339}
13340
13341ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13342 const APInt &DemandedElts,
13343 bool AllowUndefs) {
13344 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13345 return CN;
13346
13347 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13348 BitVector UndefElements;
13349 ConstantFPSDNode *CN =
13350 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13351 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13352 if (CN && (UndefElements.none() || AllowUndefs))
13353 return CN;
13354 }
13355
13356 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13357 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13358 return CN;
13359
13360 return nullptr;
13361}
13362
13363bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13364 // TODO: may want to use peekThroughBitcast() here.
13365 ConstantSDNode *C =
13366 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13367 return C && C->isZero();
13368}
13369
13370bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
13371 ConstantSDNode *C =
13372 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
13373 return C && C->isOne();
13374}
13375
13376bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
13377 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13378 return C && C->isExactlyValue(V: 1.0);
13379}
13380
13381bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
13382 N = peekThroughBitcasts(V: N);
13383 unsigned BitWidth = N.getScalarValueSizeInBits();
13384 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13385 return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth;
13386}
13387
13388bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
13389 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13390 return C && APInt::isSameValue(I1: C->getAPIntValue(),
13391 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
13392}
13393
13394bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
13395 N = peekThroughBitcasts(V: N);
13396 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
13397 return C && C->isZero();
13398}
13399
13400bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
13401 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13402 return C && C->isZero();
13403}
13404
13405HandleSDNode::~HandleSDNode() {
13406 DropOperands();
13407}
13408
13409MemSDNode::MemSDNode(
13410 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
13411 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
13412 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
13413 bool IsVolatile = false;
13414 bool IsNonTemporal = false;
13415 bool IsDereferenceable = true;
13416 bool IsInvariant = true;
13417 for (const MachineMemOperand *MMO : memoperands()) {
13418 IsVolatile |= MMO->isVolatile();
13419 IsNonTemporal |= MMO->isNonTemporal();
13420 IsDereferenceable &= MMO->isDereferenceable();
13421 IsInvariant &= MMO->isInvariant();
13422 }
13423 MemSDNodeBits.IsVolatile = IsVolatile;
13424 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
13425 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
13426 MemSDNodeBits.IsInvariant = IsInvariant;
13427
13428 // For the single-MMO case, we check here that the size of the memory operand
13429 // fits within the size of the MMO. This is because the MMO might indicate
13430 // only a possible address range instead of specifying the affected memory
13431 // addresses precisely.
13432 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
13433 TypeSize::isKnownLE(memvt.getStoreSize(),
13434 getMemOperand()->getSize().getValue())) &&
13435 "Size mismatch!");
13436}
13437
13438/// Profile - Gather unique data for the node.
13439///
13440void SDNode::Profile(FoldingSetNodeID &ID) const {
13441 AddNodeIDNode(ID, N: this);
13442}
13443
13444namespace {
13445
13446 struct EVTArray {
13447 std::vector<EVT> VTs;
13448
13449 EVTArray() {
13450 VTs.reserve(n: MVT::VALUETYPE_SIZE);
13451 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
13452 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
13453 }
13454 };
13455
13456} // end anonymous namespace
13457
13458/// getValueTypeList - Return a pointer to the specified value type.
13459///
13460const EVT *SDNode::getValueTypeList(MVT VT) {
13461 static EVTArray SimpleVTArray;
13462
13463 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
13464 return &SimpleVTArray.VTs[VT.SimpleTy];
13465}
13466
13467/// hasAnyUseOfValue - Return true if there are any use of the indicated
13468/// value. This method ignores uses of other values defined by this operation.
13469bool SDNode::hasAnyUseOfValue(unsigned Value) const {
13470 assert(Value < getNumValues() && "Bad value!");
13471
13472 for (SDUse &U : uses())
13473 if (U.getResNo() == Value)
13474 return true;
13475
13476 return false;
13477}
13478
13479/// isOnlyUserOf - Return true if this node is the only use of N.
13480bool SDNode::isOnlyUserOf(const SDNode *N) const {
13481 bool Seen = false;
13482 for (const SDNode *User : N->users()) {
13483 if (User == this)
13484 Seen = true;
13485 else
13486 return false;
13487 }
13488
13489 return Seen;
13490}
13491
13492/// Return true if the only users of N are contained in Nodes.
13493bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
13494 bool Seen = false;
13495 for (const SDNode *User : N->users()) {
13496 if (llvm::is_contained(Range&: Nodes, Element: User))
13497 Seen = true;
13498 else
13499 return false;
13500 }
13501
13502 return Seen;
13503}
13504
13505/// Return true if the referenced return value is an operand of N.
13506bool SDValue::isOperandOf(const SDNode *N) const {
13507 return is_contained(Range: N->op_values(), Element: *this);
13508}
13509
13510bool SDNode::isOperandOf(const SDNode *N) const {
13511 return any_of(Range: N->op_values(),
13512 P: [this](SDValue Op) { return this == Op.getNode(); });
13513}
13514
13515/// reachesChainWithoutSideEffects - Return true if this operand (which must
13516/// be a chain) reaches the specified operand without crossing any
13517/// side-effecting instructions on any chain path. In practice, this looks
13518/// through token factors and non-volatile loads. In order to remain efficient,
13519/// this only looks a couple of nodes in, it does not do an exhaustive search.
13520///
13521/// Note that we only need to examine chains when we're searching for
13522/// side-effects; SelectionDAG requires that all side-effects are represented
13523/// by chains, even if another operand would force a specific ordering. This
13524/// constraint is necessary to allow transformations like splitting loads.
13525bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
13526 unsigned Depth) const {
13527 if (*this == Dest) return true;
13528
13529 // Don't search too deeply, we just want to be able to see through
13530 // TokenFactor's etc.
13531 if (Depth == 0) return false;
13532
13533 // If this is a token factor, all inputs to the TF happen in parallel.
13534 if (getOpcode() == ISD::TokenFactor) {
13535 // First, try a shallow search.
13536 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
13537 // We found the chain we want as an operand of this TokenFactor.
13538 // Essentially, we reach the chain without side-effects if we could
13539 // serialize the TokenFactor into a simple chain of operations with
13540 // Dest as the last operation. This is automatically true if the
13541 // chain has one use: there are no other ordering constraints.
13542 // If the chain has more than one use, we give up: some other
13543 // use of Dest might force a side-effect between Dest and the current
13544 // node.
13545 if (Dest.hasOneUse())
13546 return true;
13547 }
13548 // Next, try a deep search: check whether every operand of the TokenFactor
13549 // reaches Dest.
13550 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
13551 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
13552 });
13553 }
13554
13555 // Loads don't have side effects, look through them.
13556 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
13557 if (Ld->isUnordered())
13558 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
13559 }
13560 return false;
13561}
13562
13563bool SDNode::hasPredecessor(const SDNode *N) const {
13564 SmallPtrSet<const SDNode *, 32> Visited;
13565 SmallVector<const SDNode *, 16> Worklist;
13566 Worklist.push_back(Elt: this);
13567 return hasPredecessorHelper(N, Visited, Worklist);
13568}
13569
13570void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
13571 this->Flags &= Flags;
13572}
13573
13574SDValue
13575SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
13576 ArrayRef<ISD::NodeType> CandidateBinOps,
13577 bool AllowPartials) {
13578 // The pattern must end in an extract from index 0.
13579 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
13580 !isNullConstant(V: Extract->getOperand(Num: 1)))
13581 return SDValue();
13582
13583 // Match against one of the candidate binary ops.
13584 SDValue Op = Extract->getOperand(Num: 0);
13585 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
13586 return Op.getOpcode() == unsigned(BinOp);
13587 }))
13588 return SDValue();
13589
13590 // Floating-point reductions may require relaxed constraints on the final step
13591 // of the reduction because they may reorder intermediate operations.
13592 unsigned CandidateBinOp = Op.getOpcode();
13593 if (Op.getValueType().isFloatingPoint()) {
13594 SDNodeFlags Flags = Op->getFlags();
13595 switch (CandidateBinOp) {
13596 case ISD::FADD:
13597 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
13598 return SDValue();
13599 break;
13600 default:
13601 llvm_unreachable("Unhandled FP opcode for binop reduction");
13602 }
13603 }
13604
13605 // Matching failed - attempt to see if we did enough stages that a partial
13606 // reduction from a subvector is possible.
13607 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
13608 if (!AllowPartials || !Op)
13609 return SDValue();
13610 EVT OpVT = Op.getValueType();
13611 EVT OpSVT = OpVT.getScalarType();
13612 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
13613 if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
13614 return SDValue();
13615 BinOp = (ISD::NodeType)CandidateBinOp;
13616 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
13617 };
13618
13619 // At each stage, we're looking for something that looks like:
13620 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
13621 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
13622 // i32 undef, i32 undef, i32 undef, i32 undef>
13623 // %a = binop <8 x i32> %op, %s
13624 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
13625 // we expect something like:
13626 // <4,5,6,7,u,u,u,u>
13627 // <2,3,u,u,u,u,u,u>
13628 // <1,u,u,u,u,u,u,u>
13629 // While a partial reduction match would be:
13630 // <2,3,u,u,u,u,u,u>
13631 // <1,u,u,u,u,u,u,u>
13632 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
13633 SDValue PrevOp;
13634 for (unsigned i = 0; i < Stages; ++i) {
13635 unsigned MaskEnd = (1 << i);
13636
13637 if (Op.getOpcode() != CandidateBinOp)
13638 return PartialReduction(PrevOp, MaskEnd);
13639
13640 SDValue Op0 = Op.getOperand(i: 0);
13641 SDValue Op1 = Op.getOperand(i: 1);
13642
13643 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
13644 if (Shuffle) {
13645 Op = Op1;
13646 } else {
13647 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
13648 Op = Op0;
13649 }
13650
13651 // The first operand of the shuffle should be the same as the other operand
13652 // of the binop.
13653 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
13654 return PartialReduction(PrevOp, MaskEnd);
13655
13656 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
13657 for (int Index = 0; Index < (int)MaskEnd; ++Index)
13658 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
13659 return PartialReduction(PrevOp, MaskEnd);
13660
13661 PrevOp = Op;
13662 }
13663
13664 // Handle subvector reductions, which tend to appear after the shuffle
13665 // reduction stages.
13666 while (Op.getOpcode() == CandidateBinOp) {
13667 unsigned NumElts = Op.getValueType().getVectorNumElements();
13668 SDValue Op0 = Op.getOperand(i: 0);
13669 SDValue Op1 = Op.getOperand(i: 1);
13670 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13671 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13672 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
13673 break;
13674 SDValue Src = Op0.getOperand(i: 0);
13675 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
13676 if (NumSrcElts != (2 * NumElts))
13677 break;
13678 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
13679 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
13680 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
13681 Op0.getConstantOperandAPInt(i: 1) == NumElts))
13682 break;
13683 Op = Src;
13684 }
13685
13686 BinOp = (ISD::NodeType)CandidateBinOp;
13687 return Op;
13688}
13689
13690SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
13691 EVT VT = N->getValueType(ResNo: 0);
13692 EVT EltVT = VT.getVectorElementType();
13693 unsigned NE = VT.getVectorNumElements();
13694
13695 SDLoc dl(N);
13696
13697 // If ResNE is 0, fully unroll the vector op.
13698 if (ResNE == 0)
13699 ResNE = NE;
13700 else if (NE > ResNE)
13701 NE = ResNE;
13702
13703 if (N->getNumValues() == 2) {
13704 SmallVector<SDValue, 8> Scalars0, Scalars1;
13705 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13706 EVT VT1 = N->getValueType(ResNo: 1);
13707 EVT EltVT1 = VT1.getVectorElementType();
13708
13709 unsigned i;
13710 for (i = 0; i != NE; ++i) {
13711 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13712 SDValue Operand = N->getOperand(Num: j);
13713 EVT OperandVT = Operand.getValueType();
13714
13715 // A vector operand; extract a single element.
13716 EVT OperandEltVT = OperandVT.getVectorElementType();
13717 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13718 }
13719
13720 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
13721 Scalars0.push_back(Elt: EltOp);
13722 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
13723 }
13724
13725 for (; i < ResNE; ++i) {
13726 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
13727 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
13728 }
13729
13730 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
13731 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, NumElements: ResNE);
13732 SDValue Vec0 = getBuildVector(VT: VecVT, DL: dl, Ops: Scalars0);
13733 SDValue Vec1 = getBuildVector(VT: VecVT1, DL: dl, Ops: Scalars1);
13734 return getMergeValues(Ops: {Vec0, Vec1}, dl);
13735 }
13736
13737 assert(N->getNumValues() == 1 &&
13738 "Can't unroll a vector with multiple results!");
13739
13740 SmallVector<SDValue, 8> Scalars;
13741 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13742
13743 unsigned i;
13744 for (i= 0; i != NE; ++i) {
13745 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13746 SDValue Operand = N->getOperand(Num: j);
13747 EVT OperandVT = Operand.getValueType();
13748 if (OperandVT.isVector()) {
13749 // A vector operand; extract a single element.
13750 EVT OperandEltVT = OperandVT.getVectorElementType();
13751 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13752 } else {
13753 // A scalar operand; just use it as is.
13754 Operands[j] = Operand;
13755 }
13756 }
13757
13758 switch (N->getOpcode()) {
13759 default: {
13760 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
13761 Flags: N->getFlags()));
13762 break;
13763 }
13764 case ISD::VSELECT:
13765 Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
13766 break;
13767 case ISD::SHL:
13768 case ISD::SRA:
13769 case ISD::SRL:
13770 case ISD::ROTL:
13771 case ISD::ROTR:
13772 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
13773 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
13774 Op: Operands[1])));
13775 break;
13776 case ISD::SIGN_EXTEND_INREG: {
13777 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
13778 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
13779 N1: Operands[0],
13780 N2: getValueType(VT: ExtVT)));
13781 break;
13782 }
13783 case ISD::ADDRSPACECAST: {
13784 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
13785 Scalars.push_back(Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0],
13786 SrcAS: ASC->getSrcAddressSpace(),
13787 DestAS: ASC->getDestAddressSpace()));
13788 break;
13789 }
13790 }
13791 }
13792
13793 for (; i < ResNE; ++i)
13794 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
13795
13796 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
13797 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
13798}
13799
13800std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
13801 SDNode *N, unsigned ResNE) {
13802 unsigned Opcode = N->getOpcode();
13803 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
13804 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
13805 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
13806 "Expected an overflow opcode");
13807
13808 EVT ResVT = N->getValueType(ResNo: 0);
13809 EVT OvVT = N->getValueType(ResNo: 1);
13810 EVT ResEltVT = ResVT.getVectorElementType();
13811 EVT OvEltVT = OvVT.getVectorElementType();
13812 SDLoc dl(N);
13813
13814 // If ResNE is 0, fully unroll the vector op.
13815 unsigned NE = ResVT.getVectorNumElements();
13816 if (ResNE == 0)
13817 ResNE = NE;
13818 else if (NE > ResNE)
13819 NE = ResNE;
13820
13821 SmallVector<SDValue, 8> LHSScalars;
13822 SmallVector<SDValue, 8> RHSScalars;
13823 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
13824 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
13825
13826 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
13827 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
13828 SmallVector<SDValue, 8> ResScalars;
13829 SmallVector<SDValue, 8> OvScalars;
13830 for (unsigned i = 0; i < NE; ++i) {
13831 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
13832 SDValue Ov =
13833 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
13834 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
13835 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
13836
13837 ResScalars.push_back(Elt: Res);
13838 OvScalars.push_back(Elt: Ov);
13839 }
13840
13841 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
13842 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
13843
13844 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
13845 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
13846 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
13847 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
13848}
13849
13850bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
13851 LoadSDNode *Base,
13852 unsigned Bytes,
13853 int Dist) const {
13854 if (LD->isVolatile() || Base->isVolatile())
13855 return false;
13856 // TODO: probably too restrictive for atomics, revisit
13857 if (!LD->isSimple())
13858 return false;
13859 if (LD->isIndexed() || Base->isIndexed())
13860 return false;
13861 if (LD->getChain() != Base->getChain())
13862 return false;
13863 EVT VT = LD->getMemoryVT();
13864 if (VT.getSizeInBits() / 8 != Bytes)
13865 return false;
13866
13867 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
13868 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
13869
13870 int64_t Offset = 0;
13871 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
13872 return (Dist * (int64_t)Bytes == Offset);
13873 return false;
13874}
13875
13876/// InferPtrAlignment - Infer alignment of a load / store address. Return
13877/// std::nullopt if it cannot be inferred.
13878MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
13879 // If this is a GlobalAddress + cst, return the alignment.
13880 const GlobalValue *GV = nullptr;
13881 int64_t GVOffset = 0;
13882 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
13883 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
13884 KnownBits Known(PtrWidth);
13885 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
13886 unsigned AlignBits = Known.countMinTrailingZeros();
13887 if (AlignBits)
13888 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
13889 }
13890
13891 // If this is a direct reference to a stack slot, use information about the
13892 // stack slot's alignment.
13893 int FrameIdx = INT_MIN;
13894 int64_t FrameOffset = 0;
13895 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
13896 FrameIdx = FI->getIndex();
13897 } else if (isBaseWithConstantOffset(Op: Ptr) &&
13898 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
13899 // Handle FI+Cst
13900 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
13901 FrameOffset = Ptr.getConstantOperandVal(i: 1);
13902 }
13903
13904 if (FrameIdx != INT_MIN) {
13905 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
13906 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
13907 }
13908
13909 return std::nullopt;
13910}
13911
13912/// Split the scalar node with EXTRACT_ELEMENT using the provided
13913/// VTs and return the low/high part.
13914std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
13915 const SDLoc &DL,
13916 const EVT &LoVT,
13917 const EVT &HiVT) {
13918 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
13919 "Split node must be a scalar type");
13920 SDValue Lo =
13921 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
13922 SDValue Hi =
13923 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
13924 return std::make_pair(x&: Lo, y&: Hi);
13925}
13926
13927/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
13928/// which is split (or expanded) into two not necessarily identical pieces.
13929std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
13930 // Currently all types are split in half.
13931 EVT LoVT, HiVT;
13932 if (!VT.isVector())
13933 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
13934 else
13935 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
13936
13937 return std::make_pair(x&: LoVT, y&: HiVT);
13938}
13939
13940/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
13941/// type, dependent on an enveloping VT that has been split into two identical
13942/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
13943std::pair<EVT, EVT>
13944SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
13945 bool *HiIsEmpty) const {
13946 EVT EltTp = VT.getVectorElementType();
13947 // Examples:
13948 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
13949 // custom VL=9 with enveloping VL=8/8 yields 8/1
13950 // custom VL=10 with enveloping VL=8/8 yields 8/2
13951 // etc.
13952 ElementCount VTNumElts = VT.getVectorElementCount();
13953 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
13954 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
13955 "Mixing fixed width and scalable vectors when enveloping a type");
13956 EVT LoVT, HiVT;
13957 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
13958 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
13959 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
13960 *HiIsEmpty = false;
13961 } else {
13962 // Flag that hi type has zero storage size, but return split envelop type
13963 // (this would be easier if vector types with zero elements were allowed).
13964 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
13965 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
13966 *HiIsEmpty = true;
13967 }
13968 return std::make_pair(x&: LoVT, y&: HiVT);
13969}
13970
13971/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
13972/// low/high part.
13973std::pair<SDValue, SDValue>
13974SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
13975 const EVT &HiVT) {
13976 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
13977 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
13978 "Splitting vector with an invalid mixture of fixed and scalable "
13979 "vector types");
13980 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
13981 N.getValueType().getVectorMinNumElements() &&
13982 "More vector elements requested than available!");
13983 SDValue Lo, Hi;
13984 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
13985 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
13986 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
13987 // IDX with the runtime scaling factor of the result vector type. For
13988 // fixed-width result vectors, that runtime scaling factor is 1.
13989 Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
13990 N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
13991 return std::make_pair(x&: Lo, y&: Hi);
13992}
13993
13994std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
13995 const SDLoc &DL) {
13996 // Split the vector length parameter.
13997 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
13998 EVT VT = N.getValueType();
13999 assert(VecVT.getVectorElementCount().isKnownEven() &&
14000 "Expecting the mask to be an evenly-sized vector");
14001 SDValue HalfNumElts = getElementCount(
14002 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14003 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14004 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14005 return std::make_pair(x&: Lo, y&: Hi);
14006}
14007
14008/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14009SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14010 EVT VT = N.getValueType();
14011 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14012 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14013 return getInsertSubvector(DL, Vec: getUNDEF(VT: WideVT), SubVec: N, Idx: 0);
14014}
14015
14016void SelectionDAG::ExtractVectorElements(SDValue Op,
14017 SmallVectorImpl<SDValue> &Args,
14018 unsigned Start, unsigned Count,
14019 EVT EltVT) {
14020 EVT VT = Op.getValueType();
14021 if (Count == 0)
14022 Count = VT.getVectorNumElements();
14023 if (EltVT == EVT())
14024 EltVT = VT.getVectorElementType();
14025 SDLoc SL(Op);
14026 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14027 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14028 }
14029}
14030
14031// getAddressSpace - Return the address space this GlobalAddress belongs to.
14032unsigned GlobalAddressSDNode::getAddressSpace() const {
14033 return getGlobal()->getType()->getAddressSpace();
14034}
14035
14036Type *ConstantPoolSDNode::getType() const {
14037 if (isMachineConstantPoolEntry())
14038 return Val.MachineCPVal->getType();
14039 return Val.ConstVal->getType();
14040}
14041
14042bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14043 unsigned &SplatBitSize,
14044 bool &HasAnyUndefs,
14045 unsigned MinSplatBits,
14046 bool IsBigEndian) const {
14047 EVT VT = getValueType(ResNo: 0);
14048 assert(VT.isVector() && "Expected a vector type");
14049 unsigned VecWidth = VT.getSizeInBits();
14050 if (MinSplatBits > VecWidth)
14051 return false;
14052
14053 // FIXME: The widths are based on this node's type, but build vectors can
14054 // truncate their operands.
14055 SplatValue = APInt(VecWidth, 0);
14056 SplatUndef = APInt(VecWidth, 0);
14057
14058 // Get the bits. Bits with undefined values (when the corresponding element
14059 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14060 // in SplatValue. If any of the values are not constant, give up and return
14061 // false.
14062 unsigned int NumOps = getNumOperands();
14063 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14064 unsigned EltWidth = VT.getScalarSizeInBits();
14065
14066 for (unsigned j = 0; j < NumOps; ++j) {
14067 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14068 SDValue OpVal = getOperand(Num: i);
14069 unsigned BitPos = j * EltWidth;
14070
14071 if (OpVal.isUndef())
14072 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14073 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14074 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14075 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14076 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14077 else
14078 return false;
14079 }
14080
14081 // The build_vector is all constants or undefs. Find the smallest element
14082 // size that splats the vector.
14083 HasAnyUndefs = (SplatUndef != 0);
14084
14085 // FIXME: This does not work for vectors with elements less than 8 bits.
14086 while (VecWidth > 8) {
14087 // If we can't split in half, stop here.
14088 if (VecWidth & 1)
14089 break;
14090
14091 unsigned HalfSize = VecWidth / 2;
14092 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14093 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14094 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14095 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14096
14097 // If the two halves do not match (ignoring undef bits), stop here.
14098 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14099 MinSplatBits > HalfSize)
14100 break;
14101
14102 SplatValue = HighValue | LowValue;
14103 SplatUndef = HighUndef & LowUndef;
14104
14105 VecWidth = HalfSize;
14106 }
14107
14108 // FIXME: The loop above only tries to split in halves. But if the input
14109 // vector for example is <3 x i16> it wouldn't be able to detect a
14110 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14111 // optimizations. I guess that back in the days when this helper was created
14112 // vectors normally was power-of-2 sized.
14113
14114 SplatBitSize = VecWidth;
14115 return true;
14116}
14117
14118SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14119 BitVector *UndefElements) const {
14120 unsigned NumOps = getNumOperands();
14121 if (UndefElements) {
14122 UndefElements->clear();
14123 UndefElements->resize(N: NumOps);
14124 }
14125 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14126 if (!DemandedElts)
14127 return SDValue();
14128 SDValue Splatted;
14129 for (unsigned i = 0; i != NumOps; ++i) {
14130 if (!DemandedElts[i])
14131 continue;
14132 SDValue Op = getOperand(Num: i);
14133 if (Op.isUndef()) {
14134 if (UndefElements)
14135 (*UndefElements)[i] = true;
14136 } else if (!Splatted) {
14137 Splatted = Op;
14138 } else if (Splatted != Op) {
14139 return SDValue();
14140 }
14141 }
14142
14143 if (!Splatted) {
14144 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14145 assert(getOperand(FirstDemandedIdx).isUndef() &&
14146 "Can only have a splat without a constant for all undefs.");
14147 return getOperand(Num: FirstDemandedIdx);
14148 }
14149
14150 return Splatted;
14151}
14152
14153SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14154 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14155 return getSplatValue(DemandedElts, UndefElements);
14156}
14157
14158bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14159 SmallVectorImpl<SDValue> &Sequence,
14160 BitVector *UndefElements) const {
14161 unsigned NumOps = getNumOperands();
14162 Sequence.clear();
14163 if (UndefElements) {
14164 UndefElements->clear();
14165 UndefElements->resize(N: NumOps);
14166 }
14167 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14168 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14169 return false;
14170
14171 // Set the undefs even if we don't find a sequence (like getSplatValue).
14172 if (UndefElements)
14173 for (unsigned I = 0; I != NumOps; ++I)
14174 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14175 (*UndefElements)[I] = true;
14176
14177 // Iteratively widen the sequence length looking for repetitions.
14178 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14179 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14180 for (unsigned I = 0; I != NumOps; ++I) {
14181 if (!DemandedElts[I])
14182 continue;
14183 SDValue &SeqOp = Sequence[I % SeqLen];
14184 SDValue Op = getOperand(Num: I);
14185 if (Op.isUndef()) {
14186 if (!SeqOp)
14187 SeqOp = Op;
14188 continue;
14189 }
14190 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14191 Sequence.clear();
14192 break;
14193 }
14194 SeqOp = Op;
14195 }
14196 if (!Sequence.empty())
14197 return true;
14198 }
14199
14200 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14201 return false;
14202}
14203
14204bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14205 BitVector *UndefElements) const {
14206 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14207 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14208}
14209
14210ConstantSDNode *
14211BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14212 BitVector *UndefElements) const {
14213 return dyn_cast_or_null<ConstantSDNode>(
14214 Val: getSplatValue(DemandedElts, UndefElements));
14215}
14216
14217ConstantSDNode *
14218BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14219 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14220}
14221
14222ConstantFPSDNode *
14223BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14224 BitVector *UndefElements) const {
14225 return dyn_cast_or_null<ConstantFPSDNode>(
14226 Val: getSplatValue(DemandedElts, UndefElements));
14227}
14228
14229ConstantFPSDNode *
14230BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14231 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14232}
14233
14234int32_t
14235BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14236 uint32_t BitWidth) const {
14237 if (ConstantFPSDNode *CN =
14238 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14239 bool IsExact;
14240 APSInt IntVal(BitWidth);
14241 const APFloat &APF = CN->getValueAPF();
14242 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14243 APFloat::opOK ||
14244 !IsExact)
14245 return -1;
14246
14247 return IntVal.exactLogBase2();
14248 }
14249 return -1;
14250}
14251
14252bool BuildVectorSDNode::getConstantRawBits(
14253 bool IsLittleEndian, unsigned DstEltSizeInBits,
14254 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14255 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14256 if (!isConstant())
14257 return false;
14258
14259 unsigned NumSrcOps = getNumOperands();
14260 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14261 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14262 "Invalid bitcast scale");
14263
14264 // Extract raw src bits.
14265 SmallVector<APInt> SrcBitElements(NumSrcOps,
14266 APInt::getZero(numBits: SrcEltSizeInBits));
14267 BitVector SrcUndeElements(NumSrcOps, false);
14268
14269 for (unsigned I = 0; I != NumSrcOps; ++I) {
14270 SDValue Op = getOperand(Num: I);
14271 if (Op.isUndef()) {
14272 SrcUndeElements.set(I);
14273 continue;
14274 }
14275 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14276 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14277 assert((CInt || CFP) && "Unknown constant");
14278 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14279 : CFP->getValueAPF().bitcastToAPInt();
14280 }
14281
14282 // Recast to dst width.
14283 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14284 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14285 return true;
14286}
14287
14288void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14289 unsigned DstEltSizeInBits,
14290 SmallVectorImpl<APInt> &DstBitElements,
14291 ArrayRef<APInt> SrcBitElements,
14292 BitVector &DstUndefElements,
14293 const BitVector &SrcUndefElements) {
14294 unsigned NumSrcOps = SrcBitElements.size();
14295 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14296 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14297 "Invalid bitcast scale");
14298 assert(NumSrcOps == SrcUndefElements.size() &&
14299 "Vector size mismatch");
14300
14301 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14302 DstUndefElements.clear();
14303 DstUndefElements.resize(N: NumDstOps, t: false);
14304 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14305
14306 // Concatenate src elements constant bits together into dst element.
14307 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14308 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14309 for (unsigned I = 0; I != NumDstOps; ++I) {
14310 DstUndefElements.set(I);
14311 APInt &DstBits = DstBitElements[I];
14312 for (unsigned J = 0; J != Scale; ++J) {
14313 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14314 if (SrcUndefElements[Idx])
14315 continue;
14316 DstUndefElements.reset(Idx: I);
14317 const APInt &SrcBits = SrcBitElements[Idx];
14318 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14319 "Illegal constant bitwidths");
14320 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14321 }
14322 }
14323 return;
14324 }
14325
14326 // Split src element constant bits into dst elements.
14327 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14328 for (unsigned I = 0; I != NumSrcOps; ++I) {
14329 if (SrcUndefElements[I]) {
14330 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14331 continue;
14332 }
14333 const APInt &SrcBits = SrcBitElements[I];
14334 for (unsigned J = 0; J != Scale; ++J) {
14335 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14336 APInt &DstBits = DstBitElements[Idx];
14337 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14338 }
14339 }
14340}
14341
14342bool BuildVectorSDNode::isConstant() const {
14343 for (const SDValue &Op : op_values()) {
14344 unsigned Opc = Op.getOpcode();
14345 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14346 return false;
14347 }
14348 return true;
14349}
14350
14351std::optional<std::pair<APInt, APInt>>
14352BuildVectorSDNode::isArithmeticSequence() const {
14353 unsigned NumOps = getNumOperands();
14354 if (NumOps < 2)
14355 return std::nullopt;
14356
14357 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14358 APInt Start, Stride;
14359 int FirstIdx = -1, SecondIdx = -1;
14360
14361 // Find the first two non-undef constant elements to determine Start and
14362 // Stride, then verify all remaining elements match the sequence.
14363 for (unsigned I = 0; I < NumOps; ++I) {
14364 SDValue Op = getOperand(Num: I);
14365 if (Op->isUndef())
14366 continue;
14367 if (!isa<ConstantSDNode>(Val: Op))
14368 return std::nullopt;
14369
14370 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
14371 if (FirstIdx < 0) {
14372 FirstIdx = I;
14373 Start = Val;
14374 } else if (SecondIdx < 0) {
14375 SecondIdx = I;
14376 // Compute stride using modular arithmetic. Simple division would handle
14377 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
14378 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
14379 // Note that modular arithmetic is agnostic to signed/unsigned.
14380 unsigned IdxDiff = I - FirstIdx;
14381 APInt ValDiff = Val - Start;
14382
14383 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
14384 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
14385 if (ValDiff.countr_zero() < CommonPow2Bits)
14386 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
14387 IdxDiff >>= CommonPow2Bits;
14388 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
14389
14390 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
14391 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
14392 // one, but we could try all candidates to handle more cases.
14393 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
14394 if (Stride.isZero())
14395 return std::nullopt;
14396
14397 // Step 3: Adjust Start based on the first defined element's index.
14398 Start -= Stride * FirstIdx;
14399 } else {
14400 // Verify this element matches the sequence.
14401 if (Val != Start + Stride * I)
14402 return std::nullopt;
14403 }
14404 }
14405
14406 // Need at least two defined elements.
14407 if (SecondIdx < 0)
14408 return std::nullopt;
14409
14410 return std::make_pair(x&: Start, y&: Stride);
14411}
14412
14413bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
14414 // Find the first non-undef value in the shuffle mask.
14415 unsigned i, e;
14416 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
14417 /* search */;
14418
14419 // If all elements are undefined, this shuffle can be considered a splat
14420 // (although it should eventually get simplified away completely).
14421 if (i == e)
14422 return true;
14423
14424 // Make sure all remaining elements are either undef or the same as the first
14425 // non-undef value.
14426 for (int Idx = Mask[i]; i != e; ++i)
14427 if (Mask[i] >= 0 && Mask[i] != Idx)
14428 return false;
14429 return true;
14430}
14431
14432// Returns true if it is a constant integer BuildVector or constant integer,
14433// possibly hidden by a bitcast.
14434bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
14435 SDValue N, bool AllowOpaques) const {
14436 N = peekThroughBitcasts(V: N);
14437
14438 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
14439 return AllowOpaques || !C->isOpaque();
14440
14441 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
14442 return true;
14443
14444 // Treat a GlobalAddress supporting constant offset folding as a
14445 // constant integer.
14446 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
14447 if (GA->getOpcode() == ISD::GlobalAddress &&
14448 TLI->isOffsetFoldingLegal(GA))
14449 return true;
14450
14451 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14452 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
14453 return true;
14454 return false;
14455}
14456
14457// Returns true if it is a constant float BuildVector or constant float.
14458bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
14459 if (isa<ConstantFPSDNode>(Val: N))
14460 return true;
14461
14462 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
14463 return true;
14464
14465 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14466 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
14467 return true;
14468
14469 return false;
14470}
14471
14472std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
14473 ConstantSDNode *Const =
14474 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
14475 if (!Const)
14476 return std::nullopt;
14477
14478 EVT VT = N->getValueType(ResNo: 0);
14479 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
14480 switch (TLI->getBooleanContents(Type: N.getValueType())) {
14481 case TargetLowering::ZeroOrOneBooleanContent:
14482 if (CVal.isOne())
14483 return true;
14484 if (CVal.isZero())
14485 return false;
14486 return std::nullopt;
14487 case TargetLowering::ZeroOrNegativeOneBooleanContent:
14488 if (CVal.isAllOnes())
14489 return true;
14490 if (CVal.isZero())
14491 return false;
14492 return std::nullopt;
14493 case TargetLowering::UndefinedBooleanContent:
14494 return CVal[0];
14495 }
14496 llvm_unreachable("Unknown BooleanContent enum");
14497}
14498
14499void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
14500 assert(!Node->OperandList && "Node already has operands");
14501 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
14502 "too many operands to fit into SDNode");
14503 SDUse *Ops = OperandRecycler.allocate(
14504 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
14505
14506 bool IsDivergent = false;
14507 for (unsigned I = 0; I != Vals.size(); ++I) {
14508 Ops[I].setUser(Node);
14509 Ops[I].setInitial(Vals[I]);
14510 EVT VT = Ops[I].getValueType();
14511
14512 // Skip Chain. It does not carry divergence.
14513 if (VT != MVT::Other &&
14514 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
14515 Ops[I].getNode()->isDivergent()) {
14516 IsDivergent = true;
14517 }
14518 }
14519 Node->NumOperands = Vals.size();
14520 Node->OperandList = Ops;
14521 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
14522 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
14523 Node->SDNodeBits.IsDivergent = IsDivergent;
14524 }
14525 checkForCycles(N: Node);
14526}
14527
14528SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
14529 SmallVectorImpl<SDValue> &Vals) {
14530 size_t Limit = SDNode::getMaxNumOperands();
14531 while (Vals.size() > Limit) {
14532 unsigned SliceIdx = Vals.size() - Limit;
14533 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
14534 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
14535 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
14536 Vals.emplace_back(Args&: NewTF);
14537 }
14538 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
14539}
14540
14541SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
14542 EVT VT, SDNodeFlags Flags) {
14543 switch (Opcode) {
14544 default:
14545 return SDValue();
14546 case ISD::ADD:
14547 case ISD::OR:
14548 case ISD::XOR:
14549 case ISD::UMAX:
14550 return getConstant(Val: 0, DL, VT);
14551 case ISD::MUL:
14552 return getConstant(Val: 1, DL, VT);
14553 case ISD::AND:
14554 case ISD::UMIN:
14555 return getAllOnesConstant(DL, VT);
14556 case ISD::SMAX:
14557 return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
14558 case ISD::SMIN:
14559 return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
14560 case ISD::FADD:
14561 // If flags allow, prefer positive zero since it's generally cheaper
14562 // to materialize on most targets.
14563 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
14564 case ISD::FMUL:
14565 return getConstantFP(Val: 1.0, DL, VT);
14566 case ISD::FMINNUM:
14567 case ISD::FMAXNUM: {
14568 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
14569 const fltSemantics &Semantics = VT.getFltSemantics();
14570 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
14571 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
14572 APFloat::getLargest(Sem: Semantics);
14573 if (Opcode == ISD::FMAXNUM)
14574 NeutralAF.changeSign();
14575
14576 return getConstantFP(V: NeutralAF, DL, VT);
14577 }
14578 case ISD::FMINIMUM:
14579 case ISD::FMAXIMUM: {
14580 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
14581 const fltSemantics &Semantics = VT.getFltSemantics();
14582 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
14583 : APFloat::getLargest(Sem: Semantics);
14584 if (Opcode == ISD::FMAXIMUM)
14585 NeutralAF.changeSign();
14586
14587 return getConstantFP(V: NeutralAF, DL, VT);
14588 }
14589
14590 }
14591}
14592
14593/// Helper used to make a call to a library function that has one argument of
14594/// pointer type.
14595///
14596/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
14597/// used to get or set floating-point state. They have one argument of pointer
14598/// type, which points to the memory region containing bits of the
14599/// floating-point state. The value returned by such function is ignored in the
14600/// created call.
14601///
14602/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
14603/// \param Ptr Pointer used to save/load state.
14604/// \param InChain Ingoing token chain.
14605/// \returns Outgoing chain token.
14606SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
14607 SDValue InChain,
14608 const SDLoc &DLoc) {
14609 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
14610 TargetLowering::ArgListTy Args;
14611 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
14612 RTLIB::LibcallImpl LibcallImpl =
14613 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
14614 if (LibcallImpl == RTLIB::Unsupported)
14615 reportFatalUsageError(reason: "emitting call to unsupported libcall");
14616
14617 SDValue Callee =
14618 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
14619 TargetLowering::CallLoweringInfo CLI(*this);
14620 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
14621 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
14622 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
14623 return TLI->LowerCallTo(CLI).second;
14624}
14625
14626void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
14627 assert(From && To && "Invalid SDNode; empty source SDValue?");
14628 auto I = SDEI.find(Val: From);
14629 if (I == SDEI.end())
14630 return;
14631
14632 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
14633 // the iterator, hence the need to make a copy to prevent a use-after-free.
14634 NodeExtraInfo NEI = I->second;
14635 if (LLVM_LIKELY(!NEI.PCSections)) {
14636 // No deep copy required for the types of extra info set.
14637 //
14638 // FIXME: Investigate if other types of extra info also need deep copy. This
14639 // depends on the types of nodes they can be attached to: if some extra info
14640 // is only ever attached to nodes where a replacement To node is always the
14641 // node where later use and propagation of the extra info has the intended
14642 // semantics, no deep copy is required.
14643 SDEI[To] = std::move(NEI);
14644 return;
14645 }
14646
14647 const SDNode *EntrySDN = getEntryNode().getNode();
14648
14649 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
14650 // through the replacement of From with To. Otherwise, replacements of a node
14651 // (From) with more complex nodes (To and its operands) may result in lost
14652 // extra info where the root node (To) is insignificant in further propagating
14653 // and using extra info when further lowering to MIR.
14654 //
14655 // In the first step pre-populate the visited set with the nodes reachable
14656 // from the old From node. This avoids copying NodeExtraInfo to parts of the
14657 // DAG that is not new and should be left untouched.
14658 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
14659 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
14660 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
14661 if (MaxDepth == 0) {
14662 // Remember this node in case we need to increase MaxDepth and continue
14663 // populating FromReach from this node.
14664 Leafs.emplace_back(Args&: N);
14665 return;
14666 }
14667 if (!FromReach.insert(V: N).second)
14668 return;
14669 for (const SDValue &Op : N->op_values())
14670 Self(Self, Op.getNode(), MaxDepth - 1);
14671 };
14672
14673 // Copy extra info to To and all its transitive operands (that are new).
14674 SmallPtrSet<const SDNode *, 8> Visited;
14675 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
14676 if (FromReach.contains(V: N))
14677 return true;
14678 if (!Visited.insert(Ptr: N).second)
14679 return true;
14680 if (EntrySDN == N)
14681 return false;
14682 for (const SDValue &Op : N->op_values()) {
14683 if (N == To && Op.getNode() == EntrySDN) {
14684 // Special case: New node's operand is the entry node; just need to
14685 // copy extra info to new node.
14686 break;
14687 }
14688 if (!Self(Self, Op.getNode()))
14689 return false;
14690 }
14691 // Copy only if entry node was not reached.
14692 SDEI[N] = std::move(NEI);
14693 return true;
14694 };
14695
14696 // We first try with a lower MaxDepth, assuming that the path to common
14697 // operands between From and To is relatively short. This significantly
14698 // improves performance in the common case. The initial MaxDepth is big
14699 // enough to avoid retry in the common case; the last MaxDepth is large
14700 // enough to avoid having to use the fallback below (and protects from
14701 // potential stack exhaustion from recursion).
14702 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
14703 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
14704 // StartFrom is the previous (or initial) set of leafs reachable at the
14705 // previous maximum depth.
14706 SmallVector<const SDNode *> StartFrom;
14707 std::swap(LHS&: StartFrom, RHS&: Leafs);
14708 for (const SDNode *N : StartFrom)
14709 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
14710 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
14711 return;
14712 // This should happen very rarely (reached the entry node).
14713 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
14714 assert(!Leafs.empty());
14715 }
14716
14717 // This should not happen - but if it did, that means the subgraph reachable
14718 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
14719 // could not visit all reachable common operands. Consequently, we were able
14720 // to reach the entry node.
14721 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
14722 assert(false && "From subgraph too complex - increase max. MaxDepth?");
14723 // Best-effort fallback if assertions disabled.
14724 SDEI[To] = std::move(NEI);
14725}
14726
14727#ifndef NDEBUG
14728static void checkForCyclesHelper(const SDNode *N,
14729 SmallPtrSetImpl<const SDNode*> &Visited,
14730 SmallPtrSetImpl<const SDNode*> &Checked,
14731 const llvm::SelectionDAG *DAG) {
14732 // If this node has already been checked, don't check it again.
14733 if (Checked.count(N))
14734 return;
14735
14736 // If a node has already been visited on this depth-first walk, reject it as
14737 // a cycle.
14738 if (!Visited.insert(N).second) {
14739 errs() << "Detected cycle in SelectionDAG\n";
14740 dbgs() << "Offending node:\n";
14741 N->dumprFull(DAG); dbgs() << "\n";
14742 abort();
14743 }
14744
14745 for (const SDValue &Op : N->op_values())
14746 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
14747
14748 Checked.insert(N);
14749 Visited.erase(N);
14750}
14751#endif
14752
14753void llvm::checkForCycles(const llvm::SDNode *N,
14754 const llvm::SelectionDAG *DAG,
14755 bool force) {
14756#ifndef NDEBUG
14757 bool check = force;
14758#ifdef EXPENSIVE_CHECKS
14759 check = true;
14760#endif // EXPENSIVE_CHECKS
14761 if (check) {
14762 assert(N && "Checking nonexistent SDNode");
14763 SmallPtrSet<const SDNode*, 32> visited;
14764 SmallPtrSet<const SDNode*, 32> checked;
14765 checkForCyclesHelper(N, visited, checked, DAG);
14766 }
14767#endif // !NDEBUG
14768}
14769
14770void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
14771 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
14772}
14773