1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/FunctionLoweringInfo.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineConstantPool.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/RuntimeLibcallUtil.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetFrameLowering.h"
46#include "llvm/CodeGen/TargetLowering.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGen/ValueTypes.h"
50#include "llvm/CodeGenTypes/MachineValueType.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfoMetadata.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
61#include "llvm/Support/Casting.h"
62#include "llvm/Support/CodeGen.h"
63#include "llvm/Support/Compiler.h"
64#include "llvm/Support/Debug.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/MathExtras.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/TargetParser/Triple.h"
72#include "llvm/Transforms/Utils/SizeOpts.h"
73#include <algorithm>
74#include <cassert>
75#include <cstdint>
76#include <cstdlib>
77#include <limits>
78#include <optional>
79#include <string>
80#include <utility>
81#include <vector>
82
83using namespace llvm;
84using namespace llvm::SDPatternMatch;
85
86/// makeVTList - Return an instance of the SDVTList struct initialized with the
87/// specified members.
88static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
89 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
90 return Res;
91}
92
93// Default null implementations of the callbacks.
94void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
95void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
96void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
97
98void SelectionDAG::DAGNodeDeletedListener::anchor() {}
99void SelectionDAG::DAGNodeInsertedListener::anchor() {}
100
101#define DEBUG_TYPE "selectiondag"
102
103static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
104 cl::Hidden, cl::init(Val: true),
105 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
106
107static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
108 cl::desc("Number limit for gluing ld/st of memcpy."),
109 cl::Hidden, cl::init(Val: 0));
110
111static cl::opt<unsigned>
112 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
113 cl::desc("DAG combiner limit number of steps when searching DAG "
114 "for predecessor nodes"));
115
116static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
117 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
118}
119
120unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
121
122//===----------------------------------------------------------------------===//
123// ConstantFPSDNode Class
124//===----------------------------------------------------------------------===//
125
126/// isExactlyValue - We don't rely on operator== working on double values, as
127/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
128/// As such, this method can be used to do an exact bit-for-bit comparison of
129/// two floating point values.
130bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
131 return getValueAPF().bitwiseIsEqual(RHS: V);
132}
133
134bool ConstantFPSDNode::isValueValidForType(EVT VT,
135 const APFloat& Val) {
136 assert(VT.isFloatingPoint() && "Can only convert between FP types");
137
138 // convert modifies in place, so make a copy.
139 APFloat Val2 = APFloat(Val);
140 bool losesInfo;
141 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
142 losesInfo: &losesInfo);
143 return !losesInfo;
144}
145
146//===----------------------------------------------------------------------===//
147// ISD Namespace
148//===----------------------------------------------------------------------===//
149
150bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
151 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
152 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
153 unsigned EltSize =
154 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
155 SplatVal = OptAPInt->trunc(width: EltSize);
156 return true;
157 }
158 }
159
160 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
161 if (!BV)
162 return false;
163
164 APInt SplatUndef;
165 unsigned SplatBitSize;
166 bool HasUndefs;
167 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
168 // Endianness does not matter here. We are checking for a splat given the
169 // element size of the vector, and if we find such a splat for little endian
170 // layout, then that should be valid also for big endian (as the full vector
171 // size is known to be a multiple of the element size).
172 const bool IsBigEndian = false;
173 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
174 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
175 EltSize == SplatBitSize;
176}
177
178// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
179// specializations of the more general isConstantSplatVector()?
180
181bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
182 // Look through a bit convert.
183 while (N->getOpcode() == ISD::BITCAST)
184 N = N->getOperand(Num: 0).getNode();
185
186 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
187 APInt SplatVal;
188 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
189 }
190
191 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
192
193 unsigned i = 0, e = N->getNumOperands();
194
195 // Skip over all of the undef values.
196 while (i != e && N->getOperand(Num: i).isUndef())
197 ++i;
198
199 // Do not accept an all-undef vector.
200 if (i == e) return false;
201
202 // Do not accept build_vectors that aren't all constants or which have non-~0
203 // elements. We have to be a bit careful here, as the type of the constant
204 // may not be the same as the type of the vector elements due to type
205 // legalization (the elements are promoted to a legal type for the target and
206 // a vector of a type may be legal when the base element type is not).
207 // We only want to check enough bits to cover the vector elements, because
208 // we care if the resultant vector is all ones, not whether the individual
209 // constants are.
210 SDValue NotZero = N->getOperand(Num: i);
211 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
212 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
213 if (OptAPInt->countr_one() < EltSize)
214 return false;
215 } else
216 return false;
217
218 // Okay, we have at least one ~0 value, check to see if the rest match or are
219 // undefs. Even with the above element type twiddling, this should be OK, as
220 // the same type legalization should have applied to all the elements.
221 for (++i; i != e; ++i)
222 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
223 return false;
224 return true;
225}
226
227bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
228 // Look through a bit convert.
229 while (N->getOpcode() == ISD::BITCAST)
230 N = N->getOperand(Num: 0).getNode();
231
232 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
233 APInt SplatVal;
234 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
235 }
236
237 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
238
239 bool IsAllUndef = true;
240 for (const SDValue &Op : N->op_values()) {
241 if (Op.isUndef())
242 continue;
243 IsAllUndef = false;
244 // Do not accept build_vectors that aren't all constants or which have non-0
245 // elements. We have to be a bit careful here, as the type of the constant
246 // may not be the same as the type of the vector elements due to type
247 // legalization (the elements are promoted to a legal type for the target
248 // and a vector of a type may be legal when the base element type is not).
249 // We only want to check enough bits to cover the vector elements, because
250 // we care if the resultant vector is all zeros, not whether the individual
251 // constants are.
252 if (auto OptAPInt = Op->bitcastToAPInt()) {
253 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
254 if (OptAPInt->countr_zero() < EltSize)
255 return false;
256 } else
257 return false;
258 }
259
260 // Do not accept an all-undef vector.
261 if (IsAllUndef)
262 return false;
263 return true;
264}
265
266bool ISD::isBuildVectorAllOnes(const SDNode *N) {
267 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
268}
269
270bool ISD::isBuildVectorAllZeros(const SDNode *N) {
271 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
272}
273
274bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
275 if (N->getOpcode() != ISD::BUILD_VECTOR)
276 return false;
277
278 for (const SDValue &Op : N->op_values()) {
279 if (Op.isUndef())
280 continue;
281 if (!isa<ConstantSDNode>(Val: Op))
282 return false;
283 }
284 return true;
285}
286
287bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
288 if (N->getOpcode() != ISD::BUILD_VECTOR)
289 return false;
290
291 for (const SDValue &Op : N->op_values()) {
292 if (Op.isUndef())
293 continue;
294 if (!isa<ConstantFPSDNode>(Val: Op))
295 return false;
296 }
297 return true;
298}
299
300bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
301 bool Signed) {
302 assert(N->getValueType(0).isVector() && "Expected a vector!");
303
304 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
305 if (EltSize <= NewEltSize)
306 return false;
307
308 if (N->getOpcode() == ISD::ZERO_EXTEND) {
309 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
310 NewEltSize) &&
311 !Signed;
312 }
313 if (N->getOpcode() == ISD::SIGN_EXTEND) {
314 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
315 NewEltSize) &&
316 Signed;
317 }
318 if (N->getOpcode() != ISD::BUILD_VECTOR)
319 return false;
320
321 for (const SDValue &Op : N->op_values()) {
322 if (Op.isUndef())
323 continue;
324 if (!isa<ConstantSDNode>(Val: Op))
325 return false;
326
327 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
328 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
329 return false;
330 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
331 return false;
332 }
333
334 return true;
335}
336
337bool ISD::allOperandsUndef(const SDNode *N) {
338 // Return false if the node has no operands.
339 // This is "logically inconsistent" with the definition of "all" but
340 // is probably the desired behavior.
341 if (N->getNumOperands() == 0)
342 return false;
343 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
344}
345
346bool ISD::isFreezeUndef(const SDNode *N) {
347 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
348}
349
350template <typename ConstNodeType>
351bool ISD::matchUnaryPredicateImpl(SDValue Op,
352 std::function<bool(ConstNodeType *)> Match,
353 bool AllowUndefs, bool AllowTruncation) {
354 // FIXME: Add support for scalar UNDEF cases?
355 if (auto *C = dyn_cast<ConstNodeType>(Op))
356 return Match(C);
357
358 // FIXME: Add support for vector UNDEF cases?
359 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
360 ISD::SPLAT_VECTOR != Op.getOpcode())
361 return false;
362
363 EVT SVT = Op.getValueType().getScalarType();
364 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
365 if (AllowUndefs && Op.getOperand(i).isUndef()) {
366 if (!Match(nullptr))
367 return false;
368 continue;
369 }
370
371 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
372 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
373 !Match(Cst))
374 return false;
375 }
376 return true;
377}
378// Build used template types.
379template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
380 SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);
381template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
382 SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);
383
384bool ISD::matchBinaryPredicate(
385 SDValue LHS, SDValue RHS,
386 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
387 bool AllowUndefs, bool AllowTypeMismatch) {
388 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
389 return false;
390
391 // TODO: Add support for scalar UNDEF cases?
392 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
393 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
394 return Match(LHSCst, RHSCst);
395
396 // TODO: Add support for vector UNDEF cases?
397 if (LHS.getOpcode() != RHS.getOpcode() ||
398 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
399 LHS.getOpcode() != ISD::SPLAT_VECTOR))
400 return false;
401
402 EVT SVT = LHS.getValueType().getScalarType();
403 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
404 SDValue LHSOp = LHS.getOperand(i);
405 SDValue RHSOp = RHS.getOperand(i);
406 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
407 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
408 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
409 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
410 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
411 return false;
412 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
413 LHSOp.getValueType() != RHSOp.getValueType()))
414 return false;
415 if (!Match(LHSCst, RHSCst))
416 return false;
417 }
418 return true;
419}
420
421ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
422 switch (MinMaxOpc) {
423 default:
424 llvm_unreachable("unrecognized opcode");
425 case ISD::UMIN:
426 return ISD::UMAX;
427 case ISD::UMAX:
428 return ISD::UMIN;
429 case ISD::SMIN:
430 return ISD::SMAX;
431 case ISD::SMAX:
432 return ISD::SMIN;
433 }
434}
435
436ISD::NodeType ISD::getOppositeSignednessMinMaxOpcode(unsigned MinMaxOpc) {
437 switch (MinMaxOpc) {
438 default:
439 llvm_unreachable("unrecognized min/max opcode");
440 case ISD::SMIN:
441 return ISD::UMIN;
442 case ISD::SMAX:
443 return ISD::UMAX;
444 case ISD::UMIN:
445 return ISD::SMIN;
446 case ISD::UMAX:
447 return ISD::SMAX;
448 }
449}
450
451ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
452 switch (VecReduceOpcode) {
453 default:
454 llvm_unreachable("Expected VECREDUCE opcode");
455 case ISD::VECREDUCE_FADD:
456 case ISD::VECREDUCE_SEQ_FADD:
457 case ISD::VP_REDUCE_FADD:
458 case ISD::VP_REDUCE_SEQ_FADD:
459 return ISD::FADD;
460 case ISD::VECREDUCE_FMUL:
461 case ISD::VECREDUCE_SEQ_FMUL:
462 case ISD::VP_REDUCE_FMUL:
463 case ISD::VP_REDUCE_SEQ_FMUL:
464 return ISD::FMUL;
465 case ISD::VECREDUCE_ADD:
466 case ISD::VP_REDUCE_ADD:
467 return ISD::ADD;
468 case ISD::VECREDUCE_MUL:
469 case ISD::VP_REDUCE_MUL:
470 return ISD::MUL;
471 case ISD::VECREDUCE_AND:
472 case ISD::VP_REDUCE_AND:
473 return ISD::AND;
474 case ISD::VECREDUCE_OR:
475 case ISD::VP_REDUCE_OR:
476 return ISD::OR;
477 case ISD::VECREDUCE_XOR:
478 case ISD::VP_REDUCE_XOR:
479 return ISD::XOR;
480 case ISD::VECREDUCE_SMAX:
481 case ISD::VP_REDUCE_SMAX:
482 return ISD::SMAX;
483 case ISD::VECREDUCE_SMIN:
484 case ISD::VP_REDUCE_SMIN:
485 return ISD::SMIN;
486 case ISD::VECREDUCE_UMAX:
487 case ISD::VP_REDUCE_UMAX:
488 return ISD::UMAX;
489 case ISD::VECREDUCE_UMIN:
490 case ISD::VP_REDUCE_UMIN:
491 return ISD::UMIN;
492 case ISD::VECREDUCE_FMAX:
493 case ISD::VP_REDUCE_FMAX:
494 return ISD::FMAXNUM;
495 case ISD::VECREDUCE_FMIN:
496 case ISD::VP_REDUCE_FMIN:
497 return ISD::FMINNUM;
498 case ISD::VECREDUCE_FMAXIMUM:
499 case ISD::VP_REDUCE_FMAXIMUM:
500 return ISD::FMAXIMUM;
501 case ISD::VECREDUCE_FMINIMUM:
502 case ISD::VP_REDUCE_FMINIMUM:
503 return ISD::FMINIMUM;
504 }
505}
506
507bool ISD::isVPOpcode(unsigned Opcode) {
508 switch (Opcode) {
509 default:
510 return false;
511#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
512 case ISD::VPSD: \
513 return true;
514#include "llvm/IR/VPIntrinsics.def"
515 }
516}
517
518bool ISD::isVPBinaryOp(unsigned Opcode) {
519 switch (Opcode) {
520 default:
521 break;
522#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
523#define VP_PROPERTY_BINARYOP return true;
524#define END_REGISTER_VP_SDNODE(VPSD) break;
525#include "llvm/IR/VPIntrinsics.def"
526 }
527 return false;
528}
529
530bool ISD::isVPReduction(unsigned Opcode) {
531 switch (Opcode) {
532 default:
533 return false;
534 case ISD::VP_REDUCE_ADD:
535 case ISD::VP_REDUCE_MUL:
536 case ISD::VP_REDUCE_AND:
537 case ISD::VP_REDUCE_OR:
538 case ISD::VP_REDUCE_XOR:
539 case ISD::VP_REDUCE_SMAX:
540 case ISD::VP_REDUCE_SMIN:
541 case ISD::VP_REDUCE_UMAX:
542 case ISD::VP_REDUCE_UMIN:
543 case ISD::VP_REDUCE_FMAX:
544 case ISD::VP_REDUCE_FMIN:
545 case ISD::VP_REDUCE_FMAXIMUM:
546 case ISD::VP_REDUCE_FMINIMUM:
547 case ISD::VP_REDUCE_FADD:
548 case ISD::VP_REDUCE_FMUL:
549 case ISD::VP_REDUCE_SEQ_FADD:
550 case ISD::VP_REDUCE_SEQ_FMUL:
551 return true;
552 }
553}
554
555/// The operand position of the vector mask.
556std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
557 switch (Opcode) {
558 default:
559 return std::nullopt;
560#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
561 case ISD::VPSD: \
562 return MASKPOS;
563#include "llvm/IR/VPIntrinsics.def"
564 }
565}
566
567/// The operand position of the explicit vector length parameter.
568std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
569 switch (Opcode) {
570 default:
571 return std::nullopt;
572#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
573 case ISD::VPSD: \
574 return EVLPOS;
575#include "llvm/IR/VPIntrinsics.def"
576 }
577}
578
579std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
580 bool hasFPExcept) {
581 // FIXME: Return strict opcodes in case of fp exceptions.
582 switch (VPOpcode) {
583 default:
584 return std::nullopt;
585#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
586#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
587#define END_REGISTER_VP_SDNODE(VPOPC) break;
588#include "llvm/IR/VPIntrinsics.def"
589 }
590 return std::nullopt;
591}
592
593std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
594 switch (Opcode) {
595 default:
596 return std::nullopt;
597#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
598#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
599#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
600#include "llvm/IR/VPIntrinsics.def"
601 }
602}
603
604ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
605 switch (ExtType) {
606 case ISD::EXTLOAD:
607 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
608 case ISD::SEXTLOAD:
609 return ISD::SIGN_EXTEND;
610 case ISD::ZEXTLOAD:
611 return ISD::ZERO_EXTEND;
612 default:
613 break;
614 }
615
616 llvm_unreachable("Invalid LoadExtType");
617}
618
619ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
620 // To perform this operation, we just need to swap the L and G bits of the
621 // operation.
622 unsigned OldL = (Operation >> 2) & 1;
623 unsigned OldG = (Operation >> 1) & 1;
624 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
625 (OldL << 1) | // New G bit
626 (OldG << 2)); // New L bit.
627}
628
629static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
630 unsigned Operation = Op;
631 if (isIntegerLike)
632 Operation ^= 7; // Flip L, G, E bits, but not U.
633 else
634 Operation ^= 15; // Flip all of the condition bits.
635
636 if (Operation > ISD::SETTRUE2)
637 Operation &= ~8; // Don't let N and U bits get set.
638
639 return ISD::CondCode(Operation);
640}
641
642ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
643 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
644}
645
646ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
647 bool isIntegerLike) {
648 return getSetCCInverseImpl(Op, isIntegerLike);
649}
650
651/// For an integer comparison, return 1 if the comparison is a signed operation
652/// and 2 if the result is an unsigned comparison. Return zero if the operation
653/// does not depend on the sign of the input (setne and seteq).
654static int isSignedOp(ISD::CondCode Opcode) {
655 switch (Opcode) {
656 default: llvm_unreachable("Illegal integer setcc operation!");
657 case ISD::SETEQ:
658 case ISD::SETNE: return 0;
659 case ISD::SETLT:
660 case ISD::SETLE:
661 case ISD::SETGT:
662 case ISD::SETGE: return 1;
663 case ISD::SETULT:
664 case ISD::SETULE:
665 case ISD::SETUGT:
666 case ISD::SETUGE: return 2;
667 }
668}
669
670ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
671 EVT Type) {
672 bool IsInteger = Type.isInteger();
673 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
674 // Cannot fold a signed integer setcc with an unsigned integer setcc.
675 return ISD::SETCC_INVALID;
676
677 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
678
679 // If the N and U bits get set, then the resultant comparison DOES suddenly
680 // care about orderedness, and it is true when ordered.
681 if (Op > ISD::SETTRUE2)
682 Op &= ~16; // Clear the U bit if the N bit is set.
683
684 // Canonicalize illegal integer setcc's.
685 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
686 Op = ISD::SETNE;
687
688 return ISD::CondCode(Op);
689}
690
691ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
692 EVT Type) {
693 bool IsInteger = Type.isInteger();
694 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
695 // Cannot fold a signed setcc with an unsigned setcc.
696 return ISD::SETCC_INVALID;
697
698 // Combine all of the condition bits.
699 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
700
701 // Canonicalize illegal integer setcc's.
702 if (IsInteger) {
703 switch (Result) {
704 default: break;
705 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
706 case ISD::SETOEQ: // SETEQ & SETU[LG]E
707 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
708 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
709 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
710 }
711 }
712
713 return Result;
714}
715
716//===----------------------------------------------------------------------===//
717// SDNode Profile Support
718//===----------------------------------------------------------------------===//
719
720/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
721static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
722 ID.AddInteger(I: OpC);
723}
724
725/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
726/// solely with their pointer.
727static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
728 ID.AddPointer(Ptr: VTList.VTs);
729}
730
731/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
732static void AddNodeIDOperands(FoldingSetNodeID &ID,
733 ArrayRef<SDValue> Ops) {
734 for (const auto &Op : Ops) {
735 ID.AddPointer(Ptr: Op.getNode());
736 ID.AddInteger(I: Op.getResNo());
737 }
738}
739
740/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
741static void AddNodeIDOperands(FoldingSetNodeID &ID,
742 ArrayRef<SDUse> Ops) {
743 for (const auto &Op : Ops) {
744 ID.AddPointer(Ptr: Op.getNode());
745 ID.AddInteger(I: Op.getResNo());
746 }
747}
748
749static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
750 SDVTList VTList, ArrayRef<SDValue> OpList) {
751 AddNodeIDOpcode(ID, OpC);
752 AddNodeIDValueTypes(ID, VTList);
753 AddNodeIDOperands(ID, Ops: OpList);
754}
755
756/// If this is an SDNode with special info, add this info to the NodeID data.
757static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
758 switch (N->getOpcode()) {
759 case ISD::TargetExternalSymbol:
760 case ISD::ExternalSymbol:
761 case ISD::MCSymbol:
762 llvm_unreachable("Should only be used on nodes with operands");
763 default: break; // Normal nodes don't need extra info.
764 case ISD::TargetConstant:
765 case ISD::Constant: {
766 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
767 ID.AddPointer(Ptr: C->getConstantIntValue());
768 ID.AddBoolean(B: C->isOpaque());
769 break;
770 }
771 case ISD::TargetConstantFP:
772 case ISD::ConstantFP:
773 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
774 break;
775 case ISD::TargetGlobalAddress:
776 case ISD::GlobalAddress:
777 case ISD::TargetGlobalTLSAddress:
778 case ISD::GlobalTLSAddress: {
779 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
780 ID.AddPointer(Ptr: GA->getGlobal());
781 ID.AddInteger(I: GA->getOffset());
782 ID.AddInteger(I: GA->getTargetFlags());
783 break;
784 }
785 case ISD::BasicBlock:
786 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
787 break;
788 case ISD::Register:
789 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
790 break;
791 case ISD::RegisterMask:
792 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
793 break;
794 case ISD::SRCVALUE:
795 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
796 break;
797 case ISD::FrameIndex:
798 case ISD::TargetFrameIndex:
799 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
800 break;
801 case ISD::PSEUDO_PROBE:
802 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
803 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
804 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
805 break;
806 case ISD::JumpTable:
807 case ISD::TargetJumpTable:
808 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
809 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
810 break;
811 case ISD::ConstantPool:
812 case ISD::TargetConstantPool: {
813 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
814 ID.AddInteger(I: CP->getAlign().value());
815 ID.AddInteger(I: CP->getOffset());
816 if (CP->isMachineConstantPoolEntry())
817 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
818 else
819 ID.AddPointer(Ptr: CP->getConstVal());
820 ID.AddInteger(I: CP->getTargetFlags());
821 break;
822 }
823 case ISD::TargetIndex: {
824 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
825 ID.AddInteger(I: TI->getIndex());
826 ID.AddInteger(I: TI->getOffset());
827 ID.AddInteger(I: TI->getTargetFlags());
828 break;
829 }
830 case ISD::LOAD: {
831 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
832 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
833 ID.AddInteger(I: LD->getRawSubclassData());
834 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
835 ID.AddInteger(I: LD->getMemOperand()->getFlags());
836 break;
837 }
838 case ISD::STORE: {
839 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
840 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
841 ID.AddInteger(I: ST->getRawSubclassData());
842 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
843 ID.AddInteger(I: ST->getMemOperand()->getFlags());
844 break;
845 }
846 case ISD::VP_LOAD: {
847 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
848 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
849 ID.AddInteger(I: ELD->getRawSubclassData());
850 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
851 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
852 break;
853 }
854 case ISD::VP_LOAD_FF: {
855 const auto *LD = cast<VPLoadFFSDNode>(Val: N);
856 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
857 ID.AddInteger(I: LD->getRawSubclassData());
858 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
859 ID.AddInteger(I: LD->getMemOperand()->getFlags());
860 break;
861 }
862 case ISD::VP_STORE: {
863 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
864 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
865 ID.AddInteger(I: EST->getRawSubclassData());
866 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
867 ID.AddInteger(I: EST->getMemOperand()->getFlags());
868 break;
869 }
870 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
871 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
872 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
873 ID.AddInteger(I: SLD->getRawSubclassData());
874 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
875 break;
876 }
877 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
878 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
879 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
880 ID.AddInteger(I: SST->getRawSubclassData());
881 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
882 break;
883 }
884 case ISD::VP_GATHER: {
885 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
886 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
887 ID.AddInteger(I: EG->getRawSubclassData());
888 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
889 ID.AddInteger(I: EG->getMemOperand()->getFlags());
890 break;
891 }
892 case ISD::VP_SCATTER: {
893 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
894 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
895 ID.AddInteger(I: ES->getRawSubclassData());
896 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
897 ID.AddInteger(I: ES->getMemOperand()->getFlags());
898 break;
899 }
900 case ISD::MLOAD: {
901 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
902 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
903 ID.AddInteger(I: MLD->getRawSubclassData());
904 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
905 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
906 break;
907 }
908 case ISD::MSTORE: {
909 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
910 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
911 ID.AddInteger(I: MST->getRawSubclassData());
912 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
913 ID.AddInteger(I: MST->getMemOperand()->getFlags());
914 break;
915 }
916 case ISD::MGATHER: {
917 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
918 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
919 ID.AddInteger(I: MG->getRawSubclassData());
920 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
921 ID.AddInteger(I: MG->getMemOperand()->getFlags());
922 break;
923 }
924 case ISD::MSCATTER: {
925 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
926 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
927 ID.AddInteger(I: MS->getRawSubclassData());
928 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
929 ID.AddInteger(I: MS->getMemOperand()->getFlags());
930 break;
931 }
932 case ISD::ATOMIC_CMP_SWAP:
933 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
934 case ISD::ATOMIC_SWAP:
935 case ISD::ATOMIC_LOAD_ADD:
936 case ISD::ATOMIC_LOAD_SUB:
937 case ISD::ATOMIC_LOAD_AND:
938 case ISD::ATOMIC_LOAD_CLR:
939 case ISD::ATOMIC_LOAD_OR:
940 case ISD::ATOMIC_LOAD_XOR:
941 case ISD::ATOMIC_LOAD_NAND:
942 case ISD::ATOMIC_LOAD_MIN:
943 case ISD::ATOMIC_LOAD_MAX:
944 case ISD::ATOMIC_LOAD_UMIN:
945 case ISD::ATOMIC_LOAD_UMAX:
946 case ISD::ATOMIC_LOAD:
947 case ISD::ATOMIC_STORE: {
948 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
949 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
950 ID.AddInteger(I: AT->getRawSubclassData());
951 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
952 ID.AddInteger(I: AT->getMemOperand()->getFlags());
953 break;
954 }
955 case ISD::VECTOR_SHUFFLE: {
956 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
957 for (int M : Mask)
958 ID.AddInteger(I: M);
959 break;
960 }
961 case ISD::ADDRSPACECAST: {
962 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
963 ID.AddInteger(I: ASC->getSrcAddressSpace());
964 ID.AddInteger(I: ASC->getDestAddressSpace());
965 break;
966 }
967 case ISD::TargetBlockAddress:
968 case ISD::BlockAddress: {
969 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
970 ID.AddPointer(Ptr: BA->getBlockAddress());
971 ID.AddInteger(I: BA->getOffset());
972 ID.AddInteger(I: BA->getTargetFlags());
973 break;
974 }
975 case ISD::AssertAlign:
976 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
977 break;
978 case ISD::PREFETCH:
979 case ISD::INTRINSIC_VOID:
980 case ISD::INTRINSIC_W_CHAIN:
981 // Handled by MemIntrinsicSDNode check after the switch.
982 break;
983 case ISD::MDNODE_SDNODE:
984 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
985 break;
986 } // end switch (N->getOpcode())
987
988 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
989 // to check.
990 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
991 ID.AddInteger(I: MN->getRawSubclassData());
992 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
993 for (const MachineMemOperand *MMO : MN->memoperands()) {
994 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
995 ID.AddInteger(I: MMO->getFlags());
996 }
997 }
998}
999
1000/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
1001/// data.
1002static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
1003 AddNodeIDOpcode(ID, OpC: N->getOpcode());
1004 // Add the return value info.
1005 AddNodeIDValueTypes(ID, VTList: N->getVTList());
1006 // Add the operand info.
1007 AddNodeIDOperands(ID, Ops: N->ops());
1008
1009 // Handle SDNode leafs with special info.
1010 AddNodeIDCustom(ID, N);
1011}
1012
1013//===----------------------------------------------------------------------===//
1014// SelectionDAG Class
1015//===----------------------------------------------------------------------===//
1016
1017/// doNotCSE - Return true if CSE should not be performed for this node.
1018static bool doNotCSE(SDNode *N) {
1019 if (N->getValueType(ResNo: 0) == MVT::Glue)
1020 return true; // Never CSE anything that produces a glue result.
1021
1022 switch (N->getOpcode()) {
1023 default: break;
1024 case ISD::HANDLENODE:
1025 case ISD::EH_LABEL:
1026 return true; // Never CSE these nodes.
1027 }
1028
1029 // Check that remaining values produced are not flags.
1030 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1031 if (N->getValueType(ResNo: i) == MVT::Glue)
1032 return true; // Never CSE anything that produces a glue result.
1033
1034 return false;
1035}
1036
1037/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1038/// SelectionDAG.
1039void SelectionDAG::RemoveDeadNodes() {
1040 // Create a dummy node (which is not added to allnodes), that adds a reference
1041 // to the root node, preventing it from being deleted.
1042 HandleSDNode Dummy(getRoot());
1043
1044 SmallVector<SDNode*, 128> DeadNodes;
1045
1046 // Add all obviously-dead nodes to the DeadNodes worklist.
1047 for (SDNode &Node : allnodes())
1048 if (Node.use_empty())
1049 DeadNodes.push_back(Elt: &Node);
1050
1051 RemoveDeadNodes(DeadNodes);
1052
1053 // If the root changed (e.g. it was a dead load, update the root).
1054 setRoot(Dummy.getValue());
1055}
1056
1057/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1058/// given list, and any nodes that become unreachable as a result.
1059void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1060
1061 // Process the worklist, deleting the nodes and adding their uses to the
1062 // worklist.
1063 while (!DeadNodes.empty()) {
1064 SDNode *N = DeadNodes.pop_back_val();
1065 // Skip to next node if we've already managed to delete the node. This could
1066 // happen if replacing a node causes a node previously added to the node to
1067 // be deleted.
1068 if (N->getOpcode() == ISD::DELETED_NODE)
1069 continue;
1070
1071 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1072 DUL->NodeDeleted(N, nullptr);
1073
1074 // Take the node out of the appropriate CSE map.
1075 RemoveNodeFromCSEMaps(N);
1076
1077 // Next, brutally remove the operand list. This is safe to do, as there are
1078 // no cycles in the graph.
1079 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1080 SDUse &Use = *I++;
1081 SDNode *Operand = Use.getNode();
1082 Use.set(SDValue());
1083
1084 // Now that we removed this operand, see if there are no uses of it left.
1085 if (Operand->use_empty())
1086 DeadNodes.push_back(Elt: Operand);
1087 }
1088
1089 DeallocateNode(N);
1090 }
1091}
1092
1093void SelectionDAG::RemoveDeadNode(SDNode *N){
1094 SmallVector<SDNode*, 16> DeadNodes(1, N);
1095
1096 // Create a dummy node that adds a reference to the root node, preventing
1097 // it from being deleted. (This matters if the root is an operand of the
1098 // dead node.)
1099 HandleSDNode Dummy(getRoot());
1100
1101 RemoveDeadNodes(DeadNodes);
1102}
1103
1104void SelectionDAG::DeleteNode(SDNode *N) {
1105 // First take this out of the appropriate CSE map.
1106 RemoveNodeFromCSEMaps(N);
1107
1108 // Finally, remove uses due to operands of this node, remove from the
1109 // AllNodes list, and delete the node.
1110 DeleteNodeNotInCSEMaps(N);
1111}
1112
1113void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1114 assert(N->getIterator() != AllNodes.begin() &&
1115 "Cannot delete the entry node!");
1116 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1117
1118 // Drop all of the operands and decrement used node's use counts.
1119 N->DropOperands();
1120
1121 DeallocateNode(N);
1122}
1123
1124void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1125 assert(!(V->isVariadic() && isParameter));
1126 if (isParameter)
1127 ByvalParmDbgValues.push_back(Elt: V);
1128 else
1129 DbgValues.push_back(Elt: V);
1130 for (const SDNode *Node : V->getSDNodes())
1131 if (Node)
1132 DbgValMap[Node].push_back(Elt: V);
1133}
1134
1135void SDDbgInfo::erase(const SDNode *Node) {
1136 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1137 if (I == DbgValMap.end())
1138 return;
1139 for (auto &Val: I->second)
1140 Val->setIsInvalidated();
1141 DbgValMap.erase(I);
1142}
1143
1144void SelectionDAG::DeallocateNode(SDNode *N) {
1145 // If we have operands, deallocate them.
1146 removeOperands(Node: N);
1147
1148 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1149
1150 // Set the opcode to DELETED_NODE to help catch bugs when node
1151 // memory is reallocated.
1152 // FIXME: There are places in SDag that have grown a dependency on the opcode
1153 // value in the released node.
1154 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1155 N->NodeType = ISD::DELETED_NODE;
1156
1157 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1158 // them and forget about that node.
1159 DbgInfo->erase(Node: N);
1160
1161 // Invalidate extra info.
1162 SDEI.erase(Val: N);
1163}
1164
1165#ifndef NDEBUG
1166/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1167void SelectionDAG::verifyNode(SDNode *N) const {
1168 switch (N->getOpcode()) {
1169 default:
1170 if (N->isTargetOpcode())
1171 getSelectionDAGInfo().verifyTargetNode(*this, N);
1172 break;
1173 case ISD::BUILD_PAIR: {
1174 EVT VT = N->getValueType(0);
1175 assert(N->getNumValues() == 1 && "Too many results!");
1176 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1177 "Wrong return type!");
1178 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1179 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1180 "Mismatched operand types!");
1181 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1182 "Wrong operand type!");
1183 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1184 "Wrong return type size");
1185 break;
1186 }
1187 case ISD::BUILD_VECTOR: {
1188 assert(N->getNumValues() == 1 && "Too many results!");
1189 assert(N->getValueType(0).isVector() && "Wrong return type!");
1190 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1191 "Wrong number of operands!");
1192 EVT EltVT = N->getValueType(0).getVectorElementType();
1193 for (const SDUse &Op : N->ops()) {
1194 assert((Op.getValueType() == EltVT ||
1195 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1196 EltVT.bitsLE(Op.getValueType()))) &&
1197 "Wrong operand type!");
1198 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1199 "Operands must all have the same type");
1200 }
1201 break;
1202 }
1203 case ISD::SADDO:
1204 case ISD::UADDO:
1205 case ISD::SSUBO:
1206 case ISD::USUBO:
1207 assert(N->getNumValues() == 2 && "Wrong number of results!");
1208 assert(N->getVTList().NumVTs == 2 && N->getNumOperands() == 2 &&
1209 "Invalid add/sub overflow op!");
1210 assert(N->getVTList().VTs[0].isInteger() &&
1211 N->getVTList().VTs[1].isInteger() &&
1212 N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1213 N->getOperand(0).getValueType() == N->getVTList().VTs[0] &&
1214 "Binary operator types must match!");
1215 break;
1216 }
1217}
1218#endif // NDEBUG
1219
1220/// Insert a newly allocated node into the DAG.
1221///
1222/// Handles insertion into the all nodes list and CSE map, as well as
1223/// verification and other common operations when a new node is allocated.
1224void SelectionDAG::InsertNode(SDNode *N) {
1225 AllNodes.push_back(val: N);
1226#ifndef NDEBUG
1227 N->PersistentId = NextPersistentId++;
1228 verifyNode(N);
1229#endif
1230 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1231 DUL->NodeInserted(N);
1232}
1233
1234/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1235/// correspond to it. This is useful when we're about to delete or repurpose
1236/// the node. We don't want future request for structurally identical nodes
1237/// to return N anymore.
1238bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1239 bool Erased = false;
1240 switch (N->getOpcode()) {
1241 case ISD::HANDLENODE: return false; // noop.
1242 case ISD::CONDCODE:
1243 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1244 "Cond code doesn't exist!");
1245 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1246 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1247 break;
1248 case ISD::ExternalSymbol:
1249 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1250 break;
1251 case ISD::TargetExternalSymbol: {
1252 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1253 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1254 ESN->getSymbol(), ESN->getTargetFlags()));
1255 break;
1256 }
1257 case ISD::MCSymbol: {
1258 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1259 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1260 break;
1261 }
1262 case ISD::VALUETYPE: {
1263 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1264 if (VT.isExtended()) {
1265 Erased = ExtendedValueTypeNodes.erase(x: VT);
1266 } else {
1267 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1268 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1269 }
1270 break;
1271 }
1272 default:
1273 // Remove it from the CSE Map.
1274 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1275 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1276 Erased = CSEMap.RemoveNode(N);
1277 break;
1278 }
1279#ifndef NDEBUG
1280 // Verify that the node was actually in one of the CSE maps, unless it has a
1281 // glue result (which cannot be CSE'd) or is one of the special cases that are
1282 // not subject to CSE.
1283 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1284 !N->isMachineOpcode() && !doNotCSE(N)) {
1285 N->dump(this);
1286 dbgs() << "\n";
1287 llvm_unreachable("Node is not in map!");
1288 }
1289#endif
1290 return Erased;
1291}
1292
1293/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1294/// maps and modified in place. Add it back to the CSE maps, unless an identical
1295/// node already exists, in which case transfer all its users to the existing
1296/// node. This transfer can potentially trigger recursive merging.
1297void
1298SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1299 // For node types that aren't CSE'd, just act as if no identical node
1300 // already exists.
1301 if (!doNotCSE(N)) {
1302 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1303 if (Existing != N) {
1304 // If there was already an existing matching node, use ReplaceAllUsesWith
1305 // to replace the dead one with the existing one. This can cause
1306 // recursive merging of other unrelated nodes down the line.
1307 Existing->intersectFlagsWith(Flags: N->getFlags());
1308 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing))
1309 MemNode->refineRanges(NewMMOs: cast<MemSDNode>(Val: N)->memoperands());
1310 ReplaceAllUsesWith(From: N, To: Existing);
1311
1312 // N is now dead. Inform the listeners and delete it.
1313 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1314 DUL->NodeDeleted(N, Existing);
1315 DeleteNodeNotInCSEMaps(N);
1316 return;
1317 }
1318 }
1319
1320 // If the node doesn't already exist, we updated it. Inform listeners.
1321 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1322 DUL->NodeUpdated(N);
1323}
1324
1325/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1326/// were replaced with those specified. If this node is never memoized,
1327/// return null, otherwise return a pointer to the slot it would take. If a
1328/// node already exists with these operands, the slot will be non-null.
1329SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1330 void *&InsertPos) {
1331 if (doNotCSE(N))
1332 return nullptr;
1333
1334 SDValue Ops[] = { Op };
1335 FoldingSetNodeID ID;
1336 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1337 AddNodeIDCustom(ID, N);
1338 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1339 if (Node)
1340 Node->intersectFlagsWith(Flags: N->getFlags());
1341 return Node;
1342}
1343
1344/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1345/// were replaced with those specified. If this node is never memoized,
1346/// return null, otherwise return a pointer to the slot it would take. If a
1347/// node already exists with these operands, the slot will be non-null.
1348SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1349 SDValue Op1, SDValue Op2,
1350 void *&InsertPos) {
1351 if (doNotCSE(N))
1352 return nullptr;
1353
1354 SDValue Ops[] = { Op1, Op2 };
1355 FoldingSetNodeID ID;
1356 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1357 AddNodeIDCustom(ID, N);
1358 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1359 if (Node)
1360 Node->intersectFlagsWith(Flags: N->getFlags());
1361 return Node;
1362}
1363
1364/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1365/// were replaced with those specified. If this node is never memoized,
1366/// return null, otherwise return a pointer to the slot it would take. If a
1367/// node already exists with these operands, the slot will be non-null.
1368SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1369 void *&InsertPos) {
1370 if (doNotCSE(N))
1371 return nullptr;
1372
1373 FoldingSetNodeID ID;
1374 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1375 AddNodeIDCustom(ID, N);
1376 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1377 if (Node)
1378 Node->intersectFlagsWith(Flags: N->getFlags());
1379 return Node;
1380}
1381
1382Align SelectionDAG::getEVTAlign(EVT VT) const {
1383 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1384 : VT.getTypeForEVT(Context&: *getContext());
1385
1386 return getDataLayout().getABITypeAlign(Ty);
1387}
1388
1389// EntryNode could meaningfully have debug info if we can find it...
1390SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1391 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1392 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1393 Root(getEntryNode()) {
1394 InsertNode(N: &EntryNode);
1395 DbgInfo = new SDDbgInfo();
1396}
1397
1398void SelectionDAG::init(MachineFunction &NewMF,
1399 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1400 const TargetLibraryInfo *LibraryInfo,
1401 const LibcallLoweringInfo *LibcallsInfo,
1402 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1403 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
1404 FunctionVarLocs const *VarLocs) {
1405 MF = &NewMF;
1406 SDAGISelPass = PassPtr;
1407 ORE = &NewORE;
1408 TLI = getSubtarget().getTargetLowering();
1409 TSI = getSubtarget().getSelectionDAGInfo();
1410 LibInfo = LibraryInfo;
1411 Libcalls = LibcallsInfo;
1412 Context = &MF->getFunction().getContext();
1413 UA = NewUA;
1414 PSI = PSIin;
1415 BFI = BFIin;
1416 MMI = &MMIin;
1417 FnVarLocs = VarLocs;
1418}
1419
1420SelectionDAG::~SelectionDAG() {
1421 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1422 allnodes_clear();
1423 OperandRecycler.clear(OperandAllocator);
1424 delete DbgInfo;
1425}
1426
1427bool SelectionDAG::shouldOptForSize() const {
1428 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1429}
1430
1431void SelectionDAG::allnodes_clear() {
1432 assert(&*AllNodes.begin() == &EntryNode);
1433 AllNodes.remove(IT: AllNodes.begin());
1434 while (!AllNodes.empty())
1435 DeallocateNode(N: &AllNodes.front());
1436#ifndef NDEBUG
1437 NextPersistentId = 0;
1438#endif
1439}
1440
1441SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1442 void *&InsertPos) {
1443 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1444 if (N) {
1445 switch (N->getOpcode()) {
1446 default: break;
1447 case ISD::Constant:
1448 case ISD::ConstantFP:
1449 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1450 "debug location. Use another overload.");
1451 }
1452 }
1453 return N;
1454}
1455
1456SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1457 const SDLoc &DL, void *&InsertPos) {
1458 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1459 if (N) {
1460 switch (N->getOpcode()) {
1461 case ISD::Constant:
1462 case ISD::ConstantFP:
1463 // Erase debug location from the node if the node is used at several
1464 // different places. Do not propagate one location to all uses as it
1465 // will cause a worse single stepping debugging experience.
1466 if (N->getDebugLoc() != DL.getDebugLoc())
1467 N->setDebugLoc(DebugLoc());
1468 break;
1469 default:
1470 // When the node's point of use is located earlier in the instruction
1471 // sequence than its prior point of use, update its debug info to the
1472 // earlier location.
1473 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1474 N->setDebugLoc(DL.getDebugLoc());
1475 break;
1476 }
1477 }
1478 return N;
1479}
1480
1481void SelectionDAG::clear() {
1482 allnodes_clear();
1483 OperandRecycler.clear(OperandAllocator);
1484 OperandAllocator.Reset();
1485 CSEMap.clear();
1486
1487 ExtendedValueTypeNodes.clear();
1488 ExternalSymbols.clear();
1489 TargetExternalSymbols.clear();
1490 MCSymbols.clear();
1491 SDEI.clear();
1492 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1493 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1494
1495 EntryNode.UseList = nullptr;
1496 InsertNode(N: &EntryNode);
1497 Root = getEntryNode();
1498 DbgInfo->clear();
1499}
1500
1501SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1502 return VT.bitsGT(VT: Op.getValueType())
1503 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1504 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1505 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1506}
1507
1508std::pair<SDValue, SDValue>
1509SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1510 const SDLoc &DL, EVT VT) {
1511 assert(!VT.bitsEq(Op.getValueType()) &&
1512 "Strict no-op FP extend/round not allowed.");
1513 SDValue Res =
1514 VT.bitsGT(VT: Op.getValueType())
1515 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1516 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1517 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1518
1519 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1520}
1521
1522SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1523 return VT.bitsGT(VT: Op.getValueType()) ?
1524 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1525 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1526}
1527
1528SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1529 return VT.bitsGT(VT: Op.getValueType()) ?
1530 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1531 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1532}
1533
1534SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1535 return VT.bitsGT(VT: Op.getValueType()) ?
1536 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1537 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1538}
1539
1540SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1541 EVT VT) {
1542 assert(!VT.isVector());
1543 auto Type = Op.getValueType();
1544 SDValue DestOp;
1545 if (Type == VT)
1546 return Op;
1547 auto Size = Op.getValueSizeInBits();
1548 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1549 if (DestOp.getValueType() == VT)
1550 return DestOp;
1551
1552 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1553}
1554
1555SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1556 EVT VT) {
1557 assert(!VT.isVector());
1558 auto Type = Op.getValueType();
1559 SDValue DestOp;
1560 if (Type == VT)
1561 return Op;
1562 auto Size = Op.getValueSizeInBits();
1563 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1564 if (DestOp.getValueType() == VT)
1565 return DestOp;
1566
1567 return getSExtOrTrunc(Op: DestOp, DL, VT);
1568}
1569
1570SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1571 EVT VT) {
1572 assert(!VT.isVector());
1573 auto Type = Op.getValueType();
1574 SDValue DestOp;
1575 if (Type == VT)
1576 return Op;
1577 auto Size = Op.getValueSizeInBits();
1578 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1579 if (DestOp.getValueType() == VT)
1580 return DestOp;
1581
1582 return getZExtOrTrunc(Op: DestOp, DL, VT);
1583}
1584
1585SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1586 EVT OpVT) {
1587 if (VT.bitsLE(VT: Op.getValueType()))
1588 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1589
1590 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1591 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1592}
1593
1594SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1595 EVT OpVT = Op.getValueType();
1596 assert(VT.isInteger() && OpVT.isInteger() &&
1597 "Cannot getZeroExtendInReg FP types");
1598 assert(VT.isVector() == OpVT.isVector() &&
1599 "getZeroExtendInReg type should be vector iff the operand "
1600 "type is vector!");
1601 assert((!VT.isVector() ||
1602 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1603 "Vector element counts must match in getZeroExtendInReg");
1604 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1605 if (OpVT == VT)
1606 return Op;
1607 // TODO: Use computeKnownBits instead of AssertZext.
1608 if (Op.getOpcode() == ISD::AssertZext && cast<VTSDNode>(Val: Op.getOperand(i: 1))
1609 ->getVT()
1610 .getScalarType()
1611 .bitsLE(VT: VT.getScalarType()))
1612 return Op;
1613 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1614 loBitsSet: VT.getScalarSizeInBits());
1615 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1616}
1617
1618SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,
1619 SDValue EVL, const SDLoc &DL,
1620 EVT VT) {
1621 EVT OpVT = Op.getValueType();
1622 assert(VT.isInteger() && OpVT.isInteger() &&
1623 "Cannot getVPZeroExtendInReg FP types");
1624 assert(VT.isVector() && OpVT.isVector() &&
1625 "getVPZeroExtendInReg type and operand type should be vector!");
1626 assert(VT.getVectorElementCount() == OpVT.getVectorElementCount() &&
1627 "Vector element counts must match in getZeroExtendInReg");
1628 assert(VT.getScalarType().bitsLE(OpVT.getScalarType()) && "Not extending!");
1629 if (OpVT == VT)
1630 return Op;
1631 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1632 loBitsSet: VT.getScalarSizeInBits());
1633 return getNode(Opcode: ISD::VP_AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT), N3: Mask,
1634 N4: EVL);
1635}
1636
1637SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1638 // Only unsigned pointer semantics are supported right now. In the future this
1639 // might delegate to TLI to check pointer signedness.
1640 return getZExtOrTrunc(Op, DL, VT);
1641}
1642
1643SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1644 // Only unsigned pointer semantics are supported right now. In the future this
1645 // might delegate to TLI to check pointer signedness.
1646 return getZeroExtendInReg(Op, DL, VT);
1647}
1648
1649SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1650 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1651}
1652
1653/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1654SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1655 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1656}
1657
1658SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1659 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1660 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1661}
1662
1663SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1664 SDValue Mask, SDValue EVL, EVT VT) {
1665 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1666 return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1667}
1668
1669SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1670 SDValue Mask, SDValue EVL) {
1671 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1672}
1673
1674SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1675 SDValue Mask, SDValue EVL) {
1676 if (VT.bitsGT(VT: Op.getValueType()))
1677 return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1678 if (VT.bitsLT(VT: Op.getValueType()))
1679 return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1680 return Op;
1681}
1682
1683SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1684 EVT OpVT) {
1685 if (!V)
1686 return getConstant(Val: 0, DL, VT);
1687
1688 switch (TLI->getBooleanContents(Type: OpVT)) {
1689 case TargetLowering::ZeroOrOneBooleanContent:
1690 case TargetLowering::UndefinedBooleanContent:
1691 return getConstant(Val: 1, DL, VT);
1692 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1693 return getAllOnesConstant(DL, VT);
1694 }
1695 llvm_unreachable("Unexpected boolean content enum!");
1696}
1697
1698SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1699 bool isT, bool isO) {
1700 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1701 DL, VT, isTarget: isT, isOpaque: isO);
1702}
1703
1704SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1705 bool isT, bool isO) {
1706 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1707}
1708
1709SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1710 EVT VT, bool isT, bool isO) {
1711 assert(VT.isInteger() && "Cannot create FP integer constant!");
1712
1713 EVT EltVT = VT.getScalarType();
1714 const ConstantInt *Elt = &Val;
1715
1716 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1717 // to-be-splatted scalar ConstantInt.
1718 if (isa<VectorType>(Val: Elt->getType()))
1719 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1720
1721 // In some cases the vector type is legal but the element type is illegal and
1722 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1723 // inserted value (the type does not need to match the vector element type).
1724 // Any extra bits introduced will be truncated away.
1725 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1726 TargetLowering::TypePromoteInteger) {
1727 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1728 APInt NewVal;
1729 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1730 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1731 else
1732 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1733 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1734 }
1735 // In other cases the element type is illegal and needs to be expanded, for
1736 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1737 // the value into n parts and use a vector type with n-times the elements.
1738 // Then bitcast to the type requested.
1739 // Legalizing constants too early makes the DAGCombiner's job harder so we
1740 // only legalize if the DAG tells us we must produce legal types.
1741 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1742 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1743 TargetLowering::TypeExpandInteger) {
1744 const APInt &NewVal = Elt->getValue();
1745 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1746 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1747
1748 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1749 if (VT.isScalableVector() ||
1750 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1751 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1752 "Can only handle an even split!");
1753 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1754
1755 SmallVector<SDValue, 2> ScalarParts;
1756 for (unsigned i = 0; i != Parts; ++i)
1757 ScalarParts.push_back(Elt: getConstant(
1758 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1759 VT: ViaEltVT, isT, isO));
1760
1761 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1762 }
1763
1764 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1765 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1766
1767 // Check the temporary vector is the correct size. If this fails then
1768 // getTypeToTransformTo() probably returned a type whose size (in bits)
1769 // isn't a power-of-2 factor of the requested type size.
1770 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1771
1772 SmallVector<SDValue, 2> EltParts;
1773 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1774 EltParts.push_back(Elt: getConstant(
1775 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1776 VT: ViaEltVT, isT, isO));
1777
1778 // EltParts is currently in little endian order. If we actually want
1779 // big-endian order then reverse it now.
1780 if (getDataLayout().isBigEndian())
1781 std::reverse(first: EltParts.begin(), last: EltParts.end());
1782
1783 // The elements must be reversed when the element order is different
1784 // to the endianness of the elements (because the BITCAST is itself a
1785 // vector shuffle in this situation). However, we do not need any code to
1786 // perform this reversal because getConstant() is producing a vector
1787 // splat.
1788 // This situation occurs in MIPS MSA.
1789
1790 SmallVector<SDValue, 8> Ops;
1791 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1792 llvm::append_range(C&: Ops, R&: EltParts);
1793
1794 SDValue V =
1795 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1796 return V;
1797 }
1798
1799 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1800 "APInt size does not match type size!");
1801 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1802 SDVTList VTs = getVTList(VT: EltVT);
1803 FoldingSetNodeID ID;
1804 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1805 ID.AddPointer(Ptr: Elt);
1806 ID.AddBoolean(B: isO);
1807 void *IP = nullptr;
1808 SDNode *N = nullptr;
1809 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1810 if (!VT.isVector())
1811 return SDValue(N, 0);
1812
1813 if (!N) {
1814 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1815 CSEMap.InsertNode(N, InsertPos: IP);
1816 InsertNode(N);
1817 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1818 }
1819
1820 SDValue Result(N, 0);
1821 if (VT.isVector())
1822 Result = getSplat(VT, DL, Op: Result);
1823 return Result;
1824}
1825
1826SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1827 bool isT, bool isO) {
1828 unsigned Size = VT.getScalarSizeInBits();
1829 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1830}
1831
1832SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1833 bool IsOpaque) {
1834 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1835 isT: IsTarget, isO: IsOpaque);
1836}
1837
1838SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1839 bool isTarget) {
1840 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1841}
1842
1843SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1844 const SDLoc &DL) {
1845 assert(VT.isInteger() && "Shift amount is not an integer type!");
1846 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1847 return getConstant(Val, DL, VT: ShiftVT);
1848}
1849
1850SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1851 const SDLoc &DL) {
1852 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1853 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1854}
1855
1856SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1857 bool isTarget) {
1858 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1859}
1860
1861SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1862 bool isTarget) {
1863 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1864}
1865
1866SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1867 EVT VT, bool isTarget) {
1868 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1869
1870 EVT EltVT = VT.getScalarType();
1871 const ConstantFP *Elt = &V;
1872
1873 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1874 // the to-be-splatted scalar ConstantFP.
1875 if (isa<VectorType>(Val: Elt->getType()))
1876 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1877
1878 // Do the map lookup using the actual bit pattern for the floating point
1879 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1880 // we don't have issues with SNANs.
1881 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1882 SDVTList VTs = getVTList(VT: EltVT);
1883 FoldingSetNodeID ID;
1884 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1885 ID.AddPointer(Ptr: Elt);
1886 void *IP = nullptr;
1887 SDNode *N = nullptr;
1888 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1889 if (!VT.isVector())
1890 return SDValue(N, 0);
1891
1892 if (!N) {
1893 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1894 CSEMap.InsertNode(N, InsertPos: IP);
1895 InsertNode(N);
1896 }
1897
1898 SDValue Result(N, 0);
1899 if (VT.isVector())
1900 Result = getSplat(VT, DL, Op: Result);
1901 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1902 return Result;
1903}
1904
1905SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1906 bool isTarget) {
1907 EVT EltVT = VT.getScalarType();
1908 if (EltVT == MVT::f32)
1909 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1910 if (EltVT == MVT::f64)
1911 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1912 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1913 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1914 bool Ignored;
1915 APFloat APF = APFloat(Val);
1916 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1917 losesInfo: &Ignored);
1918 return getConstantFP(V: APF, DL, VT, isTarget);
1919 }
1920 llvm_unreachable("Unsupported type in getConstantFP");
1921}
1922
1923SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1924 EVT VT, int64_t Offset, bool isTargetGA,
1925 unsigned TargetFlags) {
1926 assert((TargetFlags == 0 || isTargetGA) &&
1927 "Cannot set target flags on target-independent globals");
1928
1929 // Truncate (with sign-extension) the offset value to the pointer size.
1930 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1931 if (BitWidth < 64)
1932 Offset = SignExtend64(X: Offset, B: BitWidth);
1933
1934 unsigned Opc;
1935 if (GV->isThreadLocal())
1936 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1937 else
1938 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1939
1940 SDVTList VTs = getVTList(VT);
1941 FoldingSetNodeID ID;
1942 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1943 ID.AddPointer(Ptr: GV);
1944 ID.AddInteger(I: Offset);
1945 ID.AddInteger(I: TargetFlags);
1946 void *IP = nullptr;
1947 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1948 return SDValue(E, 0);
1949
1950 auto *N = newSDNode<GlobalAddressSDNode>(
1951 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1952 CSEMap.InsertNode(N, InsertPos: IP);
1953 InsertNode(N);
1954 return SDValue(N, 0);
1955}
1956
1957SDValue SelectionDAG::getDeactivationSymbol(const GlobalValue *GV) {
1958 SDVTList VTs = getVTList(VT: MVT::Untyped);
1959 FoldingSetNodeID ID;
1960 AddNodeIDNode(ID, OpC: ISD::DEACTIVATION_SYMBOL, VTList: VTs, OpList: {});
1961 ID.AddPointer(Ptr: GV);
1962 void *IP = nullptr;
1963 if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
1964 return SDValue(E, 0);
1965
1966 auto *N = newSDNode<DeactivationSymbolSDNode>(Args&: GV, Args&: VTs);
1967 CSEMap.InsertNode(N, InsertPos: IP);
1968 InsertNode(N);
1969 return SDValue(N, 0);
1970}
1971
1972SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1973 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1974 SDVTList VTs = getVTList(VT);
1975 FoldingSetNodeID ID;
1976 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1977 ID.AddInteger(I: FI);
1978 void *IP = nullptr;
1979 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1980 return SDValue(E, 0);
1981
1982 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1983 CSEMap.InsertNode(N, InsertPos: IP);
1984 InsertNode(N);
1985 return SDValue(N, 0);
1986}
1987
1988SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1989 unsigned TargetFlags) {
1990 assert((TargetFlags == 0 || isTarget) &&
1991 "Cannot set target flags on target-independent jump tables");
1992 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1993 SDVTList VTs = getVTList(VT);
1994 FoldingSetNodeID ID;
1995 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1996 ID.AddInteger(I: JTI);
1997 ID.AddInteger(I: TargetFlags);
1998 void *IP = nullptr;
1999 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2000 return SDValue(E, 0);
2001
2002 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
2003 CSEMap.InsertNode(N, InsertPos: IP);
2004 InsertNode(N);
2005 return SDValue(N, 0);
2006}
2007
2008SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
2009 const SDLoc &DL) {
2010 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
2011 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Other, N1: Chain,
2012 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
2013}
2014
2015SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
2016 MaybeAlign Alignment, int Offset,
2017 bool isTarget, unsigned TargetFlags) {
2018 assert((TargetFlags == 0 || isTarget) &&
2019 "Cannot set target flags on target-independent globals");
2020 if (!Alignment)
2021 Alignment = shouldOptForSize()
2022 ? getDataLayout().getABITypeAlign(Ty: C->getType())
2023 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
2024 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2025 SDVTList VTs = getVTList(VT);
2026 FoldingSetNodeID ID;
2027 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2028 ID.AddInteger(I: Alignment->value());
2029 ID.AddInteger(I: Offset);
2030 ID.AddPointer(Ptr: C);
2031 ID.AddInteger(I: TargetFlags);
2032 void *IP = nullptr;
2033 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2034 return SDValue(E, 0);
2035
2036 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2037 Args&: TargetFlags);
2038 CSEMap.InsertNode(N, InsertPos: IP);
2039 InsertNode(N);
2040 SDValue V = SDValue(N, 0);
2041 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
2042 return V;
2043}
2044
2045SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
2046 MaybeAlign Alignment, int Offset,
2047 bool isTarget, unsigned TargetFlags) {
2048 assert((TargetFlags == 0 || isTarget) &&
2049 "Cannot set target flags on target-independent globals");
2050 if (!Alignment)
2051 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
2052 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2053 SDVTList VTs = getVTList(VT);
2054 FoldingSetNodeID ID;
2055 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2056 ID.AddInteger(I: Alignment->value());
2057 ID.AddInteger(I: Offset);
2058 C->addSelectionDAGCSEId(ID);
2059 ID.AddInteger(I: TargetFlags);
2060 void *IP = nullptr;
2061 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2062 return SDValue(E, 0);
2063
2064 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2065 Args&: TargetFlags);
2066 CSEMap.InsertNode(N, InsertPos: IP);
2067 InsertNode(N);
2068 return SDValue(N, 0);
2069}
2070
2071SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2072 FoldingSetNodeID ID;
2073 AddNodeIDNode(ID, OpC: ISD::BasicBlock, VTList: getVTList(VT: MVT::Other), OpList: {});
2074 ID.AddPointer(Ptr: MBB);
2075 void *IP = nullptr;
2076 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2077 return SDValue(E, 0);
2078
2079 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2080 CSEMap.InsertNode(N, InsertPos: IP);
2081 InsertNode(N);
2082 return SDValue(N, 0);
2083}
2084
2085SDValue SelectionDAG::getValueType(EVT VT) {
2086 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2087 ValueTypeNodes.size())
2088 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2089
2090 SDNode *&N = VT.isExtended() ?
2091 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2092
2093 if (N) return SDValue(N, 0);
2094 N = newSDNode<VTSDNode>(Args&: VT);
2095 InsertNode(N);
2096 return SDValue(N, 0);
2097}
2098
2099SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2100 SDNode *&N = ExternalSymbols[Sym];
2101 if (N) return SDValue(N, 0);
2102 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2103 InsertNode(N);
2104 return SDValue(N, 0);
2105}
2106
2107SDValue SelectionDAG::getExternalSymbol(RTLIB::LibcallImpl Libcall, EVT VT) {
2108 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2109 return getExternalSymbol(Sym: SymName.data(), VT);
2110}
2111
2112SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2113 SDNode *&N = MCSymbols[Sym];
2114 if (N)
2115 return SDValue(N, 0);
2116 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2117 InsertNode(N);
2118 return SDValue(N, 0);
2119}
2120
2121SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2122 unsigned TargetFlags) {
2123 SDNode *&N =
2124 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2125 if (N) return SDValue(N, 0);
2126 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2127 InsertNode(N);
2128 return SDValue(N, 0);
2129}
2130
2131SDValue SelectionDAG::getTargetExternalSymbol(RTLIB::LibcallImpl Libcall,
2132 EVT VT, unsigned TargetFlags) {
2133 StringRef SymName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(CallImpl: Libcall);
2134 return getTargetExternalSymbol(Sym: SymName.data(), VT, TargetFlags);
2135}
2136
2137SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2138 if ((unsigned)Cond >= CondCodeNodes.size())
2139 CondCodeNodes.resize(new_size: Cond+1);
2140
2141 if (!CondCodeNodes[Cond]) {
2142 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2143 CondCodeNodes[Cond] = N;
2144 InsertNode(N);
2145 }
2146
2147 return SDValue(CondCodeNodes[Cond], 0);
2148}
2149
2150SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm) {
2151 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2152 "APInt size does not match type size!");
2153
2154 if (MulImm == 0)
2155 return getConstant(Val: 0, DL, VT);
2156
2157 const MachineFunction &MF = getMachineFunction();
2158 const Function &F = MF.getFunction();
2159 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2160 if (const APInt *C = CR.getSingleElement())
2161 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2162
2163 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2164}
2165
2166/// \returns a value of type \p VT that represents the runtime value of \p
2167/// Quantity, i.e. scaled by vscale if it's scalable, or a fixed constant
2168/// otherwise. Quantity should be a FixedOrScalableQuantity, i.e. ElementCount
2169/// or TypeSize.
2170template <typename Ty>
2171static SDValue getFixedOrScalableQuantity(SelectionDAG &DAG, const SDLoc &DL,
2172 EVT VT, Ty Quantity) {
2173 if (Quantity.isScalable())
2174 return DAG.getVScale(
2175 DL, VT, MulImm: APInt(VT.getSizeInBits(), Quantity.getKnownMinValue()));
2176
2177 return DAG.getConstant(Quantity.getKnownMinValue(), DL, VT);
2178}
2179
2180SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT,
2181 ElementCount EC) {
2182 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: EC);
2183}
2184
2185SDValue SelectionDAG::getTypeSize(const SDLoc &DL, EVT VT, TypeSize TS) {
2186 return getFixedOrScalableQuantity(DAG&: *this, DL, VT, Quantity: TS);
2187}
2188
2189SDValue SelectionDAG::getMaskFromElementCount(const SDLoc &DL, EVT DataVT,
2190 ElementCount EC) {
2191 EVT IdxVT = TLI->getVectorIdxTy(DL: getDataLayout());
2192 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: DataVT);
2193 return getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: MaskVT,
2194 N1: getConstant(Val: 0, DL, VT: IdxVT), N2: getElementCount(DL, VT: IdxVT, EC));
2195}
2196
2197SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2198 APInt One(ResVT.getScalarSizeInBits(), 1);
2199 return getStepVector(DL, ResVT, StepVal: One);
2200}
2201
2202SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2203 const APInt &StepVal) {
2204 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2205 if (ResVT.isScalableVector())
2206 return getNode(
2207 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2208 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2209
2210 SmallVector<SDValue, 16> OpsStepConstants;
2211 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2212 OpsStepConstants.push_back(
2213 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2214 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2215}
2216
2217/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2218/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2219static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2220 std::swap(a&: N1, b&: N2);
2221 ShuffleVectorSDNode::commuteMask(Mask: M);
2222}
2223
2224SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2225 SDValue N2, ArrayRef<int> Mask) {
2226 assert(VT.getVectorNumElements() == Mask.size() &&
2227 "Must have the same number of vector elements as mask elements!");
2228 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2229 "Invalid VECTOR_SHUFFLE");
2230
2231 // Canonicalize shuffle undef, undef -> undef
2232 if (N1.isUndef() && N2.isUndef())
2233 return getUNDEF(VT);
2234
2235 // Validate that all indices in Mask are within the range of the elements
2236 // input to the shuffle.
2237 int NElts = Mask.size();
2238 assert(llvm::all_of(Mask,
2239 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2240 "Index out of range");
2241
2242 // Copy the mask so we can do any needed cleanup.
2243 SmallVector<int, 8> MaskVec(Mask);
2244
2245 // Canonicalize shuffle v, v -> v, undef
2246 if (N1 == N2) {
2247 N2 = getUNDEF(VT);
2248 for (int i = 0; i != NElts; ++i)
2249 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2250 }
2251
2252 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2253 if (N1.isUndef())
2254 commuteShuffle(N1, N2, M: MaskVec);
2255
2256 if (TLI->hasVectorBlend()) {
2257 // If shuffling a splat, try to blend the splat instead. We do this here so
2258 // that even when this arises during lowering we don't have to re-handle it.
2259 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2260 BitVector UndefElements;
2261 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2262 if (!Splat)
2263 return;
2264
2265 for (int i = 0; i < NElts; ++i) {
2266 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2267 continue;
2268
2269 // If this input comes from undef, mark it as such.
2270 if (UndefElements[MaskVec[i] - Offset]) {
2271 MaskVec[i] = -1;
2272 continue;
2273 }
2274
2275 // If we can blend a non-undef lane, use that instead.
2276 if (!UndefElements[i])
2277 MaskVec[i] = i + Offset;
2278 }
2279 };
2280 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2281 BlendSplat(N1BV, 0);
2282 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2283 BlendSplat(N2BV, NElts);
2284 }
2285
2286 // Canonicalize all index into lhs, -> shuffle lhs, undef
2287 // Canonicalize all index into rhs, -> shuffle rhs, undef
2288 bool AllLHS = true, AllRHS = true;
2289 bool N2Undef = N2.isUndef();
2290 for (int i = 0; i != NElts; ++i) {
2291 if (MaskVec[i] >= NElts) {
2292 if (N2Undef)
2293 MaskVec[i] = -1;
2294 else
2295 AllLHS = false;
2296 } else if (MaskVec[i] >= 0) {
2297 AllRHS = false;
2298 }
2299 }
2300 if (AllLHS && AllRHS)
2301 return getUNDEF(VT);
2302 if (AllLHS && !N2Undef)
2303 N2 = getUNDEF(VT);
2304 if (AllRHS) {
2305 N1 = getUNDEF(VT);
2306 commuteShuffle(N1, N2, M: MaskVec);
2307 }
2308 // Reset our undef status after accounting for the mask.
2309 N2Undef = N2.isUndef();
2310 // Re-check whether both sides ended up undef.
2311 if (N1.isUndef() && N2Undef)
2312 return getUNDEF(VT);
2313
2314 // If Identity shuffle return that node.
2315 bool Identity = true, AllSame = true;
2316 for (int i = 0; i != NElts; ++i) {
2317 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2318 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2319 }
2320 if (Identity && NElts)
2321 return N1;
2322
2323 // Shuffling a constant splat doesn't change the result.
2324 if (N2Undef) {
2325 SDValue V = N1;
2326
2327 // Look through any bitcasts. We check that these don't change the number
2328 // (and size) of elements and just changes their types.
2329 while (V.getOpcode() == ISD::BITCAST)
2330 V = V->getOperand(Num: 0);
2331
2332 // A splat should always show up as a build vector node.
2333 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2334 BitVector UndefElements;
2335 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2336 // If this is a splat of an undef, shuffling it is also undef.
2337 if (Splat && Splat.isUndef())
2338 return getUNDEF(VT);
2339
2340 bool SameNumElts =
2341 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2342
2343 // We only have a splat which can skip shuffles if there is a splatted
2344 // value and no undef lanes rearranged by the shuffle.
2345 if (Splat && UndefElements.none()) {
2346 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2347 // number of elements match or the value splatted is a zero constant.
2348 if (SameNumElts || isNullConstant(V: Splat))
2349 return N1;
2350 }
2351
2352 // If the shuffle itself creates a splat, build the vector directly.
2353 if (AllSame && SameNumElts) {
2354 EVT BuildVT = BV->getValueType(ResNo: 0);
2355 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2356 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2357
2358 // We may have jumped through bitcasts, so the type of the
2359 // BUILD_VECTOR may not match the type of the shuffle.
2360 if (BuildVT != VT)
2361 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2362 return NewBV;
2363 }
2364 }
2365 }
2366
2367 SDVTList VTs = getVTList(VT);
2368 FoldingSetNodeID ID;
2369 SDValue Ops[2] = { N1, N2 };
2370 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: VTs, OpList: Ops);
2371 for (int i = 0; i != NElts; ++i)
2372 ID.AddInteger(I: MaskVec[i]);
2373
2374 void* IP = nullptr;
2375 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2376 return SDValue(E, 0);
2377
2378 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2379 // SDNode doesn't have access to it. This memory will be "leaked" when
2380 // the node is deallocated, but recovered when the NodeAllocator is released.
2381 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2382 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2383
2384 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2385 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2386 createOperands(Node: N, Vals: Ops);
2387
2388 CSEMap.InsertNode(N, InsertPos: IP);
2389 InsertNode(N);
2390 SDValue V = SDValue(N, 0);
2391 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2392 return V;
2393}
2394
2395SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2396 EVT VT = SV.getValueType(ResNo: 0);
2397 SmallVector<int, 8> MaskVec(SV.getMask());
2398 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2399
2400 SDValue Op0 = SV.getOperand(Num: 0);
2401 SDValue Op1 = SV.getOperand(Num: 1);
2402 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2403}
2404
2405SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2406 SDVTList VTs = getVTList(VT);
2407 FoldingSetNodeID ID;
2408 AddNodeIDNode(ID, OpC: ISD::Register, VTList: VTs, OpList: {});
2409 ID.AddInteger(I: Reg.id());
2410 void *IP = nullptr;
2411 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2412 return SDValue(E, 0);
2413
2414 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2415 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2416 CSEMap.InsertNode(N, InsertPos: IP);
2417 InsertNode(N);
2418 return SDValue(N, 0);
2419}
2420
2421SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2422 FoldingSetNodeID ID;
2423 AddNodeIDNode(ID, OpC: ISD::RegisterMask, VTList: getVTList(VT: MVT::Untyped), OpList: {});
2424 ID.AddPointer(Ptr: RegMask);
2425 void *IP = nullptr;
2426 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2427 return SDValue(E, 0);
2428
2429 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2430 CSEMap.InsertNode(N, InsertPos: IP);
2431 InsertNode(N);
2432 return SDValue(N, 0);
2433}
2434
2435SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2436 MCSymbol *Label) {
2437 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2438}
2439
2440SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2441 SDValue Root, MCSymbol *Label) {
2442 FoldingSetNodeID ID;
2443 SDValue Ops[] = { Root };
2444 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT: MVT::Other), OpList: Ops);
2445 ID.AddPointer(Ptr: Label);
2446 void *IP = nullptr;
2447 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2448 return SDValue(E, 0);
2449
2450 auto *N =
2451 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2452 createOperands(Node: N, Vals: Ops);
2453
2454 CSEMap.InsertNode(N, InsertPos: IP);
2455 InsertNode(N);
2456 return SDValue(N, 0);
2457}
2458
2459SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2460 int64_t Offset, bool isTarget,
2461 unsigned TargetFlags) {
2462 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2463 SDVTList VTs = getVTList(VT);
2464
2465 FoldingSetNodeID ID;
2466 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2467 ID.AddPointer(Ptr: BA);
2468 ID.AddInteger(I: Offset);
2469 ID.AddInteger(I: TargetFlags);
2470 void *IP = nullptr;
2471 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2472 return SDValue(E, 0);
2473
2474 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2475 CSEMap.InsertNode(N, InsertPos: IP);
2476 InsertNode(N);
2477 return SDValue(N, 0);
2478}
2479
2480SDValue SelectionDAG::getSrcValue(const Value *V) {
2481 FoldingSetNodeID ID;
2482 AddNodeIDNode(ID, OpC: ISD::SRCVALUE, VTList: getVTList(VT: MVT::Other), OpList: {});
2483 ID.AddPointer(Ptr: V);
2484
2485 void *IP = nullptr;
2486 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2487 return SDValue(E, 0);
2488
2489 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2490 CSEMap.InsertNode(N, InsertPos: IP);
2491 InsertNode(N);
2492 return SDValue(N, 0);
2493}
2494
2495SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2496 FoldingSetNodeID ID;
2497 AddNodeIDNode(ID, OpC: ISD::MDNODE_SDNODE, VTList: getVTList(VT: MVT::Other), OpList: {});
2498 ID.AddPointer(Ptr: MD);
2499
2500 void *IP = nullptr;
2501 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2502 return SDValue(E, 0);
2503
2504 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2505 CSEMap.InsertNode(N, InsertPos: IP);
2506 InsertNode(N);
2507 return SDValue(N, 0);
2508}
2509
2510SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2511 if (VT == V.getValueType())
2512 return V;
2513
2514 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2515}
2516
2517SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2518 unsigned SrcAS, unsigned DestAS) {
2519 SDVTList VTs = getVTList(VT);
2520 SDValue Ops[] = {Ptr};
2521 FoldingSetNodeID ID;
2522 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: VTs, OpList: Ops);
2523 ID.AddInteger(I: SrcAS);
2524 ID.AddInteger(I: DestAS);
2525
2526 void *IP = nullptr;
2527 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2528 return SDValue(E, 0);
2529
2530 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2531 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2532 createOperands(Node: N, Vals: Ops);
2533
2534 CSEMap.InsertNode(N, InsertPos: IP);
2535 InsertNode(N);
2536 return SDValue(N, 0);
2537}
2538
2539SDValue SelectionDAG::getFreeze(SDValue V) {
2540 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2541}
2542
2543SDValue SelectionDAG::getFreeze(SDValue V, const APInt &DemandedElts,
2544 bool PoisonOnly) {
2545 if (isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts, PoisonOnly))
2546 return V;
2547 return getFreeze(V);
2548}
2549
2550/// getShiftAmountOperand - Return the specified value casted to
2551/// the target's desired shift amount type.
2552SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2553 EVT OpTy = Op.getValueType();
2554 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2555 if (OpTy == ShTy || OpTy.isVector()) return Op;
2556
2557 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2558}
2559
2560SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2561 SDLoc dl(Node);
2562 const TargetLowering &TLI = getTargetLoweringInfo();
2563 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2564 EVT VT = Node->getValueType(ResNo: 0);
2565 SDValue Tmp1 = Node->getOperand(Num: 0);
2566 SDValue Tmp2 = Node->getOperand(Num: 1);
2567 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2568
2569 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2570 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2571 SDValue VAList = VAListLoad;
2572
2573 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2574 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2575 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2576
2577 VAList = getNode(
2578 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2579 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2580 }
2581
2582 // Increment the pointer, VAList, to the next vaarg
2583 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2584 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2585 Ty: VT.getTypeForEVT(Context&: *getContext())),
2586 DL: dl, VT: VAList.getValueType()));
2587 // Store the incremented VAList to the legalized pointer
2588 Tmp1 =
2589 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2590 // Load the actual argument out of the pointer VAList
2591 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2592}
2593
2594SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2595 SDLoc dl(Node);
2596 const TargetLowering &TLI = getTargetLoweringInfo();
2597 // This defaults to loading a pointer from the input and storing it to the
2598 // output, returning the chain.
2599 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2600 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2601 SDValue Tmp1 =
2602 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2603 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2604 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2605 PtrInfo: MachinePointerInfo(VD));
2606}
2607
2608Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2609 const DataLayout &DL = getDataLayout();
2610 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2611 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2612
2613 if (TLI->isTypeLegal(VT) || !VT.isVector())
2614 return RedAlign;
2615
2616 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2617 const Align StackAlign = TFI->getStackAlign();
2618
2619 // See if we can choose a smaller ABI alignment in cases where it's an
2620 // illegal vector type that will get broken down.
2621 if (RedAlign > StackAlign) {
2622 EVT IntermediateVT;
2623 MVT RegisterVT;
2624 unsigned NumIntermediates;
2625 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2626 NumIntermediates, RegisterVT);
2627 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2628 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2629 if (RedAlign2 < RedAlign)
2630 RedAlign = RedAlign2;
2631
2632 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2633 // If the stack is not realignable, the alignment should be limited to the
2634 // StackAlignment
2635 RedAlign = std::min(a: RedAlign, b: StackAlign);
2636 }
2637
2638 return RedAlign;
2639}
2640
2641SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2642 MachineFrameInfo &MFI = MF->getFrameInfo();
2643 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2644 int StackID = 0;
2645 if (Bytes.isScalable())
2646 StackID = TFI->getStackIDForScalableVectors();
2647 // The stack id gives an indication of whether the object is scalable or
2648 // not, so it's safe to pass in the minimum size here.
2649 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2650 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2651 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2652}
2653
2654SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2655 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2656 Align StackAlign =
2657 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2658 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2659}
2660
2661SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2662 TypeSize VT1Size = VT1.getStoreSize();
2663 TypeSize VT2Size = VT2.getStoreSize();
2664 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2665 "Don't know how to choose the maximum size when creating a stack "
2666 "temporary");
2667 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2668 ? VT1Size
2669 : VT2Size;
2670
2671 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2672 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2673 const DataLayout &DL = getDataLayout();
2674 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2675 return CreateStackTemporary(Bytes, Alignment: Align);
2676}
2677
2678SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2679 ISD::CondCode Cond, const SDLoc &dl,
2680 SDNodeFlags Flags) {
2681 EVT OpVT = N1.getValueType();
2682
2683 auto GetUndefBooleanConstant = [&]() {
2684 if (VT.getScalarType() == MVT::i1 ||
2685 TLI->getBooleanContents(Type: OpVT) ==
2686 TargetLowering::UndefinedBooleanContent)
2687 return getUNDEF(VT);
2688 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2689 // so we cannot use getUNDEF(). Return zero instead.
2690 return getConstant(Val: 0, DL: dl, VT);
2691 };
2692
2693 // These setcc operations always fold.
2694 switch (Cond) {
2695 default: break;
2696 case ISD::SETFALSE:
2697 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2698 case ISD::SETTRUE:
2699 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2700
2701 case ISD::SETOEQ:
2702 case ISD::SETOGT:
2703 case ISD::SETOGE:
2704 case ISD::SETOLT:
2705 case ISD::SETOLE:
2706 case ISD::SETONE:
2707 case ISD::SETO:
2708 case ISD::SETUO:
2709 case ISD::SETUEQ:
2710 case ISD::SETUNE:
2711 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2712 break;
2713 }
2714
2715 if (OpVT.isInteger()) {
2716 // For EQ and NE, we can always pick a value for the undef to make the
2717 // predicate pass or fail, so we can return undef.
2718 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2719 // icmp eq/ne X, undef -> undef.
2720 if ((N1.isUndef() || N2.isUndef()) &&
2721 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2722 return GetUndefBooleanConstant();
2723
2724 // If both operands are undef, we can return undef for int comparison.
2725 // icmp undef, undef -> undef.
2726 if (N1.isUndef() && N2.isUndef())
2727 return GetUndefBooleanConstant();
2728
2729 // icmp X, X -> true/false
2730 // icmp X, undef -> true/false because undef could be X.
2731 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2732 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2733 }
2734
2735 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2736 const APInt &C2 = N2C->getAPIntValue();
2737 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2738 const APInt &C1 = N1C->getAPIntValue();
2739
2740 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2741 DL: dl, VT, OpVT);
2742 }
2743 }
2744
2745 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2746 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2747
2748 if (N1CFP && N2CFP) {
2749 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2750 switch (Cond) {
2751 default: break;
2752 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2753 return GetUndefBooleanConstant();
2754 [[fallthrough]];
2755 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2756 OpVT);
2757 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2758 return GetUndefBooleanConstant();
2759 [[fallthrough]];
2760 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2761 R==APFloat::cmpLessThan, DL: dl, VT,
2762 OpVT);
2763 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2764 return GetUndefBooleanConstant();
2765 [[fallthrough]];
2766 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2767 OpVT);
2768 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2769 return GetUndefBooleanConstant();
2770 [[fallthrough]];
2771 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2772 VT, OpVT);
2773 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2774 return GetUndefBooleanConstant();
2775 [[fallthrough]];
2776 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2777 R==APFloat::cmpEqual, DL: dl, VT,
2778 OpVT);
2779 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2780 return GetUndefBooleanConstant();
2781 [[fallthrough]];
2782 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2783 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2784 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2785 OpVT);
2786 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2787 OpVT);
2788 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2789 R==APFloat::cmpEqual, DL: dl, VT,
2790 OpVT);
2791 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2792 OpVT);
2793 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2794 R==APFloat::cmpLessThan, DL: dl, VT,
2795 OpVT);
2796 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2797 R==APFloat::cmpUnordered, DL: dl, VT,
2798 OpVT);
2799 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2800 VT, OpVT);
2801 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2802 OpVT);
2803 }
2804 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2805 // Ensure that the constant occurs on the RHS.
2806 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2807 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2808 return SDValue();
2809 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond, /*Chian=*/Chain: {},
2810 /*IsSignaling=*/false, Flags);
2811 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2812 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2813 // If an operand is known to be a nan (or undef that could be a nan), we can
2814 // fold it.
2815 // Choosing NaN for the undef will always make unordered comparison succeed
2816 // and ordered comparison fails.
2817 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2818 switch (ISD::getUnorderedFlavor(Cond)) {
2819 default:
2820 llvm_unreachable("Unknown flavor!");
2821 case 0: // Known false.
2822 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2823 case 1: // Known true.
2824 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2825 case 2: // Undefined.
2826 return GetUndefBooleanConstant();
2827 }
2828 }
2829
2830 // Could not fold it.
2831 return SDValue();
2832}
2833
2834/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2835/// use this predicate to simplify operations downstream.
2836bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2837 unsigned BitWidth = Op.getScalarValueSizeInBits();
2838 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2839}
2840
2841// TODO: Should have argument to specify if sign bit of nan is ignorable.
2842bool SelectionDAG::SignBitIsZeroFP(SDValue Op, unsigned Depth) const {
2843 if (Depth >= MaxRecursionDepth)
2844 return false; // Limit search depth.
2845
2846 unsigned Opc = Op.getOpcode();
2847 switch (Opc) {
2848 case ISD::FABS:
2849 return true;
2850 case ISD::AssertNoFPClass: {
2851 FPClassTest NoFPClass =
2852 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
2853
2854 const FPClassTest TestMask = fcNan | fcNegative;
2855 return (NoFPClass & TestMask) == TestMask;
2856 }
2857 case ISD::ARITH_FENCE:
2858 return SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2859 case ISD::FEXP:
2860 case ISD::FEXP2:
2861 case ISD::FEXP10:
2862 return Op->getFlags().hasNoNaNs();
2863 case ISD::FMINNUM:
2864 case ISD::FMINNUM_IEEE:
2865 case ISD::FMINIMUM:
2866 case ISD::FMINIMUMNUM:
2867 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2868 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2869 case ISD::FMAXNUM:
2870 case ISD::FMAXNUM_IEEE:
2871 case ISD::FMAXIMUM:
2872 case ISD::FMAXIMUMNUM:
2873 // TODO: If we can ignore the sign bit of nans, only one side being known 0
2874 // is sufficient.
2875 return SignBitIsZeroFP(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
2876 SignBitIsZeroFP(Op: Op.getOperand(i: 0), Depth: Depth + 1);
2877 default:
2878 return false;
2879 }
2880
2881 llvm_unreachable("covered opcode switch");
2882}
2883
2884/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2885/// this predicate to simplify operations downstream. Mask is known to be zero
2886/// for bits that V cannot have.
2887bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2888 unsigned Depth) const {
2889 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2890}
2891
2892/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2893/// DemandedElts. We use this predicate to simplify operations downstream.
2894/// Mask is known to be zero for bits that V cannot have.
2895bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2896 const APInt &DemandedElts,
2897 unsigned Depth) const {
2898 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2899}
2900
2901/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2902/// DemandedElts. We use this predicate to simplify operations downstream.
2903bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2904 unsigned Depth /* = 0 */) const {
2905 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2906}
2907
2908/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2909bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2910 unsigned Depth) const {
2911 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2912}
2913
2914APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2915 const APInt &DemandedElts,
2916 unsigned Depth) const {
2917 EVT VT = Op.getValueType();
2918 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2919
2920 unsigned NumElts = VT.getVectorNumElements();
2921 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2922
2923 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2924 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2925 if (!DemandedElts[EltIdx])
2926 continue; // Don't query elements that are not demanded.
2927 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2928 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2929 KnownZeroElements.setBit(EltIdx);
2930 }
2931 return KnownZeroElements;
2932}
2933
2934/// isSplatValue - Return true if the vector V has the same value
2935/// across all DemandedElts. For scalable vectors, we don't know the
2936/// number of lanes at compile time. Instead, we use a 1 bit APInt
2937/// to represent a conservative value for all lanes; that is, that
2938/// one bit value is implicitly splatted across all lanes.
2939bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2940 APInt &UndefElts, unsigned Depth) const {
2941 unsigned Opcode = V.getOpcode();
2942 EVT VT = V.getValueType();
2943 assert(VT.isVector() && "Vector type expected");
2944 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2945 "scalable demanded bits are ignored");
2946
2947 if (!DemandedElts)
2948 return false; // No demanded elts, better to assume we don't know anything.
2949
2950 if (Depth >= MaxRecursionDepth)
2951 return false; // Limit search depth.
2952
2953 // Deal with some common cases here that work for both fixed and scalable
2954 // vector types.
2955 switch (Opcode) {
2956 case ISD::SPLAT_VECTOR:
2957 UndefElts = V.getOperand(i: 0).isUndef()
2958 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
2959 : APInt(DemandedElts.getBitWidth(), 0);
2960 return true;
2961 case ISD::ADD:
2962 case ISD::SUB:
2963 case ISD::AND:
2964 case ISD::XOR:
2965 case ISD::OR: {
2966 APInt UndefLHS, UndefRHS;
2967 SDValue LHS = V.getOperand(i: 0);
2968 SDValue RHS = V.getOperand(i: 1);
2969 // Only recognize splats with the same demanded undef elements for both
2970 // operands, otherwise we might fail to handle binop-specific undef
2971 // handling.
2972 // e.g. (and undef, 0) -> 0 etc.
2973 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
2974 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
2975 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
2976 UndefElts = UndefLHS | UndefRHS;
2977 return true;
2978 }
2979 return false;
2980 }
2981 case ISD::ABS:
2982 case ISD::TRUNCATE:
2983 case ISD::SIGN_EXTEND:
2984 case ISD::ZERO_EXTEND:
2985 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
2986 default:
2987 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2988 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2989 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
2990 Depth);
2991 break;
2992 }
2993
2994 // We don't support other cases than those above for scalable vectors at
2995 // the moment.
2996 if (VT.isScalableVector())
2997 return false;
2998
2999 unsigned NumElts = VT.getVectorNumElements();
3000 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3001 UndefElts = APInt::getZero(numBits: NumElts);
3002
3003 switch (Opcode) {
3004 case ISD::BUILD_VECTOR: {
3005 SDValue Scl;
3006 for (unsigned i = 0; i != NumElts; ++i) {
3007 SDValue Op = V.getOperand(i);
3008 if (Op.isUndef()) {
3009 UndefElts.setBit(i);
3010 continue;
3011 }
3012 if (!DemandedElts[i])
3013 continue;
3014 if (Scl && Scl != Op)
3015 return false;
3016 Scl = Op;
3017 }
3018 return true;
3019 }
3020 case ISD::VECTOR_SHUFFLE: {
3021 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3022 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3023 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3024 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3025 for (int i = 0; i != (int)NumElts; ++i) {
3026 int M = Mask[i];
3027 if (M < 0) {
3028 UndefElts.setBit(i);
3029 continue;
3030 }
3031 if (!DemandedElts[i])
3032 continue;
3033 if (M < (int)NumElts)
3034 DemandedLHS.setBit(M);
3035 else
3036 DemandedRHS.setBit(M - NumElts);
3037 }
3038
3039 // If we aren't demanding either op, assume there's no splat.
3040 // If we are demanding both ops, assume there's no splat.
3041 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3042 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3043 return false;
3044
3045 // See if the demanded elts of the source op is a splat or we only demand
3046 // one element, which should always be a splat.
3047 // TODO: Handle source ops splats with undefs.
3048 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3049 APInt SrcUndefs;
3050 return (SrcElts.popcount() == 1) ||
3051 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3052 (SrcElts & SrcUndefs).isZero());
3053 };
3054 if (!DemandedLHS.isZero())
3055 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3056 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3057 }
3058 case ISD::EXTRACT_SUBVECTOR: {
3059 // Offset the demanded elts by the subvector index.
3060 SDValue Src = V.getOperand(i: 0);
3061 // We don't support scalable vectors at the moment.
3062 if (Src.getValueType().isScalableVector())
3063 return false;
3064 uint64_t Idx = V.getConstantOperandVal(i: 1);
3065 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3066 APInt UndefSrcElts;
3067 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3068 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3069 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3070 return true;
3071 }
3072 break;
3073 }
3074 case ISD::ANY_EXTEND_VECTOR_INREG:
3075 case ISD::SIGN_EXTEND_VECTOR_INREG:
3076 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3077 // Widen the demanded elts by the src element count.
3078 SDValue Src = V.getOperand(i: 0);
3079 // We don't support scalable vectors at the moment.
3080 if (Src.getValueType().isScalableVector())
3081 return false;
3082 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3083 APInt UndefSrcElts;
3084 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3085 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3086 UndefElts = UndefSrcElts.trunc(width: NumElts);
3087 return true;
3088 }
3089 break;
3090 }
3091 case ISD::BITCAST: {
3092 SDValue Src = V.getOperand(i: 0);
3093 EVT SrcVT = Src.getValueType();
3094 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3095 unsigned BitWidth = VT.getScalarSizeInBits();
3096
3097 // Ignore bitcasts from unsupported types.
3098 // TODO: Add fp support?
3099 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3100 break;
3101
3102 // Bitcast 'small element' vector to 'large element' vector.
3103 if ((BitWidth % SrcBitWidth) == 0) {
3104 // See if each sub element is a splat.
3105 unsigned Scale = BitWidth / SrcBitWidth;
3106 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3107 APInt ScaledDemandedElts =
3108 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3109 for (unsigned I = 0; I != Scale; ++I) {
3110 APInt SubUndefElts;
3111 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3112 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3113 SubDemandedElts &= ScaledDemandedElts;
3114 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3115 return false;
3116 // TODO: Add support for merging sub undef elements.
3117 if (!SubUndefElts.isZero())
3118 return false;
3119 }
3120 return true;
3121 }
3122 break;
3123 }
3124 }
3125
3126 return false;
3127}
3128
3129/// Helper wrapper to main isSplatValue function.
3130bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3131 EVT VT = V.getValueType();
3132 assert(VT.isVector() && "Vector type expected");
3133
3134 APInt UndefElts;
3135 // Since the number of lanes in a scalable vector is unknown at compile time,
3136 // we track one bit which is implicitly broadcast to all lanes. This means
3137 // that all lanes in a scalable vector are considered demanded.
3138 APInt DemandedElts
3139 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3140 return isSplatValue(V, DemandedElts, UndefElts) &&
3141 (AllowUndefs || !UndefElts);
3142}
3143
3144SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3145 V = peekThroughExtractSubvectors(V);
3146
3147 EVT VT = V.getValueType();
3148 unsigned Opcode = V.getOpcode();
3149 switch (Opcode) {
3150 default: {
3151 APInt UndefElts;
3152 // Since the number of lanes in a scalable vector is unknown at compile time,
3153 // we track one bit which is implicitly broadcast to all lanes. This means
3154 // that all lanes in a scalable vector are considered demanded.
3155 APInt DemandedElts
3156 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3157
3158 if (isSplatValue(V, DemandedElts, UndefElts)) {
3159 if (VT.isScalableVector()) {
3160 // DemandedElts and UndefElts are ignored for scalable vectors, since
3161 // the only supported cases are SPLAT_VECTOR nodes.
3162 SplatIdx = 0;
3163 } else {
3164 // Handle case where all demanded elements are UNDEF.
3165 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3166 SplatIdx = 0;
3167 return getUNDEF(VT);
3168 }
3169 SplatIdx = (UndefElts & DemandedElts).countr_one();
3170 }
3171 return V;
3172 }
3173 break;
3174 }
3175 case ISD::SPLAT_VECTOR:
3176 SplatIdx = 0;
3177 return V;
3178 case ISD::VECTOR_SHUFFLE: {
3179 assert(!VT.isScalableVector());
3180 // Check if this is a shuffle node doing a splat.
3181 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3182 // getTargetVShiftNode currently struggles without the splat source.
3183 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3184 if (!SVN->isSplat())
3185 break;
3186 int Idx = SVN->getSplatIndex();
3187 int NumElts = V.getValueType().getVectorNumElements();
3188 SplatIdx = Idx % NumElts;
3189 return V.getOperand(i: Idx / NumElts);
3190 }
3191 }
3192
3193 return SDValue();
3194}
3195
3196SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3197 int SplatIdx;
3198 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3199 EVT SVT = SrcVector.getValueType().getScalarType();
3200 EVT LegalSVT = SVT;
3201 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3202 if (!SVT.isInteger())
3203 return SDValue();
3204 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3205 if (LegalSVT.bitsLT(VT: SVT))
3206 return SDValue();
3207 }
3208 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3209 }
3210 return SDValue();
3211}
3212
3213std::optional<ConstantRange>
3214SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3215 unsigned Depth) const {
3216 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3217 V.getOpcode() == ISD::SRA) &&
3218 "Unknown shift node");
3219 // Shifting more than the bitwidth is not valid.
3220 unsigned BitWidth = V.getScalarValueSizeInBits();
3221
3222 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3223 const APInt &ShAmt = Cst->getAPIntValue();
3224 if (ShAmt.uge(RHS: BitWidth))
3225 return std::nullopt;
3226 return ConstantRange(ShAmt);
3227 }
3228
3229 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3230 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3231 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3232 if (!DemandedElts[i])
3233 continue;
3234 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3235 if (!SA) {
3236 MinAmt = MaxAmt = nullptr;
3237 break;
3238 }
3239 const APInt &ShAmt = SA->getAPIntValue();
3240 if (ShAmt.uge(RHS: BitWidth))
3241 return std::nullopt;
3242 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3243 MinAmt = &ShAmt;
3244 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3245 MaxAmt = &ShAmt;
3246 }
3247 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3248 "Failed to find matching min/max shift amounts");
3249 if (MinAmt && MaxAmt)
3250 return ConstantRange(*MinAmt, *MaxAmt + 1);
3251 }
3252
3253 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3254 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3255 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3256 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3257 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3258
3259 return std::nullopt;
3260}
3261
3262std::optional<unsigned>
3263SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3264 unsigned Depth) const {
3265 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3266 V.getOpcode() == ISD::SRA) &&
3267 "Unknown shift node");
3268 if (std::optional<ConstantRange> AmtRange =
3269 getValidShiftAmountRange(V, DemandedElts, Depth))
3270 if (const APInt *ShAmt = AmtRange->getSingleElement())
3271 return ShAmt->getZExtValue();
3272 return std::nullopt;
3273}
3274
3275std::optional<unsigned>
3276SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3277 EVT VT = V.getValueType();
3278 APInt DemandedElts = VT.isFixedLengthVector()
3279 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3280 : APInt(1, 1);
3281 return getValidShiftAmount(V, DemandedElts, Depth);
3282}
3283
3284std::optional<unsigned>
3285SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3286 unsigned Depth) const {
3287 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3288 V.getOpcode() == ISD::SRA) &&
3289 "Unknown shift node");
3290 if (std::optional<ConstantRange> AmtRange =
3291 getValidShiftAmountRange(V, DemandedElts, Depth))
3292 return AmtRange->getUnsignedMin().getZExtValue();
3293 return std::nullopt;
3294}
3295
3296std::optional<unsigned>
3297SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3298 EVT VT = V.getValueType();
3299 APInt DemandedElts = VT.isFixedLengthVector()
3300 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3301 : APInt(1, 1);
3302 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3303}
3304
3305std::optional<unsigned>
3306SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3307 unsigned Depth) const {
3308 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3309 V.getOpcode() == ISD::SRA) &&
3310 "Unknown shift node");
3311 if (std::optional<ConstantRange> AmtRange =
3312 getValidShiftAmountRange(V, DemandedElts, Depth))
3313 return AmtRange->getUnsignedMax().getZExtValue();
3314 return std::nullopt;
3315}
3316
3317std::optional<unsigned>
3318SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3319 EVT VT = V.getValueType();
3320 APInt DemandedElts = VT.isFixedLengthVector()
3321 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3322 : APInt(1, 1);
3323 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3324}
3325
3326/// Determine which bits of Op are known to be either zero or one and return
3327/// them in Known. For vectors, the known bits are those that are shared by
3328/// every vector element.
3329KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3330 EVT VT = Op.getValueType();
3331
3332 // Since the number of lanes in a scalable vector is unknown at compile time,
3333 // we track one bit which is implicitly broadcast to all lanes. This means
3334 // that all lanes in a scalable vector are considered demanded.
3335 APInt DemandedElts = VT.isFixedLengthVector()
3336 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3337 : APInt(1, 1);
3338 return computeKnownBits(Op, DemandedElts, Depth);
3339}
3340
3341/// Determine which bits of Op are known to be either zero or one and return
3342/// them in Known. The DemandedElts argument allows us to only collect the known
3343/// bits that are shared by the requested vector elements.
3344KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3345 unsigned Depth) const {
3346 unsigned BitWidth = Op.getScalarValueSizeInBits();
3347
3348 KnownBits Known(BitWidth); // Don't know anything.
3349
3350 if (auto OptAPInt = Op->bitcastToAPInt()) {
3351 // We know all of the bits for a constant!
3352 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3353 }
3354
3355 if (Depth >= MaxRecursionDepth)
3356 return Known; // Limit search depth.
3357
3358 KnownBits Known2;
3359 unsigned NumElts = DemandedElts.getBitWidth();
3360 assert((!Op.getValueType().isScalableVector() || NumElts == 1) &&
3361 "DemandedElts for scalable vectors must be 1 to represent all lanes");
3362 assert((!Op.getValueType().isFixedLengthVector() ||
3363 NumElts == Op.getValueType().getVectorNumElements()) &&
3364 "Unexpected vector size");
3365
3366 if (!DemandedElts)
3367 return Known; // No demanded elts, better to assume we don't know anything.
3368
3369 unsigned Opcode = Op.getOpcode();
3370 switch (Opcode) {
3371 case ISD::MERGE_VALUES:
3372 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3373 Depth: Depth + 1);
3374 case ISD::SPLAT_VECTOR: {
3375 SDValue SrcOp = Op.getOperand(i: 0);
3376 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3377 "Expected SPLAT_VECTOR implicit truncation");
3378 // Implicitly truncate the bits to match the official semantics of
3379 // SPLAT_VECTOR.
3380 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3381 break;
3382 }
3383 case ISD::SPLAT_VECTOR_PARTS: {
3384 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3385 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3386 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3387 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3388 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3389 }
3390 break;
3391 }
3392 case ISD::STEP_VECTOR: {
3393 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3394
3395 if (Step.isPowerOf2())
3396 Known.Zero.setLowBits(Step.logBase2());
3397
3398 const Function &F = getMachineFunction().getFunction();
3399
3400 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3401 break;
3402 const APInt MinNumElts =
3403 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3404
3405 bool Overflow;
3406 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3407 .getUnsignedMax()
3408 .umul_ov(RHS: MinNumElts, Overflow);
3409 if (Overflow)
3410 break;
3411
3412 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3413 if (Overflow)
3414 break;
3415
3416 Known.Zero.setHighBits(MaxValue.countl_zero());
3417 break;
3418 }
3419 case ISD::BUILD_VECTOR:
3420 assert(!Op.getValueType().isScalableVector());
3421 // Collect the known bits that are shared by every demanded vector element.
3422 Known.setAllConflict();
3423 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3424 if (!DemandedElts[i])
3425 continue;
3426
3427 SDValue SrcOp = Op.getOperand(i);
3428 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3429
3430 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3431 if (SrcOp.getValueSizeInBits() != BitWidth) {
3432 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3433 "Expected BUILD_VECTOR implicit truncation");
3434 Known2 = Known2.trunc(BitWidth);
3435 }
3436
3437 // Known bits are the values that are shared by every demanded element.
3438 Known = Known.intersectWith(RHS: Known2);
3439
3440 // If we don't know any bits, early out.
3441 if (Known.isUnknown())
3442 break;
3443 }
3444 break;
3445 case ISD::VECTOR_COMPRESS: {
3446 SDValue Vec = Op.getOperand(i: 0);
3447 SDValue PassThru = Op.getOperand(i: 2);
3448 Known = computeKnownBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
3449 // If we don't know any bits, early out.
3450 if (Known.isUnknown())
3451 break;
3452 Known2 = computeKnownBits(Op: Vec, Depth: Depth + 1);
3453 Known = Known.intersectWith(RHS: Known2);
3454 break;
3455 }
3456 case ISD::VECTOR_SHUFFLE: {
3457 assert(!Op.getValueType().isScalableVector());
3458 // Collect the known bits that are shared by every vector element referenced
3459 // by the shuffle.
3460 APInt DemandedLHS, DemandedRHS;
3461 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3462 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3463 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3464 DemandedLHS, DemandedRHS))
3465 break;
3466
3467 // Known bits are the values that are shared by every demanded element.
3468 Known.setAllConflict();
3469 if (!!DemandedLHS) {
3470 SDValue LHS = Op.getOperand(i: 0);
3471 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3472 Known = Known.intersectWith(RHS: Known2);
3473 }
3474 // If we don't know any bits, early out.
3475 if (Known.isUnknown())
3476 break;
3477 if (!!DemandedRHS) {
3478 SDValue RHS = Op.getOperand(i: 1);
3479 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3480 Known = Known.intersectWith(RHS: Known2);
3481 }
3482 break;
3483 }
3484 case ISD::VSCALE: {
3485 const Function &F = getMachineFunction().getFunction();
3486 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3487 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3488 break;
3489 }
3490 case ISD::CONCAT_VECTORS: {
3491 if (Op.getValueType().isScalableVector())
3492 break;
3493 // Split DemandedElts and test each of the demanded subvectors.
3494 Known.setAllConflict();
3495 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3496 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3497 unsigned NumSubVectors = Op.getNumOperands();
3498 for (unsigned i = 0; i != NumSubVectors; ++i) {
3499 APInt DemandedSub =
3500 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3501 if (!!DemandedSub) {
3502 SDValue Sub = Op.getOperand(i);
3503 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3504 Known = Known.intersectWith(RHS: Known2);
3505 }
3506 // If we don't know any bits, early out.
3507 if (Known.isUnknown())
3508 break;
3509 }
3510 break;
3511 }
3512 case ISD::INSERT_SUBVECTOR: {
3513 if (Op.getValueType().isScalableVector())
3514 break;
3515 // Demand any elements from the subvector and the remainder from the src its
3516 // inserted into.
3517 SDValue Src = Op.getOperand(i: 0);
3518 SDValue Sub = Op.getOperand(i: 1);
3519 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3520 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3521 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3522 APInt DemandedSrcElts = DemandedElts;
3523 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3524
3525 Known.setAllConflict();
3526 if (!!DemandedSubElts) {
3527 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3528 if (Known.isUnknown())
3529 break; // early-out.
3530 }
3531 if (!!DemandedSrcElts) {
3532 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3533 Known = Known.intersectWith(RHS: Known2);
3534 }
3535 break;
3536 }
3537 case ISD::EXTRACT_SUBVECTOR: {
3538 // Offset the demanded elts by the subvector index.
3539 SDValue Src = Op.getOperand(i: 0);
3540
3541 APInt DemandedSrcElts;
3542 if (Src.getValueType().isScalableVector())
3543 DemandedSrcElts = APInt(1, 1); // <=> 'demand all elements'
3544 else {
3545 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3546 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3547 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3548 }
3549 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3550 break;
3551 }
3552 case ISD::SCALAR_TO_VECTOR: {
3553 if (Op.getValueType().isScalableVector())
3554 break;
3555 // We know about scalar_to_vector as much as we know about it source,
3556 // which becomes the first element of otherwise unknown vector.
3557 if (DemandedElts != 1)
3558 break;
3559
3560 SDValue N0 = Op.getOperand(i: 0);
3561 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3562 if (N0.getValueSizeInBits() != BitWidth)
3563 Known = Known.trunc(BitWidth);
3564
3565 break;
3566 }
3567 case ISD::BITCAST: {
3568 if (Op.getValueType().isScalableVector())
3569 break;
3570
3571 SDValue N0 = Op.getOperand(i: 0);
3572 EVT SubVT = N0.getValueType();
3573 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3574
3575 // Ignore bitcasts from unsupported types.
3576 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3577 break;
3578
3579 // Fast handling of 'identity' bitcasts.
3580 if (BitWidth == SubBitWidth) {
3581 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3582 break;
3583 }
3584
3585 bool IsLE = getDataLayout().isLittleEndian();
3586
3587 // Bitcast 'small element' vector to 'large element' scalar/vector.
3588 if ((BitWidth % SubBitWidth) == 0) {
3589 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3590
3591 // Collect known bits for the (larger) output by collecting the known
3592 // bits from each set of sub elements and shift these into place.
3593 // We need to separately call computeKnownBits for each set of
3594 // sub elements as the knownbits for each is likely to be different.
3595 unsigned SubScale = BitWidth / SubBitWidth;
3596 APInt SubDemandedElts(NumElts * SubScale, 0);
3597 for (unsigned i = 0; i != NumElts; ++i)
3598 if (DemandedElts[i])
3599 SubDemandedElts.setBit(i * SubScale);
3600
3601 for (unsigned i = 0; i != SubScale; ++i) {
3602 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3603 Depth: Depth + 1);
3604 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3605 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3606 }
3607 }
3608
3609 // Bitcast 'large element' scalar/vector to 'small element' vector.
3610 if ((SubBitWidth % BitWidth) == 0) {
3611 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3612
3613 // Collect known bits for the (smaller) output by collecting the known
3614 // bits from the overlapping larger input elements and extracting the
3615 // sub sections we actually care about.
3616 unsigned SubScale = SubBitWidth / BitWidth;
3617 APInt SubDemandedElts =
3618 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3619 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3620
3621 Known.setAllConflict();
3622 for (unsigned i = 0; i != NumElts; ++i)
3623 if (DemandedElts[i]) {
3624 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3625 unsigned Offset = (Shifts % SubScale) * BitWidth;
3626 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3627 // If we don't know any bits, early out.
3628 if (Known.isUnknown())
3629 break;
3630 }
3631 }
3632 break;
3633 }
3634 case ISD::AND:
3635 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3636 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3637
3638 Known &= Known2;
3639 break;
3640 case ISD::OR:
3641 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3642 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3643
3644 Known |= Known2;
3645 break;
3646 case ISD::XOR:
3647 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3648 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3649
3650 Known ^= Known2;
3651 break;
3652 case ISD::MUL: {
3653 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3654 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3655 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3656 // TODO: SelfMultiply can be poison, but not undef.
3657 if (SelfMultiply)
3658 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3659 Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
3660 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3661
3662 // If the multiplication is known not to overflow, the product of a number
3663 // with itself is non-negative. Only do this if we didn't already computed
3664 // the opposite value for the sign bit.
3665 if (Op->getFlags().hasNoSignedWrap() &&
3666 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3667 !Known.isNegative())
3668 Known.makeNonNegative();
3669 break;
3670 }
3671 case ISD::MULHU: {
3672 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3673 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3674 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3675 break;
3676 }
3677 case ISD::MULHS: {
3678 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3679 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3680 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3681 break;
3682 }
3683 case ISD::ABDU: {
3684 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3685 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3686 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3687 break;
3688 }
3689 case ISD::ABDS: {
3690 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3691 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3692 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3693 unsigned SignBits1 =
3694 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3695 if (SignBits1 == 1)
3696 break;
3697 unsigned SignBits0 =
3698 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3699 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3700 break;
3701 }
3702 case ISD::UMUL_LOHI: {
3703 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3704 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3705 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3706 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3707 if (Op.getResNo() == 0)
3708 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3709 else
3710 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3711 break;
3712 }
3713 case ISD::SMUL_LOHI: {
3714 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3715 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3716 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3717 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3718 if (Op.getResNo() == 0)
3719 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3720 else
3721 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3722 break;
3723 }
3724 case ISD::AVGFLOORU: {
3725 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3726 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3727 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3728 break;
3729 }
3730 case ISD::AVGCEILU: {
3731 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3732 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3733 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3734 break;
3735 }
3736 case ISD::AVGFLOORS: {
3737 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3738 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3739 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3740 break;
3741 }
3742 case ISD::AVGCEILS: {
3743 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3744 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3745 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3746 break;
3747 }
3748 case ISD::SELECT:
3749 case ISD::VSELECT:
3750 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3751 // If we don't know any bits, early out.
3752 if (Known.isUnknown())
3753 break;
3754 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3755
3756 // Only known if known in both the LHS and RHS.
3757 Known = Known.intersectWith(RHS: Known2);
3758 break;
3759 case ISD::SELECT_CC:
3760 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3761 // If we don't know any bits, early out.
3762 if (Known.isUnknown())
3763 break;
3764 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3765
3766 // Only known if known in both the LHS and RHS.
3767 Known = Known.intersectWith(RHS: Known2);
3768 break;
3769 case ISD::SMULO:
3770 case ISD::UMULO:
3771 if (Op.getResNo() != 1)
3772 break;
3773 // The boolean result conforms to getBooleanContents.
3774 // If we know the result of a setcc has the top bits zero, use this info.
3775 // We know that we have an integer-based boolean since these operations
3776 // are only available for integer.
3777 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3778 TargetLowering::ZeroOrOneBooleanContent &&
3779 BitWidth > 1)
3780 Known.Zero.setBitsFrom(1);
3781 break;
3782 case ISD::SETCC:
3783 case ISD::SETCCCARRY:
3784 case ISD::STRICT_FSETCC:
3785 case ISD::STRICT_FSETCCS: {
3786 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3787 // If we know the result of a setcc has the top bits zero, use this info.
3788 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3789 TargetLowering::ZeroOrOneBooleanContent &&
3790 BitWidth > 1)
3791 Known.Zero.setBitsFrom(1);
3792 break;
3793 }
3794 case ISD::SHL: {
3795 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3796 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3797
3798 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3799 bool NSW = Op->getFlags().hasNoSignedWrap();
3800
3801 bool ShAmtNonZero = Known2.isNonZero();
3802
3803 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3804
3805 // Minimum shift low bits are known zero.
3806 if (std::optional<unsigned> ShMinAmt =
3807 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3808 Known.Zero.setLowBits(*ShMinAmt);
3809 break;
3810 }
3811 case ISD::SRL:
3812 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3813 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3814 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3815 Exact: Op->getFlags().hasExact());
3816
3817 // Minimum shift high bits are known zero.
3818 if (std::optional<unsigned> ShMinAmt =
3819 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3820 Known.Zero.setHighBits(*ShMinAmt);
3821 break;
3822 case ISD::SRA:
3823 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3824 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3825 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3826 Exact: Op->getFlags().hasExact());
3827 break;
3828 case ISD::ROTL:
3829 case ISD::ROTR:
3830 if (ConstantSDNode *C =
3831 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
3832 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3833
3834 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3835
3836 // Canonicalize to ROTR.
3837 if (Opcode == ISD::ROTL && Amt != 0)
3838 Amt = BitWidth - Amt;
3839
3840 Known.Zero = Known.Zero.rotr(rotateAmt: Amt);
3841 Known.One = Known.One.rotr(rotateAmt: Amt);
3842 }
3843 break;
3844 case ISD::FSHL:
3845 case ISD::FSHR:
3846 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3847 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3848
3849 // For fshl, 0-shift returns the 1st arg.
3850 // For fshr, 0-shift returns the 2nd arg.
3851 if (Amt == 0) {
3852 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3853 DemandedElts, Depth: Depth + 1);
3854 break;
3855 }
3856
3857 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3858 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3859 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3860 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3861 if (Opcode == ISD::FSHL) {
3862 Known <<= Amt;
3863 Known2 >>= BitWidth - Amt;
3864 } else {
3865 Known <<= BitWidth - Amt;
3866 Known2 >>= Amt;
3867 }
3868 Known = Known.unionWith(RHS: Known2);
3869 }
3870 break;
3871 case ISD::SHL_PARTS:
3872 case ISD::SRA_PARTS:
3873 case ISD::SRL_PARTS: {
3874 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3875
3876 // Collect lo/hi source values and concatenate.
3877 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3878 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3879 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3880 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3881 Known = Known2.concat(Lo: Known);
3882
3883 // Collect shift amount.
3884 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3885
3886 if (Opcode == ISD::SHL_PARTS)
3887 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3888 else if (Opcode == ISD::SRA_PARTS)
3889 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3890 else // if (Opcode == ISD::SRL_PARTS)
3891 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3892
3893 // TODO: Minimum shift low/high bits are known zero.
3894
3895 if (Op.getResNo() == 0)
3896 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3897 else
3898 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3899 break;
3900 }
3901 case ISD::SIGN_EXTEND_INREG: {
3902 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3903 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3904 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3905 break;
3906 }
3907 case ISD::CTTZ:
3908 case ISD::CTTZ_ZERO_UNDEF: {
3909 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3910 // If we have a known 1, its position is our upper bound.
3911 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3912 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3913 Known.Zero.setBitsFrom(LowBits);
3914 break;
3915 }
3916 case ISD::CTLZ:
3917 case ISD::CTLZ_ZERO_UNDEF: {
3918 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3919 // If we have a known 1, its position is our upper bound.
3920 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3921 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3922 Known.Zero.setBitsFrom(LowBits);
3923 break;
3924 }
3925 case ISD::CTLS: {
3926 unsigned MinRedundantSignBits =
3927 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1;
3928 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
3929 APInt(BitWidth, BitWidth));
3930 Known = Range.toKnownBits();
3931 break;
3932 }
3933 case ISD::CTPOP: {
3934 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3935 // If we know some of the bits are zero, they can't be one.
3936 unsigned PossibleOnes = Known2.countMaxPopulation();
3937 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3938 break;
3939 }
3940 case ISD::PARITY: {
3941 // Parity returns 0 everywhere but the LSB.
3942 Known.Zero.setBitsFrom(1);
3943 break;
3944 }
3945 case ISD::CLMUL: {
3946 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3947 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3948 Known = KnownBits::clmul(LHS: Known, RHS: Known2);
3949 break;
3950 }
3951 case ISD::MGATHER:
3952 case ISD::MLOAD: {
3953 ISD::LoadExtType ETy =
3954 (Opcode == ISD::MGATHER)
3955 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3956 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3957 if (ETy == ISD::ZEXTLOAD) {
3958 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3959 KnownBits Known0(MemVT.getScalarSizeInBits());
3960 return Known0.zext(BitWidth);
3961 }
3962 break;
3963 }
3964 case ISD::LOAD: {
3965 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3966 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3967 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3968 // Determine any common known bits from the loaded constant pool value.
3969 Type *CstTy = Cst->getType();
3970 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3971 !Op.getValueType().isScalableVector()) {
3972 // If its a vector splat, then we can (quickly) reuse the scalar path.
3973 // NOTE: We assume all elements match and none are UNDEF.
3974 if (CstTy->isVectorTy()) {
3975 if (const Constant *Splat = Cst->getSplatValue()) {
3976 Cst = Splat;
3977 CstTy = Cst->getType();
3978 }
3979 }
3980 // TODO - do we need to handle different bitwidths?
3981 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3982 // Iterate across all vector elements finding common known bits.
3983 Known.setAllConflict();
3984 for (unsigned i = 0; i != NumElts; ++i) {
3985 if (!DemandedElts[i])
3986 continue;
3987 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
3988 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
3989 const APInt &Value = CInt->getValue();
3990 Known.One &= Value;
3991 Known.Zero &= ~Value;
3992 continue;
3993 }
3994 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
3995 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3996 Known.One &= Value;
3997 Known.Zero &= ~Value;
3998 continue;
3999 }
4000 }
4001 Known.One.clearAllBits();
4002 Known.Zero.clearAllBits();
4003 break;
4004 }
4005 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4006 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4007 Known = KnownBits::makeConstant(C: CInt->getValue());
4008 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4009 Known =
4010 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4011 }
4012 }
4013 }
4014 } else if (Op.getResNo() == 0) {
4015 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4016 KnownBits KnownScalarMemory(ScalarMemorySize);
4017 if (const MDNode *MD = LD->getRanges())
4018 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4019
4020 // Extend the Known bits from memory to the size of the scalar result.
4021 if (ISD::isZEXTLoad(N: Op.getNode()))
4022 Known = KnownScalarMemory.zext(BitWidth);
4023 else if (ISD::isSEXTLoad(N: Op.getNode()))
4024 Known = KnownScalarMemory.sext(BitWidth);
4025 else if (ISD::isEXTLoad(N: Op.getNode()))
4026 Known = KnownScalarMemory.anyext(BitWidth);
4027 else
4028 Known = KnownScalarMemory;
4029 assert(Known.getBitWidth() == BitWidth);
4030 return Known;
4031 }
4032 break;
4033 }
4034 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4035 if (Op.getValueType().isScalableVector())
4036 break;
4037 EVT InVT = Op.getOperand(i: 0).getValueType();
4038 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4039 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4040 Known = Known.zext(BitWidth);
4041 break;
4042 }
4043 case ISD::ZERO_EXTEND: {
4044 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4045 Known = Known.zext(BitWidth);
4046 break;
4047 }
4048 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4049 if (Op.getValueType().isScalableVector())
4050 break;
4051 EVT InVT = Op.getOperand(i: 0).getValueType();
4052 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4053 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4054 // If the sign bit is known to be zero or one, then sext will extend
4055 // it to the top bits, else it will just zext.
4056 Known = Known.sext(BitWidth);
4057 break;
4058 }
4059 case ISD::SIGN_EXTEND: {
4060 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4061 // If the sign bit is known to be zero or one, then sext will extend
4062 // it to the top bits, else it will just zext.
4063 Known = Known.sext(BitWidth);
4064 break;
4065 }
4066 case ISD::ANY_EXTEND_VECTOR_INREG: {
4067 if (Op.getValueType().isScalableVector())
4068 break;
4069 EVT InVT = Op.getOperand(i: 0).getValueType();
4070 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4071 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4072 Known = Known.anyext(BitWidth);
4073 break;
4074 }
4075 case ISD::ANY_EXTEND: {
4076 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4077 Known = Known.anyext(BitWidth);
4078 break;
4079 }
4080 case ISD::TRUNCATE: {
4081 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4082 Known = Known.trunc(BitWidth);
4083 break;
4084 }
4085 case ISD::TRUNCATE_SSAT_S: {
4086 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4087 Known = Known.truncSSat(BitWidth);
4088 break;
4089 }
4090 case ISD::TRUNCATE_SSAT_U: {
4091 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4092 Known = Known.truncSSatU(BitWidth);
4093 break;
4094 }
4095 case ISD::TRUNCATE_USAT_U: {
4096 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4097 Known = Known.truncUSat(BitWidth);
4098 break;
4099 }
4100 case ISD::AssertZext: {
4101 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4102 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4103 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4104 Known.Zero |= (~InMask);
4105 Known.One &= (~Known.Zero);
4106 break;
4107 }
4108 case ISD::AssertAlign: {
4109 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4110 assert(LogOfAlign != 0);
4111
4112 // TODO: Should use maximum with source
4113 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4114 // well as clearing one bits.
4115 Known.Zero.setLowBits(LogOfAlign);
4116 Known.One.clearLowBits(loBits: LogOfAlign);
4117 break;
4118 }
4119 case ISD::AssertNoFPClass: {
4120 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4121
4122 FPClassTest NoFPClass =
4123 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
4124 const FPClassTest NegativeTestMask = fcNan | fcNegative;
4125 if ((NoFPClass & NegativeTestMask) == NegativeTestMask) {
4126 // Cannot be negative.
4127 Known.makeNonNegative();
4128 }
4129
4130 const FPClassTest PositiveTestMask = fcNan | fcPositive;
4131 if ((NoFPClass & PositiveTestMask) == PositiveTestMask) {
4132 // Cannot be positive.
4133 Known.makeNegative();
4134 }
4135
4136 break;
4137 }
4138 case ISD::FGETSIGN:
4139 // All bits are zero except the low bit.
4140 Known.Zero.setBitsFrom(1);
4141 break;
4142 case ISD::ADD:
4143 case ISD::SUB: {
4144 SDNodeFlags Flags = Op.getNode()->getFlags();
4145 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4146 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4147 Known = KnownBits::computeForAddSub(
4148 Add: Op.getOpcode() == ISD::ADD, NSW: Flags.hasNoSignedWrap(),
4149 NUW: Flags.hasNoUnsignedWrap(), LHS: Known, RHS: Known2);
4150 break;
4151 }
4152 case ISD::USUBO:
4153 case ISD::SSUBO:
4154 case ISD::USUBO_CARRY:
4155 case ISD::SSUBO_CARRY:
4156 if (Op.getResNo() == 1) {
4157 // If we know the result of a setcc has the top bits zero, use this info.
4158 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4159 TargetLowering::ZeroOrOneBooleanContent &&
4160 BitWidth > 1)
4161 Known.Zero.setBitsFrom(1);
4162 break;
4163 }
4164 [[fallthrough]];
4165 case ISD::SUBC: {
4166 assert(Op.getResNo() == 0 &&
4167 "We only compute knownbits for the difference here.");
4168
4169 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4170 KnownBits Borrow(1);
4171 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4172 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4173 // Borrow has bit width 1
4174 Borrow = Borrow.trunc(BitWidth: 1);
4175 } else {
4176 Borrow.setAllZero();
4177 }
4178
4179 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4180 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4181 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4182 break;
4183 }
4184 case ISD::UADDO:
4185 case ISD::SADDO:
4186 case ISD::UADDO_CARRY:
4187 case ISD::SADDO_CARRY:
4188 if (Op.getResNo() == 1) {
4189 // If we know the result of a setcc has the top bits zero, use this info.
4190 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4191 TargetLowering::ZeroOrOneBooleanContent &&
4192 BitWidth > 1)
4193 Known.Zero.setBitsFrom(1);
4194 break;
4195 }
4196 [[fallthrough]];
4197 case ISD::ADDC:
4198 case ISD::ADDE: {
4199 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4200
4201 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4202 KnownBits Carry(1);
4203 if (Opcode == ISD::ADDE)
4204 // Can't track carry from glue, set carry to unknown.
4205 Carry.resetAll();
4206 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4207 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4208 // Carry has bit width 1
4209 Carry = Carry.trunc(BitWidth: 1);
4210 } else {
4211 Carry.setAllZero();
4212 }
4213
4214 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4215 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4216 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4217 break;
4218 }
4219 case ISD::UDIV: {
4220 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4221 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4222 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4223 break;
4224 }
4225 case ISD::SDIV: {
4226 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4227 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4228 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4229 break;
4230 }
4231 case ISD::SREM: {
4232 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4233 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4234 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4235 break;
4236 }
4237 case ISD::UREM: {
4238 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4239 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4240 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4241 break;
4242 }
4243 case ISD::EXTRACT_ELEMENT: {
4244 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4245 const unsigned Index = Op.getConstantOperandVal(i: 1);
4246 const unsigned EltBitWidth = Op.getValueSizeInBits();
4247
4248 // Remove low part of known bits mask
4249 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4250 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4251
4252 // Remove high part of known bit mask
4253 Known = Known.trunc(BitWidth: EltBitWidth);
4254 break;
4255 }
4256 case ISD::EXTRACT_VECTOR_ELT: {
4257 SDValue InVec = Op.getOperand(i: 0);
4258 SDValue EltNo = Op.getOperand(i: 1);
4259 EVT VecVT = InVec.getValueType();
4260 // computeKnownBits not yet implemented for scalable vectors.
4261 if (VecVT.isScalableVector())
4262 break;
4263 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4264 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4265
4266 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4267 // anything about the extended bits.
4268 if (BitWidth > EltBitWidth)
4269 Known = Known.trunc(BitWidth: EltBitWidth);
4270
4271 // If we know the element index, just demand that vector element, else for
4272 // an unknown element index, ignore DemandedElts and demand them all.
4273 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4274 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4275 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4276 DemandedSrcElts =
4277 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4278
4279 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4280 if (BitWidth > EltBitWidth)
4281 Known = Known.anyext(BitWidth);
4282 break;
4283 }
4284 case ISD::INSERT_VECTOR_ELT: {
4285 if (Op.getValueType().isScalableVector())
4286 break;
4287
4288 // If we know the element index, split the demand between the
4289 // source vector and the inserted element, otherwise assume we need
4290 // the original demanded vector elements and the value.
4291 SDValue InVec = Op.getOperand(i: 0);
4292 SDValue InVal = Op.getOperand(i: 1);
4293 SDValue EltNo = Op.getOperand(i: 2);
4294 bool DemandedVal = true;
4295 APInt DemandedVecElts = DemandedElts;
4296 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4297 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4298 unsigned EltIdx = CEltNo->getZExtValue();
4299 DemandedVal = !!DemandedElts[EltIdx];
4300 DemandedVecElts.clearBit(BitPosition: EltIdx);
4301 }
4302 Known.setAllConflict();
4303 if (DemandedVal) {
4304 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4305 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4306 }
4307 if (!!DemandedVecElts) {
4308 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4309 Known = Known.intersectWith(RHS: Known2);
4310 }
4311 break;
4312 }
4313 case ISD::BITREVERSE: {
4314 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4315 Known = Known2.reverseBits();
4316 break;
4317 }
4318 case ISD::BSWAP: {
4319 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4320 Known = Known2.byteSwap();
4321 break;
4322 }
4323 case ISD::ABS: {
4324 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4325 Known = Known2.abs();
4326 Known.Zero.setHighBits(
4327 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4328 break;
4329 }
4330 case ISD::USUBSAT: {
4331 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4332 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4333 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4334 break;
4335 }
4336 case ISD::UMIN: {
4337 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4338 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4339 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4340 break;
4341 }
4342 case ISD::UMAX: {
4343 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4344 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4345 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4346 break;
4347 }
4348 case ISD::SMIN:
4349 case ISD::SMAX: {
4350 // If we have a clamp pattern, we know that the number of sign bits will be
4351 // the minimum of the clamp min/max range.
4352 bool IsMax = (Opcode == ISD::SMAX);
4353 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4354 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4355 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4356 CstHigh =
4357 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4358 if (CstLow && CstHigh) {
4359 if (!IsMax)
4360 std::swap(a&: CstLow, b&: CstHigh);
4361
4362 const APInt &ValueLow = CstLow->getAPIntValue();
4363 const APInt &ValueHigh = CstHigh->getAPIntValue();
4364 if (ValueLow.sle(RHS: ValueHigh)) {
4365 unsigned LowSignBits = ValueLow.getNumSignBits();
4366 unsigned HighSignBits = ValueHigh.getNumSignBits();
4367 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4368 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4369 Known.One.setHighBits(MinSignBits);
4370 break;
4371 }
4372 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4373 Known.Zero.setHighBits(MinSignBits);
4374 break;
4375 }
4376 }
4377 }
4378
4379 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4380 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4381 if (IsMax)
4382 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4383 else
4384 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4385
4386 // For SMAX, if CstLow is non-negative we know the result will be
4387 // non-negative and thus all sign bits are 0.
4388 // TODO: There's an equivalent of this for smin with negative constant for
4389 // known ones.
4390 if (IsMax && CstLow) {
4391 const APInt &ValueLow = CstLow->getAPIntValue();
4392 if (ValueLow.isNonNegative()) {
4393 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4394 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4395 }
4396 }
4397
4398 break;
4399 }
4400 case ISD::UINT_TO_FP: {
4401 Known.makeNonNegative();
4402 break;
4403 }
4404 case ISD::SINT_TO_FP: {
4405 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4406 if (Known2.isNonNegative())
4407 Known.makeNonNegative();
4408 else if (Known2.isNegative())
4409 Known.makeNegative();
4410 break;
4411 }
4412 case ISD::FP_TO_UINT_SAT: {
4413 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4414 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4415 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4416 break;
4417 }
4418 case ISD::ATOMIC_LOAD: {
4419 // If we are looking at the loaded value.
4420 if (Op.getResNo() == 0) {
4421 auto *AT = cast<AtomicSDNode>(Val&: Op);
4422 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4423 KnownBits KnownScalarMemory(ScalarMemorySize);
4424 if (const MDNode *MD = AT->getRanges())
4425 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4426
4427 switch (AT->getExtensionType()) {
4428 case ISD::ZEXTLOAD:
4429 Known = KnownScalarMemory.zext(BitWidth);
4430 break;
4431 case ISD::SEXTLOAD:
4432 Known = KnownScalarMemory.sext(BitWidth);
4433 break;
4434 case ISD::EXTLOAD:
4435 switch (TLI->getExtendForAtomicOps()) {
4436 case ISD::ZERO_EXTEND:
4437 Known = KnownScalarMemory.zext(BitWidth);
4438 break;
4439 case ISD::SIGN_EXTEND:
4440 Known = KnownScalarMemory.sext(BitWidth);
4441 break;
4442 default:
4443 Known = KnownScalarMemory.anyext(BitWidth);
4444 break;
4445 }
4446 break;
4447 case ISD::NON_EXTLOAD:
4448 Known = KnownScalarMemory;
4449 break;
4450 }
4451 assert(Known.getBitWidth() == BitWidth);
4452 }
4453 break;
4454 }
4455 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4456 if (Op.getResNo() == 1) {
4457 // The boolean result conforms to getBooleanContents.
4458 // If we know the result of a setcc has the top bits zero, use this info.
4459 // We know that we have an integer-based boolean since these operations
4460 // are only available for integer.
4461 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4462 TargetLowering::ZeroOrOneBooleanContent &&
4463 BitWidth > 1)
4464 Known.Zero.setBitsFrom(1);
4465 break;
4466 }
4467 [[fallthrough]];
4468 case ISD::ATOMIC_CMP_SWAP:
4469 case ISD::ATOMIC_SWAP:
4470 case ISD::ATOMIC_LOAD_ADD:
4471 case ISD::ATOMIC_LOAD_SUB:
4472 case ISD::ATOMIC_LOAD_AND:
4473 case ISD::ATOMIC_LOAD_CLR:
4474 case ISD::ATOMIC_LOAD_OR:
4475 case ISD::ATOMIC_LOAD_XOR:
4476 case ISD::ATOMIC_LOAD_NAND:
4477 case ISD::ATOMIC_LOAD_MIN:
4478 case ISD::ATOMIC_LOAD_MAX:
4479 case ISD::ATOMIC_LOAD_UMIN:
4480 case ISD::ATOMIC_LOAD_UMAX: {
4481 // If we are looking at the loaded value.
4482 if (Op.getResNo() == 0) {
4483 auto *AT = cast<AtomicSDNode>(Val&: Op);
4484 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4485
4486 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4487 Known.Zero.setBitsFrom(MemBits);
4488 }
4489 break;
4490 }
4491 case ISD::FrameIndex:
4492 case ISD::TargetFrameIndex:
4493 TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4494 Known, MF: getMachineFunction());
4495 break;
4496
4497 default:
4498 if (Opcode < ISD::BUILTIN_OP_END)
4499 break;
4500 [[fallthrough]];
4501 case ISD::INTRINSIC_WO_CHAIN:
4502 case ISD::INTRINSIC_W_CHAIN:
4503 case ISD::INTRINSIC_VOID:
4504 // TODO: Probably okay to remove after audit; here to reduce change size
4505 // in initial enablement patch for scalable vectors
4506 if (Op.getValueType().isScalableVector())
4507 break;
4508
4509 // Allow the target to implement this method for its nodes.
4510 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4511 break;
4512 }
4513
4514 return Known;
4515}
4516
4517/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4518static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4519 switch (OR) {
4520 case ConstantRange::OverflowResult::MayOverflow:
4521 return SelectionDAG::OFK_Sometime;
4522 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4523 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4524 return SelectionDAG::OFK_Always;
4525 case ConstantRange::OverflowResult::NeverOverflows:
4526 return SelectionDAG::OFK_Never;
4527 }
4528 llvm_unreachable("Unknown OverflowResult");
4529}
4530
4531SelectionDAG::OverflowKind
4532SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4533 // X + 0 never overflow
4534 if (isNullConstant(V: N1))
4535 return OFK_Never;
4536
4537 // If both operands each have at least two sign bits, the addition
4538 // cannot overflow.
4539 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4540 return OFK_Never;
4541
4542 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4543 return OFK_Sometime;
4544}
4545
4546SelectionDAG::OverflowKind
4547SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4548 // X + 0 never overflow
4549 if (isNullConstant(V: N1))
4550 return OFK_Never;
4551
4552 // mulhi + 1 never overflow
4553 KnownBits N1Known = computeKnownBits(Op: N1);
4554 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4555 N1Known.getMaxValue().ult(RHS: 2))
4556 return OFK_Never;
4557
4558 KnownBits N0Known = computeKnownBits(Op: N0);
4559 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4560 N0Known.getMaxValue().ult(RHS: 2))
4561 return OFK_Never;
4562
4563 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4564 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4565 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4566 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4567}
4568
4569SelectionDAG::OverflowKind
4570SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4571 // X - 0 never overflow
4572 if (isNullConstant(V: N1))
4573 return OFK_Never;
4574
4575 // If both operands each have at least two sign bits, the subtraction
4576 // cannot overflow.
4577 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4578 return OFK_Never;
4579
4580 KnownBits N0Known = computeKnownBits(Op: N0);
4581 KnownBits N1Known = computeKnownBits(Op: N1);
4582 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4583 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4584 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4585}
4586
4587SelectionDAG::OverflowKind
4588SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4589 // X - 0 never overflow
4590 if (isNullConstant(V: N1))
4591 return OFK_Never;
4592
4593 KnownBits N0Known = computeKnownBits(Op: N0);
4594 KnownBits N1Known = computeKnownBits(Op: N1);
4595 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4596 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4597 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4598}
4599
4600SelectionDAG::OverflowKind
4601SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4602 // X * 0 and X * 1 never overflow.
4603 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4604 return OFK_Never;
4605
4606 KnownBits N0Known = computeKnownBits(Op: N0);
4607 KnownBits N1Known = computeKnownBits(Op: N1);
4608 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4609 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4610 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4611}
4612
4613SelectionDAG::OverflowKind
4614SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4615 // X * 0 and X * 1 never overflow.
4616 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4617 return OFK_Never;
4618
4619 // Get the size of the result.
4620 unsigned BitWidth = N0.getScalarValueSizeInBits();
4621
4622 // Sum of the sign bits.
4623 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4624
4625 // If we have enough sign bits, then there's no overflow.
4626 if (SignBits > BitWidth + 1)
4627 return OFK_Never;
4628
4629 if (SignBits == BitWidth + 1) {
4630 // The overflow occurs when the true multiplication of the
4631 // the operands is the minimum negative number.
4632 KnownBits N0Known = computeKnownBits(Op: N0);
4633 KnownBits N1Known = computeKnownBits(Op: N1);
4634 // If one of the operands is non-negative, then there's no
4635 // overflow.
4636 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4637 return OFK_Never;
4638 }
4639
4640 return OFK_Sometime;
4641}
4642
4643bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, bool OrZero,
4644 unsigned Depth) const {
4645 EVT VT = Val.getValueType();
4646
4647 // Since the number of lanes in a scalable vector is unknown at compile time,
4648 // we track one bit which is implicitly broadcast to all lanes. This means
4649 // that all lanes in a scalable vector are considered demanded.
4650 APInt DemandedElts = VT.isFixedLengthVector()
4651 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4652 : APInt(1, 1);
4653
4654 return isKnownToBeAPowerOfTwo(Val, DemandedElts, OrZero, Depth);
4655}
4656
4657bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val,
4658 const APInt &DemandedElts,
4659 bool OrZero, unsigned Depth) const {
4660 if (Depth >= MaxRecursionDepth)
4661 return false; // Limit search depth.
4662
4663 EVT OpVT = Val.getValueType();
4664 unsigned BitWidth = OpVT.getScalarSizeInBits();
4665 [[maybe_unused]] unsigned NumElts = DemandedElts.getBitWidth();
4666 assert((!OpVT.isScalableVector() || NumElts == 1) &&
4667 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4668 assert(
4669 (!OpVT.isFixedLengthVector() || NumElts == OpVT.getVectorNumElements()) &&
4670 "Unexpected vector size");
4671
4672 auto IsPowerOfTwoOrZero = [BitWidth, OrZero](const ConstantSDNode *C) {
4673 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
4674 return (OrZero && V.isZero()) || V.isPowerOf2();
4675 };
4676
4677 // Is the constant a known power of 2 or zero?
4678 if (ISD::matchUnaryPredicate(Op: Val, Match: IsPowerOfTwoOrZero))
4679 return true;
4680
4681 switch (Val.getOpcode()) {
4682 case ISD::BUILD_VECTOR:
4683 // Are all operands of a build vector constant powers of two or zero?
4684 if (all_of(Range: enumerate(First: Val->ops()), P: [&](auto P) {
4685 auto *C = dyn_cast<ConstantSDNode>(P.value());
4686 return !DemandedElts[P.index()] || (C && IsPowerOfTwoOrZero(C));
4687 }))
4688 return true;
4689 break;
4690
4691 case ISD::SPLAT_VECTOR:
4692 // Is the operand of a splat vector a constant power of two?
4693 if (auto *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4694 if (IsPowerOfTwoOrZero(C))
4695 return true;
4696 break;
4697
4698 case ISD::EXTRACT_VECTOR_ELT: {
4699 SDValue InVec = Val.getOperand(i: 0);
4700 SDValue EltNo = Val.getOperand(i: 1);
4701 EVT VecVT = InVec.getValueType();
4702
4703 // Skip scalable vectors or implicit extensions.
4704 if (VecVT.isScalableVector() ||
4705 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
4706 break;
4707
4708 // If we know the element index, just demand that vector element, else for
4709 // an unknown element index, ignore DemandedElts and demand them all.
4710 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4711 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4712 APInt DemandedSrcElts =
4713 ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts)
4714 ? APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue())
4715 : APInt::getAllOnes(numBits: NumSrcElts);
4716 return isKnownToBeAPowerOfTwo(Val: InVec, DemandedElts: DemandedSrcElts, OrZero, Depth: Depth + 1);
4717 }
4718
4719 case ISD::AND: {
4720 // Looking for `x & -x` pattern:
4721 // If x == 0:
4722 // x & -x -> 0
4723 // If x != 0:
4724 // x & -x -> non-zero pow2
4725 // so if we find the pattern return whether we know `x` is non-zero.
4726 SDValue X;
4727 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))))
4728 return OrZero || isKnownNeverZero(Op: X, DemandedElts, Depth);
4729 break;
4730 }
4731
4732 case ISD::SHL: {
4733 // A left-shift of a constant one will have exactly one bit set because
4734 // shifting the bit off the end is undefined.
4735 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4736 if (C && C->getAPIntValue() == 1)
4737 return true;
4738 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4739 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4740 Depth: Depth + 1);
4741 }
4742
4743 case ISD::SRL: {
4744 // A logical right-shift of a constant sign-bit will have exactly
4745 // one bit set.
4746 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0), DemandedElts);
4747 if (C && C->getAPIntValue().isSignMask())
4748 return true;
4749 return (OrZero || isKnownNeverZero(Op: Val, DemandedElts, Depth)) &&
4750 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4751 Depth: Depth + 1);
4752 }
4753
4754 case ISD::ROTL:
4755 case ISD::ROTR:
4756 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4757 Depth: Depth + 1);
4758 case ISD::BSWAP:
4759 case ISD::BITREVERSE:
4760 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts, OrZero,
4761 Depth: Depth + 1);
4762
4763 case ISD::SMIN:
4764 case ISD::SMAX:
4765 case ISD::UMIN:
4766 case ISD::UMAX:
4767 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), /*OrZero=*/false,
4768 Depth: Depth + 1) &&
4769 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4770 Depth: Depth + 1);
4771
4772 case ISD::SELECT:
4773 case ISD::VSELECT:
4774 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), DemandedElts, OrZero,
4775 Depth: Depth + 1) &&
4776 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts, OrZero,
4777 Depth: Depth + 1);
4778
4779 case ISD::ZERO_EXTEND:
4780 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4781 Depth: Depth + 1);
4782
4783 case ISD::VSCALE:
4784 // vscale(power-of-two) is a power-of-two
4785 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), /*OrZero=*/false,
4786 Depth: Depth + 1);
4787
4788 case ISD::VECTOR_SHUFFLE: {
4789 assert(!Val.getValueType().isScalableVector());
4790 // Demanded elements with undef shuffle mask elements are unknown
4791 // - we cannot guarantee they are a power of two, so return false.
4792 APInt DemandedLHS, DemandedRHS;
4793 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val);
4794 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4795 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4796 DemandedLHS, DemandedRHS))
4797 return false;
4798
4799 // All demanded elements from LHS must be known power of two.
4800 if (!!DemandedLHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), DemandedElts: DemandedLHS,
4801 OrZero, Depth: Depth + 1))
4802 return false;
4803
4804 // All demanded elements from RHS must be known power of two.
4805 if (!!DemandedRHS && !isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), DemandedElts: DemandedRHS,
4806 OrZero, Depth: Depth + 1))
4807 return false;
4808
4809 return true;
4810 }
4811 }
4812
4813 // More could be done here, though the above checks are enough
4814 // to handle some common cases.
4815 return false;
4816}
4817
4818bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4819 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4820 return C1->getValueAPF().getExactLog2Abs() >= 0;
4821
4822 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4823 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), OrZero: Depth + 1);
4824
4825 return false;
4826}
4827
4828unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4829 EVT VT = Op.getValueType();
4830
4831 // Since the number of lanes in a scalable vector is unknown at compile time,
4832 // we track one bit which is implicitly broadcast to all lanes. This means
4833 // that all lanes in a scalable vector are considered demanded.
4834 APInt DemandedElts = VT.isFixedLengthVector()
4835 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4836 : APInt(1, 1);
4837 return ComputeNumSignBits(Op, DemandedElts, Depth);
4838}
4839
4840unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4841 unsigned Depth) const {
4842 EVT VT = Op.getValueType();
4843 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4844 unsigned VTBits = VT.getScalarSizeInBits();
4845 unsigned NumElts = DemandedElts.getBitWidth();
4846 unsigned Tmp, Tmp2;
4847 unsigned FirstAnswer = 1;
4848
4849 assert((!VT.isScalableVector() || NumElts == 1) &&
4850 "DemandedElts for scalable vectors must be 1 to represent all lanes");
4851
4852 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4853 const APInt &Val = C->getAPIntValue();
4854 return Val.getNumSignBits();
4855 }
4856
4857 if (Depth >= MaxRecursionDepth)
4858 return 1; // Limit search depth.
4859
4860 if (!DemandedElts)
4861 return 1; // No demanded elts, better to assume we don't know anything.
4862
4863 unsigned Opcode = Op.getOpcode();
4864 switch (Opcode) {
4865 default: break;
4866 case ISD::AssertSext:
4867 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4868 return VTBits-Tmp+1;
4869 case ISD::AssertZext:
4870 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4871 return VTBits-Tmp;
4872 case ISD::FREEZE:
4873 if (isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
4874 /*PoisonOnly=*/false))
4875 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4876 break;
4877 case ISD::MERGE_VALUES:
4878 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4879 Depth: Depth + 1);
4880 case ISD::SPLAT_VECTOR: {
4881 // Check if the sign bits of source go down as far as the truncated value.
4882 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4883 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4884 if (NumSrcSignBits > (NumSrcBits - VTBits))
4885 return NumSrcSignBits - (NumSrcBits - VTBits);
4886 break;
4887 }
4888 case ISD::BUILD_VECTOR:
4889 assert(!VT.isScalableVector());
4890 Tmp = VTBits;
4891 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4892 if (!DemandedElts[i])
4893 continue;
4894
4895 SDValue SrcOp = Op.getOperand(i);
4896 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4897 // for constant nodes to ensure we only look at the sign bits.
4898 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4899 APInt T = C->getAPIntValue().trunc(width: VTBits);
4900 Tmp2 = T.getNumSignBits();
4901 } else {
4902 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4903
4904 if (SrcOp.getValueSizeInBits() != VTBits) {
4905 assert(SrcOp.getValueSizeInBits() > VTBits &&
4906 "Expected BUILD_VECTOR implicit truncation");
4907 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4908 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4909 }
4910 }
4911 Tmp = std::min(a: Tmp, b: Tmp2);
4912 }
4913 return Tmp;
4914
4915 case ISD::VECTOR_COMPRESS: {
4916 SDValue Vec = Op.getOperand(i: 0);
4917 SDValue PassThru = Op.getOperand(i: 2);
4918 Tmp = ComputeNumSignBits(Op: PassThru, DemandedElts, Depth: Depth + 1);
4919 if (Tmp == 1)
4920 return 1;
4921 Tmp2 = ComputeNumSignBits(Op: Vec, Depth: Depth + 1);
4922 Tmp = std::min(a: Tmp, b: Tmp2);
4923 return Tmp;
4924 }
4925
4926 case ISD::VECTOR_SHUFFLE: {
4927 // Collect the minimum number of sign bits that are shared by every vector
4928 // element referenced by the shuffle.
4929 APInt DemandedLHS, DemandedRHS;
4930 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4931 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4932 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4933 DemandedLHS, DemandedRHS))
4934 return 1;
4935
4936 Tmp = std::numeric_limits<unsigned>::max();
4937 if (!!DemandedLHS)
4938 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
4939 if (!!DemandedRHS) {
4940 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
4941 Tmp = std::min(a: Tmp, b: Tmp2);
4942 }
4943 // If we don't know anything, early out and try computeKnownBits fall-back.
4944 if (Tmp == 1)
4945 break;
4946 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4947 return Tmp;
4948 }
4949
4950 case ISD::BITCAST: {
4951 if (VT.isScalableVector())
4952 break;
4953 SDValue N0 = Op.getOperand(i: 0);
4954 EVT SrcVT = N0.getValueType();
4955 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4956
4957 // Ignore bitcasts from unsupported types..
4958 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4959 break;
4960
4961 // Fast handling of 'identity' bitcasts.
4962 if (VTBits == SrcBits)
4963 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
4964
4965 bool IsLE = getDataLayout().isLittleEndian();
4966
4967 // Bitcast 'large element' scalar/vector to 'small element' vector.
4968 if ((SrcBits % VTBits) == 0) {
4969 assert(VT.isVector() && "Expected bitcast to vector");
4970
4971 unsigned Scale = SrcBits / VTBits;
4972 APInt SrcDemandedElts =
4973 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
4974
4975 // Fast case - sign splat can be simply split across the small elements.
4976 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
4977 if (Tmp == SrcBits)
4978 return VTBits;
4979
4980 // Slow case - determine how far the sign extends into each sub-element.
4981 Tmp2 = VTBits;
4982 for (unsigned i = 0; i != NumElts; ++i)
4983 if (DemandedElts[i]) {
4984 unsigned SubOffset = i % Scale;
4985 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4986 SubOffset = SubOffset * VTBits;
4987 if (Tmp <= SubOffset)
4988 return 1;
4989 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
4990 }
4991 return Tmp2;
4992 }
4993 break;
4994 }
4995
4996 case ISD::FP_TO_SINT_SAT:
4997 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4998 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4999 return VTBits - Tmp + 1;
5000 case ISD::SIGN_EXTEND:
5001 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
5002 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
5003 case ISD::SIGN_EXTEND_INREG:
5004 // Max of the input and what this extends.
5005 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
5006 Tmp = VTBits-Tmp+1;
5007 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5008 return std::max(a: Tmp, b: Tmp2);
5009 case ISD::SIGN_EXTEND_VECTOR_INREG: {
5010 if (VT.isScalableVector())
5011 break;
5012 SDValue Src = Op.getOperand(i: 0);
5013 EVT SrcVT = Src.getValueType();
5014 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
5015 Tmp = VTBits - SrcVT.getScalarSizeInBits();
5016 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
5017 }
5018 case ISD::SRA:
5019 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5020 // SRA X, C -> adds C sign bits.
5021 if (std::optional<unsigned> ShAmt =
5022 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
5023 Tmp = std::min(a: Tmp + *ShAmt, b: VTBits);
5024 return Tmp;
5025 case ISD::SHL:
5026 if (std::optional<ConstantRange> ShAmtRange =
5027 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
5028 unsigned MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
5029 unsigned MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
5030 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
5031 // shifted out, then we can compute the number of sign bits for the
5032 // operand being extended. A future improvement could be to pass along the
5033 // "shifted left by" information in the recursive calls to
5034 // ComputeKnownSignBits. Allowing us to handle this more generically.
5035 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
5036 SDValue Ext = Op.getOperand(i: 0);
5037 EVT ExtVT = Ext.getValueType();
5038 SDValue Extendee = Ext.getOperand(i: 0);
5039 EVT ExtendeeVT = Extendee.getValueType();
5040 unsigned SizeDifference =
5041 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
5042 if (SizeDifference <= MinShAmt) {
5043 Tmp = SizeDifference +
5044 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
5045 if (MaxShAmt < Tmp)
5046 return Tmp - MaxShAmt;
5047 }
5048 }
5049 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
5050 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5051 if (MaxShAmt < Tmp)
5052 return Tmp - MaxShAmt;
5053 }
5054 break;
5055 case ISD::AND:
5056 case ISD::OR:
5057 case ISD::XOR: // NOT is handled here.
5058 // Logical binary ops preserve the number of sign bits at the worst.
5059 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
5060 if (Tmp != 1) {
5061 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5062 FirstAnswer = std::min(a: Tmp, b: Tmp2);
5063 // We computed what we know about the sign bits as our first
5064 // answer. Now proceed to the generic code that uses
5065 // computeKnownBits, and pick whichever answer is better.
5066 }
5067 break;
5068
5069 case ISD::SELECT:
5070 case ISD::VSELECT:
5071 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
5072 if (Tmp == 1) return 1; // Early out.
5073 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5074 return std::min(a: Tmp, b: Tmp2);
5075 case ISD::SELECT_CC:
5076 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
5077 if (Tmp == 1) return 1; // Early out.
5078 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
5079 return std::min(a: Tmp, b: Tmp2);
5080
5081 case ISD::SMIN:
5082 case ISD::SMAX: {
5083 // If we have a clamp pattern, we know that the number of sign bits will be
5084 // the minimum of the clamp min/max range.
5085 bool IsMax = (Opcode == ISD::SMAX);
5086 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
5087 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
5088 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
5089 CstHigh =
5090 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
5091 if (CstLow && CstHigh) {
5092 if (!IsMax)
5093 std::swap(a&: CstLow, b&: CstHigh);
5094 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
5095 Tmp = CstLow->getAPIntValue().getNumSignBits();
5096 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
5097 return std::min(a: Tmp, b: Tmp2);
5098 }
5099 }
5100
5101 // Fallback - just get the minimum number of sign bits of the operands.
5102 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5103 if (Tmp == 1)
5104 return 1; // Early out.
5105 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5106 return std::min(a: Tmp, b: Tmp2);
5107 }
5108 case ISD::UMIN:
5109 case ISD::UMAX:
5110 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5111 if (Tmp == 1)
5112 return 1; // Early out.
5113 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5114 return std::min(a: Tmp, b: Tmp2);
5115 case ISD::SSUBO_CARRY:
5116 case ISD::USUBO_CARRY:
5117 // sub_carry(x,x,c) -> 0/-1 (sext carry)
5118 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
5119 return VTBits;
5120 [[fallthrough]];
5121 case ISD::SADDO:
5122 case ISD::UADDO:
5123 case ISD::SADDO_CARRY:
5124 case ISD::UADDO_CARRY:
5125 case ISD::SSUBO:
5126 case ISD::USUBO:
5127 case ISD::SMULO:
5128 case ISD::UMULO:
5129 if (Op.getResNo() != 1)
5130 break;
5131 // The boolean result conforms to getBooleanContents. Fall through.
5132 // If setcc returns 0/-1, all bits are sign bits.
5133 // We know that we have an integer-based boolean since these operations
5134 // are only available for integer.
5135 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
5136 TargetLowering::ZeroOrNegativeOneBooleanContent)
5137 return VTBits;
5138 break;
5139 case ISD::SETCC:
5140 case ISD::SETCCCARRY:
5141 case ISD::STRICT_FSETCC:
5142 case ISD::STRICT_FSETCCS: {
5143 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5144 // If setcc returns 0/-1, all bits are sign bits.
5145 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5146 TargetLowering::ZeroOrNegativeOneBooleanContent)
5147 return VTBits;
5148 break;
5149 }
5150 case ISD::ROTL:
5151 case ISD::ROTR:
5152 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5153
5154 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
5155 if (Tmp == VTBits)
5156 return VTBits;
5157
5158 if (ConstantSDNode *C =
5159 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
5160 unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
5161
5162 // Handle rotate right by N like a rotate left by 32-N.
5163 if (Opcode == ISD::ROTR)
5164 RotAmt = (VTBits - RotAmt) % VTBits;
5165
5166 // If we aren't rotating out all of the known-in sign bits, return the
5167 // number that are left. This handles rotl(sext(x), 1) for example.
5168 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
5169 }
5170 break;
5171 case ISD::ADD:
5172 case ISD::ADDC:
5173 // TODO: Move Operand 1 check before Operand 0 check
5174 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5175 if (Tmp == 1) return 1; // Early out.
5176
5177 // Special case decrementing a value (ADD X, -1):
5178 if (ConstantSDNode *CRHS =
5179 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5180 if (CRHS->isAllOnes()) {
5181 KnownBits Known =
5182 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5183
5184 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5185 // sign bits set.
5186 if ((Known.Zero | 1).isAllOnes())
5187 return VTBits;
5188
5189 // If we are subtracting one from a positive number, there is no carry
5190 // out of the result.
5191 if (Known.isNonNegative())
5192 return Tmp;
5193 }
5194
5195 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5196 if (Tmp2 == 1) return 1; // Early out.
5197
5198 // Add can have at most one carry bit. Thus we know that the output
5199 // is, at worst, one more bit than the inputs.
5200 return std::min(a: Tmp, b: Tmp2) - 1;
5201 case ISD::SUB:
5202 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5203 if (Tmp2 == 1) return 1; // Early out.
5204
5205 // Handle NEG.
5206 if (ConstantSDNode *CLHS =
5207 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5208 if (CLHS->isZero()) {
5209 KnownBits Known =
5210 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5211 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5212 // sign bits set.
5213 if ((Known.Zero | 1).isAllOnes())
5214 return VTBits;
5215
5216 // If the input is known to be positive (the sign bit is known clear),
5217 // the output of the NEG has the same number of sign bits as the input.
5218 if (Known.isNonNegative())
5219 return Tmp2;
5220
5221 // Otherwise, we treat this like a SUB.
5222 }
5223
5224 // Sub can have at most one carry bit. Thus we know that the output
5225 // is, at worst, one more bit than the inputs.
5226 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5227 if (Tmp == 1) return 1; // Early out.
5228 return std::min(a: Tmp, b: Tmp2) - 1;
5229 case ISD::MUL: {
5230 // The output of the Mul can be at most twice the valid bits in the inputs.
5231 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5232 if (SignBitsOp0 == 1)
5233 break;
5234 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5235 if (SignBitsOp1 == 1)
5236 break;
5237 unsigned OutValidBits =
5238 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5239 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5240 }
5241 case ISD::AVGCEILS:
5242 case ISD::AVGFLOORS:
5243 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5244 if (Tmp == 1)
5245 return 1; // Early out.
5246 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5247 return std::min(a: Tmp, b: Tmp2);
5248 case ISD::SREM:
5249 // The sign bit is the LHS's sign bit, except when the result of the
5250 // remainder is zero. The magnitude of the result should be less than or
5251 // equal to the magnitude of the LHS. Therefore, the result should have
5252 // at least as many sign bits as the left hand side.
5253 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5254 case ISD::TRUNCATE: {
5255 // Check if the sign bits of source go down as far as the truncated value.
5256 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5257 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5258 if (NumSrcSignBits > (NumSrcBits - VTBits))
5259 return NumSrcSignBits - (NumSrcBits - VTBits);
5260 break;
5261 }
5262 case ISD::EXTRACT_ELEMENT: {
5263 if (VT.isScalableVector())
5264 break;
5265 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5266 const int BitWidth = Op.getValueSizeInBits();
5267 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5268
5269 // Get reverse index (starting from 1), Op1 value indexes elements from
5270 // little end. Sign starts at big end.
5271 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5272
5273 // If the sign portion ends in our element the subtraction gives correct
5274 // result. Otherwise it gives either negative or > bitwidth result
5275 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 1, hi: BitWidth);
5276 }
5277 case ISD::INSERT_VECTOR_ELT: {
5278 if (VT.isScalableVector())
5279 break;
5280 // If we know the element index, split the demand between the
5281 // source vector and the inserted element, otherwise assume we need
5282 // the original demanded vector elements and the value.
5283 SDValue InVec = Op.getOperand(i: 0);
5284 SDValue InVal = Op.getOperand(i: 1);
5285 SDValue EltNo = Op.getOperand(i: 2);
5286 bool DemandedVal = true;
5287 APInt DemandedVecElts = DemandedElts;
5288 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5289 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5290 unsigned EltIdx = CEltNo->getZExtValue();
5291 DemandedVal = !!DemandedElts[EltIdx];
5292 DemandedVecElts.clearBit(BitPosition: EltIdx);
5293 }
5294 Tmp = std::numeric_limits<unsigned>::max();
5295 if (DemandedVal) {
5296 // TODO - handle implicit truncation of inserted elements.
5297 if (InVal.getScalarValueSizeInBits() != VTBits)
5298 break;
5299 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5300 Tmp = std::min(a: Tmp, b: Tmp2);
5301 }
5302 if (!!DemandedVecElts) {
5303 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5304 Tmp = std::min(a: Tmp, b: Tmp2);
5305 }
5306 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5307 return Tmp;
5308 }
5309 case ISD::EXTRACT_VECTOR_ELT: {
5310 SDValue InVec = Op.getOperand(i: 0);
5311 SDValue EltNo = Op.getOperand(i: 1);
5312 EVT VecVT = InVec.getValueType();
5313 // ComputeNumSignBits not yet implemented for scalable vectors.
5314 if (VecVT.isScalableVector())
5315 break;
5316 const unsigned BitWidth = Op.getValueSizeInBits();
5317 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5318 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5319
5320 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5321 // anything about sign bits. But if the sizes match we can derive knowledge
5322 // about sign bits from the vector operand.
5323 if (BitWidth != EltBitWidth)
5324 break;
5325
5326 // If we know the element index, just demand that vector element, else for
5327 // an unknown element index, ignore DemandedElts and demand them all.
5328 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5329 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5330 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5331 DemandedSrcElts =
5332 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5333
5334 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5335 }
5336 case ISD::EXTRACT_SUBVECTOR: {
5337 // Offset the demanded elts by the subvector index.
5338 SDValue Src = Op.getOperand(i: 0);
5339
5340 APInt DemandedSrcElts;
5341 if (Src.getValueType().isScalableVector())
5342 DemandedSrcElts = APInt(1, 1);
5343 else {
5344 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5345 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5346 DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5347 }
5348 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5349 }
5350 case ISD::CONCAT_VECTORS: {
5351 if (VT.isScalableVector())
5352 break;
5353 // Determine the minimum number of sign bits across all demanded
5354 // elts of the input vectors. Early out if the result is already 1.
5355 Tmp = std::numeric_limits<unsigned>::max();
5356 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5357 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5358 unsigned NumSubVectors = Op.getNumOperands();
5359 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5360 APInt DemandedSub =
5361 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5362 if (!DemandedSub)
5363 continue;
5364 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5365 Tmp = std::min(a: Tmp, b: Tmp2);
5366 }
5367 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5368 return Tmp;
5369 }
5370 case ISD::INSERT_SUBVECTOR: {
5371 if (VT.isScalableVector())
5372 break;
5373 // Demand any elements from the subvector and the remainder from the src its
5374 // inserted into.
5375 SDValue Src = Op.getOperand(i: 0);
5376 SDValue Sub = Op.getOperand(i: 1);
5377 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5378 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5379 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5380 APInt DemandedSrcElts = DemandedElts;
5381 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5382
5383 Tmp = std::numeric_limits<unsigned>::max();
5384 if (!!DemandedSubElts) {
5385 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5386 if (Tmp == 1)
5387 return 1; // early-out
5388 }
5389 if (!!DemandedSrcElts) {
5390 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5391 Tmp = std::min(a: Tmp, b: Tmp2);
5392 }
5393 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5394 return Tmp;
5395 }
5396 case ISD::LOAD: {
5397 // If we are looking at the loaded value of the SDNode.
5398 if (Op.getResNo() != 0)
5399 break;
5400
5401 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5402 if (const MDNode *Ranges = LD->getRanges()) {
5403 if (DemandedElts != 1)
5404 break;
5405
5406 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5407 if (VTBits > CR.getBitWidth()) {
5408 switch (LD->getExtensionType()) {
5409 case ISD::SEXTLOAD:
5410 CR = CR.signExtend(BitWidth: VTBits);
5411 break;
5412 case ISD::ZEXTLOAD:
5413 CR = CR.zeroExtend(BitWidth: VTBits);
5414 break;
5415 default:
5416 break;
5417 }
5418 }
5419
5420 if (VTBits != CR.getBitWidth())
5421 break;
5422 return std::min(a: CR.getSignedMin().getNumSignBits(),
5423 b: CR.getSignedMax().getNumSignBits());
5424 }
5425
5426 unsigned ExtType = LD->getExtensionType();
5427 switch (ExtType) {
5428 default:
5429 break;
5430 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5431 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5432 return VTBits - Tmp + 1;
5433 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5434 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5435 return VTBits - Tmp;
5436 case ISD::NON_EXTLOAD:
5437 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5438 // We only need to handle vectors - computeKnownBits should handle
5439 // scalar cases.
5440 Type *CstTy = Cst->getType();
5441 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5442 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5443 VTBits == CstTy->getScalarSizeInBits()) {
5444 Tmp = VTBits;
5445 for (unsigned i = 0; i != NumElts; ++i) {
5446 if (!DemandedElts[i])
5447 continue;
5448 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5449 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5450 const APInt &Value = CInt->getValue();
5451 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5452 continue;
5453 }
5454 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5455 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5456 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5457 continue;
5458 }
5459 }
5460 // Unknown type. Conservatively assume no bits match sign bit.
5461 return 1;
5462 }
5463 return Tmp;
5464 }
5465 }
5466 break;
5467 }
5468
5469 break;
5470 }
5471 case ISD::ATOMIC_CMP_SWAP:
5472 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5473 case ISD::ATOMIC_SWAP:
5474 case ISD::ATOMIC_LOAD_ADD:
5475 case ISD::ATOMIC_LOAD_SUB:
5476 case ISD::ATOMIC_LOAD_AND:
5477 case ISD::ATOMIC_LOAD_CLR:
5478 case ISD::ATOMIC_LOAD_OR:
5479 case ISD::ATOMIC_LOAD_XOR:
5480 case ISD::ATOMIC_LOAD_NAND:
5481 case ISD::ATOMIC_LOAD_MIN:
5482 case ISD::ATOMIC_LOAD_MAX:
5483 case ISD::ATOMIC_LOAD_UMIN:
5484 case ISD::ATOMIC_LOAD_UMAX:
5485 case ISD::ATOMIC_LOAD: {
5486 auto *AT = cast<AtomicSDNode>(Val&: Op);
5487 // If we are looking at the loaded value.
5488 if (Op.getResNo() == 0) {
5489 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5490 if (Tmp == VTBits)
5491 return 1; // early-out
5492
5493 // For atomic_load, prefer to use the extension type.
5494 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5495 switch (AT->getExtensionType()) {
5496 default:
5497 break;
5498 case ISD::SEXTLOAD:
5499 return VTBits - Tmp + 1;
5500 case ISD::ZEXTLOAD:
5501 return VTBits - Tmp;
5502 }
5503 }
5504
5505 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5506 return VTBits - Tmp + 1;
5507 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5508 return VTBits - Tmp;
5509 }
5510 break;
5511 }
5512 }
5513
5514 // Allow the target to implement this method for its nodes.
5515 if (Opcode >= ISD::BUILTIN_OP_END ||
5516 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5517 Opcode == ISD::INTRINSIC_W_CHAIN ||
5518 Opcode == ISD::INTRINSIC_VOID) {
5519 // TODO: This can probably be removed once target code is audited. This
5520 // is here purely to reduce patch size and review complexity.
5521 if (!VT.isScalableVector()) {
5522 unsigned NumBits =
5523 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5524 if (NumBits > 1)
5525 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5526 }
5527 }
5528
5529 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5530 // use this information.
5531 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5532 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5533}
5534
5535unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5536 unsigned Depth) const {
5537 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5538 return Op.getScalarValueSizeInBits() - SignBits + 1;
5539}
5540
5541unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5542 const APInt &DemandedElts,
5543 unsigned Depth) const {
5544 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5545 return Op.getScalarValueSizeInBits() - SignBits + 1;
5546}
5547
5548bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
5549 unsigned Depth) const {
5550 // Early out for FREEZE.
5551 if (Op.getOpcode() == ISD::FREEZE)
5552 return true;
5553
5554 EVT VT = Op.getValueType();
5555 APInt DemandedElts = VT.isFixedLengthVector()
5556 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5557 : APInt(1, 1);
5558 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
5559}
5560
5561bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5562 const APInt &DemandedElts,
5563 bool PoisonOnly,
5564 unsigned Depth) const {
5565 unsigned Opcode = Op.getOpcode();
5566
5567 // Early out for FREEZE.
5568 if (Opcode == ISD::FREEZE)
5569 return true;
5570
5571 if (Depth >= MaxRecursionDepth)
5572 return false; // Limit search depth.
5573
5574 if (isIntOrFPConstant(V: Op))
5575 return true;
5576
5577 switch (Opcode) {
5578 case ISD::CONDCODE:
5579 case ISD::VALUETYPE:
5580 case ISD::FrameIndex:
5581 case ISD::TargetFrameIndex:
5582 case ISD::CopyFromReg:
5583 return true;
5584
5585 case ISD::POISON:
5586 return false;
5587
5588 case ISD::UNDEF:
5589 return PoisonOnly;
5590
5591 case ISD::BUILD_VECTOR:
5592 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5593 // this shouldn't affect the result.
5594 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5595 if (!DemandedElts[i])
5596 continue;
5597 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly,
5598 Depth: Depth + 1))
5599 return false;
5600 }
5601 return true;
5602
5603 case ISD::EXTRACT_SUBVECTOR: {
5604 SDValue Src = Op.getOperand(i: 0);
5605 if (Src.getValueType().isScalableVector())
5606 break;
5607 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5608 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5609 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5610 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5611 Depth: Depth + 1);
5612 }
5613
5614 case ISD::INSERT_SUBVECTOR: {
5615 if (Op.getValueType().isScalableVector())
5616 break;
5617 SDValue Src = Op.getOperand(i: 0);
5618 SDValue Sub = Op.getOperand(i: 1);
5619 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5620 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5621 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5622 APInt DemandedSrcElts = DemandedElts;
5623 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5624
5625 if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
5626 Op: Sub, DemandedElts: DemandedSubElts, PoisonOnly, Depth: Depth + 1))
5627 return false;
5628 if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
5629 Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly, Depth: Depth + 1))
5630 return false;
5631 return true;
5632 }
5633
5634 case ISD::EXTRACT_VECTOR_ELT: {
5635 SDValue Src = Op.getOperand(i: 0);
5636 auto *IndexC = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5637 EVT SrcVT = Src.getValueType();
5638 if (SrcVT.isFixedLengthVector() && IndexC &&
5639 IndexC->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5640 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5641 BitNo: IndexC->getZExtValue());
5642 return isGuaranteedNotToBeUndefOrPoison(Op: Src, DemandedElts: DemandedSrcElts, PoisonOnly,
5643 Depth: Depth + 1);
5644 }
5645 break;
5646 }
5647
5648 case ISD::INSERT_VECTOR_ELT: {
5649 SDValue InVec = Op.getOperand(i: 0);
5650 SDValue InVal = Op.getOperand(i: 1);
5651 SDValue EltNo = Op.getOperand(i: 2);
5652 EVT VT = InVec.getValueType();
5653 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
5654 if (IndexC && VT.isFixedLengthVector() &&
5655 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements())) {
5656 if (DemandedElts[IndexC->getZExtValue()] &&
5657 !isGuaranteedNotToBeUndefOrPoison(Op: InVal, PoisonOnly, Depth: Depth + 1))
5658 return false;
5659 APInt InVecDemandedElts = DemandedElts;
5660 InVecDemandedElts.clearBit(BitPosition: IndexC->getZExtValue());
5661 if (!!InVecDemandedElts &&
5662 !isGuaranteedNotToBeUndefOrPoison(
5663 Op: peekThroughInsertVectorElt(V: InVec, DemandedElts: InVecDemandedElts),
5664 DemandedElts: InVecDemandedElts, PoisonOnly, Depth: Depth + 1))
5665 return false;
5666 return true;
5667 }
5668 break;
5669 }
5670
5671 case ISD::SCALAR_TO_VECTOR:
5672 // Check upper (known undef) elements.
5673 if (DemandedElts.ugt(RHS: 1) && !PoisonOnly)
5674 return false;
5675 // Check element zero.
5676 if (DemandedElts[0] && !isGuaranteedNotToBeUndefOrPoison(
5677 Op: Op.getOperand(i: 0), PoisonOnly, Depth: Depth + 1))
5678 return false;
5679 return true;
5680
5681 case ISD::SPLAT_VECTOR:
5682 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), PoisonOnly,
5683 Depth: Depth + 1);
5684
5685 case ISD::VECTOR_SHUFFLE: {
5686 APInt DemandedLHS, DemandedRHS;
5687 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5688 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5689 DemandedElts, DemandedLHS, DemandedRHS,
5690 /*AllowUndefElts=*/false))
5691 return false;
5692 if (!DemandedLHS.isZero() &&
5693 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS,
5694 PoisonOnly, Depth: Depth + 1))
5695 return false;
5696 if (!DemandedRHS.isZero() &&
5697 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS,
5698 PoisonOnly, Depth: Depth + 1))
5699 return false;
5700 return true;
5701 }
5702
5703 case ISD::SHL:
5704 case ISD::SRL:
5705 case ISD::SRA:
5706 // Shift amount operand is checked by canCreateUndefOrPoison. So it is
5707 // enough to check operand 0 if Op can't create undef/poison.
5708 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5709 /*ConsiderFlags*/ true, Depth) &&
5710 isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts,
5711 PoisonOnly, Depth: Depth + 1);
5712
5713 case ISD::BSWAP:
5714 case ISD::CTPOP:
5715 case ISD::BITREVERSE:
5716 case ISD::AND:
5717 case ISD::OR:
5718 case ISD::XOR:
5719 case ISD::ADD:
5720 case ISD::SUB:
5721 case ISD::MUL:
5722 case ISD::SADDSAT:
5723 case ISD::UADDSAT:
5724 case ISD::SSUBSAT:
5725 case ISD::USUBSAT:
5726 case ISD::SSHLSAT:
5727 case ISD::USHLSAT:
5728 case ISD::SMIN:
5729 case ISD::SMAX:
5730 case ISD::UMIN:
5731 case ISD::UMAX:
5732 case ISD::ZERO_EXTEND:
5733 case ISD::SIGN_EXTEND:
5734 case ISD::ANY_EXTEND:
5735 case ISD::TRUNCATE:
5736 case ISD::VSELECT: {
5737 // If Op can't create undef/poison and none of its operands are undef/poison
5738 // then Op is never undef/poison. A difference from the more common check
5739 // below, outside the switch, is that we handle elementwise operations for
5740 // which the DemandedElts mask is valid for all operands here.
5741 return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
5742 /*ConsiderFlags*/ true, Depth) &&
5743 all_of(Range: Op->ops(), P: [&](SDValue V) {
5744 return isGuaranteedNotToBeUndefOrPoison(Op: V, DemandedElts,
5745 PoisonOnly, Depth: Depth + 1);
5746 });
5747 }
5748
5749 // TODO: Search for noundef attributes from library functions.
5750
5751 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5752
5753 default:
5754 // Allow the target to implement this method for its nodes.
5755 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5756 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5757 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5758 Op, DemandedElts, DAG: *this, PoisonOnly, Depth);
5759 break;
5760 }
5761
5762 // If Op can't create undef/poison and none of its operands are undef/poison
5763 // then Op is never undef/poison.
5764 // NOTE: TargetNodes can handle this in themselves in
5765 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5766 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5767 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5768 Depth) &&
5769 all_of(Range: Op->ops(), P: [&](SDValue V) {
5770 return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1);
5771 });
5772}
5773
5774bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
5775 bool ConsiderFlags,
5776 unsigned Depth) const {
5777 EVT VT = Op.getValueType();
5778 APInt DemandedElts = VT.isFixedLengthVector()
5779 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5780 : APInt(1, 1);
5781 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5782 Depth);
5783}
5784
5785bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5786 bool PoisonOnly, bool ConsiderFlags,
5787 unsigned Depth) const {
5788 if (ConsiderFlags && Op->hasPoisonGeneratingFlags())
5789 return true;
5790
5791 unsigned Opcode = Op.getOpcode();
5792 switch (Opcode) {
5793 case ISD::AssertSext:
5794 case ISD::AssertZext:
5795 case ISD::AssertAlign:
5796 case ISD::AssertNoFPClass:
5797 // Assertion nodes can create poison if the assertion fails.
5798 return true;
5799
5800 case ISD::FREEZE:
5801 case ISD::CONCAT_VECTORS:
5802 case ISD::INSERT_SUBVECTOR:
5803 case ISD::EXTRACT_SUBVECTOR:
5804 case ISD::SADDSAT:
5805 case ISD::UADDSAT:
5806 case ISD::SSUBSAT:
5807 case ISD::USUBSAT:
5808 case ISD::MULHU:
5809 case ISD::MULHS:
5810 case ISD::AVGFLOORS:
5811 case ISD::AVGFLOORU:
5812 case ISD::AVGCEILS:
5813 case ISD::AVGCEILU:
5814 case ISD::ABDU:
5815 case ISD::ABDS:
5816 case ISD::SMIN:
5817 case ISD::SMAX:
5818 case ISD::SCMP:
5819 case ISD::UMIN:
5820 case ISD::UMAX:
5821 case ISD::UCMP:
5822 case ISD::AND:
5823 case ISD::XOR:
5824 case ISD::ROTL:
5825 case ISD::ROTR:
5826 case ISD::FSHL:
5827 case ISD::FSHR:
5828 case ISD::BSWAP:
5829 case ISD::CTTZ:
5830 case ISD::CTLZ:
5831 case ISD::CTLS:
5832 case ISD::CTPOP:
5833 case ISD::BITREVERSE:
5834 case ISD::PARITY:
5835 case ISD::SIGN_EXTEND:
5836 case ISD::TRUNCATE:
5837 case ISD::SIGN_EXTEND_INREG:
5838 case ISD::SIGN_EXTEND_VECTOR_INREG:
5839 case ISD::ZERO_EXTEND_VECTOR_INREG:
5840 case ISD::BITCAST:
5841 case ISD::BUILD_VECTOR:
5842 case ISD::BUILD_PAIR:
5843 case ISD::SPLAT_VECTOR:
5844 case ISD::FABS:
5845 return false;
5846
5847 case ISD::ABS:
5848 // ISD::ABS defines abs(INT_MIN) -> INT_MIN and never generates poison.
5849 // Different to Intrinsic::abs.
5850 return false;
5851
5852 case ISD::ADDC:
5853 case ISD::SUBC:
5854 case ISD::ADDE:
5855 case ISD::SUBE:
5856 case ISD::SADDO:
5857 case ISD::SSUBO:
5858 case ISD::SMULO:
5859 case ISD::SADDO_CARRY:
5860 case ISD::SSUBO_CARRY:
5861 case ISD::UADDO:
5862 case ISD::USUBO:
5863 case ISD::UMULO:
5864 case ISD::UADDO_CARRY:
5865 case ISD::USUBO_CARRY:
5866 // No poison on result or overflow flags.
5867 return false;
5868
5869 case ISD::SELECT_CC:
5870 case ISD::SETCC: {
5871 // Integer setcc cannot create undef or poison.
5872 if (Op.getOperand(i: 0).getValueType().isInteger())
5873 return false;
5874
5875 // FP compares are more complicated. They can create poison for nan/infinity
5876 // based on options and flags. The options and flags also cause special
5877 // nonan condition codes to be used. Those condition codes may be preserved
5878 // even if the nonan flag is dropped somewhere.
5879 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
5880 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
5881 return (unsigned)CCCode & 0x10U;
5882 }
5883
5884 case ISD::OR:
5885 case ISD::ZERO_EXTEND:
5886 case ISD::SELECT:
5887 case ISD::VSELECT:
5888 case ISD::ADD:
5889 case ISD::SUB:
5890 case ISD::MUL:
5891 case ISD::FNEG:
5892 case ISD::FADD:
5893 case ISD::FSUB:
5894 case ISD::FMUL:
5895 case ISD::FDIV:
5896 case ISD::FREM:
5897 case ISD::FCOPYSIGN:
5898 case ISD::FMA:
5899 case ISD::FMAD:
5900 case ISD::FMULADD:
5901 case ISD::FP_EXTEND:
5902 case ISD::FP_TO_SINT_SAT:
5903 case ISD::FP_TO_UINT_SAT:
5904 case ISD::TRUNCATE_SSAT_S:
5905 case ISD::TRUNCATE_SSAT_U:
5906 case ISD::TRUNCATE_USAT_U:
5907 // No poison except from flags (which is handled above)
5908 return false;
5909
5910 case ISD::SHL:
5911 case ISD::SRL:
5912 case ISD::SRA:
5913 // If the max shift amount isn't in range, then the shift can
5914 // create poison.
5915 return !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
5916
5917 case ISD::CTTZ_ZERO_UNDEF:
5918 case ISD::CTLZ_ZERO_UNDEF:
5919 // If the amount is zero then the result will be poison.
5920 // TODO: Add isKnownNeverZero DemandedElts handling.
5921 return !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5922
5923 case ISD::SCALAR_TO_VECTOR:
5924 // Check if we demand any upper (undef) elements.
5925 return !PoisonOnly && DemandedElts.ugt(RHS: 1);
5926
5927 case ISD::INSERT_VECTOR_ELT:
5928 case ISD::EXTRACT_VECTOR_ELT: {
5929 // Ensure that the element index is in bounds.
5930 EVT VecVT = Op.getOperand(i: 0).getValueType();
5931 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
5932 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
5933 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5934 }
5935
5936 case ISD::VECTOR_SHUFFLE: {
5937 // Check for any demanded shuffle element that is undef.
5938 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5939 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
5940 if (Elt < 0 && DemandedElts[Idx])
5941 return true;
5942 return false;
5943 }
5944
5945 case ISD::VECTOR_COMPRESS:
5946 return false;
5947
5948 default:
5949 // Allow the target to implement this method for its nodes.
5950 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5951 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5952 return TLI->canCreateUndefOrPoisonForTargetNode(
5953 Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth);
5954 break;
5955 }
5956
5957 // Be conservative and return true.
5958 return true;
5959}
5960
5961bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
5962 unsigned Opcode = Op.getOpcode();
5963 if (Opcode == ISD::OR)
5964 return Op->getFlags().hasDisjoint() ||
5965 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
5966 if (Opcode == ISD::XOR)
5967 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
5968 return false;
5969}
5970
5971bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5972 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
5973 (Op.isAnyAdd() || isADDLike(Op));
5974}
5975
5976bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
5977 unsigned Depth) const {
5978 EVT VT = Op.getValueType();
5979
5980 // Since the number of lanes in a scalable vector is unknown at compile time,
5981 // we track one bit which is implicitly broadcast to all lanes. This means
5982 // that all lanes in a scalable vector are considered demanded.
5983 APInt DemandedElts = VT.isFixedLengthVector()
5984 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5985 : APInt(1, 1);
5986
5987 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
5988}
5989
5990bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
5991 bool SNaN, unsigned Depth) const {
5992 assert(!DemandedElts.isZero() && "No demanded elements");
5993
5994 // If we're told that NaNs won't happen, assume they won't.
5995 if (Op->getFlags().hasNoNaNs())
5996 return true;
5997
5998 if (Depth >= MaxRecursionDepth)
5999 return false; // Limit search depth.
6000
6001 // If the value is a constant, we can obviously see if it is a NaN or not.
6002 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
6003 return !C->getValueAPF().isNaN() ||
6004 (SNaN && !C->getValueAPF().isSignaling());
6005 }
6006
6007 unsigned Opcode = Op.getOpcode();
6008 switch (Opcode) {
6009 case ISD::FADD:
6010 case ISD::FSUB:
6011 case ISD::FMUL:
6012 case ISD::FDIV:
6013 case ISD::FREM:
6014 case ISD::FSIN:
6015 case ISD::FCOS:
6016 case ISD::FTAN:
6017 case ISD::FASIN:
6018 case ISD::FACOS:
6019 case ISD::FATAN:
6020 case ISD::FATAN2:
6021 case ISD::FSINH:
6022 case ISD::FCOSH:
6023 case ISD::FTANH:
6024 case ISD::FMA:
6025 case ISD::FMULADD:
6026 case ISD::FMAD: {
6027 if (SNaN)
6028 return true;
6029 // TODO: Need isKnownNeverInfinity
6030 return false;
6031 }
6032 case ISD::FCANONICALIZE:
6033 case ISD::FEXP:
6034 case ISD::FEXP2:
6035 case ISD::FEXP10:
6036 case ISD::FTRUNC:
6037 case ISD::FFLOOR:
6038 case ISD::FCEIL:
6039 case ISD::FROUND:
6040 case ISD::FROUNDEVEN:
6041 case ISD::LROUND:
6042 case ISD::LLROUND:
6043 case ISD::FRINT:
6044 case ISD::LRINT:
6045 case ISD::LLRINT:
6046 case ISD::FNEARBYINT:
6047 case ISD::FLDEXP: {
6048 if (SNaN)
6049 return true;
6050 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6051 }
6052 case ISD::FABS:
6053 case ISD::FNEG:
6054 case ISD::FCOPYSIGN: {
6055 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6056 }
6057 case ISD::SELECT:
6058 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
6059 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
6060 case ISD::FP_EXTEND:
6061 case ISD::FP_ROUND: {
6062 if (SNaN)
6063 return true;
6064 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6065 }
6066 case ISD::SINT_TO_FP:
6067 case ISD::UINT_TO_FP:
6068 return true;
6069 case ISD::FSQRT: // Need is known positive
6070 case ISD::FLOG:
6071 case ISD::FLOG2:
6072 case ISD::FLOG10:
6073 case ISD::FPOWI:
6074 case ISD::FPOW: {
6075 if (SNaN)
6076 return true;
6077 // TODO: Refine on operand
6078 return false;
6079 }
6080 case ISD::FMINNUM:
6081 case ISD::FMAXNUM:
6082 case ISD::FMINIMUMNUM:
6083 case ISD::FMAXIMUMNUM: {
6084 // Only one needs to be known not-nan, since it will be returned if the
6085 // other ends up being one.
6086 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
6087 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6088 }
6089 case ISD::FMINNUM_IEEE:
6090 case ISD::FMAXNUM_IEEE: {
6091 if (SNaN)
6092 return true;
6093 // This can return a NaN if either operand is an sNaN, or if both operands
6094 // are NaN.
6095 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6096 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
6097 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
6098 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
6099 }
6100 case ISD::FMINIMUM:
6101 case ISD::FMAXIMUM: {
6102 // TODO: Does this quiet or return the origina NaN as-is?
6103 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
6104 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
6105 }
6106 case ISD::EXTRACT_VECTOR_ELT: {
6107 SDValue Src = Op.getOperand(i: 0);
6108 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
6109 EVT SrcVT = Src.getValueType();
6110 if (SrcVT.isFixedLengthVector() && Idx &&
6111 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
6112 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
6113 BitNo: Idx->getZExtValue());
6114 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6115 }
6116 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6117 }
6118 case ISD::EXTRACT_SUBVECTOR: {
6119 SDValue Src = Op.getOperand(i: 0);
6120 if (Src.getValueType().isFixedLengthVector()) {
6121 unsigned Idx = Op.getConstantOperandVal(i: 1);
6122 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
6123 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
6124 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6125 }
6126 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
6127 }
6128 case ISD::INSERT_SUBVECTOR: {
6129 SDValue BaseVector = Op.getOperand(i: 0);
6130 SDValue SubVector = Op.getOperand(i: 1);
6131 EVT BaseVectorVT = BaseVector.getValueType();
6132 if (BaseVectorVT.isFixedLengthVector()) {
6133 unsigned Idx = Op.getConstantOperandVal(i: 2);
6134 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
6135 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
6136
6137 // Clear/Extract the bits at the position where the subvector will be
6138 // inserted.
6139 APInt DemandedMask =
6140 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
6141 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
6142 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
6143
6144 bool NeverNaN = true;
6145 if (!DemandedSrcElts.isZero())
6146 NeverNaN &=
6147 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
6148 if (NeverNaN && !DemandedSubElts.isZero())
6149 NeverNaN &=
6150 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
6151 return NeverNaN;
6152 }
6153 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
6154 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
6155 }
6156 case ISD::BUILD_VECTOR: {
6157 unsigned NumElts = Op.getNumOperands();
6158 for (unsigned I = 0; I != NumElts; ++I)
6159 if (DemandedElts[I] &&
6160 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
6161 return false;
6162 return true;
6163 }
6164 case ISD::SPLAT_VECTOR:
6165 return isKnownNeverNaN(Op: Op.getOperand(i: 0), SNaN, Depth: Depth + 1);
6166 case ISD::AssertNoFPClass: {
6167 FPClassTest NoFPClass =
6168 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
6169 if ((NoFPClass & fcNan) == fcNan)
6170 return true;
6171 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
6172 return true;
6173 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
6174 }
6175 default:
6176 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
6177 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
6178 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
6179 Depth);
6180 }
6181
6182 return false;
6183 }
6184}
6185
6186bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
6187 assert(Op.getValueType().isFloatingPoint() &&
6188 "Floating point type expected");
6189
6190 // If the value is a constant, we can obviously see if it is a zero or not.
6191 return ISD::matchUnaryFpPredicate(
6192 Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); });
6193}
6194
6195bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
6196 EVT VT = Op.getValueType();
6197
6198 // Since the number of lanes in a scalable vector is unknown at compile time,
6199 // we track one bit which is implicitly broadcast to all lanes. This means
6200 // that all lanes in a scalable vector are considered demanded.
6201 APInt DemandedElts = VT.isFixedLengthVector()
6202 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
6203 : APInt(1, 1);
6204
6205 return isKnownNeverZero(Op, DemandedElts, Depth);
6206}
6207
6208bool SelectionDAG::isKnownNeverZero(SDValue Op, const APInt &DemandedElts,
6209 unsigned Depth) const {
6210 if (Depth >= MaxRecursionDepth)
6211 return false; // Limit search depth.
6212
6213 EVT OpVT = Op.getValueType();
6214 unsigned BitWidth = OpVT.getScalarSizeInBits();
6215
6216 assert(!Op.getValueType().isFloatingPoint() &&
6217 "Floating point types unsupported - use isKnownNeverZeroFloat");
6218
6219 // If the value is a constant, we can obviously see if it is a zero or not.
6220 auto IsNeverZero = [BitWidth](const ConstantSDNode *C) {
6221 APInt V = C->getAPIntValue().zextOrTrunc(width: BitWidth);
6222 return !V.isZero();
6223 };
6224
6225 if (ISD::matchUnaryPredicate(Op, Match: IsNeverZero))
6226 return true;
6227
6228 // TODO: Recognize more cases here. Most of the cases are also incomplete to
6229 // some degree.
6230 switch (Op.getOpcode()) {
6231 default:
6232 break;
6233
6234 case ISD::BUILD_VECTOR:
6235 // Are all operands of a build vector constant non-zero?
6236 if (all_of(Range: enumerate(First: Op->ops()), P: [&](auto P) {
6237 auto *C = dyn_cast<ConstantSDNode>(P.value());
6238 return !DemandedElts[P.index()] || (C && IsNeverZero(C));
6239 }))
6240 return true;
6241 break;
6242
6243 case ISD::SPLAT_VECTOR:
6244 // Is the operand of a splat vector a constant non-zero?
6245 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 0)))
6246 if (IsNeverZero(C))
6247 return true;
6248 break;
6249
6250 case ISD::EXTRACT_VECTOR_ELT: {
6251 SDValue InVec = Op.getOperand(i: 0);
6252 SDValue EltNo = Op.getOperand(i: 1);
6253 EVT VecVT = InVec.getValueType();
6254
6255 // Skip scalable vectors or implicit extensions.
6256 if (VecVT.isScalableVector() ||
6257 OpVT.getScalarSizeInBits() != VecVT.getScalarSizeInBits())
6258 break;
6259
6260 // If we know the element index, just demand that vector element, else for
6261 // an unknown element index, ignore DemandedElts and demand them all.
6262 const unsigned NumSrcElts = VecVT.getVectorNumElements();
6263 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
6264 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
6265 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
6266 DemandedSrcElts =
6267 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
6268
6269 return isKnownNeverZero(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
6270 }
6271
6272 case ISD::OR:
6273 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6274 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6275
6276 case ISD::VSELECT:
6277 case ISD::SELECT:
6278 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6279 isKnownNeverZero(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
6280
6281 case ISD::SHL: {
6282 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6283 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6284 KnownBits ValKnown =
6285 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6286 // 1 << X is never zero.
6287 if (ValKnown.One[0])
6288 return true;
6289 // If max shift cnt of known ones is non-zero, result is non-zero.
6290 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6291 .getMaxValue();
6292 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6293 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
6294 return true;
6295 break;
6296 }
6297
6298 case ISD::UADDSAT:
6299 case ISD::UMAX:
6300 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6301 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6302
6303 case ISD::UMIN:
6304 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6305 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6306
6307 // For smin/smax: If either operand is known negative/positive
6308 // respectively we don't need the other to be known at all.
6309 case ISD::SMAX: {
6310 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6311 if (Op1.isStrictlyPositive())
6312 return true;
6313
6314 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6315 if (Op0.isStrictlyPositive())
6316 return true;
6317
6318 if (Op1.isNonZero() && Op0.isNonZero())
6319 return true;
6320
6321 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6322 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6323 }
6324 case ISD::SMIN: {
6325 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6326 if (Op1.isNegative())
6327 return true;
6328
6329 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6330 if (Op0.isNegative())
6331 return true;
6332
6333 if (Op1.isNonZero() && Op0.isNonZero())
6334 return true;
6335
6336 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) &&
6337 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6338 }
6339
6340 case ISD::ROTL:
6341 case ISD::ROTR:
6342 case ISD::BITREVERSE:
6343 case ISD::BSWAP:
6344 case ISD::CTPOP:
6345 case ISD::ABS:
6346 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6347
6348 case ISD::SRA:
6349 case ISD::SRL: {
6350 if (Op->getFlags().hasExact())
6351 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6352 KnownBits ValKnown =
6353 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6354 if (ValKnown.isNegative())
6355 return true;
6356 // If max shift cnt of known ones is non-zero, result is non-zero.
6357 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)
6358 .getMaxValue();
6359 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6360 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6361 return true;
6362 break;
6363 }
6364 case ISD::UDIV:
6365 case ISD::SDIV:
6366 // div exact can only produce a zero if the dividend is zero.
6367 // TODO: For udiv this is also true if Op1 u<= Op0
6368 if (Op->getFlags().hasExact())
6369 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6370 break;
6371
6372 case ISD::ADD:
6373 if (Op->getFlags().hasNoUnsignedWrap())
6374 if (isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1) ||
6375 isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1))
6376 return true;
6377 // TODO: There are a lot more cases we can prove for add.
6378 break;
6379
6380 case ISD::SUB: {
6381 if (isNullConstant(V: Op.getOperand(i: 0)))
6382 return isKnownNeverZero(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
6383
6384 std::optional<bool> ne = KnownBits::ne(
6385 LHS: computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1),
6386 RHS: computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1));
6387 return ne && *ne;
6388 }
6389
6390 case ISD::MUL:
6391 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6392 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6393 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6394 return true;
6395 break;
6396
6397 case ISD::ZERO_EXTEND:
6398 case ISD::SIGN_EXTEND:
6399 return isKnownNeverZero(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
6400 case ISD::VSCALE: {
6401 const Function &F = getMachineFunction().getFunction();
6402 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6403 ConstantRange CR =
6404 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6405 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6406 return true;
6407 break;
6408 }
6409 }
6410
6411 return computeKnownBits(Op, Depth).isNonZero();
6412}
6413
6414bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6415 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6416 return !C1->isNegative();
6417
6418 switch (Op.getOpcode()) {
6419 case ISD::FABS:
6420 case ISD::FEXP:
6421 case ISD::FEXP2:
6422 case ISD::FEXP10:
6423 return true;
6424 default:
6425 return false;
6426 }
6427
6428 llvm_unreachable("covered opcode switch");
6429}
6430
6431bool SelectionDAG::canIgnoreSignBitOfZero(const SDUse &Use) const {
6432 assert(Use.getValueType().isFloatingPoint());
6433 const SDNode *User = Use.getUser();
6434 if (User->getFlags().hasNoSignedZeros())
6435 return true;
6436
6437 unsigned OperandNo = Use.getOperandNo();
6438 // Check if this use is insensitive to the sign of zero
6439 switch (User->getOpcode()) {
6440 case ISD::SETCC:
6441 // Comparisons: IEEE-754 specifies +0.0 == -0.0.
6442 case ISD::FABS:
6443 // fabs always produces +0.0.
6444 return true;
6445 case ISD::FCOPYSIGN:
6446 // copysign overwrites the sign bit of the first operand.
6447 return OperandNo == 0;
6448 case ISD::FADD:
6449 case ISD::FSUB: {
6450 // Arithmetic with non-zero constants fixes the uncertainty around the
6451 // sign bit.
6452 SDValue Other = User->getOperand(Num: 1 - OperandNo);
6453 return isKnownNeverZeroFloat(Op: Other);
6454 }
6455 case ISD::FP_TO_SINT:
6456 case ISD::FP_TO_UINT:
6457 // fp-to-int conversions normalize signed zeros.
6458 return true;
6459 default:
6460 return false;
6461 }
6462}
6463
6464bool SelectionDAG::canIgnoreSignBitOfZero(SDValue Op) const {
6465 if (Op->getFlags().hasNoSignedZeros())
6466 return true;
6467 // FIXME: Limit the amount of checked uses to not introduce a compile-time
6468 // regression. Ideally, this should be implemented as a demanded-bits
6469 // optimization that stems from the users.
6470 if (Op->use_size() > 2)
6471 return false;
6472 return all_of(Range: Op->uses(),
6473 P: [&](const SDUse &Use) { return canIgnoreSignBitOfZero(Use); });
6474}
6475
6476bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6477 // Check the obvious case.
6478 if (A == B) return true;
6479
6480 // For negative and positive zero.
6481 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6482 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6483 if (CA->isZero() && CB->isZero()) return true;
6484
6485 // Otherwise they may not be equal.
6486 return false;
6487}
6488
6489// Only bits set in Mask must be negated, other bits may be arbitrary.
6490SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6491 if (isBitwiseNot(V, AllowUndefs))
6492 return V.getOperand(i: 0);
6493
6494 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6495 // bits in the non-extended part.
6496 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6497 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6498 return SDValue();
6499 SDValue ExtArg = V.getOperand(i: 0);
6500 if (ExtArg.getScalarValueSizeInBits() >=
6501 MaskC->getAPIntValue().getActiveBits() &&
6502 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6503 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6504 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6505 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6506 return SDValue();
6507}
6508
6509static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6510 // Match masked merge pattern (X & ~M) op (Y & M)
6511 // Including degenerate case (X & ~M) op M
6512 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6513 SDValue Other) {
6514 if (SDValue NotOperand =
6515 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6516 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6517 NotOperand->getOpcode() == ISD::TRUNCATE)
6518 NotOperand = NotOperand->getOperand(Num: 0);
6519
6520 if (Other == NotOperand)
6521 return true;
6522 if (Other->getOpcode() == ISD::AND)
6523 return NotOperand == Other->getOperand(Num: 0) ||
6524 NotOperand == Other->getOperand(Num: 1);
6525 }
6526 return false;
6527 };
6528
6529 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6530 A = A->getOperand(Num: 0);
6531
6532 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6533 B = B->getOperand(Num: 0);
6534
6535 if (A->getOpcode() == ISD::AND)
6536 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6537 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6538 return false;
6539}
6540
6541// FIXME: unify with llvm::haveNoCommonBitsSet.
6542bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6543 assert(A.getValueType() == B.getValueType() &&
6544 "Values must have the same type");
6545 if (haveNoCommonBitsSetCommutative(A, B) ||
6546 haveNoCommonBitsSetCommutative(A: B, B: A))
6547 return true;
6548 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6549 RHS: computeKnownBits(Op: B));
6550}
6551
6552static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6553 SelectionDAG &DAG) {
6554 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6555 return DAG.getConstant(Val: 0, DL, VT);
6556
6557 return SDValue();
6558}
6559
6560static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6561 ArrayRef<SDValue> Ops,
6562 SelectionDAG &DAG) {
6563 int NumOps = Ops.size();
6564 assert(NumOps != 0 && "Can't build an empty vector!");
6565 assert(!VT.isScalableVector() &&
6566 "BUILD_VECTOR cannot be used with scalable types");
6567 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6568 "Incorrect element count in BUILD_VECTOR!");
6569
6570 // BUILD_VECTOR of UNDEFs is UNDEF.
6571 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6572 return DAG.getUNDEF(VT);
6573
6574 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6575 SDValue IdentitySrc;
6576 bool IsIdentity = true;
6577 for (int i = 0; i != NumOps; ++i) {
6578 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6579 Ops[i].getOperand(i: 0).getValueType() != VT ||
6580 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6581 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6582 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6583 IsIdentity = false;
6584 break;
6585 }
6586 IdentitySrc = Ops[i].getOperand(i: 0);
6587 }
6588 if (IsIdentity)
6589 return IdentitySrc;
6590
6591 return SDValue();
6592}
6593
6594/// Try to simplify vector concatenation to an input value, undef, or build
6595/// vector.
6596static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6597 ArrayRef<SDValue> Ops,
6598 SelectionDAG &DAG) {
6599 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6600 assert(llvm::all_of(Ops,
6601 [Ops](SDValue Op) {
6602 return Ops[0].getValueType() == Op.getValueType();
6603 }) &&
6604 "Concatenation of vectors with inconsistent value types!");
6605 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6606 VT.getVectorElementCount() &&
6607 "Incorrect element count in vector concatenation!");
6608
6609 if (Ops.size() == 1)
6610 return Ops[0];
6611
6612 // Concat of UNDEFs is UNDEF.
6613 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6614 return DAG.getUNDEF(VT);
6615
6616 // Scan the operands and look for extract operations from a single source
6617 // that correspond to insertion at the same location via this concatenation:
6618 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6619 SDValue IdentitySrc;
6620 bool IsIdentity = true;
6621 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6622 SDValue Op = Ops[i];
6623 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6624 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6625 Op.getOperand(i: 0).getValueType() != VT ||
6626 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
6627 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
6628 IsIdentity = false;
6629 break;
6630 }
6631 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6632 "Unexpected identity source vector for concat of extracts");
6633 IdentitySrc = Op.getOperand(i: 0);
6634 }
6635 if (IsIdentity) {
6636 assert(IdentitySrc && "Failed to set source vector of extracts");
6637 return IdentitySrc;
6638 }
6639
6640 // The code below this point is only designed to work for fixed width
6641 // vectors, so we bail out for now.
6642 if (VT.isScalableVector())
6643 return SDValue();
6644
6645 // A CONCAT_VECTOR of scalar sources, such as UNDEF, BUILD_VECTOR and
6646 // single-element INSERT_VECTOR_ELT operands can be simplified to one big
6647 // BUILD_VECTOR.
6648 // FIXME: Add support for SCALAR_TO_VECTOR as well.
6649 EVT SVT = VT.getScalarType();
6650 SmallVector<SDValue, 16> Elts;
6651 for (SDValue Op : Ops) {
6652 EVT OpVT = Op.getValueType();
6653 if (Op.isUndef())
6654 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
6655 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
6656 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
6657 else if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
6658 OpVT.getVectorNumElements() == 1 &&
6659 isNullConstant(V: Op.getOperand(i: 2)))
6660 Elts.push_back(Elt: Op.getOperand(i: 1));
6661 else
6662 return SDValue();
6663 }
6664
6665 // BUILD_VECTOR requires all inputs to be of the same type, find the
6666 // maximum type and extend them all.
6667 for (SDValue Op : Elts)
6668 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
6669
6670 if (SVT.bitsGT(VT: VT.getScalarType())) {
6671 for (SDValue &Op : Elts) {
6672 if (Op.isUndef())
6673 Op = DAG.getUNDEF(VT: SVT);
6674 else
6675 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
6676 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
6677 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
6678 }
6679 }
6680
6681 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
6682 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
6683 return V;
6684}
6685
6686/// Gets or creates the specified node.
6687SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
6688 SDVTList VTs = getVTList(VT);
6689 FoldingSetNodeID ID;
6690 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: {});
6691 void *IP = nullptr;
6692 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
6693 return SDValue(E, 0);
6694
6695 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6696 CSEMap.InsertNode(N, InsertPos: IP);
6697
6698 InsertNode(N);
6699 SDValue V = SDValue(N, 0);
6700 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6701 return V;
6702}
6703
6704SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6705 SDValue N1) {
6706 SDNodeFlags Flags;
6707 if (Inserter)
6708 Flags = Inserter->getFlags();
6709 return getNode(Opcode, DL, VT, Operand: N1, Flags);
6710}
6711
6712SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6713 SDValue N1, const SDNodeFlags Flags) {
6714 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
6715
6716 // Constant fold unary operations with a vector integer or float operand.
6717 switch (Opcode) {
6718 default:
6719 // FIXME: Entirely reasonable to perform folding of other unary
6720 // operations here as the need arises.
6721 break;
6722 case ISD::FNEG:
6723 case ISD::FABS:
6724 case ISD::FCEIL:
6725 case ISD::FTRUNC:
6726 case ISD::FFLOOR:
6727 case ISD::FP_EXTEND:
6728 case ISD::FP_TO_SINT:
6729 case ISD::FP_TO_UINT:
6730 case ISD::FP_TO_FP16:
6731 case ISD::FP_TO_BF16:
6732 case ISD::TRUNCATE:
6733 case ISD::ANY_EXTEND:
6734 case ISD::ZERO_EXTEND:
6735 case ISD::SIGN_EXTEND:
6736 case ISD::UINT_TO_FP:
6737 case ISD::SINT_TO_FP:
6738 case ISD::FP16_TO_FP:
6739 case ISD::BF16_TO_FP:
6740 case ISD::BITCAST:
6741 case ISD::ABS:
6742 case ISD::BITREVERSE:
6743 case ISD::BSWAP:
6744 case ISD::CTLZ:
6745 case ISD::CTLZ_ZERO_UNDEF:
6746 case ISD::CTTZ:
6747 case ISD::CTTZ_ZERO_UNDEF:
6748 case ISD::CTPOP:
6749 case ISD::CTLS:
6750 case ISD::STEP_VECTOR: {
6751 SDValue Ops = {N1};
6752 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
6753 return Fold;
6754 }
6755 }
6756
6757 unsigned OpOpcode = N1.getNode()->getOpcode();
6758 switch (Opcode) {
6759 case ISD::STEP_VECTOR:
6760 assert(VT.isScalableVector() &&
6761 "STEP_VECTOR can only be used with scalable types");
6762 assert(OpOpcode == ISD::TargetConstant &&
6763 VT.getVectorElementType() == N1.getValueType() &&
6764 "Unexpected step operand");
6765 break;
6766 case ISD::FREEZE:
6767 assert(VT == N1.getValueType() && "Unexpected VT!");
6768 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly=*/false))
6769 return N1;
6770 break;
6771 case ISD::TokenFactor:
6772 case ISD::MERGE_VALUES:
6773 case ISD::CONCAT_VECTORS:
6774 return N1; // Factor, merge or concat of one node? No need.
6775 case ISD::BUILD_VECTOR: {
6776 // Attempt to simplify BUILD_VECTOR.
6777 SDValue Ops[] = {N1};
6778 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
6779 return V;
6780 break;
6781 }
6782 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
6783 case ISD::FP_EXTEND:
6784 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
6785 "Invalid FP cast!");
6786 if (N1.getValueType() == VT) return N1; // noop conversion.
6787 assert((!VT.isVector() || VT.getVectorElementCount() ==
6788 N1.getValueType().getVectorElementCount()) &&
6789 "Vector element count mismatch!");
6790 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
6791 if (N1.isUndef())
6792 return getUNDEF(VT);
6793 break;
6794 case ISD::FP_TO_SINT:
6795 case ISD::FP_TO_UINT:
6796 if (N1.isUndef())
6797 return getUNDEF(VT);
6798 break;
6799 case ISD::SINT_TO_FP:
6800 case ISD::UINT_TO_FP:
6801 // [us]itofp(undef) = 0, because the result value is bounded.
6802 if (N1.isUndef())
6803 return getConstantFP(Val: 0.0, DL, VT);
6804 break;
6805 case ISD::SIGN_EXTEND:
6806 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6807 "Invalid SIGN_EXTEND!");
6808 assert(VT.isVector() == N1.getValueType().isVector() &&
6809 "SIGN_EXTEND result type type should be vector iff the operand "
6810 "type is vector!");
6811 if (N1.getValueType() == VT) return N1; // noop extension
6812 assert((!VT.isVector() || VT.getVectorElementCount() ==
6813 N1.getValueType().getVectorElementCount()) &&
6814 "Vector element count mismatch!");
6815 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
6816 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
6817 SDNodeFlags Flags;
6818 if (OpOpcode == ISD::ZERO_EXTEND)
6819 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6820 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6821 transferDbgValues(From: N1, To: NewVal);
6822 return NewVal;
6823 }
6824
6825 if (OpOpcode == ISD::POISON)
6826 return getPOISON(VT);
6827
6828 if (N1.isUndef())
6829 // sext(undef) = 0, because the top bits will all be the same.
6830 return getConstant(Val: 0, DL, VT);
6831
6832 // Skip unnecessary sext_inreg pattern:
6833 // (sext (trunc x)) -> x iff the upper bits are all signbits.
6834 if (OpOpcode == ISD::TRUNCATE) {
6835 SDValue OpOp = N1.getOperand(i: 0);
6836 if (OpOp.getValueType() == VT) {
6837 unsigned NumSignExtBits =
6838 VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
6839 if (ComputeNumSignBits(Op: OpOp) > NumSignExtBits) {
6840 transferDbgValues(From: N1, To: OpOp);
6841 return OpOp;
6842 }
6843 }
6844 }
6845 break;
6846 case ISD::ZERO_EXTEND:
6847 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6848 "Invalid ZERO_EXTEND!");
6849 assert(VT.isVector() == N1.getValueType().isVector() &&
6850 "ZERO_EXTEND result type type should be vector iff the operand "
6851 "type is vector!");
6852 if (N1.getValueType() == VT) return N1; // noop extension
6853 assert((!VT.isVector() || VT.getVectorElementCount() ==
6854 N1.getValueType().getVectorElementCount()) &&
6855 "Vector element count mismatch!");
6856 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
6857 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
6858 SDNodeFlags Flags;
6859 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6860 SDValue NewVal =
6861 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
6862 transferDbgValues(From: N1, To: NewVal);
6863 return NewVal;
6864 }
6865
6866 if (OpOpcode == ISD::POISON)
6867 return getPOISON(VT);
6868
6869 if (N1.isUndef())
6870 // zext(undef) = 0, because the top bits will be zero.
6871 return getConstant(Val: 0, DL, VT);
6872
6873 // Skip unnecessary zext_inreg pattern:
6874 // (zext (trunc x)) -> x iff the upper bits are known zero.
6875 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
6876 // use to recognise zext_inreg patterns.
6877 if (OpOpcode == ISD::TRUNCATE) {
6878 SDValue OpOp = N1.getOperand(i: 0);
6879 if (OpOp.getValueType() == VT) {
6880 if (OpOp.getOpcode() != ISD::AND) {
6881 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
6882 loBit: N1.getScalarValueSizeInBits());
6883 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
6884 transferDbgValues(From: N1, To: OpOp);
6885 return OpOp;
6886 }
6887 }
6888 }
6889 }
6890 break;
6891 case ISD::ANY_EXTEND:
6892 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6893 "Invalid ANY_EXTEND!");
6894 assert(VT.isVector() == N1.getValueType().isVector() &&
6895 "ANY_EXTEND result type type should be vector iff the operand "
6896 "type is vector!");
6897 if (N1.getValueType() == VT) return N1; // noop extension
6898 assert((!VT.isVector() || VT.getVectorElementCount() ==
6899 N1.getValueType().getVectorElementCount()) &&
6900 "Vector element count mismatch!");
6901 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
6902
6903 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6904 OpOpcode == ISD::ANY_EXTEND) {
6905 SDNodeFlags Flags;
6906 if (OpOpcode == ISD::ZERO_EXTEND)
6907 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6908 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
6909 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6910 }
6911 if (N1.isUndef())
6912 return getUNDEF(VT);
6913
6914 // (ext (trunc x)) -> x
6915 if (OpOpcode == ISD::TRUNCATE) {
6916 SDValue OpOp = N1.getOperand(i: 0);
6917 if (OpOp.getValueType() == VT) {
6918 transferDbgValues(From: N1, To: OpOp);
6919 return OpOp;
6920 }
6921 }
6922 break;
6923 case ISD::TRUNCATE:
6924 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6925 "Invalid TRUNCATE!");
6926 assert(VT.isVector() == N1.getValueType().isVector() &&
6927 "TRUNCATE result type type should be vector iff the operand "
6928 "type is vector!");
6929 if (N1.getValueType() == VT) return N1; // noop truncate
6930 assert((!VT.isVector() || VT.getVectorElementCount() ==
6931 N1.getValueType().getVectorElementCount()) &&
6932 "Vector element count mismatch!");
6933 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
6934 if (OpOpcode == ISD::TRUNCATE)
6935 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6936 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6937 OpOpcode == ISD::ANY_EXTEND) {
6938 // If the source is smaller than the dest, we still need an extend.
6939 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
6940 VT: VT.getScalarType())) {
6941 SDNodeFlags Flags;
6942 if (OpOpcode == ISD::ZERO_EXTEND)
6943 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6944 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6945 }
6946 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
6947 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6948 return N1.getOperand(i: 0);
6949 }
6950 if (N1.isUndef())
6951 return getUNDEF(VT);
6952 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
6953 return getVScale(DL, VT,
6954 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
6955 break;
6956 case ISD::ANY_EXTEND_VECTOR_INREG:
6957 case ISD::ZERO_EXTEND_VECTOR_INREG:
6958 case ISD::SIGN_EXTEND_VECTOR_INREG:
6959 assert(VT.isVector() && "This DAG node is restricted to vector types.");
6960 assert(N1.getValueType().bitsLE(VT) &&
6961 "The input must be the same size or smaller than the result.");
6962 assert(VT.getVectorMinNumElements() <
6963 N1.getValueType().getVectorMinNumElements() &&
6964 "The destination vector type must have fewer lanes than the input.");
6965 break;
6966 case ISD::ABS:
6967 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
6968 if (N1.isUndef())
6969 return getConstant(Val: 0, DL, VT);
6970 break;
6971 case ISD::BSWAP:
6972 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
6973 assert((VT.getScalarSizeInBits() % 16 == 0) &&
6974 "BSWAP types must be a multiple of 16 bits!");
6975 if (N1.isUndef())
6976 return getUNDEF(VT);
6977 // bswap(bswap(X)) -> X.
6978 if (OpOpcode == ISD::BSWAP)
6979 return N1.getOperand(i: 0);
6980 break;
6981 case ISD::BITREVERSE:
6982 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
6983 if (N1.isUndef())
6984 return getUNDEF(VT);
6985 break;
6986 case ISD::BITCAST:
6987 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
6988 "Cannot BITCAST between types of different sizes!");
6989 if (VT == N1.getValueType()) return N1; // noop conversion.
6990 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
6991 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
6992 if (N1.isUndef())
6993 return getUNDEF(VT);
6994 break;
6995 case ISD::SCALAR_TO_VECTOR:
6996 assert(VT.isVector() && !N1.getValueType().isVector() &&
6997 (VT.getVectorElementType() == N1.getValueType() ||
6998 (VT.getVectorElementType().isInteger() &&
6999 N1.getValueType().isInteger() &&
7000 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7001 "Illegal SCALAR_TO_VECTOR node!");
7002 if (N1.isUndef())
7003 return getUNDEF(VT);
7004 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
7005 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
7006 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
7007 N1.getConstantOperandVal(i: 1) == 0 &&
7008 N1.getOperand(i: 0).getValueType() == VT)
7009 return N1.getOperand(i: 0);
7010 break;
7011 case ISD::FNEG:
7012 // Negation of an unknown bag of bits is still completely undefined.
7013 if (N1.isUndef())
7014 return getUNDEF(VT);
7015
7016 if (OpOpcode == ISD::FNEG) // --X -> X
7017 return N1.getOperand(i: 0);
7018 break;
7019 case ISD::FABS:
7020 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
7021 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
7022 break;
7023 case ISD::VSCALE:
7024 assert(VT == N1.getValueType() && "Unexpected VT!");
7025 break;
7026 case ISD::CTPOP:
7027 if (N1.getValueType().getScalarType() == MVT::i1)
7028 return N1;
7029 break;
7030 case ISD::CTLZ:
7031 case ISD::CTTZ:
7032 if (N1.getValueType().getScalarType() == MVT::i1)
7033 return getNOT(DL, Val: N1, VT: N1.getValueType());
7034 break;
7035 case ISD::CTLS:
7036 if (N1.getValueType().getScalarType() == MVT::i1)
7037 return getConstant(Val: 0, DL, VT);
7038 break;
7039 case ISD::VECREDUCE_ADD:
7040 if (N1.getValueType().getScalarType() == MVT::i1)
7041 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
7042 break;
7043 case ISD::VECREDUCE_SMIN:
7044 case ISD::VECREDUCE_UMAX:
7045 if (N1.getValueType().getScalarType() == MVT::i1)
7046 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
7047 break;
7048 case ISD::VECREDUCE_SMAX:
7049 case ISD::VECREDUCE_UMIN:
7050 if (N1.getValueType().getScalarType() == MVT::i1)
7051 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
7052 break;
7053 case ISD::SPLAT_VECTOR:
7054 assert(VT.isVector() && "Wrong return type!");
7055 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
7056 // that for now.
7057 assert((VT.getVectorElementType() == N1.getValueType() ||
7058 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
7059 (VT.getVectorElementType().isInteger() &&
7060 N1.getValueType().isInteger() &&
7061 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
7062 "Wrong operand type!");
7063 break;
7064 }
7065
7066 SDNode *N;
7067 SDVTList VTs = getVTList(VT);
7068 SDValue Ops[] = {N1};
7069 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
7070 FoldingSetNodeID ID;
7071 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7072 void *IP = nullptr;
7073 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7074 E->intersectFlagsWith(Flags);
7075 return SDValue(E, 0);
7076 }
7077
7078 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7079 N->setFlags(Flags);
7080 createOperands(Node: N, Vals: Ops);
7081 CSEMap.InsertNode(N, InsertPos: IP);
7082 } else {
7083 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7084 createOperands(Node: N, Vals: Ops);
7085 }
7086
7087 InsertNode(N);
7088 SDValue V = SDValue(N, 0);
7089 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7090 return V;
7091}
7092
7093static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
7094 const APInt &C2) {
7095 switch (Opcode) {
7096 case ISD::ADD: return C1 + C2;
7097 case ISD::SUB: return C1 - C2;
7098 case ISD::MUL: return C1 * C2;
7099 case ISD::AND: return C1 & C2;
7100 case ISD::OR: return C1 | C2;
7101 case ISD::XOR: return C1 ^ C2;
7102 case ISD::SHL: return C1 << C2;
7103 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
7104 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
7105 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
7106 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
7107 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
7108 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
7109 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
7110 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
7111 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
7112 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
7113 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
7114 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
7115 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
7116 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
7117 case ISD::UDIV:
7118 if (!C2.getBoolValue())
7119 break;
7120 return C1.udiv(RHS: C2);
7121 case ISD::UREM:
7122 if (!C2.getBoolValue())
7123 break;
7124 return C1.urem(RHS: C2);
7125 case ISD::SDIV:
7126 if (!C2.getBoolValue())
7127 break;
7128 return C1.sdiv(RHS: C2);
7129 case ISD::SREM:
7130 if (!C2.getBoolValue())
7131 break;
7132 return C1.srem(RHS: C2);
7133 case ISD::AVGFLOORS:
7134 return APIntOps::avgFloorS(C1, C2);
7135 case ISD::AVGFLOORU:
7136 return APIntOps::avgFloorU(C1, C2);
7137 case ISD::AVGCEILS:
7138 return APIntOps::avgCeilS(C1, C2);
7139 case ISD::AVGCEILU:
7140 return APIntOps::avgCeilU(C1, C2);
7141 case ISD::ABDS:
7142 return APIntOps::abds(A: C1, B: C2);
7143 case ISD::ABDU:
7144 return APIntOps::abdu(A: C1, B: C2);
7145 case ISD::MULHS:
7146 return APIntOps::mulhs(C1, C2);
7147 case ISD::MULHU:
7148 return APIntOps::mulhu(C1, C2);
7149 case ISD::CLMUL:
7150 return APIntOps::clmul(LHS: C1, RHS: C2);
7151 case ISD::CLMULR:
7152 return APIntOps::clmulr(LHS: C1, RHS: C2);
7153 case ISD::CLMULH:
7154 return APIntOps::clmulh(LHS: C1, RHS: C2);
7155 }
7156 return std::nullopt;
7157}
7158// Handle constant folding with UNDEF.
7159// TODO: Handle more cases.
7160static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
7161 bool IsUndef1, const APInt &C2,
7162 bool IsUndef2) {
7163 if (!(IsUndef1 || IsUndef2))
7164 return FoldValue(Opcode, C1, C2);
7165
7166 // Fold and(x, undef) -> 0
7167 // Fold mul(x, undef) -> 0
7168 if (Opcode == ISD::AND || Opcode == ISD::MUL)
7169 return APInt::getZero(numBits: C1.getBitWidth());
7170
7171 return std::nullopt;
7172}
7173
7174SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
7175 const GlobalAddressSDNode *GA,
7176 const SDNode *N2) {
7177 if (GA->getOpcode() != ISD::GlobalAddress)
7178 return SDValue();
7179 if (!TLI->isOffsetFoldingLegal(GA))
7180 return SDValue();
7181 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
7182 if (!C2)
7183 return SDValue();
7184 int64_t Offset = C2->getSExtValue();
7185 switch (Opcode) {
7186 case ISD::ADD:
7187 case ISD::PTRADD:
7188 break;
7189 case ISD::SUB: Offset = -uint64_t(Offset); break;
7190 default: return SDValue();
7191 }
7192 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
7193 Offset: GA->getOffset() + uint64_t(Offset));
7194}
7195
7196bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
7197 switch (Opcode) {
7198 case ISD::SDIV:
7199 case ISD::UDIV:
7200 case ISD::SREM:
7201 case ISD::UREM: {
7202 // If a divisor is zero/undef or any element of a divisor vector is
7203 // zero/undef, the whole op is undef.
7204 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
7205 SDValue Divisor = Ops[1];
7206 if (Divisor.isUndef() || isNullConstant(V: Divisor))
7207 return true;
7208
7209 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
7210 llvm::any_of(Range: Divisor->op_values(),
7211 P: [](SDValue V) { return V.isUndef() ||
7212 isNullConstant(V); });
7213 // TODO: Handle signed overflow.
7214 }
7215 // TODO: Handle oversized shifts.
7216 default:
7217 return false;
7218 }
7219}
7220
7221SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
7222 EVT VT, ArrayRef<SDValue> Ops,
7223 SDNodeFlags Flags) {
7224 // If the opcode is a target-specific ISD node, there's nothing we can
7225 // do here and the operand rules may not line up with the below, so
7226 // bail early.
7227 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
7228 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
7229 // foldCONCAT_VECTORS in getNode before this is called.
7230 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
7231 return SDValue();
7232
7233 unsigned NumOps = Ops.size();
7234 if (NumOps == 0)
7235 return SDValue();
7236
7237 if (isUndef(Opcode, Ops))
7238 return getUNDEF(VT);
7239
7240 // Handle unary special cases.
7241 if (NumOps == 1) {
7242 SDValue N1 = Ops[0];
7243
7244 // Constant fold unary operations with an integer constant operand. Even
7245 // opaque constant will be folded, because the folding of unary operations
7246 // doesn't create new constants with different values. Nevertheless, the
7247 // opaque flag is preserved during folding to prevent future folding with
7248 // other constants.
7249 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
7250 const APInt &Val = C->getAPIntValue();
7251 switch (Opcode) {
7252 case ISD::SIGN_EXTEND:
7253 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7254 isT: C->isTargetOpcode(), isO: C->isOpaque());
7255 case ISD::TRUNCATE:
7256 if (C->isOpaque())
7257 break;
7258 [[fallthrough]];
7259 case ISD::ZERO_EXTEND:
7260 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7261 isT: C->isTargetOpcode(), isO: C->isOpaque());
7262 case ISD::ANY_EXTEND:
7263 // Some targets like RISCV prefer to sign extend some types.
7264 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
7265 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7266 isT: C->isTargetOpcode(), isO: C->isOpaque());
7267 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
7268 isT: C->isTargetOpcode(), isO: C->isOpaque());
7269 case ISD::ABS:
7270 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
7271 isO: C->isOpaque());
7272 case ISD::BITREVERSE:
7273 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
7274 isO: C->isOpaque());
7275 case ISD::BSWAP:
7276 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
7277 isO: C->isOpaque());
7278 case ISD::CTPOP:
7279 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
7280 isO: C->isOpaque());
7281 case ISD::CTLZ:
7282 case ISD::CTLZ_ZERO_UNDEF:
7283 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
7284 isO: C->isOpaque());
7285 case ISD::CTTZ:
7286 case ISD::CTTZ_ZERO_UNDEF:
7287 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
7288 isO: C->isOpaque());
7289 case ISD::CTLS:
7290 // CTLS returns the number of extra sign bits so subtract one.
7291 return getConstant(Val: Val.getNumSignBits() - 1, DL, VT,
7292 isT: C->isTargetOpcode(), isO: C->isOpaque());
7293 case ISD::UINT_TO_FP:
7294 case ISD::SINT_TO_FP: {
7295 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
7296 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
7297 RM: APFloat::rmNearestTiesToEven);
7298 return getConstantFP(V: FPV, DL, VT);
7299 }
7300 case ISD::FP16_TO_FP:
7301 case ISD::BF16_TO_FP: {
7302 bool Ignored;
7303 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
7304 : APFloat::BFloat(),
7305 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
7306
7307 // This can return overflow, underflow, or inexact; we don't care.
7308 // FIXME need to be more flexible about rounding mode.
7309 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7310 losesInfo: &Ignored);
7311 return getConstantFP(V: FPV, DL, VT);
7312 }
7313 case ISD::STEP_VECTOR:
7314 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
7315 return V;
7316 break;
7317 case ISD::BITCAST:
7318 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
7319 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
7320 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
7321 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
7322 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
7323 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
7324 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
7325 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
7326 break;
7327 }
7328 }
7329
7330 // Constant fold unary operations with a floating point constant operand.
7331 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
7332 APFloat V = C->getValueAPF(); // make copy
7333 switch (Opcode) {
7334 case ISD::FNEG:
7335 V.changeSign();
7336 return getConstantFP(V, DL, VT);
7337 case ISD::FABS:
7338 V.clearSign();
7339 return getConstantFP(V, DL, VT);
7340 case ISD::FCEIL: {
7341 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
7342 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7343 return getConstantFP(V, DL, VT);
7344 return SDValue();
7345 }
7346 case ISD::FTRUNC: {
7347 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
7348 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7349 return getConstantFP(V, DL, VT);
7350 return SDValue();
7351 }
7352 case ISD::FFLOOR: {
7353 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
7354 if (fs == APFloat::opOK || fs == APFloat::opInexact)
7355 return getConstantFP(V, DL, VT);
7356 return SDValue();
7357 }
7358 case ISD::FP_EXTEND: {
7359 bool ignored;
7360 // This can return overflow, underflow, or inexact; we don't care.
7361 // FIXME need to be more flexible about rounding mode.
7362 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7363 losesInfo: &ignored);
7364 return getConstantFP(V, DL, VT);
7365 }
7366 case ISD::FP_TO_SINT:
7367 case ISD::FP_TO_UINT: {
7368 bool ignored;
7369 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
7370 // FIXME need to be more flexible about rounding mode.
7371 APFloat::opStatus s =
7372 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
7373 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
7374 break;
7375 return getConstant(Val: IntVal, DL, VT);
7376 }
7377 case ISD::FP_TO_FP16:
7378 case ISD::FP_TO_BF16: {
7379 bool Ignored;
7380 // This can return overflow, underflow, or inexact; we don't care.
7381 // FIXME need to be more flexible about rounding mode.
7382 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
7383 : APFloat::BFloat(),
7384 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
7385 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7386 }
7387 case ISD::BITCAST:
7388 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
7389 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7390 VT);
7391 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
7392 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
7393 VT);
7394 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
7395 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
7396 VT);
7397 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
7398 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
7399 break;
7400 }
7401 }
7402
7403 // Early-out if we failed to constant fold a bitcast.
7404 if (Opcode == ISD::BITCAST)
7405 return SDValue();
7406 }
7407
7408 // Handle binops special cases.
7409 if (NumOps == 2) {
7410 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
7411 return CFP;
7412
7413 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7414 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
7415 if (C1->isOpaque() || C2->isOpaque())
7416 return SDValue();
7417
7418 std::optional<APInt> FoldAttempt =
7419 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
7420 if (!FoldAttempt)
7421 return SDValue();
7422
7423 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
7424 assert((!Folded || !VT.isVector()) &&
7425 "Can't fold vectors ops with scalar operands");
7426 return Folded;
7427 }
7428 }
7429
7430 // fold (add Sym, c) -> Sym+c
7431 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
7432 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
7433 if (TLI->isCommutativeBinOp(Opcode))
7434 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
7435 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
7436
7437 // fold (sext_in_reg c1) -> c2
7438 if (Opcode == ISD::SIGN_EXTEND_INREG) {
7439 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
7440
7441 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
7442 unsigned FromBits = EVT.getScalarSizeInBits();
7443 Val <<= Val.getBitWidth() - FromBits;
7444 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
7445 return getConstant(Val, DL, VT: ConstantVT);
7446 };
7447
7448 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
7449 const APInt &Val = C1->getAPIntValue();
7450 return SignExtendInReg(Val, VT);
7451 }
7452
7453 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7454 SmallVector<SDValue, 8> ScalarOps;
7455 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7456 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7457 SDValue Op = Ops[0].getOperand(i: I);
7458 if (Op.isUndef()) {
7459 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7460 continue;
7461 }
7462 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7463 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7464 }
7465 return getBuildVector(VT, DL, Ops: ScalarOps);
7466 }
7467
7468 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7469 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7470 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7471 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7472 Ops[0].getOperand(i: 0).getValueType()));
7473 }
7474 }
7475
7476 // Handle fshl/fshr special cases.
7477 if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
7478 auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0]);
7479 auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1]);
7480 auto *C3 = dyn_cast<ConstantSDNode>(Val: Ops[2]);
7481
7482 if (C1 && C2 && C3) {
7483 if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
7484 return SDValue();
7485 const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
7486 &V3 = C3->getAPIntValue();
7487
7488 APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(Hi: V1, Lo: V2, Shift: V3)
7489 : APIntOps::fshr(Hi: V1, Lo: V2, Shift: V3);
7490 return getConstant(Val: FoldedVal, DL, VT);
7491 }
7492 }
7493
7494 // Handle fma/fmad special cases.
7495 if (Opcode == ISD::FMA || Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7496 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7497 assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7498 Ops[2].getValueType() == VT && "FMA types must match!");
7499 ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Val: Ops[0]);
7500 ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Val: Ops[1]);
7501 ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Val: Ops[2]);
7502 if (C1 && C2 && C3) {
7503 APFloat V1 = C1->getValueAPF();
7504 const APFloat &V2 = C2->getValueAPF();
7505 const APFloat &V3 = C3->getValueAPF();
7506 if (Opcode == ISD::FMAD || Opcode == ISD::FMULADD) {
7507 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7508 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7509 } else
7510 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7511 return getConstantFP(V: V1, DL, VT);
7512 }
7513 }
7514
7515 // This is for vector folding only from here on.
7516 if (!VT.isVector())
7517 return SDValue();
7518
7519 ElementCount NumElts = VT.getVectorElementCount();
7520
7521 // See if we can fold through any bitcasted integer ops.
7522 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
7523 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7524 (Ops[0].getOpcode() == ISD::BITCAST ||
7525 Ops[1].getOpcode() == ISD::BITCAST)) {
7526 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
7527 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
7528 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
7529 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
7530 if (BV1 && BV2 && N1.getValueType().isInteger() &&
7531 N2.getValueType().isInteger()) {
7532 bool IsLE = getDataLayout().isLittleEndian();
7533 unsigned EltBits = VT.getScalarSizeInBits();
7534 SmallVector<APInt> RawBits1, RawBits2;
7535 BitVector UndefElts1, UndefElts2;
7536 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
7537 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
7538 SmallVector<APInt> RawBits;
7539 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
7540 std::optional<APInt> Fold = FoldValueWithUndef(
7541 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
7542 if (!Fold)
7543 break;
7544 RawBits.push_back(Elt: *Fold);
7545 }
7546 if (RawBits.size() == NumElts.getFixedValue()) {
7547 // We have constant folded, but we might need to cast this again back
7548 // to the original (possibly legalized) type.
7549 EVT BVVT, BVEltVT;
7550 if (N1.getValueType() == VT) {
7551 BVVT = N1.getValueType();
7552 BVEltVT = BV1->getOperand(Num: 0).getValueType();
7553 } else {
7554 BVVT = N2.getValueType();
7555 BVEltVT = BV2->getOperand(Num: 0).getValueType();
7556 }
7557 unsigned BVEltBits = BVEltVT.getSizeInBits();
7558 SmallVector<APInt> DstBits;
7559 BitVector DstUndefs;
7560 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
7561 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
7562 SrcUndefElements: BitVector(RawBits.size(), false));
7563 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
7564 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
7565 if (DstUndefs[I])
7566 continue;
7567 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
7568 }
7569 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
7570 }
7571 }
7572 }
7573 }
7574
7575 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
7576 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
7577 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
7578 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
7579 APInt RHSVal;
7580 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
7581 APInt NewStep = Opcode == ISD::MUL
7582 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
7583 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
7584 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
7585 }
7586 }
7587
7588 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
7589 return !Op.getValueType().isVector() ||
7590 Op.getValueType().getVectorElementCount() == NumElts;
7591 };
7592
7593 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
7594 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
7595 Op.getOpcode() == ISD::BUILD_VECTOR ||
7596 Op.getOpcode() == ISD::SPLAT_VECTOR;
7597 };
7598
7599 // All operands must be vector types with the same number of elements as
7600 // the result type and must be either UNDEF or a build/splat vector
7601 // or UNDEF scalars.
7602 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
7603 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
7604 return SDValue();
7605
7606 // If we are comparing vectors, then the result needs to be a i1 boolean that
7607 // is then extended back to the legal result type depending on how booleans
7608 // are represented.
7609 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
7610 ISD::NodeType ExtendCode =
7611 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
7612 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
7613 : ISD::SIGN_EXTEND;
7614
7615 // Find legal integer scalar type for constant promotion and
7616 // ensure that its scalar size is at least as large as source.
7617 EVT LegalSVT = VT.getScalarType();
7618 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
7619 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
7620 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
7621 return SDValue();
7622 }
7623
7624 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
7625 // only have one operand to check. For fixed-length vector types we may have
7626 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
7627 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
7628
7629 // Constant fold each scalar lane separately.
7630 SmallVector<SDValue, 4> ScalarResults;
7631 for (unsigned I = 0; I != NumVectorElts; I++) {
7632 SmallVector<SDValue, 4> ScalarOps;
7633 for (SDValue Op : Ops) {
7634 EVT InSVT = Op.getValueType().getScalarType();
7635 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
7636 Op.getOpcode() != ISD::SPLAT_VECTOR) {
7637 if (Op.isUndef())
7638 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
7639 else
7640 ScalarOps.push_back(Elt: Op);
7641 continue;
7642 }
7643
7644 SDValue ScalarOp =
7645 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
7646 EVT ScalarVT = ScalarOp.getValueType();
7647
7648 // Build vector (integer) scalar operands may need implicit
7649 // truncation - do this before constant folding.
7650 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
7651 // Don't create illegally-typed nodes unless they're constants or undef
7652 // - if we fail to constant fold we can't guarantee the (dead) nodes
7653 // we're creating will be cleaned up before being visited for
7654 // legalization.
7655 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
7656 !isa<ConstantSDNode>(Val: ScalarOp) &&
7657 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
7658 TargetLowering::TypeLegal)
7659 return SDValue();
7660 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
7661 }
7662
7663 ScalarOps.push_back(Elt: ScalarOp);
7664 }
7665
7666 // Constant fold the scalar operands.
7667 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
7668
7669 // Scalar folding only succeeded if the result is a constant or UNDEF.
7670 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
7671 ScalarResult.getOpcode() != ISD::ConstantFP)
7672 return SDValue();
7673
7674 // Legalize the (integer) scalar constant if necessary. We only do
7675 // this once we know the folding succeeded, since otherwise we would
7676 // get a node with illegal type which has a user.
7677 if (LegalSVT != SVT)
7678 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
7679
7680 ScalarResults.push_back(Elt: ScalarResult);
7681 }
7682
7683 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
7684 : getBuildVector(VT, DL, Ops: ScalarResults);
7685 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
7686 return V;
7687}
7688
7689SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
7690 EVT VT, ArrayRef<SDValue> Ops) {
7691 // TODO: Add support for unary/ternary fp opcodes.
7692 if (Ops.size() != 2)
7693 return SDValue();
7694
7695 // TODO: We don't do any constant folding for strict FP opcodes here, but we
7696 // should. That will require dealing with a potentially non-default
7697 // rounding mode, checking the "opStatus" return value from the APFloat
7698 // math calculations, and possibly other variations.
7699 SDValue N1 = Ops[0];
7700 SDValue N2 = Ops[1];
7701 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
7702 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
7703 if (N1CFP && N2CFP) {
7704 APFloat C1 = N1CFP->getValueAPF(); // make copy
7705 const APFloat &C2 = N2CFP->getValueAPF();
7706 switch (Opcode) {
7707 case ISD::FADD:
7708 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7709 return getConstantFP(V: C1, DL, VT);
7710 case ISD::FSUB:
7711 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7712 return getConstantFP(V: C1, DL, VT);
7713 case ISD::FMUL:
7714 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7715 return getConstantFP(V: C1, DL, VT);
7716 case ISD::FDIV:
7717 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7718 return getConstantFP(V: C1, DL, VT);
7719 case ISD::FREM:
7720 C1.mod(RHS: C2);
7721 return getConstantFP(V: C1, DL, VT);
7722 case ISD::FCOPYSIGN:
7723 C1.copySign(RHS: C2);
7724 return getConstantFP(V: C1, DL, VT);
7725 case ISD::FMINNUM:
7726 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
7727 case ISD::FMAXNUM:
7728 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
7729 case ISD::FMINIMUM:
7730 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
7731 case ISD::FMAXIMUM:
7732 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
7733 case ISD::FMINIMUMNUM:
7734 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
7735 case ISD::FMAXIMUMNUM:
7736 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
7737 default: break;
7738 }
7739 }
7740 if (N1CFP && Opcode == ISD::FP_ROUND) {
7741 APFloat C1 = N1CFP->getValueAPF(); // make copy
7742 bool Unused;
7743 // This can return overflow, underflow, or inexact; we don't care.
7744 // FIXME need to be more flexible about rounding mode.
7745 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7746 losesInfo: &Unused);
7747 return getConstantFP(V: C1, DL, VT);
7748 }
7749
7750 switch (Opcode) {
7751 case ISD::FSUB:
7752 // -0.0 - undef --> undef (consistent with "fneg undef")
7753 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
7754 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
7755 return getUNDEF(VT);
7756 [[fallthrough]];
7757
7758 case ISD::FADD:
7759 case ISD::FMUL:
7760 case ISD::FDIV:
7761 case ISD::FREM:
7762 // If both operands are undef, the result is undef. If 1 operand is undef,
7763 // the result is NaN. This should match the behavior of the IR optimizer.
7764 if (N1.isUndef() && N2.isUndef())
7765 return getUNDEF(VT);
7766 if (N1.isUndef() || N2.isUndef())
7767 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
7768 }
7769 return SDValue();
7770}
7771
7772SDValue SelectionDAG::FoldConstantBuildVector(BuildVectorSDNode *BV,
7773 const SDLoc &DL, EVT DstEltVT) {
7774 EVT SrcEltVT = BV->getValueType(ResNo: 0).getVectorElementType();
7775
7776 // If this is already the right type, we're done.
7777 if (SrcEltVT == DstEltVT)
7778 return SDValue(BV, 0);
7779
7780 unsigned SrcBitSize = SrcEltVT.getSizeInBits();
7781 unsigned DstBitSize = DstEltVT.getSizeInBits();
7782
7783 // If this is a conversion of N elements of one type to N elements of another
7784 // type, convert each element. This handles FP<->INT cases.
7785 if (SrcBitSize == DstBitSize) {
7786 SmallVector<SDValue, 8> Ops;
7787 for (SDValue Op : BV->op_values()) {
7788 // If the vector element type is not legal, the BUILD_VECTOR operands
7789 // are promoted and implicitly truncated. Make that explicit here.
7790 if (Op.getValueType() != SrcEltVT)
7791 Op = getNode(Opcode: ISD::TRUNCATE, DL, VT: SrcEltVT, N1: Op);
7792 Ops.push_back(Elt: getBitcast(VT: DstEltVT, V: Op));
7793 }
7794 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT,
7795 NumElements: BV->getValueType(ResNo: 0).getVectorNumElements());
7796 return getBuildVector(VT, DL, Ops);
7797 }
7798
7799 // Otherwise, we're growing or shrinking the elements. To avoid having to
7800 // handle annoying details of growing/shrinking FP values, we convert them to
7801 // int first.
7802 if (SrcEltVT.isFloatingPoint()) {
7803 // Convert the input float vector to a int vector where the elements are the
7804 // same sizes.
7805 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: SrcEltVT.getSizeInBits());
7806 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7807 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7808 DstEltVT);
7809 return SDValue();
7810 }
7811
7812 // Now we know the input is an integer vector. If the output is a FP type,
7813 // convert to integer first, then to FP of the right size.
7814 if (DstEltVT.isFloatingPoint()) {
7815 EVT IntEltVT = EVT::getIntegerVT(Context&: *getContext(), BitWidth: DstEltVT.getSizeInBits());
7816 if (SDValue Tmp = FoldConstantBuildVector(BV, DL, DstEltVT: IntEltVT))
7817 return FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: Tmp), DL,
7818 DstEltVT);
7819 return SDValue();
7820 }
7821
7822 // Okay, we know the src/dst types are both integers of differing types.
7823 assert(SrcEltVT.isInteger() && DstEltVT.isInteger());
7824
7825 // Extract the constant raw bit data.
7826 BitVector UndefElements;
7827 SmallVector<APInt> RawBits;
7828 bool IsLE = getDataLayout().isLittleEndian();
7829 if (!BV->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: DstBitSize, RawBitElements&: RawBits, UndefElements))
7830 return SDValue();
7831
7832 SmallVector<SDValue, 8> Ops;
7833 for (unsigned I = 0, E = RawBits.size(); I != E; ++I) {
7834 if (UndefElements[I])
7835 Ops.push_back(Elt: getUNDEF(VT: DstEltVT));
7836 else
7837 Ops.push_back(Elt: getConstant(Val: RawBits[I], DL, VT: DstEltVT));
7838 }
7839
7840 EVT VT = EVT::getVectorVT(Context&: *getContext(), VT: DstEltVT, NumElements: Ops.size());
7841 return getBuildVector(VT, DL, Ops);
7842}
7843
7844SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
7845 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
7846
7847 // There's no need to assert on a byte-aligned pointer. All pointers are at
7848 // least byte aligned.
7849 if (A == Align(1))
7850 return Val;
7851
7852 SDVTList VTs = getVTList(VT: Val.getValueType());
7853 FoldingSetNodeID ID;
7854 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: VTs, OpList: {Val});
7855 ID.AddInteger(I: A.value());
7856
7857 void *IP = nullptr;
7858 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
7859 return SDValue(E, 0);
7860
7861 auto *N =
7862 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
7863 createOperands(Node: N, Vals: {Val});
7864
7865 CSEMap.InsertNode(N, InsertPos: IP);
7866 InsertNode(N);
7867
7868 SDValue V(N, 0);
7869 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7870 return V;
7871}
7872
7873SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7874 SDValue N1, SDValue N2) {
7875 SDNodeFlags Flags;
7876 if (Inserter)
7877 Flags = Inserter->getFlags();
7878 return getNode(Opcode, DL, VT, N1, N2, Flags);
7879}
7880
7881void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
7882 SDValue &N2) const {
7883 if (!TLI->isCommutativeBinOp(Opcode))
7884 return;
7885
7886 // Canonicalize:
7887 // binop(const, nonconst) -> binop(nonconst, const)
7888 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
7889 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
7890 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
7891 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
7892 if ((N1C && !N2C) || (N1CFP && !N2CFP))
7893 std::swap(a&: N1, b&: N2);
7894
7895 // Canonicalize:
7896 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
7897 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
7898 N2.getOpcode() == ISD::STEP_VECTOR)
7899 std::swap(a&: N1, b&: N2);
7900}
7901
7902SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7903 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
7904 assert(N1.getOpcode() != ISD::DELETED_NODE &&
7905 N2.getOpcode() != ISD::DELETED_NODE &&
7906 "Operand is DELETED_NODE!");
7907
7908 canonicalizeCommutativeBinop(Opcode, N1, N2);
7909
7910 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
7911 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
7912
7913 // Don't allow undefs in vector splats - we might be returning N2 when folding
7914 // to zero etc.
7915 ConstantSDNode *N2CV =
7916 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
7917
7918 switch (Opcode) {
7919 default: break;
7920 case ISD::TokenFactor:
7921 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
7922 N2.getValueType() == MVT::Other && "Invalid token factor!");
7923 // Fold trivial token factors.
7924 if (N1.getOpcode() == ISD::EntryToken) return N2;
7925 if (N2.getOpcode() == ISD::EntryToken) return N1;
7926 if (N1 == N2) return N1;
7927 break;
7928 case ISD::BUILD_VECTOR: {
7929 // Attempt to simplify BUILD_VECTOR.
7930 SDValue Ops[] = {N1, N2};
7931 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7932 return V;
7933 break;
7934 }
7935 case ISD::CONCAT_VECTORS: {
7936 SDValue Ops[] = {N1, N2};
7937 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
7938 return V;
7939 break;
7940 }
7941 case ISD::AND:
7942 assert(VT.isInteger() && "This operator does not apply to FP types!");
7943 assert(N1.getValueType() == N2.getValueType() &&
7944 N1.getValueType() == VT && "Binary operator types must match!");
7945 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
7946 // worth handling here.
7947 if (N2CV && N2CV->isZero())
7948 return N2;
7949 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
7950 return N1;
7951 break;
7952 case ISD::OR:
7953 case ISD::XOR:
7954 case ISD::ADD:
7955 case ISD::PTRADD:
7956 case ISD::SUB:
7957 assert(VT.isInteger() && "This operator does not apply to FP types!");
7958 assert(N1.getValueType() == N2.getValueType() &&
7959 N1.getValueType() == VT && "Binary operator types must match!");
7960 // The equal operand types requirement is unnecessarily strong for PTRADD.
7961 // However, the SelectionDAGBuilder does not generate PTRADDs with different
7962 // operand types, and we'd need to re-implement GEP's non-standard wrapping
7963 // logic everywhere where PTRADDs may be folded or combined to properly
7964 // support them. If/when we introduce pointer types to the SDAG, we will
7965 // need to relax this constraint.
7966
7967 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
7968 // it's worth handling here.
7969 if (N2CV && N2CV->isZero())
7970 return N1;
7971 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
7972 VT.getScalarType() == MVT::i1)
7973 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
7974 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
7975 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
7976 N2.getOpcode() == ISD::VSCALE) {
7977 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
7978 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
7979 return getVScale(DL, VT, MulImm: C1 + C2);
7980 }
7981 break;
7982 case ISD::MUL:
7983 assert(VT.isInteger() && "This operator does not apply to FP types!");
7984 assert(N1.getValueType() == N2.getValueType() &&
7985 N1.getValueType() == VT && "Binary operator types must match!");
7986 if (VT.getScalarType() == MVT::i1)
7987 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
7988 if (N2CV && N2CV->isZero())
7989 return N2;
7990 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7991 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
7992 const APInt &N2CImm = N2C->getAPIntValue();
7993 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
7994 }
7995 break;
7996 case ISD::UDIV:
7997 case ISD::UREM:
7998 case ISD::MULHU:
7999 case ISD::MULHS:
8000 case ISD::SDIV:
8001 case ISD::SREM:
8002 case ISD::SADDSAT:
8003 case ISD::SSUBSAT:
8004 case ISD::UADDSAT:
8005 case ISD::USUBSAT:
8006 assert(VT.isInteger() && "This operator does not apply to FP types!");
8007 assert(N1.getValueType() == N2.getValueType() &&
8008 N1.getValueType() == VT && "Binary operator types must match!");
8009 if (VT.getScalarType() == MVT::i1) {
8010 // fold (add_sat x, y) -> (or x, y) for bool types.
8011 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
8012 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8013 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
8014 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
8015 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
8016 }
8017 break;
8018 case ISD::SCMP:
8019 case ISD::UCMP:
8020 assert(N1.getValueType() == N2.getValueType() &&
8021 "Types of operands of UCMP/SCMP must match");
8022 assert(N1.getValueType().isVector() == VT.isVector() &&
8023 "Operands and return type of must both be scalars or vectors");
8024 if (VT.isVector())
8025 assert(VT.getVectorElementCount() ==
8026 N1.getValueType().getVectorElementCount() &&
8027 "Result and operands must have the same number of elements");
8028 break;
8029 case ISD::AVGFLOORS:
8030 case ISD::AVGFLOORU:
8031 case ISD::AVGCEILS:
8032 case ISD::AVGCEILU:
8033 assert(VT.isInteger() && "This operator does not apply to FP types!");
8034 assert(N1.getValueType() == N2.getValueType() &&
8035 N1.getValueType() == VT && "Binary operator types must match!");
8036 break;
8037 case ISD::ABDS:
8038 case ISD::ABDU:
8039 assert(VT.isInteger() && "This operator does not apply to FP types!");
8040 assert(N1.getValueType() == N2.getValueType() &&
8041 N1.getValueType() == VT && "Binary operator types must match!");
8042 if (VT.getScalarType() == MVT::i1)
8043 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
8044 break;
8045 case ISD::SMIN:
8046 case ISD::UMAX:
8047 assert(VT.isInteger() && "This operator does not apply to FP types!");
8048 assert(N1.getValueType() == N2.getValueType() &&
8049 N1.getValueType() == VT && "Binary operator types must match!");
8050 if (VT.getScalarType() == MVT::i1)
8051 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
8052 break;
8053 case ISD::SMAX:
8054 case ISD::UMIN:
8055 assert(VT.isInteger() && "This operator does not apply to FP types!");
8056 assert(N1.getValueType() == N2.getValueType() &&
8057 N1.getValueType() == VT && "Binary operator types must match!");
8058 if (VT.getScalarType() == MVT::i1)
8059 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
8060 break;
8061 case ISD::FADD:
8062 case ISD::FSUB:
8063 case ISD::FMUL:
8064 case ISD::FDIV:
8065 case ISD::FREM:
8066 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
8067 assert(N1.getValueType() == N2.getValueType() &&
8068 N1.getValueType() == VT && "Binary operator types must match!");
8069 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
8070 return V;
8071 break;
8072 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
8073 assert(N1.getValueType() == VT &&
8074 N1.getValueType().isFloatingPoint() &&
8075 N2.getValueType().isFloatingPoint() &&
8076 "Invalid FCOPYSIGN!");
8077 break;
8078 case ISD::SHL:
8079 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
8080 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
8081 const APInt &ShiftImm = N2C->getAPIntValue();
8082 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
8083 }
8084 [[fallthrough]];
8085 case ISD::SRA:
8086 case ISD::SRL:
8087 if (SDValue V = simplifyShift(X: N1, Y: N2))
8088 return V;
8089 [[fallthrough]];
8090 case ISD::ROTL:
8091 case ISD::ROTR:
8092 case ISD::SSHLSAT:
8093 case ISD::USHLSAT:
8094 assert(VT == N1.getValueType() &&
8095 "Shift operators return type must be the same as their first arg");
8096 assert(VT.isInteger() && N2.getValueType().isInteger() &&
8097 "Shifts only work on integers");
8098 assert((!VT.isVector() || VT == N2.getValueType()) &&
8099 "Vector shift amounts must be in the same as their first arg");
8100 // Verify that the shift amount VT is big enough to hold valid shift
8101 // amounts. This catches things like trying to shift an i1024 value by an
8102 // i8, which is easy to fall into in generic code that uses
8103 // TLI.getShiftAmount().
8104 assert(N2.getValueType().getScalarSizeInBits() >=
8105 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
8106 "Invalid use of small shift amount with oversized value!");
8107
8108 // Always fold shifts of i1 values so the code generator doesn't need to
8109 // handle them. Since we know the size of the shift has to be less than the
8110 // size of the value, the shift/rotate count is guaranteed to be zero.
8111 if (VT == MVT::i1)
8112 return N1;
8113 if (N2CV && N2CV->isZero())
8114 return N1;
8115 break;
8116 case ISD::FP_ROUND:
8117 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
8118 VT.bitsLE(N1.getValueType()) && N2C &&
8119 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
8120 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
8121 if (N1.getValueType() == VT) return N1; // noop conversion.
8122 break;
8123 case ISD::AssertNoFPClass: {
8124 assert(N1.getValueType().isFloatingPoint() &&
8125 "AssertNoFPClass is used for a non-floating type");
8126 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
8127 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
8128 assert(llvm::to_underlying(NoFPClass) <=
8129 BitmaskEnumDetail::Mask<FPClassTest>() &&
8130 "FPClassTest value too large");
8131 (void)NoFPClass;
8132 break;
8133 }
8134 case ISD::AssertSext:
8135 case ISD::AssertZext: {
8136 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8137 assert(VT == N1.getValueType() && "Not an inreg extend!");
8138 assert(VT.isInteger() && EVT.isInteger() &&
8139 "Cannot *_EXTEND_INREG FP types");
8140 assert(!EVT.isVector() &&
8141 "AssertSExt/AssertZExt type should be the vector element type "
8142 "rather than the vector type!");
8143 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
8144 if (VT.getScalarType() == EVT) return N1; // noop assertion.
8145 break;
8146 }
8147 case ISD::SIGN_EXTEND_INREG: {
8148 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
8149 assert(VT == N1.getValueType() && "Not an inreg extend!");
8150 assert(VT.isInteger() && EVT.isInteger() &&
8151 "Cannot *_EXTEND_INREG FP types");
8152 assert(EVT.isVector() == VT.isVector() &&
8153 "SIGN_EXTEND_INREG type should be vector iff the operand "
8154 "type is vector!");
8155 assert((!EVT.isVector() ||
8156 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
8157 "Vector element counts must match in SIGN_EXTEND_INREG");
8158 assert(EVT.getScalarType().bitsLE(VT.getScalarType()) && "Not extending!");
8159 if (EVT == VT) return N1; // Not actually extending
8160 break;
8161 }
8162 case ISD::FP_TO_SINT_SAT:
8163 case ISD::FP_TO_UINT_SAT: {
8164 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
8165 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
8166 assert(N1.getValueType().isVector() == VT.isVector() &&
8167 "FP_TO_*INT_SAT type should be vector iff the operand type is "
8168 "vector!");
8169 assert((!VT.isVector() || VT.getVectorElementCount() ==
8170 N1.getValueType().getVectorElementCount()) &&
8171 "Vector element counts must match in FP_TO_*INT_SAT");
8172 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
8173 "Type to saturate to must be a scalar.");
8174 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
8175 "Not extending!");
8176 break;
8177 }
8178 case ISD::EXTRACT_VECTOR_ELT:
8179 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
8180 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
8181 element type of the vector.");
8182
8183 // Extract from an undefined value or using an undefined index is undefined.
8184 if (N1.isUndef() || N2.isUndef())
8185 return getUNDEF(VT);
8186
8187 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
8188 // vectors. For scalable vectors we will provide appropriate support for
8189 // dealing with arbitrary indices.
8190 if (N2C && N1.getValueType().isFixedLengthVector() &&
8191 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
8192 return getUNDEF(VT);
8193
8194 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
8195 // expanding copies of large vectors from registers. This only works for
8196 // fixed length vectors, since we need to know the exact number of
8197 // elements.
8198 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
8199 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
8200 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
8201 return getExtractVectorElt(DL, VT,
8202 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
8203 Idx: N2C->getZExtValue() % Factor);
8204 }
8205
8206 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
8207 // lowering is expanding large vector constants.
8208 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
8209 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
8210 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
8211 N1.getValueType().isFixedLengthVector()) &&
8212 "BUILD_VECTOR used for scalable vectors");
8213 unsigned Index =
8214 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
8215 SDValue Elt = N1.getOperand(i: Index);
8216
8217 if (VT != Elt.getValueType())
8218 // If the vector element type is not legal, the BUILD_VECTOR operands
8219 // are promoted and implicitly truncated, and the result implicitly
8220 // extended. Make that explicit here.
8221 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
8222
8223 return Elt;
8224 }
8225
8226 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
8227 // operations are lowered to scalars.
8228 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
8229 // If the indices are the same, return the inserted element else
8230 // if the indices are known different, extract the element from
8231 // the original vector.
8232 SDValue N1Op2 = N1.getOperand(i: 2);
8233 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
8234
8235 if (N1Op2C && N2C) {
8236 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
8237 if (VT == N1.getOperand(i: 1).getValueType())
8238 return N1.getOperand(i: 1);
8239 if (VT.isFloatingPoint()) {
8240 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
8241 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
8242 }
8243 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
8244 }
8245 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
8246 }
8247 }
8248
8249 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
8250 // when vector types are scalarized and v1iX is legal.
8251 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
8252 // Here we are completely ignoring the extract element index (N2),
8253 // which is fine for fixed width vectors, since any index other than 0
8254 // is undefined anyway. However, this cannot be ignored for scalable
8255 // vectors - in theory we could support this, but we don't want to do this
8256 // without a profitability check.
8257 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8258 N1.getValueType().isFixedLengthVector() &&
8259 N1.getValueType().getVectorNumElements() == 1) {
8260 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
8261 N2: N1.getOperand(i: 1));
8262 }
8263 break;
8264 case ISD::EXTRACT_ELEMENT:
8265 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
8266 assert(!N1.getValueType().isVector() && !VT.isVector() &&
8267 (N1.getValueType().isInteger() == VT.isInteger()) &&
8268 N1.getValueType() != VT &&
8269 "Wrong types for EXTRACT_ELEMENT!");
8270
8271 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
8272 // 64-bit integers into 32-bit parts. Instead of building the extract of
8273 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
8274 if (N1.getOpcode() == ISD::BUILD_PAIR)
8275 return N1.getOperand(i: N2C->getZExtValue());
8276
8277 // EXTRACT_ELEMENT of a constant int is also very common.
8278 if (N1C) {
8279 unsigned ElementSize = VT.getSizeInBits();
8280 unsigned Shift = ElementSize * N2C->getZExtValue();
8281 const APInt &Val = N1C->getAPIntValue();
8282 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
8283 }
8284 break;
8285 case ISD::EXTRACT_SUBVECTOR: {
8286 EVT N1VT = N1.getValueType();
8287 assert(VT.isVector() && N1VT.isVector() &&
8288 "Extract subvector VTs must be vectors!");
8289 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
8290 "Extract subvector VTs must have the same element type!");
8291 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
8292 "Cannot extract a scalable vector from a fixed length vector!");
8293 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8294 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
8295 "Extract subvector must be from larger vector to smaller vector!");
8296 assert(N2C && "Extract subvector index must be a constant");
8297 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
8298 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
8299 N1VT.getVectorMinNumElements()) &&
8300 "Extract subvector overflow!");
8301 assert(N2C->getAPIntValue().getBitWidth() ==
8302 TLI->getVectorIdxWidth(getDataLayout()) &&
8303 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
8304 assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
8305 "Extract index is not a multiple of the output vector length");
8306
8307 // Trivial extraction.
8308 if (VT == N1VT)
8309 return N1;
8310
8311 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
8312 if (N1.isUndef())
8313 return getUNDEF(VT);
8314
8315 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
8316 // the concat have the same type as the extract.
8317 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
8318 VT == N1.getOperand(i: 0).getValueType()) {
8319 unsigned Factor = VT.getVectorMinNumElements();
8320 return N1.getOperand(i: N2C->getZExtValue() / Factor);
8321 }
8322
8323 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
8324 // during shuffle legalization.
8325 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
8326 VT == N1.getOperand(i: 1).getValueType())
8327 return N1.getOperand(i: 1);
8328 break;
8329 }
8330 }
8331
8332 if (N1.getOpcode() == ISD::POISON || N2.getOpcode() == ISD::POISON) {
8333 switch (Opcode) {
8334 case ISD::XOR:
8335 case ISD::ADD:
8336 case ISD::PTRADD:
8337 case ISD::SUB:
8338 case ISD::SIGN_EXTEND_INREG:
8339 case ISD::UDIV:
8340 case ISD::SDIV:
8341 case ISD::UREM:
8342 case ISD::SREM:
8343 case ISD::MUL:
8344 case ISD::AND:
8345 case ISD::SSUBSAT:
8346 case ISD::USUBSAT:
8347 case ISD::UMIN:
8348 case ISD::OR:
8349 case ISD::SADDSAT:
8350 case ISD::UADDSAT:
8351 case ISD::UMAX:
8352 case ISD::SMAX:
8353 case ISD::SMIN:
8354 // fold op(arg1, poison) -> poison, fold op(poison, arg2) -> poison.
8355 return N2.getOpcode() == ISD::POISON ? N2 : N1;
8356 }
8357 }
8358
8359 // Canonicalize an UNDEF to the RHS, even over a constant.
8360 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() != ISD::UNDEF) {
8361 if (TLI->isCommutativeBinOp(Opcode)) {
8362 std::swap(a&: N1, b&: N2);
8363 } else {
8364 switch (Opcode) {
8365 case ISD::PTRADD:
8366 case ISD::SUB:
8367 // fold op(undef, non_undef_arg2) -> undef.
8368 return N1;
8369 case ISD::SIGN_EXTEND_INREG:
8370 case ISD::UDIV:
8371 case ISD::SDIV:
8372 case ISD::UREM:
8373 case ISD::SREM:
8374 case ISD::SSUBSAT:
8375 case ISD::USUBSAT:
8376 // fold op(undef, non_undef_arg2) -> 0.
8377 return getConstant(Val: 0, DL, VT);
8378 }
8379 }
8380 }
8381
8382 // Fold a bunch of operators when the RHS is undef.
8383 if (N2.getOpcode() == ISD::UNDEF) {
8384 switch (Opcode) {
8385 case ISD::XOR:
8386 if (N1.getOpcode() == ISD::UNDEF)
8387 // Handle undef ^ undef -> 0 special case. This is a common
8388 // idiom (misuse).
8389 return getConstant(Val: 0, DL, VT);
8390 [[fallthrough]];
8391 case ISD::ADD:
8392 case ISD::PTRADD:
8393 case ISD::SUB:
8394 // fold op(arg1, undef) -> undef.
8395 return N2;
8396 case ISD::UDIV:
8397 case ISD::SDIV:
8398 case ISD::UREM:
8399 case ISD::SREM:
8400 // fold op(arg1, undef) -> poison.
8401 return getPOISON(VT);
8402 case ISD::MUL:
8403 case ISD::AND:
8404 case ISD::SSUBSAT:
8405 case ISD::USUBSAT:
8406 case ISD::UMIN:
8407 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> 0.
8408 return N1.getOpcode() == ISD::UNDEF ? N2 : getConstant(Val: 0, DL, VT);
8409 case ISD::OR:
8410 case ISD::SADDSAT:
8411 case ISD::UADDSAT:
8412 case ISD::UMAX:
8413 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> -1.
8414 return N1.getOpcode() == ISD::UNDEF ? N2 : getAllOnesConstant(DL, VT);
8415 case ISD::SMAX:
8416 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MAX_INT.
8417 return N1.getOpcode() == ISD::UNDEF
8418 ? N2
8419 : getConstant(
8420 Val: APInt::getSignedMaxValue(numBits: VT.getScalarSizeInBits()), DL,
8421 VT);
8422 case ISD::SMIN:
8423 // fold op(undef, undef) -> undef, fold op(arg1, undef) -> MIN_INT.
8424 return N1.getOpcode() == ISD::UNDEF
8425 ? N2
8426 : getConstant(
8427 Val: APInt::getSignedMinValue(numBits: VT.getScalarSizeInBits()), DL,
8428 VT);
8429 }
8430 }
8431
8432 // Perform trivial constant folding.
8433 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
8434 return SV;
8435
8436 // Memoize this node if possible.
8437 SDNode *N;
8438 SDVTList VTs = getVTList(VT);
8439 SDValue Ops[] = {N1, N2};
8440 if (VT != MVT::Glue) {
8441 FoldingSetNodeID ID;
8442 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8443 void *IP = nullptr;
8444 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8445 E->intersectFlagsWith(Flags);
8446 return SDValue(E, 0);
8447 }
8448
8449 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8450 N->setFlags(Flags);
8451 createOperands(Node: N, Vals: Ops);
8452 CSEMap.InsertNode(N, InsertPos: IP);
8453 } else {
8454 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8455 createOperands(Node: N, Vals: Ops);
8456 }
8457
8458 InsertNode(N);
8459 SDValue V = SDValue(N, 0);
8460 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8461 return V;
8462}
8463
8464SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8465 SDValue N1, SDValue N2, SDValue N3) {
8466 SDNodeFlags Flags;
8467 if (Inserter)
8468 Flags = Inserter->getFlags();
8469 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
8470}
8471
8472SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8473 SDValue N1, SDValue N2, SDValue N3,
8474 const SDNodeFlags Flags) {
8475 assert(N1.getOpcode() != ISD::DELETED_NODE &&
8476 N2.getOpcode() != ISD::DELETED_NODE &&
8477 N3.getOpcode() != ISD::DELETED_NODE &&
8478 "Operand is DELETED_NODE!");
8479 // Perform various simplifications.
8480 switch (Opcode) {
8481 case ISD::BUILD_VECTOR: {
8482 // Attempt to simplify BUILD_VECTOR.
8483 SDValue Ops[] = {N1, N2, N3};
8484 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
8485 return V;
8486 break;
8487 }
8488 case ISD::CONCAT_VECTORS: {
8489 SDValue Ops[] = {N1, N2, N3};
8490 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
8491 return V;
8492 break;
8493 }
8494 case ISD::SETCC: {
8495 assert(VT.isInteger() && "SETCC result type must be an integer!");
8496 assert(N1.getValueType() == N2.getValueType() &&
8497 "SETCC operands must have the same type!");
8498 assert(VT.isVector() == N1.getValueType().isVector() &&
8499 "SETCC type should be vector iff the operand type is vector!");
8500 assert((!VT.isVector() || VT.getVectorElementCount() ==
8501 N1.getValueType().getVectorElementCount()) &&
8502 "SETCC vector element counts must match!");
8503 // Use FoldSetCC to simplify SETCC's.
8504 if (SDValue V =
8505 FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL, Flags))
8506 return V;
8507 break;
8508 }
8509 case ISD::SELECT:
8510 case ISD::VSELECT:
8511 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
8512 return V;
8513 break;
8514 case ISD::VECTOR_SHUFFLE:
8515 llvm_unreachable("should use getVectorShuffle constructor!");
8516 case ISD::VECTOR_SPLICE_LEFT:
8517 if (isNullConstant(V: N3))
8518 return N1;
8519 break;
8520 case ISD::VECTOR_SPLICE_RIGHT:
8521 if (isNullConstant(V: N3))
8522 return N2;
8523 break;
8524 case ISD::INSERT_VECTOR_ELT: {
8525 assert(VT.isVector() && VT == N1.getValueType() &&
8526 "INSERT_VECTOR_ELT vector type mismatch");
8527 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
8528 "INSERT_VECTOR_ELT scalar fp/int mismatch");
8529 assert((!VT.isFloatingPoint() ||
8530 VT.getVectorElementType() == N2.getValueType()) &&
8531 "INSERT_VECTOR_ELT fp scalar type mismatch");
8532 assert((!VT.isInteger() ||
8533 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
8534 "INSERT_VECTOR_ELT int scalar size mismatch");
8535
8536 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
8537 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
8538 // for scalable vectors where we will generate appropriate code to
8539 // deal with out-of-bounds cases correctly.
8540 if (N3C && VT.isFixedLengthVector() &&
8541 N3C->getZExtValue() >= VT.getVectorNumElements())
8542 return getUNDEF(VT);
8543
8544 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
8545 if (N3.isUndef())
8546 return getUNDEF(VT);
8547
8548 // If inserting poison, just use the input vector.
8549 if (N2.getOpcode() == ISD::POISON)
8550 return N1;
8551
8552 // Inserting undef into undef/poison is still undef.
8553 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8554 return getUNDEF(VT);
8555
8556 // If the inserted element is an UNDEF, just use the input vector.
8557 // But not if skipping the insert could make the result more poisonous.
8558 if (N2.isUndef()) {
8559 if (N3C && VT.isFixedLengthVector()) {
8560 APInt EltMask =
8561 APInt::getOneBitSet(numBits: VT.getVectorNumElements(), BitNo: N3C->getZExtValue());
8562 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8563 return N1;
8564 } else if (isGuaranteedNotToBePoison(Op: N1))
8565 return N1;
8566 }
8567 break;
8568 }
8569 case ISD::INSERT_SUBVECTOR: {
8570 // If inserting poison, just use the input vector,
8571 if (N2.getOpcode() == ISD::POISON)
8572 return N1;
8573
8574 // Inserting undef into undef/poison is still undef.
8575 if (N2.getOpcode() == ISD::UNDEF && N1.isUndef())
8576 return getUNDEF(VT);
8577
8578 EVT N2VT = N2.getValueType();
8579 assert(VT == N1.getValueType() &&
8580 "Dest and insert subvector source types must match!");
8581 assert(VT.isVector() && N2VT.isVector() &&
8582 "Insert subvector VTs must be vectors!");
8583 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
8584 "Insert subvector VTs must have the same element type!");
8585 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
8586 "Cannot insert a scalable vector into a fixed length vector!");
8587 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8588 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
8589 "Insert subvector must be from smaller vector to larger vector!");
8590 assert(isa<ConstantSDNode>(N3) &&
8591 "Insert subvector index must be constant");
8592 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
8593 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
8594 VT.getVectorMinNumElements()) &&
8595 "Insert subvector overflow!");
8596 assert(N3->getAsAPIntVal().getBitWidth() ==
8597 TLI->getVectorIdxWidth(getDataLayout()) &&
8598 "Constant index for INSERT_SUBVECTOR has an invalid size");
8599
8600 // Trivial insertion.
8601 if (VT == N2VT)
8602 return N2;
8603
8604 // If this is an insert of an extracted vector into an undef/poison vector,
8605 // we can just use the input to the extract. But not if skipping the
8606 // extract+insert could make the result more poisonous.
8607 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8608 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT) {
8609 if (N1.getOpcode() == ISD::POISON)
8610 return N2.getOperand(i: 0);
8611 if (VT.isFixedLengthVector() && N2VT.isFixedLengthVector()) {
8612 unsigned LoBit = N3->getAsZExtVal();
8613 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8614 APInt EltMask =
8615 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8616 if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0), DemandedElts: ~EltMask))
8617 return N2.getOperand(i: 0);
8618 } else if (isGuaranteedNotToBePoison(Op: N2.getOperand(i: 0)))
8619 return N2.getOperand(i: 0);
8620 }
8621
8622 // If the inserted subvector is UNDEF, just use the input vector.
8623 // But not if skipping the insert could make the result more poisonous.
8624 if (N2.isUndef()) {
8625 if (VT.isFixedLengthVector()) {
8626 unsigned LoBit = N3->getAsZExtVal();
8627 unsigned HiBit = LoBit + N2VT.getVectorNumElements();
8628 APInt EltMask =
8629 APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: LoBit, hiBit: HiBit);
8630 if (isGuaranteedNotToBePoison(Op: N1, DemandedElts: EltMask))
8631 return N1;
8632 } else if (isGuaranteedNotToBePoison(Op: N1))
8633 return N1;
8634 }
8635 break;
8636 }
8637 case ISD::BITCAST:
8638 // Fold bit_convert nodes from a type to themselves.
8639 if (N1.getValueType() == VT)
8640 return N1;
8641 break;
8642 case ISD::VP_TRUNCATE:
8643 case ISD::VP_SIGN_EXTEND:
8644 case ISD::VP_ZERO_EXTEND:
8645 // Don't create noop casts.
8646 if (N1.getValueType() == VT)
8647 return N1;
8648 break;
8649 case ISD::VECTOR_COMPRESS: {
8650 [[maybe_unused]] EVT VecVT = N1.getValueType();
8651 [[maybe_unused]] EVT MaskVT = N2.getValueType();
8652 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
8653 assert(VT == VecVT && "Vector and result type don't match.");
8654 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
8655 "All inputs must be vectors.");
8656 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
8657 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
8658 "Vector and mask must have same number of elements.");
8659
8660 if (N1.isUndef() || N2.isUndef())
8661 return N3;
8662
8663 break;
8664 }
8665 case ISD::PARTIAL_REDUCE_UMLA:
8666 case ISD::PARTIAL_REDUCE_SMLA:
8667 case ISD::PARTIAL_REDUCE_SUMLA:
8668 case ISD::PARTIAL_REDUCE_FMLA: {
8669 [[maybe_unused]] EVT AccVT = N1.getValueType();
8670 [[maybe_unused]] EVT Input1VT = N2.getValueType();
8671 [[maybe_unused]] EVT Input2VT = N3.getValueType();
8672 assert(Input1VT.isVector() && Input1VT == Input2VT &&
8673 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8674 "node to have the same type!");
8675 assert(VT.isVector() && VT == AccVT &&
8676 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
8677 "the same type as its result!");
8678 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
8679 AccVT.getVectorElementCount()) &&
8680 "Expected the element count of the second and third operands of the "
8681 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
8682 "element count of the first operand and the result!");
8683 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
8684 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8685 "node to have an element type which is the same as or smaller than "
8686 "the element type of the first operand and result!");
8687 break;
8688 }
8689 }
8690
8691 // Perform trivial constant folding for arithmetic operators.
8692 switch (Opcode) {
8693 case ISD::FMA:
8694 case ISD::FMAD:
8695 case ISD::SETCC:
8696 case ISD::FSHL:
8697 case ISD::FSHR:
8698 if (SDValue SV =
8699 FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2, N3}, Flags))
8700 return SV;
8701 break;
8702 }
8703
8704 // Memoize node if it doesn't produce a glue result.
8705 SDNode *N;
8706 SDVTList VTs = getVTList(VT);
8707 SDValue Ops[] = {N1, N2, N3};
8708 if (VT != MVT::Glue) {
8709 FoldingSetNodeID ID;
8710 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8711 void *IP = nullptr;
8712 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8713 E->intersectFlagsWith(Flags);
8714 return SDValue(E, 0);
8715 }
8716
8717 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8718 N->setFlags(Flags);
8719 createOperands(Node: N, Vals: Ops);
8720 CSEMap.InsertNode(N, InsertPos: IP);
8721 } else {
8722 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8723 createOperands(Node: N, Vals: Ops);
8724 }
8725
8726 InsertNode(N);
8727 SDValue V = SDValue(N, 0);
8728 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8729 return V;
8730}
8731
8732SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8733 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8734 const SDNodeFlags Flags) {
8735 SDValue Ops[] = { N1, N2, N3, N4 };
8736 return getNode(Opcode, DL, VT, Ops, Flags);
8737}
8738
8739SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8740 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
8741 SDNodeFlags Flags;
8742 if (Inserter)
8743 Flags = Inserter->getFlags();
8744 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
8745}
8746
8747SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8748 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8749 SDValue N5, const SDNodeFlags Flags) {
8750 SDValue Ops[] = { N1, N2, N3, N4, N5 };
8751 return getNode(Opcode, DL, VT, Ops, Flags);
8752}
8753
8754SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8755 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8756 SDValue N5) {
8757 SDNodeFlags Flags;
8758 if (Inserter)
8759 Flags = Inserter->getFlags();
8760 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
8761}
8762
8763/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
8764/// the incoming stack arguments to be loaded from the stack.
8765SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
8766 SmallVector<SDValue, 8> ArgChains;
8767
8768 // Include the original chain at the beginning of the list. When this is
8769 // used by target LowerCall hooks, this helps legalize find the
8770 // CALLSEQ_BEGIN node.
8771 ArgChains.push_back(Elt: Chain);
8772
8773 // Add a chain value for each stack argument.
8774 for (SDNode *U : getEntryNode().getNode()->users())
8775 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
8776 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
8777 if (FI->getIndex() < 0)
8778 ArgChains.push_back(Elt: SDValue(L, 1));
8779
8780 // Build a tokenfactor for all the chains.
8781 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
8782}
8783
8784/// getMemsetValue - Vectorized representation of the memset value
8785/// operand.
8786static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
8787 const SDLoc &dl) {
8788 assert(!Value.isUndef());
8789
8790 unsigned NumBits = VT.getScalarSizeInBits();
8791 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
8792 assert(C->getAPIntValue().getBitWidth() == 8);
8793 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
8794 if (VT.isInteger()) {
8795 bool IsOpaque = VT.getSizeInBits() > 64 ||
8796 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
8797 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
8798 }
8799 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
8800 }
8801
8802 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
8803 EVT IntVT = VT.getScalarType();
8804 if (!IntVT.isInteger())
8805 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
8806
8807 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
8808 if (NumBits > 8) {
8809 // Use a multiplication with 0x010101... to extend the input to the
8810 // required length.
8811 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
8812 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
8813 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
8814 }
8815
8816 if (VT != Value.getValueType() && !VT.isInteger())
8817 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
8818 if (VT != Value.getValueType())
8819 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
8820
8821 return Value;
8822}
8823
8824/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
8825/// used when a memcpy is turned into a memset when the source is a constant
8826/// string ptr.
8827static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
8828 const TargetLowering &TLI,
8829 const ConstantDataArraySlice &Slice) {
8830 // Handle vector with all elements zero.
8831 if (Slice.Array == nullptr) {
8832 if (VT.isInteger())
8833 return DAG.getConstant(Val: 0, DL: dl, VT);
8834 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
8835 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
8836 }
8837
8838 assert(!VT.isVector() && "Can't handle vector type here!");
8839 unsigned NumVTBits = VT.getSizeInBits();
8840 unsigned NumVTBytes = NumVTBits / 8;
8841 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
8842
8843 APInt Val(NumVTBits, 0);
8844 if (DAG.getDataLayout().isLittleEndian()) {
8845 for (unsigned i = 0; i != NumBytes; ++i)
8846 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
8847 } else {
8848 for (unsigned i = 0; i != NumBytes; ++i)
8849 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
8850 }
8851
8852 // If the "cost" of materializing the integer immediate is less than the cost
8853 // of a load, then it is cost effective to turn the load into the immediate.
8854 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
8855 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
8856 return DAG.getConstant(Val, DL: dl, VT);
8857 return SDValue();
8858}
8859
8860SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
8861 const SDLoc &DL,
8862 const SDNodeFlags Flags) {
8863 SDValue Index = getTypeSize(DL, VT: Base.getValueType(), TS: Offset);
8864 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
8865}
8866
8867SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
8868 const SDLoc &DL,
8869 const SDNodeFlags Flags) {
8870 assert(Offset.getValueType().isInteger());
8871 EVT BasePtrVT = Ptr.getValueType();
8872 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
8873 PtrVT: BasePtrVT))
8874 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
8875 // InBounds only applies to PTRADD, don't set it if we generate ADD.
8876 SDNodeFlags AddFlags = Flags;
8877 AddFlags.setInBounds(false);
8878 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags: AddFlags);
8879}
8880
8881/// Returns true if memcpy source is constant data.
8882static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
8883 uint64_t SrcDelta = 0;
8884 GlobalAddressSDNode *G = nullptr;
8885 if (Src.getOpcode() == ISD::GlobalAddress)
8886 G = cast<GlobalAddressSDNode>(Val&: Src);
8887 else if (Src->isAnyAdd() &&
8888 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
8889 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
8890 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
8891 SrcDelta = Src.getConstantOperandVal(i: 1);
8892 }
8893 if (!G)
8894 return false;
8895
8896 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
8897 Offset: SrcDelta + G->getOffset());
8898}
8899
8900static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
8901 SelectionDAG &DAG) {
8902 // On Darwin, -Os means optimize for size without hurting performance, so
8903 // only really optimize for size when -Oz (MinSize) is used.
8904 if (MF.getTarget().getTargetTriple().isOSDarwin())
8905 return MF.getFunction().hasMinSize();
8906 return DAG.shouldOptForSize();
8907}
8908
8909static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
8910 SmallVector<SDValue, 32> &OutChains, unsigned From,
8911 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
8912 SmallVector<SDValue, 16> &OutStoreChains) {
8913 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
8914 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
8915 SmallVector<SDValue, 16> GluedLoadChains;
8916 for (unsigned i = From; i < To; ++i) {
8917 OutChains.push_back(Elt: OutLoadChains[i]);
8918 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
8919 }
8920
8921 // Chain for all loads.
8922 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
8923 Ops: GluedLoadChains);
8924
8925 for (unsigned i = From; i < To; ++i) {
8926 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
8927 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
8928 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
8929 MMO: ST->getMemOperand());
8930 OutChains.push_back(Elt: NewStore);
8931 }
8932}
8933
8934static SDValue getMemcpyLoadsAndStores(
8935 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
8936 uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline,
8937 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
8938 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
8939 // Turn a memcpy of undef to nop.
8940 // FIXME: We need to honor volatile even is Src is undef.
8941 if (Src.isUndef())
8942 return Chain;
8943
8944 // Expand memcpy to a series of load and store ops if the size operand falls
8945 // below a certain threshold.
8946 // TODO: In the AlwaysInline case, if the size is big then generate a loop
8947 // rather than maybe a humongous number of loads and stores.
8948 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8949 const DataLayout &DL = DAG.getDataLayout();
8950 LLVMContext &C = *DAG.getContext();
8951 std::vector<EVT> MemOps;
8952 bool DstAlignCanChange = false;
8953 MachineFunction &MF = DAG.getMachineFunction();
8954 MachineFrameInfo &MFI = MF.getFrameInfo();
8955 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8956 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
8957 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
8958 DstAlignCanChange = true;
8959 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
8960 if (!SrcAlign || Alignment > *SrcAlign)
8961 SrcAlign = Alignment;
8962 assert(SrcAlign && "SrcAlign must be set");
8963 ConstantDataArraySlice Slice;
8964 // If marked as volatile, perform a copy even when marked as constant.
8965 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
8966 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
8967 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
8968 const MemOp Op = isZeroConstant
8969 ? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment,
8970 /*IsZeroMemset*/ true, IsVolatile: isVol)
8971 : MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment,
8972 SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
8973 if (!TLI.findOptimalMemOpLowering(
8974 Context&: C, MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
8975 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
8976 return SDValue();
8977
8978 if (DstAlignCanChange) {
8979 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
8980 Align NewAlign = DL.getABITypeAlign(Ty);
8981
8982 // Don't promote to an alignment that would require dynamic stack
8983 // realignment which may conflict with optimizations such as tail call
8984 // optimization.
8985 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
8986 if (!TRI->hasStackRealignment(MF))
8987 if (MaybeAlign StackAlign = DL.getStackAlignment())
8988 NewAlign = std::min(a: NewAlign, b: *StackAlign);
8989
8990 if (NewAlign > Alignment) {
8991 // Give the stack frame object a larger alignment if needed.
8992 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
8993 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
8994 Alignment = NewAlign;
8995 }
8996 }
8997
8998 // Prepare AAInfo for loads/stores after lowering this memcpy.
8999 AAMDNodes NewAAInfo = AAInfo;
9000 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9001
9002 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
9003 bool isConstant =
9004 BatchAA && SrcVal &&
9005 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
9006
9007 MachineMemOperand::Flags MMOFlags =
9008 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9009 SmallVector<SDValue, 16> OutLoadChains;
9010 SmallVector<SDValue, 16> OutStoreChains;
9011 SmallVector<SDValue, 32> OutChains;
9012 unsigned NumMemOps = MemOps.size();
9013 uint64_t SrcOff = 0, DstOff = 0;
9014 for (unsigned i = 0; i != NumMemOps; ++i) {
9015 EVT VT = MemOps[i];
9016 unsigned VTSize = VT.getSizeInBits() / 8;
9017 SDValue Value, Store;
9018
9019 if (VTSize > Size) {
9020 // Issuing an unaligned load / store pair that overlaps with the previous
9021 // pair. Adjust the offset accordingly.
9022 assert(i == NumMemOps-1 && i != 0);
9023 SrcOff -= VTSize - Size;
9024 DstOff -= VTSize - Size;
9025 }
9026
9027 if (CopyFromConstant &&
9028 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
9029 // It's unlikely a store of a vector immediate can be done in a single
9030 // instruction. It would require a load from a constantpool first.
9031 // We only handle zero vectors here.
9032 // FIXME: Handle other cases where store of vector immediate is done in
9033 // a single instruction.
9034 ConstantDataArraySlice SubSlice;
9035 if (SrcOff < Slice.Length) {
9036 SubSlice = Slice;
9037 SubSlice.move(Delta: SrcOff);
9038 } else {
9039 // This is an out-of-bounds access and hence UB. Pretend we read zero.
9040 SubSlice.Array = nullptr;
9041 SubSlice.Offset = 0;
9042 SubSlice.Length = VTSize;
9043 }
9044 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
9045 if (Value.getNode()) {
9046 Store = DAG.getStore(
9047 Chain, dl, Val: Value,
9048 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9049 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
9050 OutChains.push_back(Elt: Store);
9051 }
9052 }
9053
9054 if (!Store.getNode()) {
9055 // The type might not be legal for the target. This should only happen
9056 // if the type is smaller than a legal type, as on PPC, so the right
9057 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
9058 // to Load/Store if NVT==VT.
9059 // FIXME does the case above also need this?
9060 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
9061 assert(NVT.bitsGE(VT));
9062
9063 bool isDereferenceable =
9064 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9065 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9066 if (isDereferenceable)
9067 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9068 if (isConstant)
9069 SrcMMOFlags |= MachineMemOperand::MOInvariant;
9070
9071 Value = DAG.getExtLoad(
9072 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
9073 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9074 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
9075 Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9076 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
9077
9078 Store = DAG.getTruncStore(
9079 Chain, dl, Val: Value,
9080 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9081 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo);
9082 OutStoreChains.push_back(Elt: Store);
9083 }
9084 SrcOff += VTSize;
9085 DstOff += VTSize;
9086 Size -= VTSize;
9087 }
9088
9089 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
9090 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
9091 unsigned NumLdStInMemcpy = OutStoreChains.size();
9092
9093 if (NumLdStInMemcpy) {
9094 // It may be that memcpy might be converted to memset if it's memcpy
9095 // of constants. In such a case, we won't have loads and stores, but
9096 // just stores. In the absence of loads, there is nothing to gang up.
9097 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
9098 // If target does not care, just leave as it.
9099 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
9100 OutChains.push_back(Elt: OutLoadChains[i]);
9101 OutChains.push_back(Elt: OutStoreChains[i]);
9102 }
9103 } else {
9104 // Ld/St less than/equal limit set by target.
9105 if (NumLdStInMemcpy <= GluedLdStLimit) {
9106 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
9107 To: NumLdStInMemcpy, OutLoadChains,
9108 OutStoreChains);
9109 } else {
9110 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
9111 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
9112 unsigned GlueIter = 0;
9113
9114 // Residual ld/st.
9115 if (RemainingLdStInMemcpy) {
9116 chainLoadsAndStoresForMemcpy(
9117 DAG, dl, OutChains, From: NumLdStInMemcpy - RemainingLdStInMemcpy,
9118 To: NumLdStInMemcpy, OutLoadChains, OutStoreChains);
9119 }
9120
9121 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
9122 unsigned IndexFrom = NumLdStInMemcpy - RemainingLdStInMemcpy -
9123 GlueIter - GluedLdStLimit;
9124 unsigned IndexTo = NumLdStInMemcpy - RemainingLdStInMemcpy - GlueIter;
9125 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
9126 OutLoadChains, OutStoreChains);
9127 GlueIter += GluedLdStLimit;
9128 }
9129 }
9130 }
9131 }
9132 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9133}
9134
9135static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
9136 SDValue Chain, SDValue Dst, SDValue Src,
9137 uint64_t Size, Align Alignment,
9138 bool isVol, bool AlwaysInline,
9139 MachinePointerInfo DstPtrInfo,
9140 MachinePointerInfo SrcPtrInfo,
9141 const AAMDNodes &AAInfo) {
9142 // Turn a memmove of undef to nop.
9143 // FIXME: We need to honor volatile even is Src is undef.
9144 if (Src.isUndef())
9145 return Chain;
9146
9147 // Expand memmove to a series of load and store ops if the size operand falls
9148 // below a certain threshold.
9149 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9150 const DataLayout &DL = DAG.getDataLayout();
9151 LLVMContext &C = *DAG.getContext();
9152 std::vector<EVT> MemOps;
9153 bool DstAlignCanChange = false;
9154 MachineFunction &MF = DAG.getMachineFunction();
9155 MachineFrameInfo &MFI = MF.getFrameInfo();
9156 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9157 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9158 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9159 DstAlignCanChange = true;
9160 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
9161 if (!SrcAlign || Alignment > *SrcAlign)
9162 SrcAlign = Alignment;
9163 assert(SrcAlign && "SrcAlign must be set");
9164 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
9165 if (!TLI.findOptimalMemOpLowering(
9166 Context&: C, MemOps, Limit,
9167 Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign,
9168 /*IsVolatile*/ true),
9169 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
9170 FuncAttributes: MF.getFunction().getAttributes(), LargestVT: nullptr))
9171 return SDValue();
9172
9173 if (DstAlignCanChange) {
9174 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
9175 Align NewAlign = DL.getABITypeAlign(Ty);
9176
9177 // Don't promote to an alignment that would require dynamic stack
9178 // realignment which may conflict with optimizations such as tail call
9179 // optimization.
9180 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9181 if (!TRI->hasStackRealignment(MF))
9182 if (MaybeAlign StackAlign = DL.getStackAlignment())
9183 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9184
9185 if (NewAlign > Alignment) {
9186 // Give the stack frame object a larger alignment if needed.
9187 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9188 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9189 Alignment = NewAlign;
9190 }
9191 }
9192
9193 // Prepare AAInfo for loads/stores after lowering this memmove.
9194 AAMDNodes NewAAInfo = AAInfo;
9195 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9196
9197 MachineMemOperand::Flags MMOFlags =
9198 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
9199 uint64_t SrcOff = 0, DstOff = 0;
9200 SmallVector<SDValue, 8> LoadValues;
9201 SmallVector<SDValue, 8> LoadChains;
9202 SmallVector<SDValue, 8> OutChains;
9203 unsigned NumMemOps = MemOps.size();
9204 for (unsigned i = 0; i < NumMemOps; i++) {
9205 EVT VT = MemOps[i];
9206 unsigned VTSize = VT.getSizeInBits() / 8;
9207 SDValue Value;
9208
9209 bool isDereferenceable =
9210 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
9211 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
9212 if (isDereferenceable)
9213 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
9214
9215 Value = DAG.getLoad(
9216 VT, dl, Chain,
9217 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff)),
9218 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: *SrcAlign, MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
9219 LoadValues.push_back(Elt: Value);
9220 LoadChains.push_back(Elt: Value.getValue(R: 1));
9221 SrcOff += VTSize;
9222 }
9223 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
9224 OutChains.clear();
9225 for (unsigned i = 0; i < NumMemOps; i++) {
9226 EVT VT = MemOps[i];
9227 unsigned VTSize = VT.getSizeInBits() / 8;
9228 SDValue Store;
9229
9230 Store = DAG.getStore(
9231 Chain, dl, Val: LoadValues[i],
9232 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9233 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
9234 OutChains.push_back(Elt: Store);
9235 DstOff += VTSize;
9236 }
9237
9238 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9239}
9240
9241/// Lower the call to 'memset' intrinsic function into a series of store
9242/// operations.
9243///
9244/// \param DAG Selection DAG where lowered code is placed.
9245/// \param dl Link to corresponding IR location.
9246/// \param Chain Control flow dependency.
9247/// \param Dst Pointer to destination memory location.
9248/// \param Src Value of byte to write into the memory.
9249/// \param Size Number of bytes to write.
9250/// \param Alignment Alignment of the destination in bytes.
9251/// \param isVol True if destination is volatile.
9252/// \param AlwaysInline Makes sure no function call is generated.
9253/// \param DstPtrInfo IR information on the memory pointer.
9254/// \returns New head in the control flow, if lowering was successful, empty
9255/// SDValue otherwise.
9256///
9257/// The function tries to replace 'llvm.memset' intrinsic with several store
9258/// operations and value calculation code. This is usually profitable for small
9259/// memory size or when the semantic requires inlining.
9260static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
9261 SDValue Chain, SDValue Dst, SDValue Src,
9262 uint64_t Size, Align Alignment, bool isVol,
9263 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
9264 const AAMDNodes &AAInfo) {
9265 // Turn a memset of undef to nop.
9266 // FIXME: We need to honor volatile even is Src is undef.
9267 if (Src.isUndef())
9268 return Chain;
9269
9270 // Expand memset to a series of load/store ops if the size operand
9271 // falls below a certain threshold.
9272 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9273 std::vector<EVT> MemOps;
9274 bool DstAlignCanChange = false;
9275 LLVMContext &C = *DAG.getContext();
9276 MachineFunction &MF = DAG.getMachineFunction();
9277 MachineFrameInfo &MFI = MF.getFrameInfo();
9278 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
9279 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
9280 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
9281 DstAlignCanChange = true;
9282 bool IsZeroVal = isNullConstant(V: Src);
9283 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
9284
9285 EVT LargestVT;
9286 if (!TLI.findOptimalMemOpLowering(
9287 Context&: C, MemOps, Limit,
9288 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
9289 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes(),
9290 LargestVT: &LargestVT))
9291 return SDValue();
9292
9293 if (DstAlignCanChange) {
9294 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
9295 const DataLayout &DL = DAG.getDataLayout();
9296 Align NewAlign = DL.getABITypeAlign(Ty);
9297
9298 // Don't promote to an alignment that would require dynamic stack
9299 // realignment which may conflict with optimizations such as tail call
9300 // optimization.
9301 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
9302 if (!TRI->hasStackRealignment(MF))
9303 if (MaybeAlign StackAlign = DL.getStackAlignment())
9304 NewAlign = std::min(a: NewAlign, b: *StackAlign);
9305
9306 if (NewAlign > Alignment) {
9307 // Give the stack frame object a larger alignment if needed.
9308 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
9309 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
9310 Alignment = NewAlign;
9311 }
9312 }
9313
9314 SmallVector<SDValue, 8> OutChains;
9315 uint64_t DstOff = 0;
9316 unsigned NumMemOps = MemOps.size();
9317
9318 // Find the largest store and generate the bit pattern for it.
9319 // If target didn't set LargestVT, compute it from MemOps.
9320 if (!LargestVT.isSimple()) {
9321 LargestVT = MemOps[0];
9322 for (unsigned i = 1; i < NumMemOps; i++)
9323 if (MemOps[i].bitsGT(VT: LargestVT))
9324 LargestVT = MemOps[i];
9325 }
9326 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
9327
9328 // Prepare AAInfo for loads/stores after lowering this memset.
9329 AAMDNodes NewAAInfo = AAInfo;
9330 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
9331
9332 for (unsigned i = 0; i < NumMemOps; i++) {
9333 EVT VT = MemOps[i];
9334 unsigned VTSize = VT.getSizeInBits() / 8;
9335 // The target should specify store types that exactly cover the memset size
9336 // (with the last store potentially being oversized for overlapping stores).
9337 assert(Size > 0 && "Target specified more stores than needed in "
9338 "findOptimalMemOpLowering");
9339 if (VTSize > Size) {
9340 // Issuing an unaligned load / store pair that overlaps with the previous
9341 // pair. Adjust the offset accordingly.
9342 assert(i == NumMemOps-1 && i != 0);
9343 DstOff -= VTSize - Size;
9344 }
9345
9346 // If this store is smaller than the largest store see whether we can get
9347 // the smaller value for free with a truncate or extract vector element and
9348 // then store.
9349 SDValue Value = MemSetValue;
9350 if (VT.bitsLT(VT: LargestVT)) {
9351 unsigned Index;
9352 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
9353 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
9354 if (!LargestVT.isVector() && !VT.isVector() &&
9355 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
9356 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
9357 else if (LargestVT.isVector() && !VT.isVector() &&
9358 TLI.shallExtractConstSplatVectorElementToStore(
9359 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
9360 ElemSizeInBits: VT.getSizeInBits(), Index) &&
9361 TLI.isTypeLegal(VT: SVT) &&
9362 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
9363 // Target which can combine store(extractelement VectorTy, Idx) can get
9364 // the smaller value for free.
9365 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
9366 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
9367 } else
9368 Value = getMemsetValue(Value: Src, VT, DAG, dl);
9369 }
9370 assert(Value.getValueType() == VT && "Value with wrong type.");
9371 SDValue Store = DAG.getStore(
9372 Chain, dl, Val: Value,
9373 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff)),
9374 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
9375 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
9376 AAInfo: NewAAInfo);
9377 OutChains.push_back(Elt: Store);
9378 DstOff += VT.getSizeInBits() / 8;
9379 // For oversized overlapping stores, only subtract the remaining bytes.
9380 // For normal stores, subtract the full store size.
9381 if (VTSize > Size) {
9382 Size = 0;
9383 } else {
9384 Size -= VTSize;
9385 }
9386 }
9387
9388 // After processing all stores, Size should be exactly 0. Any remaining bytes
9389 // indicate a bug in the target's findOptimalMemOpLowering implementation.
9390 assert(Size == 0 && "Target's findOptimalMemOpLowering did not specify "
9391 "stores that exactly cover the memset size");
9392
9393 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9394}
9395
9396static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
9397 unsigned AS) {
9398 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
9399 // pointer operands can be losslessly bitcasted to pointers of address space 0
9400 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
9401 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
9402 Twine(AS));
9403 }
9404}
9405
9406static bool isInTailCallPositionWrapper(const CallInst *CI,
9407 const SelectionDAG *SelDAG,
9408 bool AllowReturnsFirstArg) {
9409 if (!CI || !CI->isTailCall())
9410 return false;
9411 // TODO: Fix "returns-first-arg" determination so it doesn't depend on which
9412 // helper symbol we lower to.
9413 return isInTailCallPosition(Call: *CI, TM: SelDAG->getTarget(),
9414 ReturnsFirstArg: AllowReturnsFirstArg &&
9415 funcReturnsFirstArgOfCall(CI: *CI));
9416}
9417
9418static std::pair<SDValue, SDValue>
9419getRuntimeCallSDValueHelper(SDValue Chain, const SDLoc &dl,
9420 TargetLowering::ArgListTy &&Args,
9421 const CallInst *CI, RTLIB::Libcall Call,
9422 SelectionDAG *DAG, const TargetLowering *TLI) {
9423 RTLIB::LibcallImpl LCImpl = DAG->getLibcalls().getLibcallImpl(Call);
9424
9425 if (LCImpl == RTLIB::Unsupported)
9426 return {};
9427
9428 TargetLowering::CallLoweringInfo CLI(*DAG);
9429 bool IsTailCall =
9430 isInTailCallPositionWrapper(CI, SelDAG: DAG, /*AllowReturnsFirstArg=*/true);
9431 SDValue Callee =
9432 DAG->getExternalSymbol(Libcall: LCImpl, VT: TLI->getPointerTy(DL: DAG->getDataLayout()));
9433
9434 CLI.setDebugLoc(dl)
9435 .setChain(Chain)
9436 .setLibCallee(CC: DAG->getLibcalls().getLibcallImplCallingConv(Call: LCImpl),
9437 ResultType: CI->getType(), Target: Callee, ArgsList: std::move(Args))
9438 .setTailCall(IsTailCall);
9439
9440 return TLI->LowerCallTo(CLI);
9441}
9442
9443std::pair<SDValue, SDValue> SelectionDAG::getStrcmp(SDValue Chain,
9444 const SDLoc &dl, SDValue S1,
9445 SDValue S2,
9446 const CallInst *CI) {
9447 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9448 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9449 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9450 Call: RTLIB::STRCMP, DAG: this, TLI);
9451}
9452
9453std::pair<SDValue, SDValue> SelectionDAG::getStrstr(SDValue Chain,
9454 const SDLoc &dl, SDValue S1,
9455 SDValue S2,
9456 const CallInst *CI) {
9457 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9458 TargetLowering::ArgListTy Args = {{S1, PT}, {S2, PT}};
9459 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9460 Call: RTLIB::STRSTR, DAG: this, TLI);
9461}
9462
9463std::pair<SDValue, SDValue> SelectionDAG::getMemccpy(SDValue Chain,
9464 const SDLoc &dl,
9465 SDValue Dst, SDValue Src,
9466 SDValue C, SDValue Size,
9467 const CallInst *CI) {
9468 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9469
9470 TargetLowering::ArgListTy Args = {
9471 {Dst, PT},
9472 {Src, PT},
9473 {C, Type::getInt32Ty(C&: *getContext())},
9474 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9475 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9476 Call: RTLIB::MEMCCPY, DAG: this, TLI);
9477}
9478
9479std::pair<SDValue, SDValue>
9480SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
9481 SDValue Mem1, SDValue Size, const CallInst *CI) {
9482 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9483 TargetLowering::ArgListTy Args = {
9484 {Mem0, PT},
9485 {Mem1, PT},
9486 {Size, getDataLayout().getIntPtrType(C&: *getContext())}};
9487 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9488 Call: RTLIB::MEMCMP, DAG: this, TLI);
9489}
9490
9491std::pair<SDValue, SDValue> SelectionDAG::getStrcpy(SDValue Chain,
9492 const SDLoc &dl,
9493 SDValue Dst, SDValue Src,
9494 const CallInst *CI) {
9495 PointerType *PT = PointerType::getUnqual(C&: *getContext());
9496 TargetLowering::ArgListTy Args = {{Dst, PT}, {Src, PT}};
9497 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9498 Call: RTLIB::STRCPY, DAG: this, TLI);
9499}
9500
9501std::pair<SDValue, SDValue> SelectionDAG::getStrlen(SDValue Chain,
9502 const SDLoc &dl,
9503 SDValue Src,
9504 const CallInst *CI) {
9505 // Emit a library call.
9506 TargetLowering::ArgListTy Args = {
9507 {Src, PointerType::getUnqual(C&: *getContext())}};
9508 return getRuntimeCallSDValueHelper(Chain, dl, Args: std::move(Args), CI,
9509 Call: RTLIB::STRLEN, DAG: this, TLI);
9510}
9511
9512SDValue SelectionDAG::getMemcpy(
9513 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
9514 Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
9515 std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
9516 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
9517 BatchAAResults *BatchAA) {
9518 // Check to see if we should lower the memcpy to loads and stores first.
9519 // For cases within the target-specified limits, this is the best choice.
9520 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9521 if (ConstantSize) {
9522 // Memcpy with size zero? Just return the original chain.
9523 if (ConstantSize->isZero())
9524 return Chain;
9525
9526 SDValue Result = getMemcpyLoadsAndStores(
9527 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9528 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9529 if (Result.getNode())
9530 return Result;
9531 }
9532
9533 // Then check to see if we should lower the memcpy with target-specific
9534 // code. If the target chooses to do this, this is the next best.
9535 if (TSI) {
9536 SDValue Result = TSI->EmitTargetCodeForMemcpy(
9537 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
9538 DstPtrInfo, SrcPtrInfo);
9539 if (Result.getNode())
9540 return Result;
9541 }
9542
9543 // If we really need inline code and the target declined to provide it,
9544 // use a (potentially long) sequence of loads and stores.
9545 if (AlwaysInline) {
9546 assert(ConstantSize && "AlwaysInline requires a constant size!");
9547 return getMemcpyLoadsAndStores(
9548 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9549 isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
9550 }
9551
9552 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9553 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9554
9555 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
9556 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
9557 // respect volatile, so they may do things like read or write memory
9558 // beyond the given memory regions. But fixing this isn't easy, and most
9559 // people don't care.
9560
9561 // Emit a library call.
9562 TargetLowering::ArgListTy Args;
9563 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9564 Args.emplace_back(args&: Dst, args&: PtrTy);
9565 Args.emplace_back(args&: Src, args&: PtrTy);
9566 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9567 // FIXME: pass in SDLoc
9568 TargetLowering::CallLoweringInfo CLI(*this);
9569 bool IsTailCall = false;
9570 RTLIB::LibcallImpl MemCpyImpl = TLI->getMemcpyImpl();
9571
9572 if (OverrideTailCall.has_value()) {
9573 IsTailCall = *OverrideTailCall;
9574 } else {
9575 bool LowersToMemcpy = MemCpyImpl == RTLIB::impl_memcpy;
9576 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemcpy);
9577 }
9578
9579 CLI.setDebugLoc(dl)
9580 .setChain(Chain)
9581 .setLibCallee(
9582 CC: Libcalls->getLibcallImplCallingConv(Call: MemCpyImpl),
9583 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9584 Target: getExternalSymbol(Libcall: MemCpyImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9585 ArgsList: std::move(Args))
9586 .setDiscardResult()
9587 .setTailCall(IsTailCall);
9588
9589 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9590 return CallResult.second;
9591}
9592
9593SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
9594 SDValue Dst, SDValue Src, SDValue Size,
9595 Type *SizeTy, unsigned ElemSz,
9596 bool isTailCall,
9597 MachinePointerInfo DstPtrInfo,
9598 MachinePointerInfo SrcPtrInfo) {
9599 // Emit a library call.
9600 TargetLowering::ArgListTy Args;
9601 Type *ArgTy = getDataLayout().getIntPtrType(C&: *getContext());
9602 Args.emplace_back(args&: Dst, args&: ArgTy);
9603 Args.emplace_back(args&: Src, args&: ArgTy);
9604 Args.emplace_back(args&: Size, args&: SizeTy);
9605
9606 RTLIB::Libcall LibraryCall =
9607 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9608 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9609 if (LibcallImpl == RTLIB::Unsupported)
9610 report_fatal_error(reason: "Unsupported element size");
9611
9612 TargetLowering::CallLoweringInfo CLI(*this);
9613 CLI.setDebugLoc(dl)
9614 .setChain(Chain)
9615 .setLibCallee(
9616 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9617 ResultType: Type::getVoidTy(C&: *getContext()),
9618 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9619 ArgsList: std::move(Args))
9620 .setDiscardResult()
9621 .setTailCall(isTailCall);
9622
9623 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9624 return CallResult.second;
9625}
9626
9627SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
9628 SDValue Src, SDValue Size, Align Alignment,
9629 bool isVol, const CallInst *CI,
9630 std::optional<bool> OverrideTailCall,
9631 MachinePointerInfo DstPtrInfo,
9632 MachinePointerInfo SrcPtrInfo,
9633 const AAMDNodes &AAInfo,
9634 BatchAAResults *BatchAA) {
9635 // Check to see if we should lower the memmove to loads and stores first.
9636 // For cases within the target-specified limits, this is the best choice.
9637 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9638 if (ConstantSize) {
9639 // Memmove with size zero? Just return the original chain.
9640 if (ConstantSize->isZero())
9641 return Chain;
9642
9643 SDValue Result = getMemmoveLoadsAndStores(
9644 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
9645 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
9646 if (Result.getNode())
9647 return Result;
9648 }
9649
9650 // Then check to see if we should lower the memmove with target-specific
9651 // code. If the target chooses to do this, this is the next best.
9652 if (TSI) {
9653 SDValue Result =
9654 TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size,
9655 Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo);
9656 if (Result.getNode())
9657 return Result;
9658 }
9659
9660 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9661 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
9662
9663 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
9664 // not be safe. See memcpy above for more details.
9665
9666 // Emit a library call.
9667 TargetLowering::ArgListTy Args;
9668 Type *PtrTy = PointerType::getUnqual(C&: *getContext());
9669 Args.emplace_back(args&: Dst, args&: PtrTy);
9670 Args.emplace_back(args&: Src, args&: PtrTy);
9671 Args.emplace_back(args&: Size, args: getDataLayout().getIntPtrType(C&: *getContext()));
9672 // FIXME: pass in SDLoc
9673 TargetLowering::CallLoweringInfo CLI(*this);
9674
9675 RTLIB::LibcallImpl MemmoveImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMMOVE);
9676
9677 bool IsTailCall = false;
9678 if (OverrideTailCall.has_value()) {
9679 IsTailCall = *OverrideTailCall;
9680 } else {
9681 bool LowersToMemmove = MemmoveImpl == RTLIB::impl_memmove;
9682 IsTailCall = isInTailCallPositionWrapper(CI, SelDAG: this, AllowReturnsFirstArg: LowersToMemmove);
9683 }
9684
9685 CLI.setDebugLoc(dl)
9686 .setChain(Chain)
9687 .setLibCallee(
9688 CC: Libcalls->getLibcallImplCallingConv(Call: MemmoveImpl),
9689 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
9690 Target: getExternalSymbol(Libcall: MemmoveImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9691 ArgsList: std::move(Args))
9692 .setDiscardResult()
9693 .setTailCall(IsTailCall);
9694
9695 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
9696 return CallResult.second;
9697}
9698
9699SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
9700 SDValue Dst, SDValue Src, SDValue Size,
9701 Type *SizeTy, unsigned ElemSz,
9702 bool isTailCall,
9703 MachinePointerInfo DstPtrInfo,
9704 MachinePointerInfo SrcPtrInfo) {
9705 // Emit a library call.
9706 TargetLowering::ArgListTy Args;
9707 Type *IntPtrTy = getDataLayout().getIntPtrType(C&: *getContext());
9708 Args.emplace_back(args&: Dst, args&: IntPtrTy);
9709 Args.emplace_back(args&: Src, args&: IntPtrTy);
9710 Args.emplace_back(args&: Size, args&: SizeTy);
9711
9712 RTLIB::Libcall LibraryCall =
9713 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9714 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9715 if (LibcallImpl == RTLIB::Unsupported)
9716 report_fatal_error(reason: "Unsupported element size");
9717
9718 TargetLowering::CallLoweringInfo CLI(*this);
9719 CLI.setDebugLoc(dl)
9720 .setChain(Chain)
9721 .setLibCallee(
9722 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9723 ResultType: Type::getVoidTy(C&: *getContext()),
9724 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9725 ArgsList: std::move(Args))
9726 .setDiscardResult()
9727 .setTailCall(isTailCall);
9728
9729 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9730 return CallResult.second;
9731}
9732
9733SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
9734 SDValue Src, SDValue Size, Align Alignment,
9735 bool isVol, bool AlwaysInline,
9736 const CallInst *CI,
9737 MachinePointerInfo DstPtrInfo,
9738 const AAMDNodes &AAInfo) {
9739 // Check to see if we should lower the memset to stores first.
9740 // For cases within the target-specified limits, this is the best choice.
9741 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9742 if (ConstantSize) {
9743 // Memset with size zero? Just return the original chain.
9744 if (ConstantSize->isZero())
9745 return Chain;
9746
9747 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9748 Size: ConstantSize->getZExtValue(), Alignment,
9749 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
9750
9751 if (Result.getNode())
9752 return Result;
9753 }
9754
9755 // Then check to see if we should lower the memset with target-specific
9756 // code. If the target chooses to do this, this is the next best.
9757 if (TSI) {
9758 SDValue Result = TSI->EmitTargetCodeForMemset(
9759 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
9760 if (Result.getNode())
9761 return Result;
9762 }
9763
9764 // If we really need inline code and the target declined to provide it,
9765 // use a (potentially long) sequence of loads and stores.
9766 if (AlwaysInline) {
9767 assert(ConstantSize && "AlwaysInline requires a constant size!");
9768 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9769 Size: ConstantSize->getZExtValue(), Alignment,
9770 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
9771 assert(Result &&
9772 "getMemsetStores must return a valid sequence when AlwaysInline");
9773 return Result;
9774 }
9775
9776 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9777
9778 // Emit a library call.
9779 auto &Ctx = *getContext();
9780 const auto& DL = getDataLayout();
9781
9782 TargetLowering::CallLoweringInfo CLI(*this);
9783 // FIXME: pass in SDLoc
9784 CLI.setDebugLoc(dl).setChain(Chain);
9785
9786 RTLIB::LibcallImpl BzeroImpl = Libcalls->getLibcallImpl(Call: RTLIB::BZERO);
9787 bool UseBZero = BzeroImpl != RTLIB::Unsupported && isNullConstant(V: Src);
9788
9789 // If zeroing out and bzero is present, use it.
9790 if (UseBZero) {
9791 TargetLowering::ArgListTy Args;
9792 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
9793 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
9794 CLI.setLibCallee(
9795 CC: Libcalls->getLibcallImplCallingConv(Call: BzeroImpl), ResultType: Type::getVoidTy(C&: Ctx),
9796 Target: getExternalSymbol(Libcall: BzeroImpl, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
9797 } else {
9798 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
9799
9800 TargetLowering::ArgListTy Args;
9801 Args.emplace_back(args&: Dst, args: PointerType::getUnqual(C&: Ctx));
9802 Args.emplace_back(args&: Src, args: Src.getValueType().getTypeForEVT(Context&: Ctx));
9803 Args.emplace_back(args&: Size, args: DL.getIntPtrType(C&: Ctx));
9804 CLI.setLibCallee(CC: Libcalls->getLibcallImplCallingConv(Call: MemsetImpl),
9805 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
9806 Target: getExternalSymbol(Libcall: MemsetImpl, VT: TLI->getPointerTy(DL)),
9807 ArgsList: std::move(Args));
9808 }
9809
9810 RTLIB::LibcallImpl MemsetImpl = Libcalls->getLibcallImpl(Call: RTLIB::MEMSET);
9811 bool LowersToMemset = MemsetImpl == RTLIB::impl_memset;
9812
9813 // If we're going to use bzero, make sure not to tail call unless the
9814 // subsequent return doesn't need a value, as bzero doesn't return the first
9815 // arg unlike memset.
9816 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
9817 bool IsTailCall =
9818 CI && CI->isTailCall() &&
9819 isInTailCallPosition(Call: *CI, TM: getTarget(), ReturnsFirstArg: ReturnsFirstArg && LowersToMemset);
9820 CLI.setDiscardResult().setTailCall(IsTailCall);
9821
9822 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9823 return CallResult.second;
9824}
9825
9826SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
9827 SDValue Dst, SDValue Value, SDValue Size,
9828 Type *SizeTy, unsigned ElemSz,
9829 bool isTailCall,
9830 MachinePointerInfo DstPtrInfo) {
9831 // Emit a library call.
9832 TargetLowering::ArgListTy Args;
9833 Args.emplace_back(args&: Dst, args: getDataLayout().getIntPtrType(C&: *getContext()));
9834 Args.emplace_back(args&: Value, args: Type::getInt8Ty(C&: *getContext()));
9835 Args.emplace_back(args&: Size, args&: SizeTy);
9836
9837 RTLIB::Libcall LibraryCall =
9838 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9839 RTLIB::LibcallImpl LibcallImpl = Libcalls->getLibcallImpl(Call: LibraryCall);
9840 if (LibcallImpl == RTLIB::Unsupported)
9841 report_fatal_error(reason: "Unsupported element size");
9842
9843 TargetLowering::CallLoweringInfo CLI(*this);
9844 CLI.setDebugLoc(dl)
9845 .setChain(Chain)
9846 .setLibCallee(
9847 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
9848 ResultType: Type::getVoidTy(C&: *getContext()),
9849 Target: getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout())),
9850 ArgsList: std::move(Args))
9851 .setDiscardResult()
9852 .setTailCall(isTailCall);
9853
9854 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9855 return CallResult.second;
9856}
9857
9858SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9859 SDVTList VTList, ArrayRef<SDValue> Ops,
9860 MachineMemOperand *MMO,
9861 ISD::LoadExtType ExtType) {
9862 FoldingSetNodeID ID;
9863 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
9864 ID.AddInteger(I: MemVT.getRawBits());
9865 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
9866 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
9867 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9868 ID.AddInteger(I: MMO->getFlags());
9869 void* IP = nullptr;
9870 if (auto *E = cast_or_null<AtomicSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
9871 E->refineAlignment(NewMMO: MMO);
9872 E->refineRanges(NewMMO: MMO);
9873 return SDValue(E, 0);
9874 }
9875
9876 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
9877 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
9878 createOperands(Node: N, Vals: Ops);
9879
9880 CSEMap.InsertNode(N, InsertPos: IP);
9881 InsertNode(N);
9882 SDValue V(N, 0);
9883 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9884 return V;
9885}
9886
9887SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
9888 EVT MemVT, SDVTList VTs, SDValue Chain,
9889 SDValue Ptr, SDValue Cmp, SDValue Swp,
9890 MachineMemOperand *MMO) {
9891 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
9892 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
9893 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
9894
9895 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
9896 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9897}
9898
9899SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9900 SDValue Chain, SDValue Ptr, SDValue Val,
9901 MachineMemOperand *MMO) {
9902 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
9903 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
9904 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
9905 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
9906 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
9907 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
9908 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
9909 Opcode == ISD::ATOMIC_LOAD_FMIN ||
9910 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
9911 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
9912 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
9913 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
9914 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
9915 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
9916 Opcode == ISD::ATOMIC_STORE) &&
9917 "Invalid Atomic Op");
9918
9919 EVT VT = Val.getValueType();
9920
9921 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
9922 getVTList(VT1: VT, VT2: MVT::Other);
9923 SDValue Ops[] = {Chain, Ptr, Val};
9924 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9925}
9926
9927SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
9928 EVT MemVT, EVT VT, SDValue Chain,
9929 SDValue Ptr, MachineMemOperand *MMO) {
9930 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
9931 SDValue Ops[] = {Chain, Ptr};
9932 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
9933}
9934
9935/// getMergeValues - Create a MERGE_VALUES node from the given operands.
9936SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
9937 if (Ops.size() == 1)
9938 return Ops[0];
9939
9940 SmallVector<EVT, 4> VTs;
9941 VTs.reserve(N: Ops.size());
9942 for (const SDValue &Op : Ops)
9943 VTs.push_back(Elt: Op.getValueType());
9944 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
9945}
9946
9947SDValue SelectionDAG::getMemIntrinsicNode(
9948 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
9949 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
9950 MachineMemOperand::Flags Flags, LocationSize Size,
9951 const AAMDNodes &AAInfo) {
9952 if (Size.hasValue() && !Size.getValue())
9953 Size = LocationSize::precise(Value: MemVT.getStoreSize());
9954
9955 MachineFunction &MF = getMachineFunction();
9956 MachineMemOperand *MMO =
9957 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, AAInfo);
9958
9959 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
9960}
9961
9962SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
9963 SDVTList VTList,
9964 ArrayRef<SDValue> Ops, EVT MemVT,
9965 MachineMemOperand *MMO) {
9966 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMOs: ArrayRef(MMO));
9967}
9968
9969SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
9970 SDVTList VTList,
9971 ArrayRef<SDValue> Ops, EVT MemVT,
9972 ArrayRef<MachineMemOperand *> MMOs) {
9973 assert(!MMOs.empty() && "Must have at least one MMO");
9974 assert(
9975 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
9976 Opcode == ISD::PREFETCH ||
9977 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
9978 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
9979 "Opcode is not a memory-accessing opcode!");
9980
9981 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs;
9982 if (MMOs.size() == 1) {
9983 MemRefs = MMOs[0];
9984 } else {
9985 // Allocate: [size_t count][MMO*][MMO*]...
9986 size_t AllocSize =
9987 sizeof(size_t) + MMOs.size() * sizeof(MachineMemOperand *);
9988 void *Buffer = Allocator.Allocate(Size: AllocSize, Alignment: alignof(size_t));
9989 size_t *CountPtr = static_cast<size_t *>(Buffer);
9990 *CountPtr = MMOs.size();
9991 MachineMemOperand **Array =
9992 reinterpret_cast<MachineMemOperand **>(CountPtr + 1);
9993 llvm::copy(Range&: MMOs, Out: Array);
9994 MemRefs = Array;
9995 }
9996
9997 // Memoize the node unless it returns a glue result.
9998 MemIntrinsicSDNode *N;
9999 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10000 FoldingSetNodeID ID;
10001 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10002 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
10003 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MemRefs));
10004 ID.AddInteger(I: MemVT.getRawBits());
10005 for (const MachineMemOperand *MMO : MMOs) {
10006 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10007 ID.AddInteger(I: MMO->getFlags());
10008 }
10009 void *IP = nullptr;
10010 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10011 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMOs: MMOs);
10012 return SDValue(E, 0);
10013 }
10014
10015 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10016 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10017 createOperands(Node: N, Vals: Ops);
10018 CSEMap.InsertNode(N, InsertPos: IP);
10019 } else {
10020 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10021 Args&: VTList, Args&: MemVT, Args&: MemRefs);
10022 createOperands(Node: N, Vals: Ops);
10023 }
10024 InsertNode(N);
10025 SDValue V(N, 0);
10026 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10027 return V;
10028}
10029
10030SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
10031 SDValue Chain, int FrameIndex) {
10032 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
10033 const auto VTs = getVTList(VT: MVT::Other);
10034 SDValue Ops[2] = {
10035 Chain,
10036 getFrameIndex(FI: FrameIndex,
10037 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
10038 isTarget: true)};
10039
10040 FoldingSetNodeID ID;
10041 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10042 ID.AddInteger(I: FrameIndex);
10043 void *IP = nullptr;
10044 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10045 return SDValue(E, 0);
10046
10047 LifetimeSDNode *N =
10048 newSDNode<LifetimeSDNode>(Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs);
10049 createOperands(Node: N, Vals: Ops);
10050 CSEMap.InsertNode(N, InsertPos: IP);
10051 InsertNode(N);
10052 SDValue V(N, 0);
10053 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10054 return V;
10055}
10056
10057SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
10058 uint64_t Guid, uint64_t Index,
10059 uint32_t Attr) {
10060 const unsigned Opcode = ISD::PSEUDO_PROBE;
10061 const auto VTs = getVTList(VT: MVT::Other);
10062 SDValue Ops[] = {Chain};
10063 FoldingSetNodeID ID;
10064 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10065 ID.AddInteger(I: Guid);
10066 ID.AddInteger(I: Index);
10067 void *IP = nullptr;
10068 if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
10069 return SDValue(E, 0);
10070
10071 auto *N = newSDNode<PseudoProbeSDNode>(
10072 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
10073 createOperands(Node: N, Vals: Ops);
10074 CSEMap.InsertNode(N, InsertPos: IP);
10075 InsertNode(N);
10076 SDValue V(N, 0);
10077 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10078 return V;
10079}
10080
10081/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10082/// MachinePointerInfo record from it. This is particularly useful because the
10083/// code generator has many cases where it doesn't bother passing in a
10084/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10085static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10086 SelectionDAG &DAG, SDValue Ptr,
10087 int64_t Offset = 0) {
10088 // If this is FI+Offset, we can model it.
10089 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
10090 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
10091 FI: FI->getIndex(), Offset);
10092
10093 // If this is (FI+Offset1)+Offset2, we can model it.
10094 if (Ptr.getOpcode() != ISD::ADD ||
10095 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
10096 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
10097 return Info;
10098
10099 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
10100 return MachinePointerInfo::getFixedStack(
10101 MF&: DAG.getMachineFunction(), FI,
10102 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
10103}
10104
10105/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
10106/// MachinePointerInfo record from it. This is particularly useful because the
10107/// code generator has many cases where it doesn't bother passing in a
10108/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
10109static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
10110 SelectionDAG &DAG, SDValue Ptr,
10111 SDValue OffsetOp) {
10112 // If the 'Offset' value isn't a constant, we can't handle this.
10113 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
10114 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
10115 if (OffsetOp.isUndef())
10116 return InferPointerInfo(Info, DAG, Ptr);
10117 return Info;
10118}
10119
10120SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10121 EVT VT, const SDLoc &dl, SDValue Chain,
10122 SDValue Ptr, SDValue Offset,
10123 MachinePointerInfo PtrInfo, EVT MemVT,
10124 Align Alignment,
10125 MachineMemOperand::Flags MMOFlags,
10126 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10127 assert(Chain.getValueType() == MVT::Other &&
10128 "Invalid chain type");
10129
10130 MMOFlags |= MachineMemOperand::MOLoad;
10131 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10132 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10133 // clients.
10134 if (PtrInfo.V.isNull())
10135 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10136
10137 TypeSize Size = MemVT.getStoreSize();
10138 MachineFunction &MF = getMachineFunction();
10139 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10140 BaseAlignment: Alignment, AAInfo, Ranges);
10141 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
10142}
10143
10144SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
10145 EVT VT, const SDLoc &dl, SDValue Chain,
10146 SDValue Ptr, SDValue Offset, EVT MemVT,
10147 MachineMemOperand *MMO) {
10148 if (VT == MemVT) {
10149 ExtType = ISD::NON_EXTLOAD;
10150 } else if (ExtType == ISD::NON_EXTLOAD) {
10151 assert(VT == MemVT && "Non-extending load from different memory type!");
10152 } else {
10153 // Extending load.
10154 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
10155 "Should only be an extending load, not truncating!");
10156 assert(VT.isInteger() == MemVT.isInteger() &&
10157 "Cannot convert from FP to Int or Int -> FP!");
10158 assert(VT.isVector() == MemVT.isVector() &&
10159 "Cannot use an ext load to convert to or from a vector!");
10160 assert((!VT.isVector() ||
10161 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
10162 "Cannot use an ext load to change the number of vector elements!");
10163 }
10164
10165 assert((!MMO->getRanges() ||
10166 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
10167 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
10168 MemVT.isInteger())) &&
10169 "Range metadata and load type must match!");
10170
10171 bool Indexed = AM != ISD::UNINDEXED;
10172 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10173
10174 SDVTList VTs = Indexed ?
10175 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
10176 SDValue Ops[] = { Chain, Ptr, Offset };
10177 FoldingSetNodeID ID;
10178 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
10179 ID.AddInteger(I: MemVT.getRawBits());
10180 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
10181 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
10182 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10183 ID.AddInteger(I: MMO->getFlags());
10184 void *IP = nullptr;
10185 if (auto *E = cast_or_null<LoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10186 E->refineAlignment(NewMMO: MMO);
10187 E->refineRanges(NewMMO: MMO);
10188 return SDValue(E, 0);
10189 }
10190 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10191 Args&: ExtType, Args&: MemVT, Args&: MMO);
10192 createOperands(Node: N, Vals: Ops);
10193
10194 CSEMap.InsertNode(N, InsertPos: IP);
10195 InsertNode(N);
10196 SDValue V(N, 0);
10197 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10198 return V;
10199}
10200
10201SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10202 SDValue Ptr, MachinePointerInfo PtrInfo,
10203 MaybeAlign Alignment,
10204 MachineMemOperand::Flags MMOFlags,
10205 const AAMDNodes &AAInfo, const MDNode *Ranges) {
10206 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10207 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10208 PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
10209}
10210
10211SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10212 SDValue Ptr, MachineMemOperand *MMO) {
10213 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10214 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10215 MemVT: VT, MMO);
10216}
10217
10218SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10219 EVT VT, SDValue Chain, SDValue Ptr,
10220 MachinePointerInfo PtrInfo, EVT MemVT,
10221 MaybeAlign Alignment,
10222 MachineMemOperand::Flags MMOFlags,
10223 const AAMDNodes &AAInfo) {
10224 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10225 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
10226 MemVT, Alignment, MMOFlags, AAInfo);
10227}
10228
10229SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
10230 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
10231 MachineMemOperand *MMO) {
10232 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10233 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
10234 MemVT, MMO);
10235}
10236
10237SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
10238 SDValue Base, SDValue Offset,
10239 ISD::MemIndexedMode AM) {
10240 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
10241 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10242 // Don't propagate the invariant or dereferenceable flags.
10243 auto MMOFlags =
10244 LD->getMemOperand()->getFlags() &
10245 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10246 return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10247 Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
10248 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
10249}
10250
10251SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10252 SDValue Ptr, MachinePointerInfo PtrInfo,
10253 Align Alignment,
10254 MachineMemOperand::Flags MMOFlags,
10255 const AAMDNodes &AAInfo) {
10256 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10257
10258 MMOFlags |= MachineMemOperand::MOStore;
10259 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10260
10261 if (PtrInfo.V.isNull())
10262 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10263
10264 MachineFunction &MF = getMachineFunction();
10265 TypeSize Size = Val.getValueType().getStoreSize();
10266 MachineMemOperand *MMO =
10267 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, AAInfo);
10268 return getStore(Chain, dl, Val, Ptr, MMO);
10269}
10270
10271SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10272 SDValue Ptr, MachineMemOperand *MMO) {
10273 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10274 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
10275 AM: ISD::UNINDEXED);
10276}
10277
10278SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10279 SDValue Ptr, SDValue Offset, EVT SVT,
10280 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
10281 bool IsTruncating) {
10282 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10283 EVT VT = Val.getValueType();
10284 if (VT == SVT) {
10285 IsTruncating = false;
10286 } else if (!IsTruncating) {
10287 assert(VT == SVT && "No-truncating store from different memory type!");
10288 } else {
10289 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10290 "Should only be a truncating store, not extending!");
10291 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10292 assert(VT.isVector() == SVT.isVector() &&
10293 "Cannot use trunc store to convert to or from a vector!");
10294 assert((!VT.isVector() ||
10295 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10296 "Cannot use trunc store to change the number of vector elements!");
10297 }
10298
10299 bool Indexed = AM != ISD::UNINDEXED;
10300 assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");
10301 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10302 : getVTList(VT: MVT::Other);
10303 SDValue Ops[] = {Chain, Val, Ptr, Offset};
10304 FoldingSetNodeID ID;
10305 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
10306 ID.AddInteger(I: SVT.getRawBits());
10307 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
10308 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
10309 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10310 ID.AddInteger(I: MMO->getFlags());
10311 void *IP = nullptr;
10312 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10313 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10314 return SDValue(E, 0);
10315 }
10316 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10317 Args&: IsTruncating, Args&: SVT, Args&: MMO);
10318 createOperands(Node: N, Vals: Ops);
10319
10320 CSEMap.InsertNode(N, InsertPos: IP);
10321 InsertNode(N);
10322 SDValue V(N, 0);
10323 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10324 return V;
10325}
10326
10327SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10328 SDValue Ptr, MachinePointerInfo PtrInfo,
10329 EVT SVT, Align Alignment,
10330 MachineMemOperand::Flags MMOFlags,
10331 const AAMDNodes &AAInfo) {
10332 assert(Chain.getValueType() == MVT::Other &&
10333 "Invalid chain type");
10334
10335 MMOFlags |= MachineMemOperand::MOStore;
10336 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10337
10338 if (PtrInfo.V.isNull())
10339 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10340
10341 MachineFunction &MF = getMachineFunction();
10342 MachineMemOperand *MMO = MF.getMachineMemOperand(
10343 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10344 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
10345}
10346
10347SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
10348 SDValue Ptr, EVT SVT,
10349 MachineMemOperand *MMO) {
10350 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10351 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
10352}
10353
10354SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
10355 SDValue Base, SDValue Offset,
10356 ISD::MemIndexedMode AM) {
10357 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
10358 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
10359 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
10360 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
10361 IsTruncating: ST->isTruncatingStore());
10362}
10363
10364SDValue SelectionDAG::getLoadVP(
10365 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
10366 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
10367 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
10368 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
10369 const MDNode *Ranges, bool IsExpanding) {
10370 MMOFlags |= MachineMemOperand::MOLoad;
10371 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
10372 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
10373 // clients.
10374 if (PtrInfo.V.isNull())
10375 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
10376
10377 TypeSize Size = MemVT.getStoreSize();
10378 MachineFunction &MF = getMachineFunction();
10379 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
10380 BaseAlignment: Alignment, AAInfo, Ranges);
10381 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
10382 MMO, IsExpanding);
10383}
10384
10385SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
10386 ISD::LoadExtType ExtType, EVT VT,
10387 const SDLoc &dl, SDValue Chain, SDValue Ptr,
10388 SDValue Offset, SDValue Mask, SDValue EVL,
10389 EVT MemVT, MachineMemOperand *MMO,
10390 bool IsExpanding) {
10391 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10392 assert(Mask.getValueType().getVectorElementCount() ==
10393 VT.getVectorElementCount() &&
10394 "Vector width mismatch between mask and data");
10395
10396 bool Indexed = AM != ISD::UNINDEXED;
10397 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10398
10399 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10400 : getVTList(VT1: VT, VT2: MVT::Other);
10401 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
10402 FoldingSetNodeID ID;
10403 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
10404 ID.AddInteger(I: MemVT.getRawBits());
10405 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
10406 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10407 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10408 ID.AddInteger(I: MMO->getFlags());
10409 void *IP = nullptr;
10410 if (auto *E = cast_or_null<VPLoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
10411 E->refineAlignment(NewMMO: MMO);
10412 E->refineRanges(NewMMO: MMO);
10413 return SDValue(E, 0);
10414 }
10415 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10416 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10417 createOperands(Node: N, Vals: Ops);
10418
10419 CSEMap.InsertNode(N, InsertPos: IP);
10420 InsertNode(N);
10421 SDValue V(N, 0);
10422 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10423 return V;
10424}
10425
10426SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10427 SDValue Ptr, SDValue Mask, SDValue EVL,
10428 MachinePointerInfo PtrInfo,
10429 MaybeAlign Alignment,
10430 MachineMemOperand::Flags MMOFlags,
10431 const AAMDNodes &AAInfo, const MDNode *Ranges,
10432 bool IsExpanding) {
10433 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10434 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10435 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
10436 IsExpanding);
10437}
10438
10439SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
10440 SDValue Ptr, SDValue Mask, SDValue EVL,
10441 MachineMemOperand *MMO, bool IsExpanding) {
10442 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10443 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
10444 Mask, EVL, MemVT: VT, MMO, IsExpanding);
10445}
10446
10447SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10448 EVT VT, SDValue Chain, SDValue Ptr,
10449 SDValue Mask, SDValue EVL,
10450 MachinePointerInfo PtrInfo, EVT MemVT,
10451 MaybeAlign Alignment,
10452 MachineMemOperand::Flags MMOFlags,
10453 const AAMDNodes &AAInfo, bool IsExpanding) {
10454 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10455 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10456 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
10457 IsExpanding);
10458}
10459
10460SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
10461 EVT VT, SDValue Chain, SDValue Ptr,
10462 SDValue Mask, SDValue EVL, EVT MemVT,
10463 MachineMemOperand *MMO, bool IsExpanding) {
10464 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10465 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
10466 EVL, MemVT, MMO, IsExpanding);
10467}
10468
10469SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
10470 SDValue Base, SDValue Offset,
10471 ISD::MemIndexedMode AM) {
10472 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
10473 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
10474 // Don't propagate the invariant or dereferenceable flags.
10475 auto MMOFlags =
10476 LD->getMemOperand()->getFlags() &
10477 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
10478 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
10479 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
10480 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
10481 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
10482 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
10483}
10484
10485SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
10486 SDValue Ptr, SDValue Offset, SDValue Mask,
10487 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
10488 ISD::MemIndexedMode AM, bool IsTruncating,
10489 bool IsCompressing) {
10490 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10491 assert(Mask.getValueType().getVectorElementCount() ==
10492 Val.getValueType().getVectorElementCount() &&
10493 "Vector width mismatch between mask and data");
10494
10495 bool Indexed = AM != ISD::UNINDEXED;
10496 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10497 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10498 : getVTList(VT: MVT::Other);
10499 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
10500 FoldingSetNodeID ID;
10501 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10502 ID.AddInteger(I: MemVT.getRawBits());
10503 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10504 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10505 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10506 ID.AddInteger(I: MMO->getFlags());
10507 void *IP = nullptr;
10508 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10509 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10510 return SDValue(E, 0);
10511 }
10512 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10513 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10514 createOperands(Node: N, Vals: Ops);
10515
10516 CSEMap.InsertNode(N, InsertPos: IP);
10517 InsertNode(N);
10518 SDValue V(N, 0);
10519 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10520 return V;
10521}
10522
10523SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10524 SDValue Val, SDValue Ptr, SDValue Mask,
10525 SDValue EVL, MachinePointerInfo PtrInfo,
10526 EVT SVT, Align Alignment,
10527 MachineMemOperand::Flags MMOFlags,
10528 const AAMDNodes &AAInfo,
10529 bool IsCompressing) {
10530 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10531
10532 MMOFlags |= MachineMemOperand::MOStore;
10533 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
10534
10535 if (PtrInfo.V.isNull())
10536 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
10537
10538 MachineFunction &MF = getMachineFunction();
10539 MachineMemOperand *MMO = MF.getMachineMemOperand(
10540 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
10541 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
10542 IsCompressing);
10543}
10544
10545SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
10546 SDValue Val, SDValue Ptr, SDValue Mask,
10547 SDValue EVL, EVT SVT,
10548 MachineMemOperand *MMO,
10549 bool IsCompressing) {
10550 EVT VT = Val.getValueType();
10551
10552 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10553 if (VT == SVT)
10554 return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
10555 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10556 /*IsTruncating*/ false, IsCompressing);
10557
10558 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10559 "Should only be a truncating store, not extending!");
10560 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10561 assert(VT.isVector() == SVT.isVector() &&
10562 "Cannot use trunc store to convert to or from a vector!");
10563 assert((!VT.isVector() ||
10564 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10565 "Cannot use trunc store to change the number of vector elements!");
10566
10567 SDVTList VTs = getVTList(VT: MVT::Other);
10568 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10569 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
10570 FoldingSetNodeID ID;
10571 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10572 ID.AddInteger(I: SVT.getRawBits());
10573 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
10574 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10575 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10576 ID.AddInteger(I: MMO->getFlags());
10577 void *IP = nullptr;
10578 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10579 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10580 return SDValue(E, 0);
10581 }
10582 auto *N =
10583 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10584 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
10585 createOperands(Node: N, Vals: Ops);
10586
10587 CSEMap.InsertNode(N, InsertPos: IP);
10588 InsertNode(N);
10589 SDValue V(N, 0);
10590 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10591 return V;
10592}
10593
10594SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
10595 SDValue Base, SDValue Offset,
10596 ISD::MemIndexedMode AM) {
10597 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
10598 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
10599 SDVTList VTs = getVTList(VT1: Base.getValueType(), VT2: MVT::Other);
10600 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
10601 Offset, ST->getMask(), ST->getVectorLength()};
10602 FoldingSetNodeID ID;
10603 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
10604 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
10605 ID.AddInteger(I: ST->getRawSubclassData());
10606 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
10607 ID.AddInteger(I: ST->getMemOperand()->getFlags());
10608 void *IP = nullptr;
10609 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10610 return SDValue(E, 0);
10611
10612 auto *N = newSDNode<VPStoreSDNode>(
10613 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
10614 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
10615 createOperands(Node: N, Vals: Ops);
10616
10617 CSEMap.InsertNode(N, InsertPos: IP);
10618 InsertNode(N);
10619 SDValue V(N, 0);
10620 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10621 return V;
10622}
10623
10624SDValue SelectionDAG::getStridedLoadVP(
10625 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
10626 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
10627 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
10628 bool Indexed = AM != ISD::UNINDEXED;
10629 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
10630
10631 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
10632 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
10633 : getVTList(VT1: VT, VT2: MVT::Other);
10634 FoldingSetNodeID ID;
10635 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
10636 ID.AddInteger(I: VT.getRawBits());
10637 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
10638 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
10639 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10640
10641 void *IP = nullptr;
10642 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10643 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10644 return SDValue(E, 0);
10645 }
10646
10647 auto *N =
10648 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
10649 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
10650 createOperands(Node: N, Vals: Ops);
10651 CSEMap.InsertNode(N, InsertPos: IP);
10652 InsertNode(N);
10653 SDValue V(N, 0);
10654 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10655 return V;
10656}
10657
10658SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
10659 SDValue Ptr, SDValue Stride,
10660 SDValue Mask, SDValue EVL,
10661 MachineMemOperand *MMO,
10662 bool IsExpanding) {
10663 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10664 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
10665 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
10666}
10667
10668SDValue SelectionDAG::getExtStridedLoadVP(
10669 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
10670 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
10671 MachineMemOperand *MMO, bool IsExpanding) {
10672 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10673 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
10674 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
10675}
10676
10677SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
10678 SDValue Val, SDValue Ptr,
10679 SDValue Offset, SDValue Stride,
10680 SDValue Mask, SDValue EVL, EVT MemVT,
10681 MachineMemOperand *MMO,
10682 ISD::MemIndexedMode AM,
10683 bool IsTruncating, bool IsCompressing) {
10684 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10685 bool Indexed = AM != ISD::UNINDEXED;
10686 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
10687 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
10688 : getVTList(VT: MVT::Other);
10689 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
10690 FoldingSetNodeID ID;
10691 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10692 ID.AddInteger(I: MemVT.getRawBits());
10693 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10694 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10695 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10696 void *IP = nullptr;
10697 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10698 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10699 return SDValue(E, 0);
10700 }
10701 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10702 Args&: VTs, Args&: AM, Args&: IsTruncating,
10703 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10704 createOperands(Node: N, Vals: Ops);
10705
10706 CSEMap.InsertNode(N, InsertPos: IP);
10707 InsertNode(N);
10708 SDValue V(N, 0);
10709 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10710 return V;
10711}
10712
10713SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
10714 SDValue Val, SDValue Ptr,
10715 SDValue Stride, SDValue Mask,
10716 SDValue EVL, EVT SVT,
10717 MachineMemOperand *MMO,
10718 bool IsCompressing) {
10719 EVT VT = Val.getValueType();
10720
10721 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10722 if (VT == SVT)
10723 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
10724 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
10725 /*IsTruncating*/ false, IsCompressing);
10726
10727 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
10728 "Should only be a truncating store, not extending!");
10729 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
10730 assert(VT.isVector() == SVT.isVector() &&
10731 "Cannot use trunc store to convert to or from a vector!");
10732 assert((!VT.isVector() ||
10733 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
10734 "Cannot use trunc store to change the number of vector elements!");
10735
10736 SDVTList VTs = getVTList(VT: MVT::Other);
10737 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
10738 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
10739 FoldingSetNodeID ID;
10740 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
10741 ID.AddInteger(I: SVT.getRawBits());
10742 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
10743 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
10744 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10745 void *IP = nullptr;
10746 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10747 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10748 return SDValue(E, 0);
10749 }
10750 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
10751 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
10752 Args&: IsCompressing, Args&: SVT, Args&: MMO);
10753 createOperands(Node: N, Vals: Ops);
10754
10755 CSEMap.InsertNode(N, InsertPos: IP);
10756 InsertNode(N);
10757 SDValue V(N, 0);
10758 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10759 return V;
10760}
10761
10762SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10763 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
10764 ISD::MemIndexType IndexType) {
10765 assert(Ops.size() == 6 && "Incompatible number of operands");
10766
10767 FoldingSetNodeID ID;
10768 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
10769 ID.AddInteger(I: VT.getRawBits());
10770 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
10771 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10772 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10773 ID.AddInteger(I: MMO->getFlags());
10774 void *IP = nullptr;
10775 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10776 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10777 return SDValue(E, 0);
10778 }
10779
10780 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10781 Args&: VT, Args&: MMO, Args&: IndexType);
10782 createOperands(Node: N, Vals: Ops);
10783
10784 assert(N->getMask().getValueType().getVectorElementCount() ==
10785 N->getValueType(0).getVectorElementCount() &&
10786 "Vector width mismatch between mask and data");
10787 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10788 N->getValueType(0).getVectorElementCount().isScalable() &&
10789 "Scalable flags of index and data do not match");
10790 assert(ElementCount::isKnownGE(
10791 N->getIndex().getValueType().getVectorElementCount(),
10792 N->getValueType(0).getVectorElementCount()) &&
10793 "Vector width mismatch between index and data");
10794 assert(isa<ConstantSDNode>(N->getScale()) &&
10795 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10796 "Scale should be a constant power of 2");
10797
10798 CSEMap.InsertNode(N, InsertPos: IP);
10799 InsertNode(N);
10800 SDValue V(N, 0);
10801 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10802 return V;
10803}
10804
10805SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10806 ArrayRef<SDValue> Ops,
10807 MachineMemOperand *MMO,
10808 ISD::MemIndexType IndexType) {
10809 assert(Ops.size() == 7 && "Incompatible number of operands");
10810
10811 FoldingSetNodeID ID;
10812 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
10813 ID.AddInteger(I: VT.getRawBits());
10814 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
10815 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10816 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10817 ID.AddInteger(I: MMO->getFlags());
10818 void *IP = nullptr;
10819 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10820 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10821 return SDValue(E, 0);
10822 }
10823 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10824 Args&: VT, Args&: MMO, Args&: IndexType);
10825 createOperands(Node: N, Vals: Ops);
10826
10827 assert(N->getMask().getValueType().getVectorElementCount() ==
10828 N->getValue().getValueType().getVectorElementCount() &&
10829 "Vector width mismatch between mask and data");
10830 assert(
10831 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10832 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10833 "Scalable flags of index and data do not match");
10834 assert(ElementCount::isKnownGE(
10835 N->getIndex().getValueType().getVectorElementCount(),
10836 N->getValue().getValueType().getVectorElementCount()) &&
10837 "Vector width mismatch between index and data");
10838 assert(isa<ConstantSDNode>(N->getScale()) &&
10839 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10840 "Scale should be a constant power of 2");
10841
10842 CSEMap.InsertNode(N, InsertPos: IP);
10843 InsertNode(N);
10844 SDValue V(N, 0);
10845 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10846 return V;
10847}
10848
10849SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10850 SDValue Base, SDValue Offset, SDValue Mask,
10851 SDValue PassThru, EVT MemVT,
10852 MachineMemOperand *MMO,
10853 ISD::MemIndexedMode AM,
10854 ISD::LoadExtType ExtTy, bool isExpanding) {
10855 bool Indexed = AM != ISD::UNINDEXED;
10856 assert((Indexed || Offset.isUndef()) &&
10857 "Unindexed masked load with an offset!");
10858 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
10859 : getVTList(VT1: VT, VT2: MVT::Other);
10860 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
10861 FoldingSetNodeID ID;
10862 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
10863 ID.AddInteger(I: MemVT.getRawBits());
10864 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
10865 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
10866 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10867 ID.AddInteger(I: MMO->getFlags());
10868 void *IP = nullptr;
10869 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10870 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10871 return SDValue(E, 0);
10872 }
10873 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10874 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
10875 createOperands(Node: N, Vals: Ops);
10876
10877 CSEMap.InsertNode(N, InsertPos: IP);
10878 InsertNode(N);
10879 SDValue V(N, 0);
10880 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10881 return V;
10882}
10883
10884SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
10885 SDValue Base, SDValue Offset,
10886 ISD::MemIndexedMode AM) {
10887 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
10888 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
10889 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
10890 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
10891 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
10892 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
10893}
10894
10895SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
10896 SDValue Val, SDValue Base, SDValue Offset,
10897 SDValue Mask, EVT MemVT,
10898 MachineMemOperand *MMO,
10899 ISD::MemIndexedMode AM, bool IsTruncating,
10900 bool IsCompressing) {
10901 assert(Chain.getValueType() == MVT::Other &&
10902 "Invalid chain type");
10903 bool Indexed = AM != ISD::UNINDEXED;
10904 assert((Indexed || Offset.isUndef()) &&
10905 "Unindexed masked store with an offset!");
10906 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
10907 : getVTList(VT: MVT::Other);
10908 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
10909 FoldingSetNodeID ID;
10910 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
10911 ID.AddInteger(I: MemVT.getRawBits());
10912 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
10913 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10914 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10915 ID.AddInteger(I: MMO->getFlags());
10916 void *IP = nullptr;
10917 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10918 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10919 return SDValue(E, 0);
10920 }
10921 auto *N =
10922 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10923 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10924 createOperands(Node: N, Vals: Ops);
10925
10926 CSEMap.InsertNode(N, InsertPos: IP);
10927 InsertNode(N);
10928 SDValue V(N, 0);
10929 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10930 return V;
10931}
10932
10933SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
10934 SDValue Base, SDValue Offset,
10935 ISD::MemIndexedMode AM) {
10936 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
10937 assert(ST->getOffset().isUndef() &&
10938 "Masked store is already a indexed store!");
10939 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
10940 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
10941 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
10942}
10943
10944SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10945 ArrayRef<SDValue> Ops,
10946 MachineMemOperand *MMO,
10947 ISD::MemIndexType IndexType,
10948 ISD::LoadExtType ExtTy) {
10949 assert(Ops.size() == 6 && "Incompatible number of operands");
10950
10951 FoldingSetNodeID ID;
10952 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
10953 ID.AddInteger(I: MemVT.getRawBits());
10954 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
10955 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
10956 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10957 ID.AddInteger(I: MMO->getFlags());
10958 void *IP = nullptr;
10959 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10960 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10961 return SDValue(E, 0);
10962 }
10963
10964 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10965 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
10966 createOperands(Node: N, Vals: Ops);
10967
10968 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
10969 "Incompatible type of the PassThru value in MaskedGatherSDNode");
10970 assert(N->getMask().getValueType().getVectorElementCount() ==
10971 N->getValueType(0).getVectorElementCount() &&
10972 "Vector width mismatch between mask and data");
10973 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10974 N->getValueType(0).getVectorElementCount().isScalable() &&
10975 "Scalable flags of index and data do not match");
10976 assert(ElementCount::isKnownGE(
10977 N->getIndex().getValueType().getVectorElementCount(),
10978 N->getValueType(0).getVectorElementCount()) &&
10979 "Vector width mismatch between index and data");
10980 assert(isa<ConstantSDNode>(N->getScale()) &&
10981 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10982 "Scale should be a constant power of 2");
10983
10984 CSEMap.InsertNode(N, InsertPos: IP);
10985 InsertNode(N);
10986 SDValue V(N, 0);
10987 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10988 return V;
10989}
10990
10991SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10992 ArrayRef<SDValue> Ops,
10993 MachineMemOperand *MMO,
10994 ISD::MemIndexType IndexType,
10995 bool IsTrunc) {
10996 assert(Ops.size() == 6 && "Incompatible number of operands");
10997
10998 FoldingSetNodeID ID;
10999 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
11000 ID.AddInteger(I: MemVT.getRawBits());
11001 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
11002 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
11003 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11004 ID.AddInteger(I: MMO->getFlags());
11005 void *IP = nullptr;
11006 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11007 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11008 return SDValue(E, 0);
11009 }
11010
11011 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11012 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
11013 createOperands(Node: N, Vals: Ops);
11014
11015 assert(N->getMask().getValueType().getVectorElementCount() ==
11016 N->getValue().getValueType().getVectorElementCount() &&
11017 "Vector width mismatch between mask and data");
11018 assert(
11019 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
11020 N->getValue().getValueType().getVectorElementCount().isScalable() &&
11021 "Scalable flags of index and data do not match");
11022 assert(ElementCount::isKnownGE(
11023 N->getIndex().getValueType().getVectorElementCount(),
11024 N->getValue().getValueType().getVectorElementCount()) &&
11025 "Vector width mismatch between index and data");
11026 assert(isa<ConstantSDNode>(N->getScale()) &&
11027 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11028 "Scale should be a constant power of 2");
11029
11030 CSEMap.InsertNode(N, InsertPos: IP);
11031 InsertNode(N);
11032 SDValue V(N, 0);
11033 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11034 return V;
11035}
11036
11037SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
11038 const SDLoc &dl, ArrayRef<SDValue> Ops,
11039 MachineMemOperand *MMO,
11040 ISD::MemIndexType IndexType) {
11041 assert(Ops.size() == 7 && "Incompatible number of operands");
11042
11043 FoldingSetNodeID ID;
11044 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTList: VTs, OpList: Ops);
11045 ID.AddInteger(I: MemVT.getRawBits());
11046 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
11047 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
11048 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11049 ID.AddInteger(I: MMO->getFlags());
11050 void *IP = nullptr;
11051 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
11052 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11053 return SDValue(E, 0);
11054 }
11055
11056 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
11057 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
11058 createOperands(Node: N, Vals: Ops);
11059
11060 assert(N->getMask().getValueType().getVectorElementCount() ==
11061 N->getIndex().getValueType().getVectorElementCount() &&
11062 "Vector width mismatch between mask and data");
11063 assert(isa<ConstantSDNode>(N->getScale()) &&
11064 N->getScale()->getAsAPIntVal().isPowerOf2() &&
11065 "Scale should be a constant power of 2");
11066 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
11067
11068 CSEMap.InsertNode(N, InsertPos: IP);
11069 InsertNode(N);
11070 SDValue V(N, 0);
11071 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11072 return V;
11073}
11074
11075SDValue SelectionDAG::getLoadFFVP(EVT VT, const SDLoc &DL, SDValue Chain,
11076 SDValue Ptr, SDValue Mask, SDValue EVL,
11077 MachineMemOperand *MMO) {
11078 SDVTList VTs = getVTList(VT1: VT, VT2: EVL.getValueType(), VT3: MVT::Other);
11079 SDValue Ops[] = {Chain, Ptr, Mask, EVL};
11080 FoldingSetNodeID ID;
11081 AddNodeIDNode(ID, OpC: ISD::VP_LOAD_FF, VTList: VTs, OpList: Ops);
11082 ID.AddInteger(I: VT.getRawBits());
11083 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadFFSDNode>(IROrder: DL.getIROrder(),
11084 Args&: VTs, Args&: VT, Args&: MMO));
11085 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11086 ID.AddInteger(I: MMO->getFlags());
11087 void *IP = nullptr;
11088 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11089 cast<VPLoadFFSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
11090 return SDValue(E, 0);
11091 }
11092 auto *N = newSDNode<VPLoadFFSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs,
11093 Args&: VT, Args&: MMO);
11094 createOperands(Node: N, Vals: Ops);
11095
11096 CSEMap.InsertNode(N, InsertPos: IP);
11097 InsertNode(N);
11098 SDValue V(N, 0);
11099 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11100 return V;
11101}
11102
11103SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11104 EVT MemVT, MachineMemOperand *MMO) {
11105 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11106 SDVTList VTs = getVTList(VT: MVT::Other);
11107 SDValue Ops[] = {Chain, Ptr};
11108 FoldingSetNodeID ID;
11109 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
11110 ID.AddInteger(I: MemVT.getRawBits());
11111 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11112 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11113 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11114 ID.AddInteger(I: MMO->getFlags());
11115 void *IP = nullptr;
11116 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11117 return SDValue(E, 0);
11118
11119 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
11120 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11121 createOperands(Node: N, Vals: Ops);
11122
11123 CSEMap.InsertNode(N, InsertPos: IP);
11124 InsertNode(N);
11125 SDValue V(N, 0);
11126 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11127 return V;
11128}
11129
11130SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
11131 EVT MemVT, MachineMemOperand *MMO) {
11132 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
11133 SDVTList VTs = getVTList(VT: MVT::Other);
11134 SDValue Ops[] = {Chain, Ptr};
11135 FoldingSetNodeID ID;
11136 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
11137 ID.AddInteger(I: MemVT.getRawBits());
11138 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
11139 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
11140 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
11141 ID.AddInteger(I: MMO->getFlags());
11142 void *IP = nullptr;
11143 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
11144 return SDValue(E, 0);
11145
11146 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
11147 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
11148 createOperands(Node: N, Vals: Ops);
11149
11150 CSEMap.InsertNode(N, InsertPos: IP);
11151 InsertNode(N);
11152 SDValue V(N, 0);
11153 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11154 return V;
11155}
11156
11157SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
11158 // select undef, T, F --> T (if T is a constant), otherwise F
11159 // select, ?, undef, F --> F
11160 // select, ?, T, undef --> T
11161 if (Cond.isUndef())
11162 return isConstantValueOfAnyType(N: T) ? T : F;
11163 if (T.isUndef())
11164 return isGuaranteedNotToBePoison(Op: F) ? F : getFreeze(V: F);
11165 if (F.isUndef())
11166 return isGuaranteedNotToBePoison(Op: T) ? T : getFreeze(V: T);
11167
11168 // select true, T, F --> T
11169 // select false, T, F --> F
11170 if (auto C = isBoolConstant(N: Cond))
11171 return *C ? T : F;
11172
11173 // select ?, T, T --> T
11174 if (T == F)
11175 return T;
11176
11177 return SDValue();
11178}
11179
11180SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
11181 // shift undef, Y --> 0 (can always assume that the undef value is 0)
11182 if (X.isUndef())
11183 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
11184 // shift X, undef --> undef (because it may shift by the bitwidth)
11185 if (Y.isUndef())
11186 return getUNDEF(VT: X.getValueType());
11187
11188 // shift 0, Y --> 0
11189 // shift X, 0 --> X
11190 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
11191 return X;
11192
11193 // shift X, C >= bitwidth(X) --> undef
11194 // All vector elements must be too big (or undef) to avoid partial undefs.
11195 auto isShiftTooBig = [X](ConstantSDNode *Val) {
11196 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
11197 };
11198 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
11199 return getUNDEF(VT: X.getValueType());
11200
11201 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
11202 if (X.getValueType().getScalarType() == MVT::i1)
11203 return X;
11204
11205 return SDValue();
11206}
11207
11208SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
11209 SDNodeFlags Flags) {
11210 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
11211 // (an undef operand can be chosen to be Nan/Inf), then the result of this
11212 // operation is poison. That result can be relaxed to undef.
11213 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
11214 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
11215 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
11216 (YC && YC->getValueAPF().isNaN());
11217 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
11218 (YC && YC->getValueAPF().isInfinity());
11219
11220 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
11221 return getUNDEF(VT: X.getValueType());
11222
11223 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
11224 return getUNDEF(VT: X.getValueType());
11225
11226 if (!YC)
11227 return SDValue();
11228
11229 // X + -0.0 --> X
11230 if (Opcode == ISD::FADD)
11231 if (YC->getValueAPF().isNegZero())
11232 return X;
11233
11234 // X - +0.0 --> X
11235 if (Opcode == ISD::FSUB)
11236 if (YC->getValueAPF().isPosZero())
11237 return X;
11238
11239 // X * 1.0 --> X
11240 // X / 1.0 --> X
11241 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
11242 if (YC->getValueAPF().isExactlyValue(V: 1.0))
11243 return X;
11244
11245 // X * 0.0 --> 0.0
11246 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
11247 if (YC->getValueAPF().isZero())
11248 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
11249
11250 return SDValue();
11251}
11252
11253SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
11254 SDValue Ptr, SDValue SV, unsigned Align) {
11255 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
11256 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
11257}
11258
11259SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11260 ArrayRef<SDUse> Ops) {
11261 switch (Ops.size()) {
11262 case 0: return getNode(Opcode, DL, VT);
11263 case 1: return getNode(Opcode, DL, VT, N1: Ops[0].get());
11264 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
11265 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
11266 default: break;
11267 }
11268
11269 // Copy from an SDUse array into an SDValue array for use with
11270 // the regular getNode logic.
11271 SmallVector<SDValue, 8> NewOps(Ops);
11272 return getNode(Opcode, DL, VT, Ops: NewOps);
11273}
11274
11275SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11276 ArrayRef<SDValue> Ops) {
11277 SDNodeFlags Flags;
11278 if (Inserter)
11279 Flags = Inserter->getFlags();
11280 return getNode(Opcode, DL, VT, Ops, Flags);
11281}
11282
11283SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
11284 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11285 unsigned NumOps = Ops.size();
11286 switch (NumOps) {
11287 case 0: return getNode(Opcode, DL, VT);
11288 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
11289 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
11290 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
11291 default: break;
11292 }
11293
11294#ifndef NDEBUG
11295 for (const auto &Op : Ops)
11296 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11297 "Operand is DELETED_NODE!");
11298#endif
11299
11300 switch (Opcode) {
11301 default: break;
11302 case ISD::BUILD_VECTOR:
11303 // Attempt to simplify BUILD_VECTOR.
11304 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
11305 return V;
11306 break;
11307 case ISD::CONCAT_VECTORS:
11308 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
11309 return V;
11310 break;
11311 case ISD::SELECT_CC:
11312 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
11313 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
11314 "LHS and RHS of condition must have same type!");
11315 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11316 "True and False arms of SelectCC must have same type!");
11317 assert(Ops[2].getValueType() == VT &&
11318 "select_cc node must be of same type as true and false value!");
11319 assert((!Ops[0].getValueType().isVector() ||
11320 Ops[0].getValueType().getVectorElementCount() ==
11321 VT.getVectorElementCount()) &&
11322 "Expected select_cc with vector result to have the same sized "
11323 "comparison type!");
11324 break;
11325 case ISD::BR_CC:
11326 assert(NumOps == 5 && "BR_CC takes 5 operands!");
11327 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
11328 "LHS/RHS of comparison should match types!");
11329 break;
11330 case ISD::VP_ADD:
11331 case ISD::VP_SUB:
11332 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
11333 if (VT.getScalarType() == MVT::i1)
11334 Opcode = ISD::VP_XOR;
11335 break;
11336 case ISD::VP_MUL:
11337 // If it is VP_MUL mask operation then turn it to VP_AND
11338 if (VT.getScalarType() == MVT::i1)
11339 Opcode = ISD::VP_AND;
11340 break;
11341 case ISD::VP_REDUCE_MUL:
11342 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
11343 if (VT == MVT::i1)
11344 Opcode = ISD::VP_REDUCE_AND;
11345 break;
11346 case ISD::VP_REDUCE_ADD:
11347 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
11348 if (VT == MVT::i1)
11349 Opcode = ISD::VP_REDUCE_XOR;
11350 break;
11351 case ISD::VP_REDUCE_SMAX:
11352 case ISD::VP_REDUCE_UMIN:
11353 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
11354 // VP_REDUCE_AND.
11355 if (VT == MVT::i1)
11356 Opcode = ISD::VP_REDUCE_AND;
11357 break;
11358 case ISD::VP_REDUCE_SMIN:
11359 case ISD::VP_REDUCE_UMAX:
11360 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
11361 // VP_REDUCE_OR.
11362 if (VT == MVT::i1)
11363 Opcode = ISD::VP_REDUCE_OR;
11364 break;
11365 }
11366
11367 // Memoize nodes.
11368 SDNode *N;
11369 SDVTList VTs = getVTList(VT);
11370
11371 if (VT != MVT::Glue) {
11372 FoldingSetNodeID ID;
11373 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
11374 void *IP = nullptr;
11375
11376 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11377 E->intersectFlagsWith(Flags);
11378 return SDValue(E, 0);
11379 }
11380
11381 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11382 createOperands(Node: N, Vals: Ops);
11383
11384 CSEMap.InsertNode(N, InsertPos: IP);
11385 } else {
11386 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11387 createOperands(Node: N, Vals: Ops);
11388 }
11389
11390 N->setFlags(Flags);
11391 InsertNode(N);
11392 SDValue V(N, 0);
11393 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11394 return V;
11395}
11396
11397SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11398 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
11399 SDNodeFlags Flags;
11400 if (Inserter)
11401 Flags = Inserter->getFlags();
11402 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11403}
11404
11405SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11406 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops,
11407 const SDNodeFlags Flags) {
11408 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops, Flags);
11409}
11410
11411SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11412 ArrayRef<SDValue> Ops) {
11413 SDNodeFlags Flags;
11414 if (Inserter)
11415 Flags = Inserter->getFlags();
11416 return getNode(Opcode, DL, VTList, Ops, Flags);
11417}
11418
11419SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11420 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
11421 if (VTList.NumVTs == 1)
11422 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
11423
11424#ifndef NDEBUG
11425 for (const auto &Op : Ops)
11426 assert(Op.getOpcode() != ISD::DELETED_NODE &&
11427 "Operand is DELETED_NODE!");
11428#endif
11429
11430 switch (Opcode) {
11431 case ISD::SADDO:
11432 case ISD::UADDO:
11433 case ISD::SSUBO:
11434 case ISD::USUBO: {
11435 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11436 "Invalid add/sub overflow op!");
11437 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11438 Ops[0].getValueType() == Ops[1].getValueType() &&
11439 Ops[0].getValueType() == VTList.VTs[0] &&
11440 "Binary operator types must match!");
11441 SDValue N1 = Ops[0], N2 = Ops[1];
11442 canonicalizeCommutativeBinop(Opcode, N1, N2);
11443
11444 // (X +- 0) -> X with zero-overflow.
11445 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
11446 /*AllowTruncation*/ true);
11447 if (N2CV && N2CV->isZero()) {
11448 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
11449 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
11450 }
11451
11452 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
11453 VTList.VTs[1].getScalarType() == MVT::i1) {
11454 SDValue F1 = getFreeze(V: N1);
11455 SDValue F2 = getFreeze(V: N2);
11456 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
11457 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
11458 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11459 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11460 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
11461 Flags);
11462 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
11463 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
11464 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
11465 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
11466 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
11467 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
11468 Flags);
11469 }
11470 }
11471 break;
11472 }
11473 case ISD::SADDO_CARRY:
11474 case ISD::UADDO_CARRY:
11475 case ISD::SSUBO_CARRY:
11476 case ISD::USUBO_CARRY:
11477 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
11478 "Invalid add/sub overflow op!");
11479 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
11480 Ops[0].getValueType() == Ops[1].getValueType() &&
11481 Ops[0].getValueType() == VTList.VTs[0] &&
11482 Ops[2].getValueType() == VTList.VTs[1] &&
11483 "Binary operator types must match!");
11484 break;
11485 case ISD::SMUL_LOHI:
11486 case ISD::UMUL_LOHI: {
11487 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
11488 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
11489 VTList.VTs[0] == Ops[0].getValueType() &&
11490 VTList.VTs[0] == Ops[1].getValueType() &&
11491 "Binary operator types must match!");
11492 // Constant fold.
11493 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
11494 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
11495 if (LHS && RHS) {
11496 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
11497 unsigned OutWidth = Width * 2;
11498 APInt Val = LHS->getAPIntValue();
11499 APInt Mul = RHS->getAPIntValue();
11500 if (Opcode == ISD::SMUL_LOHI) {
11501 Val = Val.sext(width: OutWidth);
11502 Mul = Mul.sext(width: OutWidth);
11503 } else {
11504 Val = Val.zext(width: OutWidth);
11505 Mul = Mul.zext(width: OutWidth);
11506 }
11507 Val *= Mul;
11508
11509 SDValue Hi =
11510 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
11511 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
11512 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
11513 }
11514 break;
11515 }
11516 case ISD::FFREXP: {
11517 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
11518 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
11519 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
11520
11521 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
11522 int FrexpExp;
11523 APFloat FrexpMant =
11524 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
11525 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
11526 SDValue Result1 = getSignedConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0,
11527 DL, VT: VTList.VTs[1]);
11528 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
11529 }
11530
11531 break;
11532 }
11533 case ISD::STRICT_FP_EXTEND:
11534 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
11535 "Invalid STRICT_FP_EXTEND!");
11536 assert(VTList.VTs[0].isFloatingPoint() &&
11537 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
11538 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11539 "STRICT_FP_EXTEND result type should be vector iff the operand "
11540 "type is vector!");
11541 assert((!VTList.VTs[0].isVector() ||
11542 VTList.VTs[0].getVectorElementCount() ==
11543 Ops[1].getValueType().getVectorElementCount()) &&
11544 "Vector element count mismatch!");
11545 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
11546 "Invalid fpext node, dst <= src!");
11547 break;
11548 case ISD::STRICT_FP_ROUND:
11549 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
11550 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
11551 "STRICT_FP_ROUND result type should be vector iff the operand "
11552 "type is vector!");
11553 assert((!VTList.VTs[0].isVector() ||
11554 VTList.VTs[0].getVectorElementCount() ==
11555 Ops[1].getValueType().getVectorElementCount()) &&
11556 "Vector element count mismatch!");
11557 assert(VTList.VTs[0].isFloatingPoint() &&
11558 Ops[1].getValueType().isFloatingPoint() &&
11559 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
11560 Ops[2].getOpcode() == ISD::TargetConstant &&
11561 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
11562 "Invalid STRICT_FP_ROUND!");
11563 break;
11564 }
11565
11566 // Memoize the node unless it returns a glue result.
11567 SDNode *N;
11568 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
11569 FoldingSetNodeID ID;
11570 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
11571 void *IP = nullptr;
11572 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11573 E->intersectFlagsWith(Flags);
11574 return SDValue(E, 0);
11575 }
11576
11577 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11578 createOperands(Node: N, Vals: Ops);
11579 CSEMap.InsertNode(N, InsertPos: IP);
11580 } else {
11581 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
11582 createOperands(Node: N, Vals: Ops);
11583 }
11584
11585 N->setFlags(Flags);
11586 InsertNode(N);
11587 SDValue V(N, 0);
11588 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
11589 return V;
11590}
11591
11592SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
11593 SDVTList VTList) {
11594 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
11595}
11596
11597SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11598 SDValue N1) {
11599 SDValue Ops[] = { N1 };
11600 return getNode(Opcode, DL, VTList, Ops);
11601}
11602
11603SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11604 SDValue N1, SDValue N2) {
11605 SDValue Ops[] = { N1, N2 };
11606 return getNode(Opcode, DL, VTList, Ops);
11607}
11608
11609SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11610 SDValue N1, SDValue N2, SDValue N3) {
11611 SDValue Ops[] = { N1, N2, N3 };
11612 return getNode(Opcode, DL, VTList, Ops);
11613}
11614
11615SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11616 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
11617 SDValue Ops[] = { N1, N2, N3, N4 };
11618 return getNode(Opcode, DL, VTList, Ops);
11619}
11620
11621SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
11622 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
11623 SDValue N5) {
11624 SDValue Ops[] = { N1, N2, N3, N4, N5 };
11625 return getNode(Opcode, DL, VTList, Ops);
11626}
11627
11628SDVTList SelectionDAG::getVTList(EVT VT) {
11629 if (!VT.isExtended())
11630 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
11631
11632 return makeVTList(VTs: &(*EVTs.insert(x: VT).first), NumVTs: 1);
11633}
11634
11635SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
11636 FoldingSetNodeID ID;
11637 ID.AddInteger(I: 2U);
11638 ID.AddInteger(I: VT1.getRawBits());
11639 ID.AddInteger(I: VT2.getRawBits());
11640
11641 void *IP = nullptr;
11642 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11643 if (!Result) {
11644 EVT *Array = Allocator.Allocate<EVT>(Num: 2);
11645 Array[0] = VT1;
11646 Array[1] = VT2;
11647 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
11648 VTListMap.InsertNode(N: Result, InsertPos: IP);
11649 }
11650 return Result->getSDVTList();
11651}
11652
11653SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
11654 FoldingSetNodeID ID;
11655 ID.AddInteger(I: 3U);
11656 ID.AddInteger(I: VT1.getRawBits());
11657 ID.AddInteger(I: VT2.getRawBits());
11658 ID.AddInteger(I: VT3.getRawBits());
11659
11660 void *IP = nullptr;
11661 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11662 if (!Result) {
11663 EVT *Array = Allocator.Allocate<EVT>(Num: 3);
11664 Array[0] = VT1;
11665 Array[1] = VT2;
11666 Array[2] = VT3;
11667 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
11668 VTListMap.InsertNode(N: Result, InsertPos: IP);
11669 }
11670 return Result->getSDVTList();
11671}
11672
11673SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
11674 FoldingSetNodeID ID;
11675 ID.AddInteger(I: 4U);
11676 ID.AddInteger(I: VT1.getRawBits());
11677 ID.AddInteger(I: VT2.getRawBits());
11678 ID.AddInteger(I: VT3.getRawBits());
11679 ID.AddInteger(I: VT4.getRawBits());
11680
11681 void *IP = nullptr;
11682 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11683 if (!Result) {
11684 EVT *Array = Allocator.Allocate<EVT>(Num: 4);
11685 Array[0] = VT1;
11686 Array[1] = VT2;
11687 Array[2] = VT3;
11688 Array[3] = VT4;
11689 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
11690 VTListMap.InsertNode(N: Result, InsertPos: IP);
11691 }
11692 return Result->getSDVTList();
11693}
11694
11695SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
11696 unsigned NumVTs = VTs.size();
11697 FoldingSetNodeID ID;
11698 ID.AddInteger(I: NumVTs);
11699 for (unsigned index = 0; index < NumVTs; index++) {
11700 ID.AddInteger(I: VTs[index].getRawBits());
11701 }
11702
11703 void *IP = nullptr;
11704 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
11705 if (!Result) {
11706 EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
11707 llvm::copy(Range&: VTs, Out: Array);
11708 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
11709 VTListMap.InsertNode(N: Result, InsertPos: IP);
11710 }
11711 return Result->getSDVTList();
11712}
11713
11714
11715/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
11716/// specified operands. If the resultant node already exists in the DAG,
11717/// this does not modify the specified node, instead it returns the node that
11718/// already exists. If the resultant node does not exist in the DAG, the
11719/// input node is returned. As a degenerate case, if you specify the same
11720/// input operands as the node already has, the input node is returned.
11721SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
11722 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
11723
11724 // Check to see if there is no change.
11725 if (Op == N->getOperand(Num: 0)) return N;
11726
11727 // See if the modified node already exists.
11728 void *InsertPos = nullptr;
11729 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
11730 return Existing;
11731
11732 // Nope it doesn't. Remove the node from its current place in the maps.
11733 if (InsertPos)
11734 if (!RemoveNodeFromCSEMaps(N))
11735 InsertPos = nullptr;
11736
11737 // Now we update the operands.
11738 N->OperandList[0].set(Op);
11739
11740 updateDivergence(N);
11741 // If this gets put into a CSE map, add it.
11742 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11743 return N;
11744}
11745
11746SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
11747 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
11748
11749 // Check to see if there is no change.
11750 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
11751 return N; // No operands changed, just return the input node.
11752
11753 // See if the modified node already exists.
11754 void *InsertPos = nullptr;
11755 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
11756 return Existing;
11757
11758 // Nope it doesn't. Remove the node from its current place in the maps.
11759 if (InsertPos)
11760 if (!RemoveNodeFromCSEMaps(N))
11761 InsertPos = nullptr;
11762
11763 // Now we update the operands.
11764 if (N->OperandList[0] != Op1)
11765 N->OperandList[0].set(Op1);
11766 if (N->OperandList[1] != Op2)
11767 N->OperandList[1].set(Op2);
11768
11769 updateDivergence(N);
11770 // If this gets put into a CSE map, add it.
11771 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11772 return N;
11773}
11774
11775SDNode *SelectionDAG::
11776UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
11777 SDValue Ops[] = { Op1, Op2, Op3 };
11778 return UpdateNodeOperands(N, Ops);
11779}
11780
11781SDNode *SelectionDAG::
11782UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11783 SDValue Op3, SDValue Op4) {
11784 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
11785 return UpdateNodeOperands(N, Ops);
11786}
11787
11788SDNode *SelectionDAG::
11789UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11790 SDValue Op3, SDValue Op4, SDValue Op5) {
11791 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
11792 return UpdateNodeOperands(N, Ops);
11793}
11794
11795SDNode *SelectionDAG::
11796UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
11797 unsigned NumOps = Ops.size();
11798 assert(N->getNumOperands() == NumOps &&
11799 "Update with wrong number of operands");
11800
11801 // If no operands changed just return the input node.
11802 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
11803 return N;
11804
11805 // See if the modified node already exists.
11806 void *InsertPos = nullptr;
11807 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
11808 return Existing;
11809
11810 // Nope it doesn't. Remove the node from its current place in the maps.
11811 if (InsertPos)
11812 if (!RemoveNodeFromCSEMaps(N))
11813 InsertPos = nullptr;
11814
11815 // Now we update the operands.
11816 for (unsigned i = 0; i != NumOps; ++i)
11817 if (N->OperandList[i] != Ops[i])
11818 N->OperandList[i].set(Ops[i]);
11819
11820 updateDivergence(N);
11821 // If this gets put into a CSE map, add it.
11822 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11823 return N;
11824}
11825
11826/// DropOperands - Release the operands and set this node to have
11827/// zero operands.
11828void SDNode::DropOperands() {
11829 // Unlike the code in MorphNodeTo that does this, we don't need to
11830 // watch for dead nodes here.
11831 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
11832 SDUse &Use = *I++;
11833 Use.set(SDValue());
11834 }
11835}
11836
11837void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
11838 ArrayRef<MachineMemOperand *> NewMemRefs) {
11839 if (NewMemRefs.empty()) {
11840 N->clearMemRefs();
11841 return;
11842 }
11843
11844 // Check if we can avoid allocating by storing a single reference directly.
11845 if (NewMemRefs.size() == 1) {
11846 N->MemRefs = NewMemRefs[0];
11847 N->NumMemRefs = 1;
11848 return;
11849 }
11850
11851 MachineMemOperand **MemRefsBuffer =
11852 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
11853 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
11854 N->MemRefs = MemRefsBuffer;
11855 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
11856}
11857
11858/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
11859/// machine opcode.
11860///
11861SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11862 EVT VT) {
11863 SDVTList VTs = getVTList(VT);
11864 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11865}
11866
11867SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11868 EVT VT, SDValue Op1) {
11869 SDVTList VTs = getVTList(VT);
11870 SDValue Ops[] = { Op1 };
11871 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11872}
11873
11874SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11875 EVT VT, SDValue Op1,
11876 SDValue Op2) {
11877 SDVTList VTs = getVTList(VT);
11878 SDValue Ops[] = { Op1, Op2 };
11879 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11880}
11881
11882SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11883 EVT VT, SDValue Op1,
11884 SDValue Op2, SDValue Op3) {
11885 SDVTList VTs = getVTList(VT);
11886 SDValue Ops[] = { Op1, Op2, Op3 };
11887 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11888}
11889
11890SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11891 EVT VT, ArrayRef<SDValue> Ops) {
11892 SDVTList VTs = getVTList(VT);
11893 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11894}
11895
11896SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11897 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
11898 SDVTList VTs = getVTList(VT1, VT2);
11899 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11900}
11901
11902SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11903 EVT VT1, EVT VT2) {
11904 SDVTList VTs = getVTList(VT1, VT2);
11905 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11906}
11907
11908SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11909 EVT VT1, EVT VT2, EVT VT3,
11910 ArrayRef<SDValue> Ops) {
11911 SDVTList VTs = getVTList(VT1, VT2, VT3);
11912 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11913}
11914
11915SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11916 EVT VT1, EVT VT2,
11917 SDValue Op1, SDValue Op2) {
11918 SDVTList VTs = getVTList(VT1, VT2);
11919 SDValue Ops[] = { Op1, Op2 };
11920 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11921}
11922
11923SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11924 SDVTList VTs,ArrayRef<SDValue> Ops) {
11925 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
11926 // Reset the NodeID to -1.
11927 New->setNodeId(-1);
11928 if (New != N) {
11929 ReplaceAllUsesWith(From: N, To: New);
11930 RemoveDeadNode(N);
11931 }
11932 return New;
11933}
11934
11935/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
11936/// the line number information on the merged node since it is not possible to
11937/// preserve the information that operation is associated with multiple lines.
11938/// This will make the debugger working better at -O0, were there is a higher
11939/// probability having other instructions associated with that line.
11940///
11941/// For IROrder, we keep the smaller of the two
11942SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
11943 DebugLoc NLoc = N->getDebugLoc();
11944 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
11945 N->setDebugLoc(DebugLoc());
11946 }
11947 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
11948 N->setIROrder(Order);
11949 return N;
11950}
11951
11952/// MorphNodeTo - This *mutates* the specified node to have the specified
11953/// return type, opcode, and operands.
11954///
11955/// Note that MorphNodeTo returns the resultant node. If there is already a
11956/// node of the specified opcode and operands, it returns that node instead of
11957/// the current one. Note that the SDLoc need not be the same.
11958///
11959/// Using MorphNodeTo is faster than creating a new node and swapping it in
11960/// with ReplaceAllUsesWith both because it often avoids allocating a new
11961/// node, and because it doesn't require CSE recalculation for any of
11962/// the node's users.
11963///
11964/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
11965/// As a consequence it isn't appropriate to use from within the DAG combiner or
11966/// the legalizer which maintain worklists that would need to be updated when
11967/// deleting things.
11968SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
11969 SDVTList VTs, ArrayRef<SDValue> Ops) {
11970 // If an identical node already exists, use it.
11971 void *IP = nullptr;
11972 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
11973 FoldingSetNodeID ID;
11974 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
11975 if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
11976 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
11977 }
11978
11979 if (!RemoveNodeFromCSEMaps(N))
11980 IP = nullptr;
11981
11982 // Start the morphing.
11983 N->NodeType = Opc;
11984 N->ValueList = VTs.VTs;
11985 N->NumValues = VTs.NumVTs;
11986
11987 // Clear the operands list, updating used nodes to remove this from their
11988 // use list. Keep track of any operands that become dead as a result.
11989 SmallPtrSet<SDNode*, 16> DeadNodeSet;
11990 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
11991 SDUse &Use = *I++;
11992 SDNode *Used = Use.getNode();
11993 Use.set(SDValue());
11994 if (Used->use_empty())
11995 DeadNodeSet.insert(Ptr: Used);
11996 }
11997
11998 // For MachineNode, initialize the memory references information.
11999 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
12000 MN->clearMemRefs();
12001
12002 // Swap for an appropriately sized array from the recycler.
12003 removeOperands(Node: N);
12004 createOperands(Node: N, Vals: Ops);
12005
12006 // Delete any nodes that are still dead after adding the uses for the
12007 // new operands.
12008 if (!DeadNodeSet.empty()) {
12009 SmallVector<SDNode *, 16> DeadNodes;
12010 for (SDNode *N : DeadNodeSet)
12011 if (N->use_empty())
12012 DeadNodes.push_back(Elt: N);
12013 RemoveDeadNodes(DeadNodes);
12014 }
12015
12016 if (IP)
12017 CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
12018 return N;
12019}
12020
12021SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
12022 unsigned OrigOpc = Node->getOpcode();
12023 unsigned NewOpc;
12024 switch (OrigOpc) {
12025 default:
12026 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
12027#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12028 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
12029#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
12030 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
12031#include "llvm/IR/ConstrainedOps.def"
12032 }
12033
12034 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
12035
12036 // We're taking this node out of the chain, so we need to re-link things.
12037 SDValue InputChain = Node->getOperand(Num: 0);
12038 SDValue OutputChain = SDValue(Node, 1);
12039 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
12040
12041 SmallVector<SDValue, 3> Ops;
12042 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
12043 Ops.push_back(Elt: Node->getOperand(Num: i));
12044
12045 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
12046 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
12047
12048 // MorphNodeTo can operate in two ways: if an existing node with the
12049 // specified operands exists, it can just return it. Otherwise, it
12050 // updates the node in place to have the requested operands.
12051 if (Res == Node) {
12052 // If we updated the node in place, reset the node ID. To the isel,
12053 // this should be just like a newly allocated machine node.
12054 Res->setNodeId(-1);
12055 } else {
12056 ReplaceAllUsesWith(From: Node, To: Res);
12057 RemoveDeadNode(N: Node);
12058 }
12059
12060 return Res;
12061}
12062
12063/// getMachineNode - These are used for target selectors to create a new node
12064/// with specified return type(s), MachineInstr opcode, and operands.
12065///
12066/// Note that getMachineNode returns the resultant node. If there is already a
12067/// node of the specified opcode and operands, it returns that node instead of
12068/// the current one.
12069MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12070 EVT VT) {
12071 SDVTList VTs = getVTList(VT);
12072 return getMachineNode(Opcode, dl, VTs, Ops: {});
12073}
12074
12075MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12076 EVT VT, SDValue Op1) {
12077 SDVTList VTs = getVTList(VT);
12078 SDValue Ops[] = { Op1 };
12079 return getMachineNode(Opcode, dl, VTs, Ops);
12080}
12081
12082MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12083 EVT VT, SDValue Op1, SDValue Op2) {
12084 SDVTList VTs = getVTList(VT);
12085 SDValue Ops[] = { Op1, Op2 };
12086 return getMachineNode(Opcode, dl, VTs, Ops);
12087}
12088
12089MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12090 EVT VT, SDValue Op1, SDValue Op2,
12091 SDValue Op3) {
12092 SDVTList VTs = getVTList(VT);
12093 SDValue Ops[] = { Op1, Op2, Op3 };
12094 return getMachineNode(Opcode, dl, VTs, Ops);
12095}
12096
12097MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12098 EVT VT, ArrayRef<SDValue> Ops) {
12099 SDVTList VTs = getVTList(VT);
12100 return getMachineNode(Opcode, dl, VTs, Ops);
12101}
12102
12103MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12104 EVT VT1, EVT VT2, SDValue Op1,
12105 SDValue Op2) {
12106 SDVTList VTs = getVTList(VT1, VT2);
12107 SDValue Ops[] = { Op1, Op2 };
12108 return getMachineNode(Opcode, dl, VTs, Ops);
12109}
12110
12111MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12112 EVT VT1, EVT VT2, SDValue Op1,
12113 SDValue Op2, SDValue Op3) {
12114 SDVTList VTs = getVTList(VT1, VT2);
12115 SDValue Ops[] = { Op1, Op2, Op3 };
12116 return getMachineNode(Opcode, dl, VTs, Ops);
12117}
12118
12119MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12120 EVT VT1, EVT VT2,
12121 ArrayRef<SDValue> Ops) {
12122 SDVTList VTs = getVTList(VT1, VT2);
12123 return getMachineNode(Opcode, dl, VTs, Ops);
12124}
12125
12126MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12127 EVT VT1, EVT VT2, EVT VT3,
12128 SDValue Op1, SDValue Op2) {
12129 SDVTList VTs = getVTList(VT1, VT2, VT3);
12130 SDValue Ops[] = { Op1, Op2 };
12131 return getMachineNode(Opcode, dl, VTs, Ops);
12132}
12133
12134MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12135 EVT VT1, EVT VT2, EVT VT3,
12136 SDValue Op1, SDValue Op2,
12137 SDValue Op3) {
12138 SDVTList VTs = getVTList(VT1, VT2, VT3);
12139 SDValue Ops[] = { Op1, Op2, Op3 };
12140 return getMachineNode(Opcode, dl, VTs, Ops);
12141}
12142
12143MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12144 EVT VT1, EVT VT2, EVT VT3,
12145 ArrayRef<SDValue> Ops) {
12146 SDVTList VTs = getVTList(VT1, VT2, VT3);
12147 return getMachineNode(Opcode, dl, VTs, Ops);
12148}
12149
12150MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
12151 ArrayRef<EVT> ResultTys,
12152 ArrayRef<SDValue> Ops) {
12153 SDVTList VTs = getVTList(VTs: ResultTys);
12154 return getMachineNode(Opcode, dl, VTs, Ops);
12155}
12156
12157MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
12158 SDVTList VTs,
12159 ArrayRef<SDValue> Ops) {
12160 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
12161 MachineSDNode *N;
12162 void *IP = nullptr;
12163
12164 if (DoCSE) {
12165 FoldingSetNodeID ID;
12166 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
12167 IP = nullptr;
12168 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
12169 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
12170 }
12171 }
12172
12173 // Allocate a new MachineSDNode.
12174 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
12175 createOperands(Node: N, Vals: Ops);
12176
12177 if (DoCSE)
12178 CSEMap.InsertNode(N, InsertPos: IP);
12179
12180 InsertNode(N);
12181 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
12182 return N;
12183}
12184
12185/// getTargetExtractSubreg - A convenience function for creating
12186/// TargetOpcode::EXTRACT_SUBREG nodes.
12187SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12188 SDValue Operand) {
12189 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12190 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
12191 VT, Op1: Operand, Op2: SRIdxVal);
12192 return SDValue(Subreg, 0);
12193}
12194
12195/// getTargetInsertSubreg - A convenience function for creating
12196/// TargetOpcode::INSERT_SUBREG nodes.
12197SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
12198 SDValue Operand, SDValue Subreg) {
12199 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
12200 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
12201 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
12202 return SDValue(Result, 0);
12203}
12204
12205/// getNodeIfExists - Get the specified node if it's already available, or
12206/// else return NULL.
12207SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12208 ArrayRef<SDValue> Ops,
12209 bool AllowCommute) {
12210 SDNodeFlags Flags;
12211 if (Inserter)
12212 Flags = Inserter->getFlags();
12213 return getNodeIfExists(Opcode, VTList, Ops, Flags, AllowCommute);
12214}
12215
12216SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
12217 ArrayRef<SDValue> Ops,
12218 const SDNodeFlags Flags,
12219 bool AllowCommute) {
12220 if (VTList.VTs[VTList.NumVTs - 1] == MVT::Glue)
12221 return nullptr;
12222
12223 auto Lookup = [&](ArrayRef<SDValue> LookupOps) -> SDNode * {
12224 FoldingSetNodeID ID;
12225 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: LookupOps);
12226 void *IP = nullptr;
12227 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP)) {
12228 E->intersectFlagsWith(Flags);
12229 return E;
12230 }
12231 return nullptr;
12232 };
12233
12234 if (SDNode *Existing = Lookup(Ops))
12235 return Existing;
12236
12237 if (AllowCommute && TLI->isCommutativeBinOp(Opcode))
12238 return Lookup({Ops[1], Ops[0]});
12239
12240 return nullptr;
12241}
12242
12243/// doesNodeExist - Check if a node exists without modifying its flags.
12244bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
12245 ArrayRef<SDValue> Ops) {
12246 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
12247 FoldingSetNodeID ID;
12248 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
12249 void *IP = nullptr;
12250 if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
12251 return true;
12252 }
12253 return false;
12254}
12255
12256/// getDbgValue - Creates a SDDbgValue node.
12257///
12258/// SDNode
12259SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
12260 SDNode *N, unsigned R, bool IsIndirect,
12261 const DebugLoc &DL, unsigned O) {
12262 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12263 "Expected inlined-at fields to agree");
12264 return new (DbgInfo->getAlloc())
12265 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
12266 {}, IsIndirect, DL, O,
12267 /*IsVariadic=*/false);
12268}
12269
12270/// Constant
12271SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
12272 DIExpression *Expr,
12273 const Value *C,
12274 const DebugLoc &DL, unsigned O) {
12275 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12276 "Expected inlined-at fields to agree");
12277 return new (DbgInfo->getAlloc())
12278 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
12279 /*IsIndirect=*/false, DL, O,
12280 /*IsVariadic=*/false);
12281}
12282
12283/// FrameIndex
12284SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12285 DIExpression *Expr, unsigned FI,
12286 bool IsIndirect,
12287 const DebugLoc &DL,
12288 unsigned O) {
12289 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12290 "Expected inlined-at fields to agree");
12291 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
12292}
12293
12294/// FrameIndex with dependencies
12295SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
12296 DIExpression *Expr, unsigned FI,
12297 ArrayRef<SDNode *> Dependencies,
12298 bool IsIndirect,
12299 const DebugLoc &DL,
12300 unsigned O) {
12301 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12302 "Expected inlined-at fields to agree");
12303 return new (DbgInfo->getAlloc())
12304 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
12305 Dependencies, IsIndirect, DL, O,
12306 /*IsVariadic=*/false);
12307}
12308
12309/// VReg
12310SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
12311 Register VReg, bool IsIndirect,
12312 const DebugLoc &DL, unsigned O) {
12313 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12314 "Expected inlined-at fields to agree");
12315 return new (DbgInfo->getAlloc())
12316 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
12317 {}, IsIndirect, DL, O,
12318 /*IsVariadic=*/false);
12319}
12320
12321SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
12322 ArrayRef<SDDbgOperand> Locs,
12323 ArrayRef<SDNode *> Dependencies,
12324 bool IsIndirect, const DebugLoc &DL,
12325 unsigned O, bool IsVariadic) {
12326 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
12327 "Expected inlined-at fields to agree");
12328 return new (DbgInfo->getAlloc())
12329 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
12330 DL, O, IsVariadic);
12331}
12332
12333void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
12334 unsigned OffsetInBits, unsigned SizeInBits,
12335 bool InvalidateDbg) {
12336 SDNode *FromNode = From.getNode();
12337 SDNode *ToNode = To.getNode();
12338 assert(FromNode && ToNode && "Can't modify dbg values");
12339
12340 // PR35338
12341 // TODO: assert(From != To && "Redundant dbg value transfer");
12342 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
12343 if (From == To || FromNode == ToNode)
12344 return;
12345
12346 if (!FromNode->getHasDebugValue())
12347 return;
12348
12349 SDDbgOperand FromLocOp =
12350 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
12351 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
12352
12353 SmallVector<SDDbgValue *, 2> ClonedDVs;
12354 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
12355 if (Dbg->isInvalidated())
12356 continue;
12357
12358 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
12359
12360 // Create a new location ops vector that is equal to the old vector, but
12361 // with each instance of FromLocOp replaced with ToLocOp.
12362 bool Changed = false;
12363 auto NewLocOps = Dbg->copyLocationOps();
12364 std::replace_if(
12365 first: NewLocOps.begin(), last: NewLocOps.end(),
12366 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
12367 bool Match = Op == FromLocOp;
12368 Changed |= Match;
12369 return Match;
12370 },
12371 new_value: ToLocOp);
12372 // Ignore this SDDbgValue if we didn't find a matching location.
12373 if (!Changed)
12374 continue;
12375
12376 DIVariable *Var = Dbg->getVariable();
12377 auto *Expr = Dbg->getExpression();
12378 // If a fragment is requested, update the expression.
12379 if (SizeInBits) {
12380 // When splitting a larger (e.g., sign-extended) value whose
12381 // lower bits are described with an SDDbgValue, do not attempt
12382 // to transfer the SDDbgValue to the upper bits.
12383 if (auto FI = Expr->getFragmentInfo())
12384 if (OffsetInBits + SizeInBits > FI->SizeInBits)
12385 continue;
12386 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
12387 SizeInBits);
12388 if (!Fragment)
12389 continue;
12390 Expr = *Fragment;
12391 }
12392
12393 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
12394 // Clone the SDDbgValue and move it to To.
12395 SDDbgValue *Clone = getDbgValueList(
12396 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
12397 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
12398 IsVariadic: Dbg->isVariadic());
12399 ClonedDVs.push_back(Elt: Clone);
12400
12401 if (InvalidateDbg) {
12402 // Invalidate value and indicate the SDDbgValue should not be emitted.
12403 Dbg->setIsInvalidated();
12404 Dbg->setIsEmitted();
12405 }
12406 }
12407
12408 for (SDDbgValue *Dbg : ClonedDVs) {
12409 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
12410 "Transferred DbgValues should depend on the new SDNode");
12411 AddDbgValue(DB: Dbg, isParameter: false);
12412 }
12413}
12414
12415void SelectionDAG::salvageDebugInfo(SDNode &N) {
12416 if (!N.getHasDebugValue())
12417 return;
12418
12419 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
12420 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
12421 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
12422 return SDDbgOperand::fromNode(Node, ResNo);
12423 };
12424
12425 SmallVector<SDDbgValue *, 2> ClonedDVs;
12426 for (auto *DV : GetDbgValues(SD: &N)) {
12427 if (DV->isInvalidated())
12428 continue;
12429 switch (N.getOpcode()) {
12430 default:
12431 break;
12432 case ISD::ADD: {
12433 SDValue N0 = N.getOperand(Num: 0);
12434 SDValue N1 = N.getOperand(Num: 1);
12435 if (!isa<ConstantSDNode>(Val: N0)) {
12436 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
12437 uint64_t Offset;
12438 if (RHSConstant)
12439 Offset = N.getConstantOperandVal(Num: 1);
12440 // We are not allowed to turn indirect debug values variadic, so
12441 // don't salvage those.
12442 if (!RHSConstant && DV->isIndirect())
12443 continue;
12444
12445 // Rewrite an ADD constant node into a DIExpression. Since we are
12446 // performing arithmetic to compute the variable's *value* in the
12447 // DIExpression, we need to mark the expression with a
12448 // DW_OP_stack_value.
12449 auto *DIExpr = DV->getExpression();
12450 auto NewLocOps = DV->copyLocationOps();
12451 bool Changed = false;
12452 size_t OrigLocOpsSize = NewLocOps.size();
12453 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
12454 // We're not given a ResNo to compare against because the whole
12455 // node is going away. We know that any ISD::ADD only has one
12456 // result, so we can assume any node match is using the result.
12457 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12458 NewLocOps[i].getSDNode() != &N)
12459 continue;
12460 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12461 if (RHSConstant) {
12462 SmallVector<uint64_t, 3> ExprOps;
12463 DIExpression::appendOffset(Ops&: ExprOps, Offset);
12464 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12465 } else {
12466 // Convert to a variadic expression (if not already).
12467 // convertToVariadicExpression() returns a const pointer, so we use
12468 // a temporary const variable here.
12469 const auto *TmpDIExpr =
12470 DIExpression::convertToVariadicExpression(Expr: DIExpr);
12471 SmallVector<uint64_t, 3> ExprOps;
12472 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
12473 ExprOps.push_back(Elt: NewLocOps.size());
12474 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
12475 SDDbgOperand RHS =
12476 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
12477 NewLocOps.push_back(Elt: RHS);
12478 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
12479 }
12480 Changed = true;
12481 }
12482 (void)Changed;
12483 assert(Changed && "Salvage target doesn't use N");
12484
12485 bool IsVariadic =
12486 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
12487
12488 auto AdditionalDependencies = DV->getAdditionalDependencies();
12489 SDDbgValue *Clone = getDbgValueList(
12490 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
12491 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
12492 ClonedDVs.push_back(Elt: Clone);
12493 DV->setIsInvalidated();
12494 DV->setIsEmitted();
12495 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
12496 N0.getNode()->dumprFull(this);
12497 dbgs() << " into " << *DIExpr << '\n');
12498 }
12499 break;
12500 }
12501 case ISD::TRUNCATE: {
12502 SDValue N0 = N.getOperand(Num: 0);
12503 TypeSize FromSize = N0.getValueSizeInBits();
12504 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
12505
12506 DIExpression *DbgExpression = DV->getExpression();
12507 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
12508 auto NewLocOps = DV->copyLocationOps();
12509 bool Changed = false;
12510 for (size_t i = 0; i < NewLocOps.size(); ++i) {
12511 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
12512 NewLocOps[i].getSDNode() != &N)
12513 continue;
12514
12515 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
12516 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
12517 Changed = true;
12518 }
12519 assert(Changed && "Salvage target doesn't use N");
12520 (void)Changed;
12521
12522 SDDbgValue *Clone =
12523 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
12524 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
12525 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
12526
12527 ClonedDVs.push_back(Elt: Clone);
12528 DV->setIsInvalidated();
12529 DV->setIsEmitted();
12530 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
12531 dbgs() << " into " << *DbgExpression << '\n');
12532 break;
12533 }
12534 }
12535 }
12536
12537 for (SDDbgValue *Dbg : ClonedDVs) {
12538 assert((!Dbg->getSDNodes().empty() ||
12539 llvm::any_of(Dbg->getLocationOps(),
12540 [&](const SDDbgOperand &Op) {
12541 return Op.getKind() == SDDbgOperand::FRAMEIX;
12542 })) &&
12543 "Salvaged DbgValue should depend on a new SDNode");
12544 AddDbgValue(DB: Dbg, isParameter: false);
12545 }
12546}
12547
12548/// Creates a SDDbgLabel node.
12549SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
12550 const DebugLoc &DL, unsigned O) {
12551 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
12552 "Expected inlined-at fields to agree");
12553 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
12554}
12555
12556namespace {
12557
12558/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
12559/// pointed to by a use iterator is deleted, increment the use iterator
12560/// so that it doesn't dangle.
12561///
12562class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
12563 SDNode::use_iterator &UI;
12564 SDNode::use_iterator &UE;
12565
12566 void NodeDeleted(SDNode *N, SDNode *E) override {
12567 // Increment the iterator as needed.
12568 while (UI != UE && N == UI->getUser())
12569 ++UI;
12570 }
12571
12572public:
12573 RAUWUpdateListener(SelectionDAG &d,
12574 SDNode::use_iterator &ui,
12575 SDNode::use_iterator &ue)
12576 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
12577};
12578
12579} // end anonymous namespace
12580
12581/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12582/// This can cause recursive merging of nodes in the DAG.
12583///
12584/// This version assumes From has a single result value.
12585///
12586void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
12587 SDNode *From = FromN.getNode();
12588 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
12589 "Cannot replace with this method!");
12590 assert(From != To.getNode() && "Cannot replace uses of with self");
12591
12592 // Preserve Debug Values
12593 transferDbgValues(From: FromN, To);
12594 // Preserve extra info.
12595 copyExtraInfo(From, To: To.getNode());
12596
12597 // Iterate over all the existing uses of From. New uses will be added
12598 // to the beginning of the use list, which we avoid visiting.
12599 // This specifically avoids visiting uses of From that arise while the
12600 // replacement is happening, because any such uses would be the result
12601 // of CSE: If an existing node looks like From after one of its operands
12602 // is replaced by To, we don't want to replace of all its users with To
12603 // too. See PR3018 for more info.
12604 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12605 RAUWUpdateListener Listener(*this, UI, UE);
12606 while (UI != UE) {
12607 SDNode *User = UI->getUser();
12608
12609 // This node is about to morph, remove its old self from the CSE maps.
12610 RemoveNodeFromCSEMaps(N: User);
12611
12612 // A user can appear in a use list multiple times, and when this
12613 // happens the uses are usually next to each other in the list.
12614 // To help reduce the number of CSE recomputations, process all
12615 // the uses of this user that we can find this way.
12616 do {
12617 SDUse &Use = *UI;
12618 ++UI;
12619 Use.set(To);
12620 if (To->isDivergent() != From->isDivergent())
12621 updateDivergence(N: User);
12622 } while (UI != UE && UI->getUser() == User);
12623 // Now that we have modified User, add it back to the CSE maps. If it
12624 // already exists there, recursively merge the results together.
12625 AddModifiedNodeToCSEMaps(N: User);
12626 }
12627
12628 // If we just RAUW'd the root, take note.
12629 if (FromN == getRoot())
12630 setRoot(To);
12631}
12632
12633/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12634/// This can cause recursive merging of nodes in the DAG.
12635///
12636/// This version assumes that for each value of From, there is a
12637/// corresponding value in To in the same position with the same type.
12638///
12639void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
12640#ifndef NDEBUG
12641 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12642 assert((!From->hasAnyUseOfValue(i) ||
12643 From->getValueType(i) == To->getValueType(i)) &&
12644 "Cannot use this version of ReplaceAllUsesWith!");
12645#endif
12646
12647 // Handle the trivial case.
12648 if (From == To)
12649 return;
12650
12651 // Preserve Debug Info. Only do this if there's a use.
12652 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
12653 if (From->hasAnyUseOfValue(Value: i)) {
12654 assert((i < To->getNumValues()) && "Invalid To location");
12655 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
12656 }
12657 // Preserve extra info.
12658 copyExtraInfo(From, To);
12659
12660 // Iterate over just the existing users of From. See the comments in
12661 // the ReplaceAllUsesWith above.
12662 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12663 RAUWUpdateListener Listener(*this, UI, UE);
12664 while (UI != UE) {
12665 SDNode *User = UI->getUser();
12666
12667 // This node is about to morph, remove its old self from the CSE maps.
12668 RemoveNodeFromCSEMaps(N: User);
12669
12670 // A user can appear in a use list multiple times, and when this
12671 // happens the uses are usually next to each other in the list.
12672 // To help reduce the number of CSE recomputations, process all
12673 // the uses of this user that we can find this way.
12674 do {
12675 SDUse &Use = *UI;
12676 ++UI;
12677 Use.setNode(To);
12678 if (To->isDivergent() != From->isDivergent())
12679 updateDivergence(N: User);
12680 } while (UI != UE && UI->getUser() == User);
12681
12682 // Now that we have modified User, add it back to the CSE maps. If it
12683 // already exists there, recursively merge the results together.
12684 AddModifiedNodeToCSEMaps(N: User);
12685 }
12686
12687 // If we just RAUW'd the root, take note.
12688 if (From == getRoot().getNode())
12689 setRoot(SDValue(To, getRoot().getResNo()));
12690}
12691
12692/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
12693/// This can cause recursive merging of nodes in the DAG.
12694///
12695/// This version can replace From with any result values. To must match the
12696/// number and types of values returned by From.
12697void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
12698 if (From->getNumValues() == 1) // Handle the simple case efficiently.
12699 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
12700
12701 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
12702 // Preserve Debug Info.
12703 transferDbgValues(From: SDValue(From, i), To: To[i]);
12704 // Preserve extra info.
12705 copyExtraInfo(From, To: To[i].getNode());
12706 }
12707
12708 // Iterate over just the existing users of From. See the comments in
12709 // the ReplaceAllUsesWith above.
12710 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
12711 RAUWUpdateListener Listener(*this, UI, UE);
12712 while (UI != UE) {
12713 SDNode *User = UI->getUser();
12714
12715 // This node is about to morph, remove its old self from the CSE maps.
12716 RemoveNodeFromCSEMaps(N: User);
12717
12718 // A user can appear in a use list multiple times, and when this happens the
12719 // uses are usually next to each other in the list. To help reduce the
12720 // number of CSE and divergence recomputations, process all the uses of this
12721 // user that we can find this way.
12722 bool To_IsDivergent = false;
12723 do {
12724 SDUse &Use = *UI;
12725 const SDValue &ToOp = To[Use.getResNo()];
12726 ++UI;
12727 Use.set(ToOp);
12728 if (ToOp.getValueType() != MVT::Other)
12729 To_IsDivergent |= ToOp->isDivergent();
12730 } while (UI != UE && UI->getUser() == User);
12731
12732 if (To_IsDivergent != From->isDivergent())
12733 updateDivergence(N: User);
12734
12735 // Now that we have modified User, add it back to the CSE maps. If it
12736 // already exists there, recursively merge the results together.
12737 AddModifiedNodeToCSEMaps(N: User);
12738 }
12739
12740 // If we just RAUW'd the root, take note.
12741 if (From == getRoot().getNode())
12742 setRoot(SDValue(To[getRoot().getResNo()]));
12743}
12744
12745/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
12746/// uses of other values produced by From.getNode() alone. The Deleted
12747/// vector is handled the same way as for ReplaceAllUsesWith.
12748void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
12749 // Handle the really simple, really trivial case efficiently.
12750 if (From == To) return;
12751
12752 // Handle the simple, trivial, case efficiently.
12753 if (From.getNode()->getNumValues() == 1) {
12754 ReplaceAllUsesWith(FromN: From, To);
12755 return;
12756 }
12757
12758 // Preserve Debug Info.
12759 transferDbgValues(From, To);
12760 copyExtraInfo(From: From.getNode(), To: To.getNode());
12761
12762 // Iterate over just the existing users of From. See the comments in
12763 // the ReplaceAllUsesWith above.
12764 SDNode::use_iterator UI = From.getNode()->use_begin(),
12765 UE = From.getNode()->use_end();
12766 RAUWUpdateListener Listener(*this, UI, UE);
12767 while (UI != UE) {
12768 SDNode *User = UI->getUser();
12769 bool UserRemovedFromCSEMaps = false;
12770
12771 // A user can appear in a use list multiple times, and when this
12772 // happens the uses are usually next to each other in the list.
12773 // To help reduce the number of CSE recomputations, process all
12774 // the uses of this user that we can find this way.
12775 do {
12776 SDUse &Use = *UI;
12777
12778 // Skip uses of different values from the same node.
12779 if (Use.getResNo() != From.getResNo()) {
12780 ++UI;
12781 continue;
12782 }
12783
12784 // If this node hasn't been modified yet, it's still in the CSE maps,
12785 // so remove its old self from the CSE maps.
12786 if (!UserRemovedFromCSEMaps) {
12787 RemoveNodeFromCSEMaps(N: User);
12788 UserRemovedFromCSEMaps = true;
12789 }
12790
12791 ++UI;
12792 Use.set(To);
12793 if (To->isDivergent() != From->isDivergent())
12794 updateDivergence(N: User);
12795 } while (UI != UE && UI->getUser() == User);
12796 // We are iterating over all uses of the From node, so if a use
12797 // doesn't use the specific value, no changes are made.
12798 if (!UserRemovedFromCSEMaps)
12799 continue;
12800
12801 // Now that we have modified User, add it back to the CSE maps. If it
12802 // already exists there, recursively merge the results together.
12803 AddModifiedNodeToCSEMaps(N: User);
12804 }
12805
12806 // If we just RAUW'd the root, take note.
12807 if (From == getRoot())
12808 setRoot(To);
12809}
12810
12811namespace {
12812
12813/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
12814/// to record information about a use.
12815struct UseMemo {
12816 SDNode *User;
12817 unsigned Index;
12818 SDUse *Use;
12819};
12820
12821/// operator< - Sort Memos by User.
12822bool operator<(const UseMemo &L, const UseMemo &R) {
12823 return (intptr_t)L.User < (intptr_t)R.User;
12824}
12825
12826/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
12827/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
12828/// the node already has been taken care of recursively.
12829class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
12830 SmallVectorImpl<UseMemo> &Uses;
12831
12832 void NodeDeleted(SDNode *N, SDNode *E) override {
12833 for (UseMemo &Memo : Uses)
12834 if (Memo.User == N)
12835 Memo.User = nullptr;
12836 }
12837
12838public:
12839 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
12840 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
12841};
12842
12843} // end anonymous namespace
12844
12845/// Return true if a glue output should propagate divergence information.
12846static bool gluePropagatesDivergence(const SDNode *Node) {
12847 switch (Node->getOpcode()) {
12848 case ISD::CopyFromReg:
12849 case ISD::CopyToReg:
12850 return false;
12851 default:
12852 return true;
12853 }
12854
12855 llvm_unreachable("covered opcode switch");
12856}
12857
12858bool SelectionDAG::calculateDivergence(SDNode *N) {
12859 if (TLI->isSDNodeAlwaysUniform(N)) {
12860 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
12861 "Conflicting divergence information!");
12862 return false;
12863 }
12864 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
12865 return true;
12866 for (const auto &Op : N->ops()) {
12867 EVT VT = Op.getValueType();
12868
12869 // Skip Chain. It does not carry divergence.
12870 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
12871 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
12872 return true;
12873 }
12874 return false;
12875}
12876
12877void SelectionDAG::updateDivergence(SDNode *N) {
12878 SmallVector<SDNode *, 16> Worklist(1, N);
12879 do {
12880 N = Worklist.pop_back_val();
12881 bool IsDivergent = calculateDivergence(N);
12882 if (N->SDNodeBits.IsDivergent != IsDivergent) {
12883 N->SDNodeBits.IsDivergent = IsDivergent;
12884 llvm::append_range(C&: Worklist, R: N->users());
12885 }
12886 } while (!Worklist.empty());
12887}
12888
12889void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
12890 DenseMap<SDNode *, unsigned> Degree;
12891 Order.reserve(n: AllNodes.size());
12892 for (auto &N : allnodes()) {
12893 unsigned NOps = N.getNumOperands();
12894 Degree[&N] = NOps;
12895 if (0 == NOps)
12896 Order.push_back(x: &N);
12897 }
12898 for (size_t I = 0; I != Order.size(); ++I) {
12899 SDNode *N = Order[I];
12900 for (auto *U : N->users()) {
12901 unsigned &UnsortedOps = Degree[U];
12902 if (0 == --UnsortedOps)
12903 Order.push_back(x: U);
12904 }
12905 }
12906}
12907
12908#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
12909void SelectionDAG::VerifyDAGDivergence() {
12910 std::vector<SDNode *> TopoOrder;
12911 CreateTopologicalOrder(TopoOrder);
12912 for (auto *N : TopoOrder) {
12913 assert(calculateDivergence(N) == N->isDivergent() &&
12914 "Divergence bit inconsistency detected");
12915 }
12916}
12917#endif
12918
12919/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
12920/// uses of other values produced by From.getNode() alone. The same value
12921/// may appear in both the From and To list. The Deleted vector is
12922/// handled the same way as for ReplaceAllUsesWith.
12923void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
12924 const SDValue *To,
12925 unsigned Num){
12926 // Handle the simple, trivial case efficiently.
12927 if (Num == 1)
12928 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
12929
12930 transferDbgValues(From: *From, To: *To);
12931 copyExtraInfo(From: From->getNode(), To: To->getNode());
12932
12933 // Read up all the uses and make records of them. This helps
12934 // processing new uses that are introduced during the
12935 // replacement process.
12936 SmallVector<UseMemo, 4> Uses;
12937 for (unsigned i = 0; i != Num; ++i) {
12938 unsigned FromResNo = From[i].getResNo();
12939 SDNode *FromNode = From[i].getNode();
12940 for (SDUse &Use : FromNode->uses()) {
12941 if (Use.getResNo() == FromResNo) {
12942 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
12943 Uses.push_back(Elt: Memo);
12944 }
12945 }
12946 }
12947
12948 // Sort the uses, so that all the uses from a given User are together.
12949 llvm::sort(C&: Uses);
12950 RAUOVWUpdateListener Listener(*this, Uses);
12951
12952 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
12953 UseIndex != UseIndexEnd; ) {
12954 // We know that this user uses some value of From. If it is the right
12955 // value, update it.
12956 SDNode *User = Uses[UseIndex].User;
12957 // If the node has been deleted by recursive CSE updates when updating
12958 // another node, then just skip this entry.
12959 if (User == nullptr) {
12960 ++UseIndex;
12961 continue;
12962 }
12963
12964 // This node is about to morph, remove its old self from the CSE maps.
12965 RemoveNodeFromCSEMaps(N: User);
12966
12967 // The Uses array is sorted, so all the uses for a given User
12968 // are next to each other in the list.
12969 // To help reduce the number of CSE recomputations, process all
12970 // the uses of this user that we can find this way.
12971 do {
12972 unsigned i = Uses[UseIndex].Index;
12973 SDUse &Use = *Uses[UseIndex].Use;
12974 ++UseIndex;
12975
12976 Use.set(To[i]);
12977 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
12978
12979 // Now that we have modified User, add it back to the CSE maps. If it
12980 // already exists there, recursively merge the results together.
12981 AddModifiedNodeToCSEMaps(N: User);
12982 }
12983}
12984
12985/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
12986/// based on their topological order. It returns the maximum id and a vector
12987/// of the SDNodes* in assigned order by reference.
12988unsigned SelectionDAG::AssignTopologicalOrder() {
12989 unsigned DAGSize = 0;
12990
12991 // SortedPos tracks the progress of the algorithm. Nodes before it are
12992 // sorted, nodes after it are unsorted. When the algorithm completes
12993 // it is at the end of the list.
12994 allnodes_iterator SortedPos = allnodes_begin();
12995
12996 // Visit all the nodes. Move nodes with no operands to the front of
12997 // the list immediately. Annotate nodes that do have operands with their
12998 // operand count. Before we do this, the Node Id fields of the nodes
12999 // may contain arbitrary values. After, the Node Id fields for nodes
13000 // before SortedPos will contain the topological sort index, and the
13001 // Node Id fields for nodes At SortedPos and after will contain the
13002 // count of outstanding operands.
13003 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
13004 checkForCycles(N: &N, DAG: this);
13005 unsigned Degree = N.getNumOperands();
13006 if (Degree == 0) {
13007 // A node with no uses, add it to the result array immediately.
13008 N.setNodeId(DAGSize++);
13009 allnodes_iterator Q(&N);
13010 if (Q != SortedPos)
13011 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
13012 assert(SortedPos != AllNodes.end() && "Overran node list");
13013 ++SortedPos;
13014 } else {
13015 // Temporarily use the Node Id as scratch space for the degree count.
13016 N.setNodeId(Degree);
13017 }
13018 }
13019
13020 // Visit all the nodes. As we iterate, move nodes into sorted order,
13021 // such that by the time the end is reached all nodes will be sorted.
13022 for (SDNode &Node : allnodes()) {
13023 SDNode *N = &Node;
13024 checkForCycles(N, DAG: this);
13025 // N is in sorted position, so all its uses have one less operand
13026 // that needs to be sorted.
13027 for (SDNode *P : N->users()) {
13028 unsigned Degree = P->getNodeId();
13029 assert(Degree != 0 && "Invalid node degree");
13030 --Degree;
13031 if (Degree == 0) {
13032 // All of P's operands are sorted, so P may sorted now.
13033 P->setNodeId(DAGSize++);
13034 if (P->getIterator() != SortedPos)
13035 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
13036 assert(SortedPos != AllNodes.end() && "Overran node list");
13037 ++SortedPos;
13038 } else {
13039 // Update P's outstanding operand count.
13040 P->setNodeId(Degree);
13041 }
13042 }
13043 if (Node.getIterator() == SortedPos) {
13044#ifndef NDEBUG
13045 allnodes_iterator I(N);
13046 SDNode *S = &*++I;
13047 dbgs() << "Overran sorted position:\n";
13048 S->dumprFull(this); dbgs() << "\n";
13049 dbgs() << "Checking if this is due to cycles\n";
13050 checkForCycles(this, true);
13051#endif
13052 llvm_unreachable(nullptr);
13053 }
13054 }
13055
13056 assert(SortedPos == AllNodes.end() &&
13057 "Topological sort incomplete!");
13058 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
13059 "First node in topological sort is not the entry token!");
13060 assert(AllNodes.front().getNodeId() == 0 &&
13061 "First node in topological sort has non-zero id!");
13062 assert(AllNodes.front().getNumOperands() == 0 &&
13063 "First node in topological sort has operands!");
13064 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
13065 "Last node in topologic sort has unexpected id!");
13066 assert(AllNodes.back().use_empty() &&
13067 "Last node in topologic sort has users!");
13068 assert(DAGSize == allnodes_size() && "Node count mismatch!");
13069 return DAGSize;
13070}
13071
13072void SelectionDAG::getTopologicallyOrderedNodes(
13073 SmallVectorImpl<const SDNode *> &SortedNodes) const {
13074 SortedNodes.clear();
13075 // Node -> remaining number of outstanding operands.
13076 DenseMap<const SDNode *, unsigned> RemainingOperands;
13077
13078 // Put nodes without any operands into SortedNodes first.
13079 for (const SDNode &N : allnodes()) {
13080 checkForCycles(N: &N, DAG: this);
13081 unsigned NumOperands = N.getNumOperands();
13082 if (NumOperands == 0)
13083 SortedNodes.push_back(Elt: &N);
13084 else
13085 // Record their total number of outstanding operands.
13086 RemainingOperands[&N] = NumOperands;
13087 }
13088
13089 // A node is pushed into SortedNodes when all of its operands (predecessors in
13090 // the graph) are also in SortedNodes.
13091 for (unsigned i = 0U; i < SortedNodes.size(); ++i) {
13092 const SDNode *N = SortedNodes[i];
13093 for (const SDNode *U : N->users()) {
13094 // HandleSDNode is never part of a DAG and therefore has no entry in
13095 // RemainingOperands.
13096 if (U->getOpcode() == ISD::HANDLENODE)
13097 continue;
13098 unsigned &NumRemOperands = RemainingOperands[U];
13099 assert(NumRemOperands && "Invalid number of remaining operands");
13100 --NumRemOperands;
13101 if (!NumRemOperands)
13102 SortedNodes.push_back(Elt: U);
13103 }
13104 }
13105
13106 assert(SortedNodes.size() == AllNodes.size() && "Node count mismatch");
13107 assert(SortedNodes.front()->getOpcode() == ISD::EntryToken &&
13108 "First node in topological sort is not the entry token");
13109 assert(SortedNodes.front()->getNumOperands() == 0 &&
13110 "First node in topological sort has operands");
13111}
13112
13113/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
13114/// value is produced by SD.
13115void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
13116 for (SDNode *SD : DB->getSDNodes()) {
13117 if (!SD)
13118 continue;
13119 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
13120 SD->setHasDebugValue(true);
13121 }
13122 DbgInfo->add(V: DB, isParameter);
13123}
13124
13125void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
13126
13127SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
13128 SDValue NewMemOpChain) {
13129 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
13130 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
13131 // The new memory operation must have the same position as the old load in
13132 // terms of memory dependency. Create a TokenFactor for the old load and new
13133 // memory operation and update uses of the old load's output chain to use that
13134 // TokenFactor.
13135 if (OldChain == NewMemOpChain || OldChain.use_empty())
13136 return NewMemOpChain;
13137
13138 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
13139 N1: OldChain, N2: NewMemOpChain);
13140 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
13141 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
13142 return TokenFactor;
13143}
13144
13145SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
13146 SDValue NewMemOp) {
13147 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
13148 SDValue OldChain = SDValue(OldLoad, 1);
13149 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
13150 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
13151}
13152
13153SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
13154 Function **OutFunction) {
13155 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
13156
13157 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
13158 auto *Module = MF->getFunction().getParent();
13159 auto *Function = Module->getFunction(Name: Symbol);
13160
13161 if (OutFunction != nullptr)
13162 *OutFunction = Function;
13163
13164 if (Function != nullptr) {
13165 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
13166 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
13167 }
13168
13169 std::string ErrorStr;
13170 raw_string_ostream ErrorFormatter(ErrorStr);
13171 ErrorFormatter << "Undefined external symbol ";
13172 ErrorFormatter << '"' << Symbol << '"';
13173 report_fatal_error(reason: Twine(ErrorStr));
13174}
13175
13176//===----------------------------------------------------------------------===//
13177// SDNode Class
13178//===----------------------------------------------------------------------===//
13179
13180bool llvm::isNullConstant(SDValue V) {
13181 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13182 return Const != nullptr && Const->isZero();
13183}
13184
13185bool llvm::isNullConstantOrUndef(SDValue V) {
13186 return V.isUndef() || isNullConstant(V);
13187}
13188
13189bool llvm::isNullFPConstant(SDValue V) {
13190 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
13191 return Const != nullptr && Const->isZero() && !Const->isNegative();
13192}
13193
13194bool llvm::isAllOnesConstant(SDValue V) {
13195 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13196 return Const != nullptr && Const->isAllOnes();
13197}
13198
13199bool llvm::isOneConstant(SDValue V) {
13200 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13201 return Const != nullptr && Const->isOne();
13202}
13203
13204bool llvm::isMinSignedConstant(SDValue V) {
13205 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
13206 return Const != nullptr && Const->isMinSignedValue();
13207}
13208
13209bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
13210 unsigned OperandNo) {
13211 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
13212 // TODO: Target-specific opcodes could be added.
13213 if (auto *ConstV = isConstOrConstSplat(N: V, /*AllowUndefs*/ false,
13214 /*AllowTruncation*/ true)) {
13215 APInt Const = ConstV->getAPIntValue().trunc(width: V.getScalarValueSizeInBits());
13216 switch (Opcode) {
13217 case ISD::ADD:
13218 case ISD::OR:
13219 case ISD::XOR:
13220 case ISD::UMAX:
13221 return Const.isZero();
13222 case ISD::MUL:
13223 return Const.isOne();
13224 case ISD::AND:
13225 case ISD::UMIN:
13226 return Const.isAllOnes();
13227 case ISD::SMAX:
13228 return Const.isMinSignedValue();
13229 case ISD::SMIN:
13230 return Const.isMaxSignedValue();
13231 case ISD::SUB:
13232 case ISD::SHL:
13233 case ISD::SRA:
13234 case ISD::SRL:
13235 return OperandNo == 1 && Const.isZero();
13236 case ISD::UDIV:
13237 case ISD::SDIV:
13238 return OperandNo == 1 && Const.isOne();
13239 }
13240 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) {
13241 switch (Opcode) {
13242 case ISD::FADD:
13243 return ConstFP->isZero() &&
13244 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
13245 case ISD::FSUB:
13246 return OperandNo == 1 && ConstFP->isZero() &&
13247 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
13248 case ISD::FMUL:
13249 return ConstFP->isExactlyValue(V: 1.0);
13250 case ISD::FDIV:
13251 return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0);
13252 case ISD::FMINNUM:
13253 case ISD::FMAXNUM: {
13254 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
13255 EVT VT = V.getValueType();
13256 const fltSemantics &Semantics = VT.getFltSemantics();
13257 APFloat NeutralAF = !Flags.hasNoNaNs()
13258 ? APFloat::getQNaN(Sem: Semantics)
13259 : !Flags.hasNoInfs()
13260 ? APFloat::getInf(Sem: Semantics)
13261 : APFloat::getLargest(Sem: Semantics);
13262 if (Opcode == ISD::FMAXNUM)
13263 NeutralAF.changeSign();
13264
13265 return ConstFP->isExactlyValue(V: NeutralAF);
13266 }
13267 }
13268 }
13269 return false;
13270}
13271
13272SDValue llvm::peekThroughBitcasts(SDValue V) {
13273 while (V.getOpcode() == ISD::BITCAST)
13274 V = V.getOperand(i: 0);
13275 return V;
13276}
13277
13278SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
13279 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
13280 V = V.getOperand(i: 0);
13281 return V;
13282}
13283
13284SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
13285 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
13286 V = V.getOperand(i: 0);
13287 return V;
13288}
13289
13290SDValue llvm::peekThroughInsertVectorElt(SDValue V, const APInt &DemandedElts) {
13291 while (V.getOpcode() == ISD::INSERT_VECTOR_ELT) {
13292 SDValue InVec = V.getOperand(i: 0);
13293 SDValue EltNo = V.getOperand(i: 2);
13294 EVT VT = InVec.getValueType();
13295 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
13296 if (IndexC && VT.isFixedLengthVector() &&
13297 IndexC->getAPIntValue().ult(RHS: VT.getVectorNumElements()) &&
13298 !DemandedElts[IndexC->getZExtValue()]) {
13299 V = InVec;
13300 continue;
13301 }
13302 break;
13303 }
13304 return V;
13305}
13306
13307SDValue llvm::peekThroughTruncates(SDValue V) {
13308 while (V.getOpcode() == ISD::TRUNCATE)
13309 V = V.getOperand(i: 0);
13310 return V;
13311}
13312
13313bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
13314 if (V.getOpcode() != ISD::XOR)
13315 return false;
13316 V = peekThroughBitcasts(V: V.getOperand(i: 1));
13317 unsigned NumBits = V.getScalarValueSizeInBits();
13318 ConstantSDNode *C =
13319 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
13320 return C && (C->getAPIntValue().countr_one() >= NumBits);
13321}
13322
13323ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
13324 bool AllowTruncation) {
13325 EVT VT = N.getValueType();
13326 APInt DemandedElts = VT.isFixedLengthVector()
13327 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
13328 : APInt(1, 1);
13329 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
13330}
13331
13332ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
13333 bool AllowUndefs,
13334 bool AllowTruncation) {
13335 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
13336 return CN;
13337
13338 // SplatVectors can truncate their operands. Ignore that case here unless
13339 // AllowTruncation is set.
13340 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
13341 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
13342 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
13343 EVT CVT = CN->getValueType(ResNo: 0);
13344 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
13345 if (AllowTruncation || CVT == VecEltVT)
13346 return CN;
13347 }
13348 }
13349
13350 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13351 BitVector UndefElements;
13352 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
13353
13354 // BuildVectors can truncate their operands. Ignore that case here unless
13355 // AllowTruncation is set.
13356 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13357 if (CN && (UndefElements.none() || AllowUndefs)) {
13358 EVT CVT = CN->getValueType(ResNo: 0);
13359 EVT NSVT = N.getValueType().getScalarType();
13360 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
13361 if (AllowTruncation || (CVT == NSVT))
13362 return CN;
13363 }
13364 }
13365
13366 return nullptr;
13367}
13368
13369ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
13370 EVT VT = N.getValueType();
13371 APInt DemandedElts = VT.isFixedLengthVector()
13372 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
13373 : APInt(1, 1);
13374 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
13375}
13376
13377ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
13378 const APInt &DemandedElts,
13379 bool AllowUndefs) {
13380 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
13381 return CN;
13382
13383 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
13384 BitVector UndefElements;
13385 ConstantFPSDNode *CN =
13386 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
13387 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
13388 if (CN && (UndefElements.none() || AllowUndefs))
13389 return CN;
13390 }
13391
13392 if (N.getOpcode() == ISD::SPLAT_VECTOR)
13393 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13394 return CN;
13395
13396 return nullptr;
13397}
13398
13399bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
13400 // TODO: may want to use peekThroughBitcast() here.
13401 ConstantSDNode *C =
13402 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
13403 return C && C->isZero();
13404}
13405
13406bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
13407 ConstantSDNode *C =
13408 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
13409 return C && C->isOne();
13410}
13411
13412bool llvm::isOneOrOneSplatFP(SDValue N, bool AllowUndefs) {
13413 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13414 return C && C->isExactlyValue(V: 1.0);
13415}
13416
13417bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
13418 N = peekThroughBitcasts(V: N);
13419 unsigned BitWidth = N.getScalarValueSizeInBits();
13420 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13421 return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth;
13422}
13423
13424bool llvm::isOnesOrOnesSplat(SDValue N, bool AllowUndefs) {
13425 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
13426 return C && APInt::isSameValue(I1: C->getAPIntValue(),
13427 I2: APInt(C->getAPIntValue().getBitWidth(), 1));
13428}
13429
13430bool llvm::isZeroOrZeroSplat(SDValue N, bool AllowUndefs) {
13431 N = peekThroughBitcasts(V: N);
13432 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs, AllowTruncation: true);
13433 return C && C->isZero();
13434}
13435
13436bool llvm::isZeroOrZeroSplatFP(SDValue N, bool AllowUndefs) {
13437 ConstantFPSDNode *C = isConstOrConstSplatFP(N, AllowUndefs);
13438 return C && C->isZero();
13439}
13440
13441HandleSDNode::~HandleSDNode() {
13442 DropOperands();
13443}
13444
13445MemSDNode::MemSDNode(
13446 unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt,
13447 PointerUnion<MachineMemOperand *, MachineMemOperand **> memrefs)
13448 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MemRefs(memrefs) {
13449 bool IsVolatile = false;
13450 bool IsNonTemporal = false;
13451 bool IsDereferenceable = true;
13452 bool IsInvariant = true;
13453 for (const MachineMemOperand *MMO : memoperands()) {
13454 IsVolatile |= MMO->isVolatile();
13455 IsNonTemporal |= MMO->isNonTemporal();
13456 IsDereferenceable &= MMO->isDereferenceable();
13457 IsInvariant &= MMO->isInvariant();
13458 }
13459 MemSDNodeBits.IsVolatile = IsVolatile;
13460 MemSDNodeBits.IsNonTemporal = IsNonTemporal;
13461 MemSDNodeBits.IsDereferenceable = IsDereferenceable;
13462 MemSDNodeBits.IsInvariant = IsInvariant;
13463
13464 // For the single-MMO case, we check here that the size of the memory operand
13465 // fits within the size of the MMO. This is because the MMO might indicate
13466 // only a possible address range instead of specifying the affected memory
13467 // addresses precisely.
13468 assert((getNumMemOperands() != 1 || !getMemOperand()->getType().isValid() ||
13469 TypeSize::isKnownLE(memvt.getStoreSize(),
13470 getMemOperand()->getSize().getValue())) &&
13471 "Size mismatch!");
13472}
13473
13474/// Profile - Gather unique data for the node.
13475///
13476void SDNode::Profile(FoldingSetNodeID &ID) const {
13477 AddNodeIDNode(ID, N: this);
13478}
13479
13480namespace {
13481
13482 struct EVTArray {
13483 std::vector<EVT> VTs;
13484
13485 EVTArray() {
13486 VTs.reserve(n: MVT::VALUETYPE_SIZE);
13487 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
13488 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
13489 }
13490 };
13491
13492} // end anonymous namespace
13493
13494/// getValueTypeList - Return a pointer to the specified value type.
13495///
13496const EVT *SDNode::getValueTypeList(MVT VT) {
13497 static EVTArray SimpleVTArray;
13498
13499 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
13500 return &SimpleVTArray.VTs[VT.SimpleTy];
13501}
13502
13503/// hasAnyUseOfValue - Return true if there are any use of the indicated
13504/// value. This method ignores uses of other values defined by this operation.
13505bool SDNode::hasAnyUseOfValue(unsigned Value) const {
13506 assert(Value < getNumValues() && "Bad value!");
13507
13508 for (SDUse &U : uses())
13509 if (U.getResNo() == Value)
13510 return true;
13511
13512 return false;
13513}
13514
13515/// isOnlyUserOf - Return true if this node is the only use of N.
13516bool SDNode::isOnlyUserOf(const SDNode *N) const {
13517 bool Seen = false;
13518 for (const SDNode *User : N->users()) {
13519 if (User == this)
13520 Seen = true;
13521 else
13522 return false;
13523 }
13524
13525 return Seen;
13526}
13527
13528/// Return true if the only users of N are contained in Nodes.
13529bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
13530 bool Seen = false;
13531 for (const SDNode *User : N->users()) {
13532 if (llvm::is_contained(Range&: Nodes, Element: User))
13533 Seen = true;
13534 else
13535 return false;
13536 }
13537
13538 return Seen;
13539}
13540
13541/// Return true if the referenced return value is an operand of N.
13542bool SDValue::isOperandOf(const SDNode *N) const {
13543 return is_contained(Range: N->op_values(), Element: *this);
13544}
13545
13546bool SDNode::isOperandOf(const SDNode *N) const {
13547 return any_of(Range: N->op_values(),
13548 P: [this](SDValue Op) { return this == Op.getNode(); });
13549}
13550
13551/// reachesChainWithoutSideEffects - Return true if this operand (which must
13552/// be a chain) reaches the specified operand without crossing any
13553/// side-effecting instructions on any chain path. In practice, this looks
13554/// through token factors and non-volatile loads. In order to remain efficient,
13555/// this only looks a couple of nodes in, it does not do an exhaustive search.
13556///
13557/// Note that we only need to examine chains when we're searching for
13558/// side-effects; SelectionDAG requires that all side-effects are represented
13559/// by chains, even if another operand would force a specific ordering. This
13560/// constraint is necessary to allow transformations like splitting loads.
13561bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
13562 unsigned Depth) const {
13563 if (*this == Dest) return true;
13564
13565 // Don't search too deeply, we just want to be able to see through
13566 // TokenFactor's etc.
13567 if (Depth == 0) return false;
13568
13569 // If this is a token factor, all inputs to the TF happen in parallel.
13570 if (getOpcode() == ISD::TokenFactor) {
13571 // First, try a shallow search.
13572 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
13573 // We found the chain we want as an operand of this TokenFactor.
13574 // Essentially, we reach the chain without side-effects if we could
13575 // serialize the TokenFactor into a simple chain of operations with
13576 // Dest as the last operation. This is automatically true if the
13577 // chain has one use: there are no other ordering constraints.
13578 // If the chain has more than one use, we give up: some other
13579 // use of Dest might force a side-effect between Dest and the current
13580 // node.
13581 if (Dest.hasOneUse())
13582 return true;
13583 }
13584 // Next, try a deep search: check whether every operand of the TokenFactor
13585 // reaches Dest.
13586 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
13587 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
13588 });
13589 }
13590
13591 // Loads don't have side effects, look through them.
13592 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
13593 if (Ld->isUnordered())
13594 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
13595 }
13596 return false;
13597}
13598
13599bool SDNode::hasPredecessor(const SDNode *N) const {
13600 SmallPtrSet<const SDNode *, 32> Visited;
13601 SmallVector<const SDNode *, 16> Worklist;
13602 Worklist.push_back(Elt: this);
13603 return hasPredecessorHelper(N, Visited, Worklist);
13604}
13605
13606void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
13607 this->Flags &= Flags;
13608}
13609
13610SDValue
13611SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
13612 ArrayRef<ISD::NodeType> CandidateBinOps,
13613 bool AllowPartials) {
13614 // The pattern must end in an extract from index 0.
13615 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
13616 !isNullConstant(V: Extract->getOperand(Num: 1)))
13617 return SDValue();
13618
13619 // Match against one of the candidate binary ops.
13620 SDValue Op = Extract->getOperand(Num: 0);
13621 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
13622 return Op.getOpcode() == unsigned(BinOp);
13623 }))
13624 return SDValue();
13625
13626 // Floating-point reductions may require relaxed constraints on the final step
13627 // of the reduction because they may reorder intermediate operations.
13628 unsigned CandidateBinOp = Op.getOpcode();
13629 if (Op.getValueType().isFloatingPoint()) {
13630 SDNodeFlags Flags = Op->getFlags();
13631 switch (CandidateBinOp) {
13632 case ISD::FADD:
13633 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
13634 return SDValue();
13635 break;
13636 default:
13637 llvm_unreachable("Unhandled FP opcode for binop reduction");
13638 }
13639 }
13640
13641 // Matching failed - attempt to see if we did enough stages that a partial
13642 // reduction from a subvector is possible.
13643 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
13644 if (!AllowPartials || !Op)
13645 return SDValue();
13646 EVT OpVT = Op.getValueType();
13647 EVT OpSVT = OpVT.getScalarType();
13648 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
13649 if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
13650 return SDValue();
13651 BinOp = (ISD::NodeType)CandidateBinOp;
13652 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
13653 };
13654
13655 // At each stage, we're looking for something that looks like:
13656 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
13657 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
13658 // i32 undef, i32 undef, i32 undef, i32 undef>
13659 // %a = binop <8 x i32> %op, %s
13660 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
13661 // we expect something like:
13662 // <4,5,6,7,u,u,u,u>
13663 // <2,3,u,u,u,u,u,u>
13664 // <1,u,u,u,u,u,u,u>
13665 // While a partial reduction match would be:
13666 // <2,3,u,u,u,u,u,u>
13667 // <1,u,u,u,u,u,u,u>
13668 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
13669 SDValue PrevOp;
13670 for (unsigned i = 0; i < Stages; ++i) {
13671 unsigned MaskEnd = (1 << i);
13672
13673 if (Op.getOpcode() != CandidateBinOp)
13674 return PartialReduction(PrevOp, MaskEnd);
13675
13676 SDValue Op0 = Op.getOperand(i: 0);
13677 SDValue Op1 = Op.getOperand(i: 1);
13678
13679 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
13680 if (Shuffle) {
13681 Op = Op1;
13682 } else {
13683 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
13684 Op = Op0;
13685 }
13686
13687 // The first operand of the shuffle should be the same as the other operand
13688 // of the binop.
13689 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
13690 return PartialReduction(PrevOp, MaskEnd);
13691
13692 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
13693 for (int Index = 0; Index < (int)MaskEnd; ++Index)
13694 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
13695 return PartialReduction(PrevOp, MaskEnd);
13696
13697 PrevOp = Op;
13698 }
13699
13700 // Handle subvector reductions, which tend to appear after the shuffle
13701 // reduction stages.
13702 while (Op.getOpcode() == CandidateBinOp) {
13703 unsigned NumElts = Op.getValueType().getVectorNumElements();
13704 SDValue Op0 = Op.getOperand(i: 0);
13705 SDValue Op1 = Op.getOperand(i: 1);
13706 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13707 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
13708 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
13709 break;
13710 SDValue Src = Op0.getOperand(i: 0);
13711 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
13712 if (NumSrcElts != (2 * NumElts))
13713 break;
13714 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
13715 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
13716 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
13717 Op0.getConstantOperandAPInt(i: 1) == NumElts))
13718 break;
13719 Op = Src;
13720 }
13721
13722 BinOp = (ISD::NodeType)CandidateBinOp;
13723 return Op;
13724}
13725
13726SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
13727 EVT VT = N->getValueType(ResNo: 0);
13728 EVT EltVT = VT.getVectorElementType();
13729 unsigned NE = VT.getVectorNumElements();
13730
13731 SDLoc dl(N);
13732
13733 // If ResNE is 0, fully unroll the vector op.
13734 if (ResNE == 0)
13735 ResNE = NE;
13736 else if (NE > ResNE)
13737 NE = ResNE;
13738
13739 if (N->getNumValues() == 2) {
13740 SmallVector<SDValue, 8> Scalars0, Scalars1;
13741 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13742 EVT VT1 = N->getValueType(ResNo: 1);
13743 EVT EltVT1 = VT1.getVectorElementType();
13744
13745 unsigned i;
13746 for (i = 0; i != NE; ++i) {
13747 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13748 SDValue Operand = N->getOperand(Num: j);
13749 EVT OperandVT = Operand.getValueType();
13750
13751 // A vector operand; extract a single element.
13752 EVT OperandEltVT = OperandVT.getVectorElementType();
13753 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13754 }
13755
13756 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
13757 Scalars0.push_back(Elt: EltOp);
13758 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
13759 }
13760
13761 for (; i < ResNE; ++i) {
13762 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
13763 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
13764 }
13765
13766 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
13767 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, NumElements: ResNE);
13768 SDValue Vec0 = getBuildVector(VT: VecVT, DL: dl, Ops: Scalars0);
13769 SDValue Vec1 = getBuildVector(VT: VecVT1, DL: dl, Ops: Scalars1);
13770 return getMergeValues(Ops: {Vec0, Vec1}, dl);
13771 }
13772
13773 assert(N->getNumValues() == 1 &&
13774 "Can't unroll a vector with multiple results!");
13775
13776 SmallVector<SDValue, 8> Scalars;
13777 SmallVector<SDValue, 4> Operands(N->getNumOperands());
13778
13779 unsigned i;
13780 for (i= 0; i != NE; ++i) {
13781 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
13782 SDValue Operand = N->getOperand(Num: j);
13783 EVT OperandVT = Operand.getValueType();
13784 if (OperandVT.isVector()) {
13785 // A vector operand; extract a single element.
13786 EVT OperandEltVT = OperandVT.getVectorElementType();
13787 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
13788 } else {
13789 // A scalar operand; just use it as is.
13790 Operands[j] = Operand;
13791 }
13792 }
13793
13794 switch (N->getOpcode()) {
13795 default: {
13796 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
13797 Flags: N->getFlags()));
13798 break;
13799 }
13800 case ISD::VSELECT:
13801 Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
13802 break;
13803 case ISD::SHL:
13804 case ISD::SRA:
13805 case ISD::SRL:
13806 case ISD::ROTL:
13807 case ISD::ROTR:
13808 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
13809 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
13810 Op: Operands[1])));
13811 break;
13812 case ISD::SIGN_EXTEND_INREG: {
13813 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
13814 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
13815 N1: Operands[0],
13816 N2: getValueType(VT: ExtVT)));
13817 break;
13818 }
13819 case ISD::ADDRSPACECAST: {
13820 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
13821 Scalars.push_back(Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0],
13822 SrcAS: ASC->getSrcAddressSpace(),
13823 DestAS: ASC->getDestAddressSpace()));
13824 break;
13825 }
13826 }
13827 }
13828
13829 for (; i < ResNE; ++i)
13830 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
13831
13832 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
13833 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
13834}
13835
13836std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
13837 SDNode *N, unsigned ResNE) {
13838 unsigned Opcode = N->getOpcode();
13839 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
13840 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
13841 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
13842 "Expected an overflow opcode");
13843
13844 EVT ResVT = N->getValueType(ResNo: 0);
13845 EVT OvVT = N->getValueType(ResNo: 1);
13846 EVT ResEltVT = ResVT.getVectorElementType();
13847 EVT OvEltVT = OvVT.getVectorElementType();
13848 SDLoc dl(N);
13849
13850 // If ResNE is 0, fully unroll the vector op.
13851 unsigned NE = ResVT.getVectorNumElements();
13852 if (ResNE == 0)
13853 ResNE = NE;
13854 else if (NE > ResNE)
13855 NE = ResNE;
13856
13857 SmallVector<SDValue, 8> LHSScalars;
13858 SmallVector<SDValue, 8> RHSScalars;
13859 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
13860 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
13861
13862 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
13863 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
13864 SmallVector<SDValue, 8> ResScalars;
13865 SmallVector<SDValue, 8> OvScalars;
13866 for (unsigned i = 0; i < NE; ++i) {
13867 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
13868 SDValue Ov =
13869 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
13870 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
13871 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
13872
13873 ResScalars.push_back(Elt: Res);
13874 OvScalars.push_back(Elt: Ov);
13875 }
13876
13877 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
13878 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
13879
13880 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
13881 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
13882 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
13883 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
13884}
13885
13886bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
13887 LoadSDNode *Base,
13888 unsigned Bytes,
13889 int Dist) const {
13890 if (LD->isVolatile() || Base->isVolatile())
13891 return false;
13892 // TODO: probably too restrictive for atomics, revisit
13893 if (!LD->isSimple())
13894 return false;
13895 if (LD->isIndexed() || Base->isIndexed())
13896 return false;
13897 if (LD->getChain() != Base->getChain())
13898 return false;
13899 EVT VT = LD->getMemoryVT();
13900 if (VT.getSizeInBits() / 8 != Bytes)
13901 return false;
13902
13903 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
13904 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
13905
13906 int64_t Offset = 0;
13907 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
13908 return (Dist * (int64_t)Bytes == Offset);
13909 return false;
13910}
13911
13912/// InferPtrAlignment - Infer alignment of a load / store address. Return
13913/// std::nullopt if it cannot be inferred.
13914MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
13915 // If this is a GlobalAddress + cst, return the alignment.
13916 const GlobalValue *GV = nullptr;
13917 int64_t GVOffset = 0;
13918 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
13919 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
13920 KnownBits Known(PtrWidth);
13921 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
13922 unsigned AlignBits = Known.countMinTrailingZeros();
13923 if (AlignBits)
13924 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
13925 }
13926
13927 // If this is a direct reference to a stack slot, use information about the
13928 // stack slot's alignment.
13929 int FrameIdx = INT_MIN;
13930 int64_t FrameOffset = 0;
13931 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
13932 FrameIdx = FI->getIndex();
13933 } else if (isBaseWithConstantOffset(Op: Ptr) &&
13934 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
13935 // Handle FI+Cst
13936 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
13937 FrameOffset = Ptr.getConstantOperandVal(i: 1);
13938 }
13939
13940 if (FrameIdx != INT_MIN) {
13941 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
13942 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
13943 }
13944
13945 return std::nullopt;
13946}
13947
13948/// Split the scalar node with EXTRACT_ELEMENT using the provided
13949/// VTs and return the low/high part.
13950std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
13951 const SDLoc &DL,
13952 const EVT &LoVT,
13953 const EVT &HiVT) {
13954 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
13955 "Split node must be a scalar type");
13956 SDValue Lo =
13957 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
13958 SDValue Hi =
13959 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
13960 return std::make_pair(x&: Lo, y&: Hi);
13961}
13962
13963/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
13964/// which is split (or expanded) into two not necessarily identical pieces.
13965std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
13966 // Currently all types are split in half.
13967 EVT LoVT, HiVT;
13968 if (!VT.isVector())
13969 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
13970 else
13971 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
13972
13973 return std::make_pair(x&: LoVT, y&: HiVT);
13974}
13975
13976/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
13977/// type, dependent on an enveloping VT that has been split into two identical
13978/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
13979std::pair<EVT, EVT>
13980SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
13981 bool *HiIsEmpty) const {
13982 EVT EltTp = VT.getVectorElementType();
13983 // Examples:
13984 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
13985 // custom VL=9 with enveloping VL=8/8 yields 8/1
13986 // custom VL=10 with enveloping VL=8/8 yields 8/2
13987 // etc.
13988 ElementCount VTNumElts = VT.getVectorElementCount();
13989 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
13990 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
13991 "Mixing fixed width and scalable vectors when enveloping a type");
13992 EVT LoVT, HiVT;
13993 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
13994 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
13995 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
13996 *HiIsEmpty = false;
13997 } else {
13998 // Flag that hi type has zero storage size, but return split envelop type
13999 // (this would be easier if vector types with zero elements were allowed).
14000 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
14001 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
14002 *HiIsEmpty = true;
14003 }
14004 return std::make_pair(x&: LoVT, y&: HiVT);
14005}
14006
14007/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
14008/// low/high part.
14009std::pair<SDValue, SDValue>
14010SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
14011 const EVT &HiVT) {
14012 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
14013 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
14014 "Splitting vector with an invalid mixture of fixed and scalable "
14015 "vector types");
14016 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
14017 N.getValueType().getVectorMinNumElements() &&
14018 "More vector elements requested than available!");
14019 SDValue Lo, Hi;
14020 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
14021 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
14022 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
14023 // IDX with the runtime scaling factor of the result vector type. For
14024 // fixed-width result vectors, that runtime scaling factor is 1.
14025 Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
14026 N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
14027 return std::make_pair(x&: Lo, y&: Hi);
14028}
14029
14030std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
14031 const SDLoc &DL) {
14032 // Split the vector length parameter.
14033 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
14034 EVT VT = N.getValueType();
14035 assert(VecVT.getVectorElementCount().isKnownEven() &&
14036 "Expecting the mask to be an evenly-sized vector");
14037 SDValue HalfNumElts = getElementCount(
14038 DL, VT, EC: VecVT.getVectorElementCount().divideCoefficientBy(RHS: 2));
14039 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
14040 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
14041 return std::make_pair(x&: Lo, y&: Hi);
14042}
14043
14044/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
14045SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
14046 EVT VT = N.getValueType();
14047 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
14048 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
14049 return getInsertSubvector(DL, Vec: getUNDEF(VT: WideVT), SubVec: N, Idx: 0);
14050}
14051
14052void SelectionDAG::ExtractVectorElements(SDValue Op,
14053 SmallVectorImpl<SDValue> &Args,
14054 unsigned Start, unsigned Count,
14055 EVT EltVT) {
14056 EVT VT = Op.getValueType();
14057 if (Count == 0)
14058 Count = VT.getVectorNumElements();
14059 if (EltVT == EVT())
14060 EltVT = VT.getVectorElementType();
14061 SDLoc SL(Op);
14062 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
14063 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
14064 }
14065}
14066
14067// getAddressSpace - Return the address space this GlobalAddress belongs to.
14068unsigned GlobalAddressSDNode::getAddressSpace() const {
14069 return getGlobal()->getType()->getAddressSpace();
14070}
14071
14072Type *ConstantPoolSDNode::getType() const {
14073 if (isMachineConstantPoolEntry())
14074 return Val.MachineCPVal->getType();
14075 return Val.ConstVal->getType();
14076}
14077
14078bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
14079 unsigned &SplatBitSize,
14080 bool &HasAnyUndefs,
14081 unsigned MinSplatBits,
14082 bool IsBigEndian) const {
14083 EVT VT = getValueType(ResNo: 0);
14084 assert(VT.isVector() && "Expected a vector type");
14085 unsigned VecWidth = VT.getSizeInBits();
14086 if (MinSplatBits > VecWidth)
14087 return false;
14088
14089 // FIXME: The widths are based on this node's type, but build vectors can
14090 // truncate their operands.
14091 SplatValue = APInt(VecWidth, 0);
14092 SplatUndef = APInt(VecWidth, 0);
14093
14094 // Get the bits. Bits with undefined values (when the corresponding element
14095 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
14096 // in SplatValue. If any of the values are not constant, give up and return
14097 // false.
14098 unsigned int NumOps = getNumOperands();
14099 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
14100 unsigned EltWidth = VT.getScalarSizeInBits();
14101
14102 for (unsigned j = 0; j < NumOps; ++j) {
14103 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
14104 SDValue OpVal = getOperand(Num: i);
14105 unsigned BitPos = j * EltWidth;
14106
14107 if (OpVal.isUndef())
14108 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
14109 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
14110 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
14111 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
14112 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
14113 else
14114 return false;
14115 }
14116
14117 // The build_vector is all constants or undefs. Find the smallest element
14118 // size that splats the vector.
14119 HasAnyUndefs = (SplatUndef != 0);
14120
14121 // FIXME: This does not work for vectors with elements less than 8 bits.
14122 while (VecWidth > 8) {
14123 // If we can't split in half, stop here.
14124 if (VecWidth & 1)
14125 break;
14126
14127 unsigned HalfSize = VecWidth / 2;
14128 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14129 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
14130 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
14131 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
14132
14133 // If the two halves do not match (ignoring undef bits), stop here.
14134 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
14135 MinSplatBits > HalfSize)
14136 break;
14137
14138 SplatValue = HighValue | LowValue;
14139 SplatUndef = HighUndef & LowUndef;
14140
14141 VecWidth = HalfSize;
14142 }
14143
14144 // FIXME: The loop above only tries to split in halves. But if the input
14145 // vector for example is <3 x i16> it wouldn't be able to detect a
14146 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
14147 // optimizations. I guess that back in the days when this helper was created
14148 // vectors normally was power-of-2 sized.
14149
14150 SplatBitSize = VecWidth;
14151 return true;
14152}
14153
14154SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
14155 BitVector *UndefElements) const {
14156 unsigned NumOps = getNumOperands();
14157 if (UndefElements) {
14158 UndefElements->clear();
14159 UndefElements->resize(N: NumOps);
14160 }
14161 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14162 if (!DemandedElts)
14163 return SDValue();
14164 SDValue Splatted;
14165 for (unsigned i = 0; i != NumOps; ++i) {
14166 if (!DemandedElts[i])
14167 continue;
14168 SDValue Op = getOperand(Num: i);
14169 if (Op.isUndef()) {
14170 if (UndefElements)
14171 (*UndefElements)[i] = true;
14172 } else if (!Splatted) {
14173 Splatted = Op;
14174 } else if (Splatted != Op) {
14175 return SDValue();
14176 }
14177 }
14178
14179 if (!Splatted) {
14180 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
14181 assert(getOperand(FirstDemandedIdx).isUndef() &&
14182 "Can only have a splat without a constant for all undefs.");
14183 return getOperand(Num: FirstDemandedIdx);
14184 }
14185
14186 return Splatted;
14187}
14188
14189SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
14190 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14191 return getSplatValue(DemandedElts, UndefElements);
14192}
14193
14194bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
14195 SmallVectorImpl<SDValue> &Sequence,
14196 BitVector *UndefElements) const {
14197 unsigned NumOps = getNumOperands();
14198 Sequence.clear();
14199 if (UndefElements) {
14200 UndefElements->clear();
14201 UndefElements->resize(N: NumOps);
14202 }
14203 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
14204 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
14205 return false;
14206
14207 // Set the undefs even if we don't find a sequence (like getSplatValue).
14208 if (UndefElements)
14209 for (unsigned I = 0; I != NumOps; ++I)
14210 if (DemandedElts[I] && getOperand(Num: I).isUndef())
14211 (*UndefElements)[I] = true;
14212
14213 // Iteratively widen the sequence length looking for repetitions.
14214 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
14215 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
14216 for (unsigned I = 0; I != NumOps; ++I) {
14217 if (!DemandedElts[I])
14218 continue;
14219 SDValue &SeqOp = Sequence[I % SeqLen];
14220 SDValue Op = getOperand(Num: I);
14221 if (Op.isUndef()) {
14222 if (!SeqOp)
14223 SeqOp = Op;
14224 continue;
14225 }
14226 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
14227 Sequence.clear();
14228 break;
14229 }
14230 SeqOp = Op;
14231 }
14232 if (!Sequence.empty())
14233 return true;
14234 }
14235
14236 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
14237 return false;
14238}
14239
14240bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
14241 BitVector *UndefElements) const {
14242 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
14243 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
14244}
14245
14246ConstantSDNode *
14247BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
14248 BitVector *UndefElements) const {
14249 return dyn_cast_or_null<ConstantSDNode>(
14250 Val: getSplatValue(DemandedElts, UndefElements));
14251}
14252
14253ConstantSDNode *
14254BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
14255 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
14256}
14257
14258ConstantFPSDNode *
14259BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
14260 BitVector *UndefElements) const {
14261 return dyn_cast_or_null<ConstantFPSDNode>(
14262 Val: getSplatValue(DemandedElts, UndefElements));
14263}
14264
14265ConstantFPSDNode *
14266BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
14267 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
14268}
14269
14270int32_t
14271BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
14272 uint32_t BitWidth) const {
14273 if (ConstantFPSDNode *CN =
14274 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
14275 bool IsExact;
14276 APSInt IntVal(BitWidth);
14277 const APFloat &APF = CN->getValueAPF();
14278 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
14279 APFloat::opOK ||
14280 !IsExact)
14281 return -1;
14282
14283 return IntVal.exactLogBase2();
14284 }
14285 return -1;
14286}
14287
14288bool BuildVectorSDNode::getConstantRawBits(
14289 bool IsLittleEndian, unsigned DstEltSizeInBits,
14290 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
14291 // Early-out if this contains anything but Undef/Constant/ConstantFP.
14292 if (!isConstant())
14293 return false;
14294
14295 unsigned NumSrcOps = getNumOperands();
14296 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
14297 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14298 "Invalid bitcast scale");
14299
14300 // Extract raw src bits.
14301 SmallVector<APInt> SrcBitElements(NumSrcOps,
14302 APInt::getZero(numBits: SrcEltSizeInBits));
14303 BitVector SrcUndeElements(NumSrcOps, false);
14304
14305 for (unsigned I = 0; I != NumSrcOps; ++I) {
14306 SDValue Op = getOperand(Num: I);
14307 if (Op.isUndef()) {
14308 SrcUndeElements.set(I);
14309 continue;
14310 }
14311 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
14312 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
14313 assert((CInt || CFP) && "Unknown constant");
14314 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
14315 : CFP->getValueAPF().bitcastToAPInt();
14316 }
14317
14318 // Recast to dst width.
14319 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
14320 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
14321 return true;
14322}
14323
14324void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
14325 unsigned DstEltSizeInBits,
14326 SmallVectorImpl<APInt> &DstBitElements,
14327 ArrayRef<APInt> SrcBitElements,
14328 BitVector &DstUndefElements,
14329 const BitVector &SrcUndefElements) {
14330 unsigned NumSrcOps = SrcBitElements.size();
14331 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
14332 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
14333 "Invalid bitcast scale");
14334 assert(NumSrcOps == SrcUndefElements.size() &&
14335 "Vector size mismatch");
14336
14337 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
14338 DstUndefElements.clear();
14339 DstUndefElements.resize(N: NumDstOps, t: false);
14340 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
14341
14342 // Concatenate src elements constant bits together into dst element.
14343 if (SrcEltSizeInBits <= DstEltSizeInBits) {
14344 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
14345 for (unsigned I = 0; I != NumDstOps; ++I) {
14346 DstUndefElements.set(I);
14347 APInt &DstBits = DstBitElements[I];
14348 for (unsigned J = 0; J != Scale; ++J) {
14349 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14350 if (SrcUndefElements[Idx])
14351 continue;
14352 DstUndefElements.reset(Idx: I);
14353 const APInt &SrcBits = SrcBitElements[Idx];
14354 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
14355 "Illegal constant bitwidths");
14356 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
14357 }
14358 }
14359 return;
14360 }
14361
14362 // Split src element constant bits into dst elements.
14363 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
14364 for (unsigned I = 0; I != NumSrcOps; ++I) {
14365 if (SrcUndefElements[I]) {
14366 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
14367 continue;
14368 }
14369 const APInt &SrcBits = SrcBitElements[I];
14370 for (unsigned J = 0; J != Scale; ++J) {
14371 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
14372 APInt &DstBits = DstBitElements[Idx];
14373 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
14374 }
14375 }
14376}
14377
14378bool BuildVectorSDNode::isConstant() const {
14379 for (const SDValue &Op : op_values()) {
14380 unsigned Opc = Op.getOpcode();
14381 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
14382 return false;
14383 }
14384 return true;
14385}
14386
14387std::optional<std::pair<APInt, APInt>>
14388BuildVectorSDNode::isArithmeticSequence() const {
14389 unsigned NumOps = getNumOperands();
14390 if (NumOps < 2)
14391 return std::nullopt;
14392
14393 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
14394 APInt Start, Stride;
14395 int FirstIdx = -1, SecondIdx = -1;
14396
14397 // Find the first two non-undef constant elements to determine Start and
14398 // Stride, then verify all remaining elements match the sequence.
14399 for (unsigned I = 0; I < NumOps; ++I) {
14400 SDValue Op = getOperand(Num: I);
14401 if (Op->isUndef())
14402 continue;
14403 if (!isa<ConstantSDNode>(Val: Op))
14404 return std::nullopt;
14405
14406 APInt Val = getConstantOperandAPInt(Num: I).trunc(width: EltSize);
14407 if (FirstIdx < 0) {
14408 FirstIdx = I;
14409 Start = Val;
14410 } else if (SecondIdx < 0) {
14411 SecondIdx = I;
14412 // Compute stride using modular arithmetic. Simple division would handle
14413 // common strides (1, 2, -1, etc.), but modular inverse maximizes matches.
14414 // Example: <0, poison, poison, 0xFF> has stride 0x55 since 3*0x55 = 0xFF
14415 // Note that modular arithmetic is agnostic to signed/unsigned.
14416 unsigned IdxDiff = I - FirstIdx;
14417 APInt ValDiff = Val - Start;
14418
14419 // Step 1: Factor out common powers of 2 from IdxDiff and ValDiff.
14420 unsigned CommonPow2Bits = llvm::countr_zero(Val: IdxDiff);
14421 if (ValDiff.countr_zero() < CommonPow2Bits)
14422 return std::nullopt; // ValDiff not divisible by 2^CommonPow2Bits
14423 IdxDiff >>= CommonPow2Bits;
14424 ValDiff.lshrInPlace(ShiftAmt: CommonPow2Bits);
14425
14426 // Step 2: IdxDiff is now odd, so its inverse mod 2^EltSize exists.
14427 // TODO: There are 2^CommonPow2Bits valid strides; currently we only try
14428 // one, but we could try all candidates to handle more cases.
14429 Stride = ValDiff * APInt(EltSize, IdxDiff).multiplicativeInverse();
14430 if (Stride.isZero())
14431 return std::nullopt;
14432
14433 // Step 3: Adjust Start based on the first defined element's index.
14434 Start -= Stride * FirstIdx;
14435 } else {
14436 // Verify this element matches the sequence.
14437 if (Val != Start + Stride * I)
14438 return std::nullopt;
14439 }
14440 }
14441
14442 // Need at least two defined elements.
14443 if (SecondIdx < 0)
14444 return std::nullopt;
14445
14446 return std::make_pair(x&: Start, y&: Stride);
14447}
14448
14449bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
14450 // Find the first non-undef value in the shuffle mask.
14451 unsigned i, e;
14452 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
14453 /* search */;
14454
14455 // If all elements are undefined, this shuffle can be considered a splat
14456 // (although it should eventually get simplified away completely).
14457 if (i == e)
14458 return true;
14459
14460 // Make sure all remaining elements are either undef or the same as the first
14461 // non-undef value.
14462 for (int Idx = Mask[i]; i != e; ++i)
14463 if (Mask[i] >= 0 && Mask[i] != Idx)
14464 return false;
14465 return true;
14466}
14467
14468// Returns true if it is a constant integer BuildVector or constant integer,
14469// possibly hidden by a bitcast.
14470bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
14471 SDValue N, bool AllowOpaques) const {
14472 N = peekThroughBitcasts(V: N);
14473
14474 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
14475 return AllowOpaques || !C->isOpaque();
14476
14477 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
14478 return true;
14479
14480 // Treat a GlobalAddress supporting constant offset folding as a
14481 // constant integer.
14482 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
14483 if (GA->getOpcode() == ISD::GlobalAddress &&
14484 TLI->isOffsetFoldingLegal(GA))
14485 return true;
14486
14487 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14488 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
14489 return true;
14490 return false;
14491}
14492
14493// Returns true if it is a constant float BuildVector or constant float.
14494bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
14495 if (isa<ConstantFPSDNode>(Val: N))
14496 return true;
14497
14498 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
14499 return true;
14500
14501 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
14502 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
14503 return true;
14504
14505 return false;
14506}
14507
14508std::optional<bool> SelectionDAG::isBoolConstant(SDValue N) const {
14509 ConstantSDNode *Const =
14510 isConstOrConstSplat(N, AllowUndefs: false, /*AllowTruncation=*/true);
14511 if (!Const)
14512 return std::nullopt;
14513
14514 EVT VT = N->getValueType(ResNo: 0);
14515 const APInt CVal = Const->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
14516 switch (TLI->getBooleanContents(Type: N.getValueType())) {
14517 case TargetLowering::ZeroOrOneBooleanContent:
14518 if (CVal.isOne())
14519 return true;
14520 if (CVal.isZero())
14521 return false;
14522 return std::nullopt;
14523 case TargetLowering::ZeroOrNegativeOneBooleanContent:
14524 if (CVal.isAllOnes())
14525 return true;
14526 if (CVal.isZero())
14527 return false;
14528 return std::nullopt;
14529 case TargetLowering::UndefinedBooleanContent:
14530 return CVal[0];
14531 }
14532 llvm_unreachable("Unknown BooleanContent enum");
14533}
14534
14535void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
14536 assert(!Node->OperandList && "Node already has operands");
14537 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
14538 "too many operands to fit into SDNode");
14539 SDUse *Ops = OperandRecycler.allocate(
14540 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
14541
14542 bool IsDivergent = false;
14543 for (unsigned I = 0; I != Vals.size(); ++I) {
14544 Ops[I].setUser(Node);
14545 Ops[I].setInitial(Vals[I]);
14546 EVT VT = Ops[I].getValueType();
14547
14548 // Skip Chain. It does not carry divergence.
14549 if (VT != MVT::Other &&
14550 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
14551 Ops[I].getNode()->isDivergent()) {
14552 IsDivergent = true;
14553 }
14554 }
14555 Node->NumOperands = Vals.size();
14556 Node->OperandList = Ops;
14557 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
14558 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
14559 Node->SDNodeBits.IsDivergent = IsDivergent;
14560 }
14561 checkForCycles(N: Node);
14562}
14563
14564SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
14565 SmallVectorImpl<SDValue> &Vals) {
14566 size_t Limit = SDNode::getMaxNumOperands();
14567 while (Vals.size() > Limit) {
14568 unsigned SliceIdx = Vals.size() - Limit;
14569 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
14570 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
14571 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
14572 Vals.emplace_back(Args&: NewTF);
14573 }
14574 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
14575}
14576
14577SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
14578 EVT VT, SDNodeFlags Flags) {
14579 switch (Opcode) {
14580 default:
14581 return SDValue();
14582 case ISD::ADD:
14583 case ISD::OR:
14584 case ISD::XOR:
14585 case ISD::UMAX:
14586 return getConstant(Val: 0, DL, VT);
14587 case ISD::MUL:
14588 return getConstant(Val: 1, DL, VT);
14589 case ISD::AND:
14590 case ISD::UMIN:
14591 return getAllOnesConstant(DL, VT);
14592 case ISD::SMAX:
14593 return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
14594 case ISD::SMIN:
14595 return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
14596 case ISD::FADD:
14597 // If flags allow, prefer positive zero since it's generally cheaper
14598 // to materialize on most targets.
14599 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
14600 case ISD::FMUL:
14601 return getConstantFP(Val: 1.0, DL, VT);
14602 case ISD::FMINNUM:
14603 case ISD::FMAXNUM: {
14604 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
14605 const fltSemantics &Semantics = VT.getFltSemantics();
14606 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
14607 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
14608 APFloat::getLargest(Sem: Semantics);
14609 if (Opcode == ISD::FMAXNUM)
14610 NeutralAF.changeSign();
14611
14612 return getConstantFP(V: NeutralAF, DL, VT);
14613 }
14614 case ISD::FMINIMUM:
14615 case ISD::FMAXIMUM: {
14616 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
14617 const fltSemantics &Semantics = VT.getFltSemantics();
14618 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
14619 : APFloat::getLargest(Sem: Semantics);
14620 if (Opcode == ISD::FMAXIMUM)
14621 NeutralAF.changeSign();
14622
14623 return getConstantFP(V: NeutralAF, DL, VT);
14624 }
14625
14626 }
14627}
14628
14629/// Helper used to make a call to a library function that has one argument of
14630/// pointer type.
14631///
14632/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
14633/// used to get or set floating-point state. They have one argument of pointer
14634/// type, which points to the memory region containing bits of the
14635/// floating-point state. The value returned by such function is ignored in the
14636/// created call.
14637///
14638/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
14639/// \param Ptr Pointer used to save/load state.
14640/// \param InChain Ingoing token chain.
14641/// \returns Outgoing chain token.
14642SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
14643 SDValue InChain,
14644 const SDLoc &DLoc) {
14645 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
14646 TargetLowering::ArgListTy Args;
14647 Args.emplace_back(args&: Ptr, args: Ptr.getValueType().getTypeForEVT(Context&: *getContext()));
14648 RTLIB::LibcallImpl LibcallImpl =
14649 Libcalls->getLibcallImpl(Call: static_cast<RTLIB::Libcall>(LibFunc));
14650 if (LibcallImpl == RTLIB::Unsupported)
14651 reportFatalUsageError(reason: "emitting call to unsupported libcall");
14652
14653 SDValue Callee =
14654 getExternalSymbol(Libcall: LibcallImpl, VT: TLI->getPointerTy(DL: getDataLayout()));
14655 TargetLowering::CallLoweringInfo CLI(*this);
14656 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
14657 CC: Libcalls->getLibcallImplCallingConv(Call: LibcallImpl),
14658 ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee, ArgsList: std::move(Args));
14659 return TLI->LowerCallTo(CLI).second;
14660}
14661
14662void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
14663 assert(From && To && "Invalid SDNode; empty source SDValue?");
14664 auto I = SDEI.find(Val: From);
14665 if (I == SDEI.end())
14666 return;
14667
14668 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
14669 // the iterator, hence the need to make a copy to prevent a use-after-free.
14670 NodeExtraInfo NEI = I->second;
14671 if (LLVM_LIKELY(!NEI.PCSections)) {
14672 // No deep copy required for the types of extra info set.
14673 //
14674 // FIXME: Investigate if other types of extra info also need deep copy. This
14675 // depends on the types of nodes they can be attached to: if some extra info
14676 // is only ever attached to nodes where a replacement To node is always the
14677 // node where later use and propagation of the extra info has the intended
14678 // semantics, no deep copy is required.
14679 SDEI[To] = std::move(NEI);
14680 return;
14681 }
14682
14683 const SDNode *EntrySDN = getEntryNode().getNode();
14684
14685 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
14686 // through the replacement of From with To. Otherwise, replacements of a node
14687 // (From) with more complex nodes (To and its operands) may result in lost
14688 // extra info where the root node (To) is insignificant in further propagating
14689 // and using extra info when further lowering to MIR.
14690 //
14691 // In the first step pre-populate the visited set with the nodes reachable
14692 // from the old From node. This avoids copying NodeExtraInfo to parts of the
14693 // DAG that is not new and should be left untouched.
14694 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
14695 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
14696 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
14697 if (MaxDepth == 0) {
14698 // Remember this node in case we need to increase MaxDepth and continue
14699 // populating FromReach from this node.
14700 Leafs.emplace_back(Args&: N);
14701 return;
14702 }
14703 if (!FromReach.insert(V: N).second)
14704 return;
14705 for (const SDValue &Op : N->op_values())
14706 Self(Self, Op.getNode(), MaxDepth - 1);
14707 };
14708
14709 // Copy extra info to To and all its transitive operands (that are new).
14710 SmallPtrSet<const SDNode *, 8> Visited;
14711 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
14712 if (FromReach.contains(V: N))
14713 return true;
14714 if (!Visited.insert(Ptr: N).second)
14715 return true;
14716 if (EntrySDN == N)
14717 return false;
14718 for (const SDValue &Op : N->op_values()) {
14719 if (N == To && Op.getNode() == EntrySDN) {
14720 // Special case: New node's operand is the entry node; just need to
14721 // copy extra info to new node.
14722 break;
14723 }
14724 if (!Self(Self, Op.getNode()))
14725 return false;
14726 }
14727 // Copy only if entry node was not reached.
14728 SDEI[N] = std::move(NEI);
14729 return true;
14730 };
14731
14732 // We first try with a lower MaxDepth, assuming that the path to common
14733 // operands between From and To is relatively short. This significantly
14734 // improves performance in the common case. The initial MaxDepth is big
14735 // enough to avoid retry in the common case; the last MaxDepth is large
14736 // enough to avoid having to use the fallback below (and protects from
14737 // potential stack exhaustion from recursion).
14738 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
14739 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
14740 // StartFrom is the previous (or initial) set of leafs reachable at the
14741 // previous maximum depth.
14742 SmallVector<const SDNode *> StartFrom;
14743 std::swap(LHS&: StartFrom, RHS&: Leafs);
14744 for (const SDNode *N : StartFrom)
14745 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
14746 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
14747 return;
14748 // This should happen very rarely (reached the entry node).
14749 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
14750 assert(!Leafs.empty());
14751 }
14752
14753 // This should not happen - but if it did, that means the subgraph reachable
14754 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
14755 // could not visit all reachable common operands. Consequently, we were able
14756 // to reach the entry node.
14757 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
14758 assert(false && "From subgraph too complex - increase max. MaxDepth?");
14759 // Best-effort fallback if assertions disabled.
14760 SDEI[To] = std::move(NEI);
14761}
14762
14763#ifndef NDEBUG
14764static void checkForCyclesHelper(const SDNode *N,
14765 SmallPtrSetImpl<const SDNode*> &Visited,
14766 SmallPtrSetImpl<const SDNode*> &Checked,
14767 const llvm::SelectionDAG *DAG) {
14768 // If this node has already been checked, don't check it again.
14769 if (Checked.count(N))
14770 return;
14771
14772 // If a node has already been visited on this depth-first walk, reject it as
14773 // a cycle.
14774 if (!Visited.insert(N).second) {
14775 errs() << "Detected cycle in SelectionDAG\n";
14776 dbgs() << "Offending node:\n";
14777 N->dumprFull(DAG); dbgs() << "\n";
14778 abort();
14779 }
14780
14781 for (const SDValue &Op : N->op_values())
14782 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
14783
14784 Checked.insert(N);
14785 Visited.erase(N);
14786}
14787#endif
14788
14789void llvm::checkForCycles(const llvm::SDNode *N,
14790 const llvm::SelectionDAG *DAG,
14791 bool force) {
14792#ifndef NDEBUG
14793 bool check = force;
14794#ifdef EXPENSIVE_CHECKS
14795 check = true;
14796#endif // EXPENSIVE_CHECKS
14797 if (check) {
14798 assert(N && "Checking nonexistent SDNode");
14799 SmallPtrSet<const SDNode*, 32> visited;
14800 SmallPtrSet<const SDNode*, 32> checked;
14801 checkForCyclesHelper(N, visited, checked, DAG);
14802 }
14803#endif // !NDEBUG
14804}
14805
14806void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
14807 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
14808}
14809