1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/SelectionDAG.h"
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
23#include "llvm/ADT/SmallPtrSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/AliasAnalysis.h"
27#include "llvm/Analysis/MemoryLocation.h"
28#include "llvm/Analysis/TargetLibraryInfo.h"
29#include "llvm/Analysis/ValueTracking.h"
30#include "llvm/Analysis/VectorUtils.h"
31#include "llvm/BinaryFormat/Dwarf.h"
32#include "llvm/CodeGen/Analysis.h"
33#include "llvm/CodeGen/FunctionLoweringInfo.h"
34#include "llvm/CodeGen/ISDOpcodes.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineConstantPool.h"
37#include "llvm/CodeGen/MachineFrameInfo.h"
38#include "llvm/CodeGen/MachineFunction.h"
39#include "llvm/CodeGen/MachineMemOperand.h"
40#include "llvm/CodeGen/RuntimeLibcallUtil.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
43#include "llvm/CodeGen/SelectionDAGNodes.h"
44#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
45#include "llvm/CodeGen/TargetFrameLowering.h"
46#include "llvm/CodeGen/TargetLowering.h"
47#include "llvm/CodeGen/TargetRegisterInfo.h"
48#include "llvm/CodeGen/TargetSubtargetInfo.h"
49#include "llvm/CodeGen/ValueTypes.h"
50#include "llvm/CodeGenTypes/MachineValueType.h"
51#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfoMetadata.h"
55#include "llvm/IR/DebugLoc.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
61#include "llvm/Support/Casting.h"
62#include "llvm/Support/CodeGen.h"
63#include "llvm/Support/Compiler.h"
64#include "llvm/Support/Debug.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/MathExtras.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include "llvm/TargetParser/Triple.h"
72#include "llvm/Transforms/Utils/SizeOpts.h"
73#include <algorithm>
74#include <cassert>
75#include <cstdint>
76#include <cstdlib>
77#include <limits>
78#include <optional>
79#include <set>
80#include <string>
81#include <utility>
82#include <vector>
83
84using namespace llvm;
85using namespace llvm::SDPatternMatch;
86
87/// makeVTList - Return an instance of the SDVTList struct initialized with the
88/// specified members.
89static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
90 SDVTList Res = {.VTs: VTs, .NumVTs: NumVTs};
91 return Res;
92}
93
94// Default null implementations of the callbacks.
95void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
96void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
97void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {}
98
99void SelectionDAG::DAGNodeDeletedListener::anchor() {}
100void SelectionDAG::DAGNodeInsertedListener::anchor() {}
101
102#define DEBUG_TYPE "selectiondag"
103
104static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
105 cl::Hidden, cl::init(Val: true),
106 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
107
108static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
109 cl::desc("Number limit for gluing ld/st of memcpy."),
110 cl::Hidden, cl::init(Val: 0));
111
112static cl::opt<unsigned>
113 MaxSteps("has-predecessor-max-steps", cl::Hidden, cl::init(Val: 8192),
114 cl::desc("DAG combiner limit number of steps when searching DAG "
115 "for predecessor nodes"));
116
117static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G) {
118 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
119}
120
121unsigned SelectionDAG::getHasPredecessorMaxSteps() { return MaxSteps; }
122
123//===----------------------------------------------------------------------===//
124// ConstantFPSDNode Class
125//===----------------------------------------------------------------------===//
126
127/// isExactlyValue - We don't rely on operator== working on double values, as
128/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
129/// As such, this method can be used to do an exact bit-for-bit comparison of
130/// two floating point values.
131bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const {
132 return getValueAPF().bitwiseIsEqual(RHS: V);
133}
134
135bool ConstantFPSDNode::isValueValidForType(EVT VT,
136 const APFloat& Val) {
137 assert(VT.isFloatingPoint() && "Can only convert between FP types");
138
139 // convert modifies in place, so make a copy.
140 APFloat Val2 = APFloat(Val);
141 bool losesInfo;
142 (void)Val2.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
143 losesInfo: &losesInfo);
144 return !losesInfo;
145}
146
147//===----------------------------------------------------------------------===//
148// ISD Namespace
149//===----------------------------------------------------------------------===//
150
151bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
152 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
153 if (auto OptAPInt = N->getOperand(Num: 0)->bitcastToAPInt()) {
154 unsigned EltSize =
155 N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
156 SplatVal = OptAPInt->trunc(width: EltSize);
157 return true;
158 }
159 }
160
161 auto *BV = dyn_cast<BuildVectorSDNode>(Val: N);
162 if (!BV)
163 return false;
164
165 APInt SplatUndef;
166 unsigned SplatBitSize;
167 bool HasUndefs;
168 unsigned EltSize = N->getValueType(ResNo: 0).getVectorElementType().getSizeInBits();
169 // Endianness does not matter here. We are checking for a splat given the
170 // element size of the vector, and if we find such a splat for little endian
171 // layout, then that should be valid also for big endian (as the full vector
172 // size is known to be a multiple of the element size).
173 const bool IsBigEndian = false;
174 return BV->isConstantSplat(SplatValue&: SplatVal, SplatUndef, SplatBitSize, HasAnyUndefs&: HasUndefs,
175 MinSplatBits: EltSize, isBigEndian: IsBigEndian) &&
176 EltSize == SplatBitSize;
177}
178
179// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
180// specializations of the more general isConstantSplatVector()?
181
182bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
183 // Look through a bit convert.
184 while (N->getOpcode() == ISD::BITCAST)
185 N = N->getOperand(Num: 0).getNode();
186
187 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
188 APInt SplatVal;
189 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
190 }
191
192 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
193
194 unsigned i = 0, e = N->getNumOperands();
195
196 // Skip over all of the undef values.
197 while (i != e && N->getOperand(Num: i).isUndef())
198 ++i;
199
200 // Do not accept an all-undef vector.
201 if (i == e) return false;
202
203 // Do not accept build_vectors that aren't all constants or which have non-~0
204 // elements. We have to be a bit careful here, as the type of the constant
205 // may not be the same as the type of the vector elements due to type
206 // legalization (the elements are promoted to a legal type for the target and
207 // a vector of a type may be legal when the base element type is not).
208 // We only want to check enough bits to cover the vector elements, because
209 // we care if the resultant vector is all ones, not whether the individual
210 // constants are.
211 SDValue NotZero = N->getOperand(Num: i);
212 if (auto OptAPInt = NotZero->bitcastToAPInt()) {
213 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
214 if (OptAPInt->countr_one() < EltSize)
215 return false;
216 } else
217 return false;
218
219 // Okay, we have at least one ~0 value, check to see if the rest match or are
220 // undefs. Even with the above element type twiddling, this should be OK, as
221 // the same type legalization should have applied to all the elements.
222 for (++i; i != e; ++i)
223 if (N->getOperand(Num: i) != NotZero && !N->getOperand(Num: i).isUndef())
224 return false;
225 return true;
226}
227
228bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
229 // Look through a bit convert.
230 while (N->getOpcode() == ISD::BITCAST)
231 N = N->getOperand(Num: 0).getNode();
232
233 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
234 APInt SplatVal;
235 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
236 }
237
238 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
239
240 bool IsAllUndef = true;
241 for (const SDValue &Op : N->op_values()) {
242 if (Op.isUndef())
243 continue;
244 IsAllUndef = false;
245 // Do not accept build_vectors that aren't all constants or which have non-0
246 // elements. We have to be a bit careful here, as the type of the constant
247 // may not be the same as the type of the vector elements due to type
248 // legalization (the elements are promoted to a legal type for the target
249 // and a vector of a type may be legal when the base element type is not).
250 // We only want to check enough bits to cover the vector elements, because
251 // we care if the resultant vector is all zeros, not whether the individual
252 // constants are.
253 if (auto OptAPInt = Op->bitcastToAPInt()) {
254 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
255 if (OptAPInt->countr_zero() < EltSize)
256 return false;
257 } else
258 return false;
259 }
260
261 // Do not accept an all-undef vector.
262 if (IsAllUndef)
263 return false;
264 return true;
265}
266
267bool ISD::isBuildVectorAllOnes(const SDNode *N) {
268 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
269}
270
271bool ISD::isBuildVectorAllZeros(const SDNode *N) {
272 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
273}
274
275bool ISD::isBuildVectorOfConstantSDNodes(const SDNode *N) {
276 if (N->getOpcode() != ISD::BUILD_VECTOR)
277 return false;
278
279 for (const SDValue &Op : N->op_values()) {
280 if (Op.isUndef())
281 continue;
282 if (!isa<ConstantSDNode>(Val: Op))
283 return false;
284 }
285 return true;
286}
287
288bool ISD::isBuildVectorOfConstantFPSDNodes(const SDNode *N) {
289 if (N->getOpcode() != ISD::BUILD_VECTOR)
290 return false;
291
292 for (const SDValue &Op : N->op_values()) {
293 if (Op.isUndef())
294 continue;
295 if (!isa<ConstantFPSDNode>(Val: Op))
296 return false;
297 }
298 return true;
299}
300
301bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
302 bool Signed) {
303 assert(N->getValueType(0).isVector() && "Expected a vector!");
304
305 unsigned EltSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
306 if (EltSize <= NewEltSize)
307 return false;
308
309 if (N->getOpcode() == ISD::ZERO_EXTEND) {
310 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
311 NewEltSize) &&
312 !Signed;
313 }
314 if (N->getOpcode() == ISD::SIGN_EXTEND) {
315 return (N->getOperand(Num: 0).getValueType().getScalarSizeInBits() <=
316 NewEltSize) &&
317 Signed;
318 }
319 if (N->getOpcode() != ISD::BUILD_VECTOR)
320 return false;
321
322 for (const SDValue &Op : N->op_values()) {
323 if (Op.isUndef())
324 continue;
325 if (!isa<ConstantSDNode>(Val: Op))
326 return false;
327
328 APInt C = Op->getAsAPIntVal().trunc(width: EltSize);
329 if (Signed && C.trunc(width: NewEltSize).sext(width: EltSize) != C)
330 return false;
331 if (!Signed && C.trunc(width: NewEltSize).zext(width: EltSize) != C)
332 return false;
333 }
334
335 return true;
336}
337
338bool ISD::allOperandsUndef(const SDNode *N) {
339 // Return false if the node has no operands.
340 // This is "logically inconsistent" with the definition of "all" but
341 // is probably the desired behavior.
342 if (N->getNumOperands() == 0)
343 return false;
344 return all_of(Range: N->op_values(), P: [](SDValue Op) { return Op.isUndef(); });
345}
346
347bool ISD::isFreezeUndef(const SDNode *N) {
348 return N->getOpcode() == ISD::FREEZE && N->getOperand(Num: 0).isUndef();
349}
350
351template <typename ConstNodeType>
352bool ISD::matchUnaryPredicateImpl(SDValue Op,
353 std::function<bool(ConstNodeType *)> Match,
354 bool AllowUndefs, bool AllowTruncation) {
355 // FIXME: Add support for scalar UNDEF cases?
356 if (auto *C = dyn_cast<ConstNodeType>(Op))
357 return Match(C);
358
359 // FIXME: Add support for vector UNDEF cases?
360 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
361 ISD::SPLAT_VECTOR != Op.getOpcode())
362 return false;
363
364 EVT SVT = Op.getValueType().getScalarType();
365 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
366 if (AllowUndefs && Op.getOperand(i).isUndef()) {
367 if (!Match(nullptr))
368 return false;
369 continue;
370 }
371
372 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
373 if (!Cst || (!AllowTruncation && Cst->getValueType(0) != SVT) ||
374 !Match(Cst))
375 return false;
376 }
377 return true;
378}
379// Build used template types.
380template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
381 SDValue, std::function<bool(ConstantSDNode *)>, bool, bool);
382template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
383 SDValue, std::function<bool(ConstantFPSDNode *)>, bool, bool);
384
385bool ISD::matchBinaryPredicate(
386 SDValue LHS, SDValue RHS,
387 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
388 bool AllowUndefs, bool AllowTypeMismatch) {
389 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
390 return false;
391
392 // TODO: Add support for scalar UNDEF cases?
393 if (auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHS))
394 if (auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHS))
395 return Match(LHSCst, RHSCst);
396
397 // TODO: Add support for vector UNDEF cases?
398 if (LHS.getOpcode() != RHS.getOpcode() ||
399 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
400 LHS.getOpcode() != ISD::SPLAT_VECTOR))
401 return false;
402
403 EVT SVT = LHS.getValueType().getScalarType();
404 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
405 SDValue LHSOp = LHS.getOperand(i);
406 SDValue RHSOp = RHS.getOperand(i);
407 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
408 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
409 auto *LHSCst = dyn_cast<ConstantSDNode>(Val&: LHSOp);
410 auto *RHSCst = dyn_cast<ConstantSDNode>(Val&: RHSOp);
411 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
412 return false;
413 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
414 LHSOp.getValueType() != RHSOp.getValueType()))
415 return false;
416 if (!Match(LHSCst, RHSCst))
417 return false;
418 }
419 return true;
420}
421
422ISD::NodeType ISD::getInverseMinMaxOpcode(unsigned MinMaxOpc) {
423 switch (MinMaxOpc) {
424 default:
425 llvm_unreachable("unrecognized opcode");
426 case ISD::UMIN:
427 return ISD::UMAX;
428 case ISD::UMAX:
429 return ISD::UMIN;
430 case ISD::SMIN:
431 return ISD::SMAX;
432 case ISD::SMAX:
433 return ISD::SMIN;
434 }
435}
436
437ISD::NodeType ISD::getVecReduceBaseOpcode(unsigned VecReduceOpcode) {
438 switch (VecReduceOpcode) {
439 default:
440 llvm_unreachable("Expected VECREDUCE opcode");
441 case ISD::VECREDUCE_FADD:
442 case ISD::VECREDUCE_SEQ_FADD:
443 case ISD::VP_REDUCE_FADD:
444 case ISD::VP_REDUCE_SEQ_FADD:
445 return ISD::FADD;
446 case ISD::VECREDUCE_FMUL:
447 case ISD::VECREDUCE_SEQ_FMUL:
448 case ISD::VP_REDUCE_FMUL:
449 case ISD::VP_REDUCE_SEQ_FMUL:
450 return ISD::FMUL;
451 case ISD::VECREDUCE_ADD:
452 case ISD::VP_REDUCE_ADD:
453 return ISD::ADD;
454 case ISD::VECREDUCE_MUL:
455 case ISD::VP_REDUCE_MUL:
456 return ISD::MUL;
457 case ISD::VECREDUCE_AND:
458 case ISD::VP_REDUCE_AND:
459 return ISD::AND;
460 case ISD::VECREDUCE_OR:
461 case ISD::VP_REDUCE_OR:
462 return ISD::OR;
463 case ISD::VECREDUCE_XOR:
464 case ISD::VP_REDUCE_XOR:
465 return ISD::XOR;
466 case ISD::VECREDUCE_SMAX:
467 case ISD::VP_REDUCE_SMAX:
468 return ISD::SMAX;
469 case ISD::VECREDUCE_SMIN:
470 case ISD::VP_REDUCE_SMIN:
471 return ISD::SMIN;
472 case ISD::VECREDUCE_UMAX:
473 case ISD::VP_REDUCE_UMAX:
474 return ISD::UMAX;
475 case ISD::VECREDUCE_UMIN:
476 case ISD::VP_REDUCE_UMIN:
477 return ISD::UMIN;
478 case ISD::VECREDUCE_FMAX:
479 case ISD::VP_REDUCE_FMAX:
480 return ISD::FMAXNUM;
481 case ISD::VECREDUCE_FMIN:
482 case ISD::VP_REDUCE_FMIN:
483 return ISD::FMINNUM;
484 case ISD::VECREDUCE_FMAXIMUM:
485 case ISD::VP_REDUCE_FMAXIMUM:
486 return ISD::FMAXIMUM;
487 case ISD::VECREDUCE_FMINIMUM:
488 case ISD::VP_REDUCE_FMINIMUM:
489 return ISD::FMINIMUM;
490 }
491}
492
493bool ISD::isVPOpcode(unsigned Opcode) {
494 switch (Opcode) {
495 default:
496 return false;
497#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
498 case ISD::VPSD: \
499 return true;
500#include "llvm/IR/VPIntrinsics.def"
501 }
502}
503
504bool ISD::isVPBinaryOp(unsigned Opcode) {
505 switch (Opcode) {
506 default:
507 break;
508#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
509#define VP_PROPERTY_BINARYOP return true;
510#define END_REGISTER_VP_SDNODE(VPSD) break;
511#include "llvm/IR/VPIntrinsics.def"
512 }
513 return false;
514}
515
516bool ISD::isVPReduction(unsigned Opcode) {
517 switch (Opcode) {
518 default:
519 return false;
520 case ISD::VP_REDUCE_ADD:
521 case ISD::VP_REDUCE_MUL:
522 case ISD::VP_REDUCE_AND:
523 case ISD::VP_REDUCE_OR:
524 case ISD::VP_REDUCE_XOR:
525 case ISD::VP_REDUCE_SMAX:
526 case ISD::VP_REDUCE_SMIN:
527 case ISD::VP_REDUCE_UMAX:
528 case ISD::VP_REDUCE_UMIN:
529 case ISD::VP_REDUCE_FMAX:
530 case ISD::VP_REDUCE_FMIN:
531 case ISD::VP_REDUCE_FMAXIMUM:
532 case ISD::VP_REDUCE_FMINIMUM:
533 case ISD::VP_REDUCE_FADD:
534 case ISD::VP_REDUCE_FMUL:
535 case ISD::VP_REDUCE_SEQ_FADD:
536 case ISD::VP_REDUCE_SEQ_FMUL:
537 return true;
538 }
539}
540
541/// The operand position of the vector mask.
542std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
543 switch (Opcode) {
544 default:
545 return std::nullopt;
546#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
547 case ISD::VPSD: \
548 return MASKPOS;
549#include "llvm/IR/VPIntrinsics.def"
550 }
551}
552
553/// The operand position of the explicit vector length parameter.
554std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
555 switch (Opcode) {
556 default:
557 return std::nullopt;
558#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
559 case ISD::VPSD: \
560 return EVLPOS;
561#include "llvm/IR/VPIntrinsics.def"
562 }
563}
564
565std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
566 bool hasFPExcept) {
567 // FIXME: Return strict opcodes in case of fp exceptions.
568 switch (VPOpcode) {
569 default:
570 return std::nullopt;
571#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
572#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
573#define END_REGISTER_VP_SDNODE(VPOPC) break;
574#include "llvm/IR/VPIntrinsics.def"
575 }
576 return std::nullopt;
577}
578
579std::optional<unsigned> ISD::getVPForBaseOpcode(unsigned Opcode) {
580 switch (Opcode) {
581 default:
582 return std::nullopt;
583#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
584#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
585#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
586#include "llvm/IR/VPIntrinsics.def"
587 }
588}
589
590ISD::NodeType ISD::getExtForLoadExtType(bool IsFP, ISD::LoadExtType ExtType) {
591 switch (ExtType) {
592 case ISD::EXTLOAD:
593 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
594 case ISD::SEXTLOAD:
595 return ISD::SIGN_EXTEND;
596 case ISD::ZEXTLOAD:
597 return ISD::ZERO_EXTEND;
598 default:
599 break;
600 }
601
602 llvm_unreachable("Invalid LoadExtType");
603}
604
605ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) {
606 // To perform this operation, we just need to swap the L and G bits of the
607 // operation.
608 unsigned OldL = (Operation >> 2) & 1;
609 unsigned OldG = (Operation >> 1) & 1;
610 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
611 (OldL << 1) | // New G bit
612 (OldG << 2)); // New L bit.
613}
614
615static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike) {
616 unsigned Operation = Op;
617 if (isIntegerLike)
618 Operation ^= 7; // Flip L, G, E bits, but not U.
619 else
620 Operation ^= 15; // Flip all of the condition bits.
621
622 if (Operation > ISD::SETTRUE2)
623 Operation &= ~8; // Don't let N and U bits get set.
624
625 return ISD::CondCode(Operation);
626}
627
628ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, EVT Type) {
629 return getSetCCInverseImpl(Op, isIntegerLike: Type.isInteger());
630}
631
632ISD::CondCode ISD::GlobalISel::getSetCCInverse(ISD::CondCode Op,
633 bool isIntegerLike) {
634 return getSetCCInverseImpl(Op, isIntegerLike);
635}
636
637/// For an integer comparison, return 1 if the comparison is a signed operation
638/// and 2 if the result is an unsigned comparison. Return zero if the operation
639/// does not depend on the sign of the input (setne and seteq).
640static int isSignedOp(ISD::CondCode Opcode) {
641 switch (Opcode) {
642 default: llvm_unreachable("Illegal integer setcc operation!");
643 case ISD::SETEQ:
644 case ISD::SETNE: return 0;
645 case ISD::SETLT:
646 case ISD::SETLE:
647 case ISD::SETGT:
648 case ISD::SETGE: return 1;
649 case ISD::SETULT:
650 case ISD::SETULE:
651 case ISD::SETUGT:
652 case ISD::SETUGE: return 2;
653 }
654}
655
656ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2,
657 EVT Type) {
658 bool IsInteger = Type.isInteger();
659 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
660 // Cannot fold a signed integer setcc with an unsigned integer setcc.
661 return ISD::SETCC_INVALID;
662
663 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
664
665 // If the N and U bits get set, then the resultant comparison DOES suddenly
666 // care about orderedness, and it is true when ordered.
667 if (Op > ISD::SETTRUE2)
668 Op &= ~16; // Clear the U bit if the N bit is set.
669
670 // Canonicalize illegal integer setcc's.
671 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
672 Op = ISD::SETNE;
673
674 return ISD::CondCode(Op);
675}
676
677ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2,
678 EVT Type) {
679 bool IsInteger = Type.isInteger();
680 if (IsInteger && (isSignedOp(Opcode: Op1) | isSignedOp(Opcode: Op2)) == 3)
681 // Cannot fold a signed setcc with an unsigned setcc.
682 return ISD::SETCC_INVALID;
683
684 // Combine all of the condition bits.
685 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
686
687 // Canonicalize illegal integer setcc's.
688 if (IsInteger) {
689 switch (Result) {
690 default: break;
691 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
692 case ISD::SETOEQ: // SETEQ & SETU[LG]E
693 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
694 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
695 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
696 }
697 }
698
699 return Result;
700}
701
702//===----------------------------------------------------------------------===//
703// SDNode Profile Support
704//===----------------------------------------------------------------------===//
705
706/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
707static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
708 ID.AddInteger(I: OpC);
709}
710
711/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
712/// solely with their pointer.
713static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) {
714 ID.AddPointer(Ptr: VTList.VTs);
715}
716
717/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
718static void AddNodeIDOperands(FoldingSetNodeID &ID,
719 ArrayRef<SDValue> Ops) {
720 for (const auto &Op : Ops) {
721 ID.AddPointer(Ptr: Op.getNode());
722 ID.AddInteger(I: Op.getResNo());
723 }
724}
725
726/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
727static void AddNodeIDOperands(FoldingSetNodeID &ID,
728 ArrayRef<SDUse> Ops) {
729 for (const auto &Op : Ops) {
730 ID.AddPointer(Ptr: Op.getNode());
731 ID.AddInteger(I: Op.getResNo());
732 }
733}
734
735static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
736 SDVTList VTList, ArrayRef<SDValue> OpList) {
737 AddNodeIDOpcode(ID, OpC);
738 AddNodeIDValueTypes(ID, VTList);
739 AddNodeIDOperands(ID, Ops: OpList);
740}
741
742/// If this is an SDNode with special info, add this info to the NodeID data.
743static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
744 switch (N->getOpcode()) {
745 case ISD::TargetExternalSymbol:
746 case ISD::ExternalSymbol:
747 case ISD::MCSymbol:
748 llvm_unreachable("Should only be used on nodes with operands");
749 default: break; // Normal nodes don't need extra info.
750 case ISD::TargetConstant:
751 case ISD::Constant: {
752 const ConstantSDNode *C = cast<ConstantSDNode>(Val: N);
753 ID.AddPointer(Ptr: C->getConstantIntValue());
754 ID.AddBoolean(B: C->isOpaque());
755 break;
756 }
757 case ISD::TargetConstantFP:
758 case ISD::ConstantFP:
759 ID.AddPointer(Ptr: cast<ConstantFPSDNode>(Val: N)->getConstantFPValue());
760 break;
761 case ISD::TargetGlobalAddress:
762 case ISD::GlobalAddress:
763 case ISD::TargetGlobalTLSAddress:
764 case ISD::GlobalTLSAddress: {
765 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val: N);
766 ID.AddPointer(Ptr: GA->getGlobal());
767 ID.AddInteger(I: GA->getOffset());
768 ID.AddInteger(I: GA->getTargetFlags());
769 break;
770 }
771 case ISD::BasicBlock:
772 ID.AddPointer(Ptr: cast<BasicBlockSDNode>(Val: N)->getBasicBlock());
773 break;
774 case ISD::Register:
775 ID.AddInteger(I: cast<RegisterSDNode>(Val: N)->getReg().id());
776 break;
777 case ISD::RegisterMask:
778 ID.AddPointer(Ptr: cast<RegisterMaskSDNode>(Val: N)->getRegMask());
779 break;
780 case ISD::SRCVALUE:
781 ID.AddPointer(Ptr: cast<SrcValueSDNode>(Val: N)->getValue());
782 break;
783 case ISD::FrameIndex:
784 case ISD::TargetFrameIndex:
785 ID.AddInteger(I: cast<FrameIndexSDNode>(Val: N)->getIndex());
786 break;
787 case ISD::LIFETIME_START:
788 case ISD::LIFETIME_END:
789 if (cast<LifetimeSDNode>(Val: N)->hasOffset()) {
790 ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getSize());
791 ID.AddInteger(I: cast<LifetimeSDNode>(Val: N)->getOffset());
792 }
793 break;
794 case ISD::PSEUDO_PROBE:
795 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getGuid());
796 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getIndex());
797 ID.AddInteger(I: cast<PseudoProbeSDNode>(Val: N)->getAttributes());
798 break;
799 case ISD::JumpTable:
800 case ISD::TargetJumpTable:
801 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getIndex());
802 ID.AddInteger(I: cast<JumpTableSDNode>(Val: N)->getTargetFlags());
803 break;
804 case ISD::ConstantPool:
805 case ISD::TargetConstantPool: {
806 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val: N);
807 ID.AddInteger(I: CP->getAlign().value());
808 ID.AddInteger(I: CP->getOffset());
809 if (CP->isMachineConstantPoolEntry())
810 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
811 else
812 ID.AddPointer(Ptr: CP->getConstVal());
813 ID.AddInteger(I: CP->getTargetFlags());
814 break;
815 }
816 case ISD::TargetIndex: {
817 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(Val: N);
818 ID.AddInteger(I: TI->getIndex());
819 ID.AddInteger(I: TI->getOffset());
820 ID.AddInteger(I: TI->getTargetFlags());
821 break;
822 }
823 case ISD::LOAD: {
824 const LoadSDNode *LD = cast<LoadSDNode>(Val: N);
825 ID.AddInteger(I: LD->getMemoryVT().getRawBits());
826 ID.AddInteger(I: LD->getRawSubclassData());
827 ID.AddInteger(I: LD->getPointerInfo().getAddrSpace());
828 ID.AddInteger(I: LD->getMemOperand()->getFlags());
829 break;
830 }
831 case ISD::STORE: {
832 const StoreSDNode *ST = cast<StoreSDNode>(Val: N);
833 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
834 ID.AddInteger(I: ST->getRawSubclassData());
835 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
836 ID.AddInteger(I: ST->getMemOperand()->getFlags());
837 break;
838 }
839 case ISD::VP_LOAD: {
840 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(Val: N);
841 ID.AddInteger(I: ELD->getMemoryVT().getRawBits());
842 ID.AddInteger(I: ELD->getRawSubclassData());
843 ID.AddInteger(I: ELD->getPointerInfo().getAddrSpace());
844 ID.AddInteger(I: ELD->getMemOperand()->getFlags());
845 break;
846 }
847 case ISD::VP_STORE: {
848 const VPStoreSDNode *EST = cast<VPStoreSDNode>(Val: N);
849 ID.AddInteger(I: EST->getMemoryVT().getRawBits());
850 ID.AddInteger(I: EST->getRawSubclassData());
851 ID.AddInteger(I: EST->getPointerInfo().getAddrSpace());
852 ID.AddInteger(I: EST->getMemOperand()->getFlags());
853 break;
854 }
855 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
856 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(Val: N);
857 ID.AddInteger(I: SLD->getMemoryVT().getRawBits());
858 ID.AddInteger(I: SLD->getRawSubclassData());
859 ID.AddInteger(I: SLD->getPointerInfo().getAddrSpace());
860 break;
861 }
862 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
863 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(Val: N);
864 ID.AddInteger(I: SST->getMemoryVT().getRawBits());
865 ID.AddInteger(I: SST->getRawSubclassData());
866 ID.AddInteger(I: SST->getPointerInfo().getAddrSpace());
867 break;
868 }
869 case ISD::VP_GATHER: {
870 const VPGatherSDNode *EG = cast<VPGatherSDNode>(Val: N);
871 ID.AddInteger(I: EG->getMemoryVT().getRawBits());
872 ID.AddInteger(I: EG->getRawSubclassData());
873 ID.AddInteger(I: EG->getPointerInfo().getAddrSpace());
874 ID.AddInteger(I: EG->getMemOperand()->getFlags());
875 break;
876 }
877 case ISD::VP_SCATTER: {
878 const VPScatterSDNode *ES = cast<VPScatterSDNode>(Val: N);
879 ID.AddInteger(I: ES->getMemoryVT().getRawBits());
880 ID.AddInteger(I: ES->getRawSubclassData());
881 ID.AddInteger(I: ES->getPointerInfo().getAddrSpace());
882 ID.AddInteger(I: ES->getMemOperand()->getFlags());
883 break;
884 }
885 case ISD::MLOAD: {
886 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
887 ID.AddInteger(I: MLD->getMemoryVT().getRawBits());
888 ID.AddInteger(I: MLD->getRawSubclassData());
889 ID.AddInteger(I: MLD->getPointerInfo().getAddrSpace());
890 ID.AddInteger(I: MLD->getMemOperand()->getFlags());
891 break;
892 }
893 case ISD::MSTORE: {
894 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
895 ID.AddInteger(I: MST->getMemoryVT().getRawBits());
896 ID.AddInteger(I: MST->getRawSubclassData());
897 ID.AddInteger(I: MST->getPointerInfo().getAddrSpace());
898 ID.AddInteger(I: MST->getMemOperand()->getFlags());
899 break;
900 }
901 case ISD::MGATHER: {
902 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(Val: N);
903 ID.AddInteger(I: MG->getMemoryVT().getRawBits());
904 ID.AddInteger(I: MG->getRawSubclassData());
905 ID.AddInteger(I: MG->getPointerInfo().getAddrSpace());
906 ID.AddInteger(I: MG->getMemOperand()->getFlags());
907 break;
908 }
909 case ISD::MSCATTER: {
910 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(Val: N);
911 ID.AddInteger(I: MS->getMemoryVT().getRawBits());
912 ID.AddInteger(I: MS->getRawSubclassData());
913 ID.AddInteger(I: MS->getPointerInfo().getAddrSpace());
914 ID.AddInteger(I: MS->getMemOperand()->getFlags());
915 break;
916 }
917 case ISD::ATOMIC_CMP_SWAP:
918 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
919 case ISD::ATOMIC_SWAP:
920 case ISD::ATOMIC_LOAD_ADD:
921 case ISD::ATOMIC_LOAD_SUB:
922 case ISD::ATOMIC_LOAD_AND:
923 case ISD::ATOMIC_LOAD_CLR:
924 case ISD::ATOMIC_LOAD_OR:
925 case ISD::ATOMIC_LOAD_XOR:
926 case ISD::ATOMIC_LOAD_NAND:
927 case ISD::ATOMIC_LOAD_MIN:
928 case ISD::ATOMIC_LOAD_MAX:
929 case ISD::ATOMIC_LOAD_UMIN:
930 case ISD::ATOMIC_LOAD_UMAX:
931 case ISD::ATOMIC_LOAD:
932 case ISD::ATOMIC_STORE: {
933 const AtomicSDNode *AT = cast<AtomicSDNode>(Val: N);
934 ID.AddInteger(I: AT->getMemoryVT().getRawBits());
935 ID.AddInteger(I: AT->getRawSubclassData());
936 ID.AddInteger(I: AT->getPointerInfo().getAddrSpace());
937 ID.AddInteger(I: AT->getMemOperand()->getFlags());
938 break;
939 }
940 case ISD::VECTOR_SHUFFLE: {
941 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: N)->getMask();
942 for (int M : Mask)
943 ID.AddInteger(I: M);
944 break;
945 }
946 case ISD::ADDRSPACECAST: {
947 const AddrSpaceCastSDNode *ASC = cast<AddrSpaceCastSDNode>(Val: N);
948 ID.AddInteger(I: ASC->getSrcAddressSpace());
949 ID.AddInteger(I: ASC->getDestAddressSpace());
950 break;
951 }
952 case ISD::TargetBlockAddress:
953 case ISD::BlockAddress: {
954 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Val: N);
955 ID.AddPointer(Ptr: BA->getBlockAddress());
956 ID.AddInteger(I: BA->getOffset());
957 ID.AddInteger(I: BA->getTargetFlags());
958 break;
959 }
960 case ISD::AssertAlign:
961 ID.AddInteger(I: cast<AssertAlignSDNode>(Val: N)->getAlign().value());
962 break;
963 case ISD::PREFETCH:
964 case ISD::INTRINSIC_VOID:
965 case ISD::INTRINSIC_W_CHAIN:
966 // Handled by MemIntrinsicSDNode check after the switch.
967 break;
968 case ISD::MDNODE_SDNODE:
969 ID.AddPointer(Ptr: cast<MDNodeSDNode>(Val: N)->getMD());
970 break;
971 } // end switch (N->getOpcode())
972
973 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
974 // to check.
975 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
976 ID.AddInteger(I: MN->getRawSubclassData());
977 ID.AddInteger(I: MN->getPointerInfo().getAddrSpace());
978 ID.AddInteger(I: MN->getMemOperand()->getFlags());
979 ID.AddInteger(I: MN->getMemoryVT().getRawBits());
980 }
981}
982
983/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
984/// data.
985static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
986 AddNodeIDOpcode(ID, OpC: N->getOpcode());
987 // Add the return value info.
988 AddNodeIDValueTypes(ID, VTList: N->getVTList());
989 // Add the operand info.
990 AddNodeIDOperands(ID, Ops: N->ops());
991
992 // Handle SDNode leafs with special info.
993 AddNodeIDCustom(ID, N);
994}
995
996//===----------------------------------------------------------------------===//
997// SelectionDAG Class
998//===----------------------------------------------------------------------===//
999
1000/// doNotCSE - Return true if CSE should not be performed for this node.
1001static bool doNotCSE(SDNode *N) {
1002 if (N->getValueType(ResNo: 0) == MVT::Glue)
1003 return true; // Never CSE anything that produces a glue result.
1004
1005 switch (N->getOpcode()) {
1006 default: break;
1007 case ISD::HANDLENODE:
1008 case ISD::EH_LABEL:
1009 return true; // Never CSE these nodes.
1010 }
1011
1012 // Check that remaining values produced are not flags.
1013 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
1014 if (N->getValueType(ResNo: i) == MVT::Glue)
1015 return true; // Never CSE anything that produces a glue result.
1016
1017 return false;
1018}
1019
1020/// RemoveDeadNodes - This method deletes all unreachable nodes in the
1021/// SelectionDAG.
1022void SelectionDAG::RemoveDeadNodes() {
1023 // Create a dummy node (which is not added to allnodes), that adds a reference
1024 // to the root node, preventing it from being deleted.
1025 HandleSDNode Dummy(getRoot());
1026
1027 SmallVector<SDNode*, 128> DeadNodes;
1028
1029 // Add all obviously-dead nodes to the DeadNodes worklist.
1030 for (SDNode &Node : allnodes())
1031 if (Node.use_empty())
1032 DeadNodes.push_back(Elt: &Node);
1033
1034 RemoveDeadNodes(DeadNodes);
1035
1036 // If the root changed (e.g. it was a dead load, update the root).
1037 setRoot(Dummy.getValue());
1038}
1039
1040/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1041/// given list, and any nodes that become unreachable as a result.
1042void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) {
1043
1044 // Process the worklist, deleting the nodes and adding their uses to the
1045 // worklist.
1046 while (!DeadNodes.empty()) {
1047 SDNode *N = DeadNodes.pop_back_val();
1048 // Skip to next node if we've already managed to delete the node. This could
1049 // happen if replacing a node causes a node previously added to the node to
1050 // be deleted.
1051 if (N->getOpcode() == ISD::DELETED_NODE)
1052 continue;
1053
1054 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1055 DUL->NodeDeleted(N, nullptr);
1056
1057 // Take the node out of the appropriate CSE map.
1058 RemoveNodeFromCSEMaps(N);
1059
1060 // Next, brutally remove the operand list. This is safe to do, as there are
1061 // no cycles in the graph.
1062 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1063 SDUse &Use = *I++;
1064 SDNode *Operand = Use.getNode();
1065 Use.set(SDValue());
1066
1067 // Now that we removed this operand, see if there are no uses of it left.
1068 if (Operand->use_empty())
1069 DeadNodes.push_back(Elt: Operand);
1070 }
1071
1072 DeallocateNode(N);
1073 }
1074}
1075
1076void SelectionDAG::RemoveDeadNode(SDNode *N){
1077 SmallVector<SDNode*, 16> DeadNodes(1, N);
1078
1079 // Create a dummy node that adds a reference to the root node, preventing
1080 // it from being deleted. (This matters if the root is an operand of the
1081 // dead node.)
1082 HandleSDNode Dummy(getRoot());
1083
1084 RemoveDeadNodes(DeadNodes);
1085}
1086
1087void SelectionDAG::DeleteNode(SDNode *N) {
1088 // First take this out of the appropriate CSE map.
1089 RemoveNodeFromCSEMaps(N);
1090
1091 // Finally, remove uses due to operands of this node, remove from the
1092 // AllNodes list, and delete the node.
1093 DeleteNodeNotInCSEMaps(N);
1094}
1095
1096void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1097 assert(N->getIterator() != AllNodes.begin() &&
1098 "Cannot delete the entry node!");
1099 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1100
1101 // Drop all of the operands and decrement used node's use counts.
1102 N->DropOperands();
1103
1104 DeallocateNode(N);
1105}
1106
1107void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1108 assert(!(V->isVariadic() && isParameter));
1109 if (isParameter)
1110 ByvalParmDbgValues.push_back(Elt: V);
1111 else
1112 DbgValues.push_back(Elt: V);
1113 for (const SDNode *Node : V->getSDNodes())
1114 if (Node)
1115 DbgValMap[Node].push_back(Elt: V);
1116}
1117
1118void SDDbgInfo::erase(const SDNode *Node) {
1119 DbgValMapType::iterator I = DbgValMap.find(Val: Node);
1120 if (I == DbgValMap.end())
1121 return;
1122 for (auto &Val: I->second)
1123 Val->setIsInvalidated();
1124 DbgValMap.erase(I);
1125}
1126
1127void SelectionDAG::DeallocateNode(SDNode *N) {
1128 // If we have operands, deallocate them.
1129 removeOperands(Node: N);
1130
1131 NodeAllocator.Deallocate(E: AllNodes.remove(IT: N));
1132
1133 // Set the opcode to DELETED_NODE to help catch bugs when node
1134 // memory is reallocated.
1135 // FIXME: There are places in SDag that have grown a dependency on the opcode
1136 // value in the released node.
1137 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1138 N->NodeType = ISD::DELETED_NODE;
1139
1140 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1141 // them and forget about that node.
1142 DbgInfo->erase(Node: N);
1143
1144 // Invalidate extra info.
1145 SDEI.erase(Val: N);
1146}
1147
1148#ifndef NDEBUG
1149/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1150void SelectionDAG::verifyNode(SDNode *N) const {
1151 switch (N->getOpcode()) {
1152 default:
1153 if (N->isTargetOpcode())
1154 getSelectionDAGInfo().verifyTargetNode(*this, N);
1155 break;
1156 case ISD::BUILD_PAIR: {
1157 EVT VT = N->getValueType(0);
1158 assert(N->getNumValues() == 1 && "Too many results!");
1159 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1160 "Wrong return type!");
1161 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1162 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1163 "Mismatched operand types!");
1164 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1165 "Wrong operand type!");
1166 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1167 "Wrong return type size");
1168 break;
1169 }
1170 case ISD::BUILD_VECTOR: {
1171 assert(N->getNumValues() == 1 && "Too many results!");
1172 assert(N->getValueType(0).isVector() && "Wrong return type!");
1173 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1174 "Wrong number of operands!");
1175 EVT EltVT = N->getValueType(0).getVectorElementType();
1176 for (const SDUse &Op : N->ops()) {
1177 assert((Op.getValueType() == EltVT ||
1178 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1179 EltVT.bitsLE(Op.getValueType()))) &&
1180 "Wrong operand type!");
1181 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1182 "Operands must all have the same type");
1183 }
1184 break;
1185 }
1186 }
1187}
1188#endif // NDEBUG
1189
1190/// Insert a newly allocated node into the DAG.
1191///
1192/// Handles insertion into the all nodes list and CSE map, as well as
1193/// verification and other common operations when a new node is allocated.
1194void SelectionDAG::InsertNode(SDNode *N) {
1195 AllNodes.push_back(val: N);
1196#ifndef NDEBUG
1197 N->PersistentId = NextPersistentId++;
1198 verifyNode(N);
1199#endif
1200 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1201 DUL->NodeInserted(N);
1202}
1203
1204/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1205/// correspond to it. This is useful when we're about to delete or repurpose
1206/// the node. We don't want future request for structurally identical nodes
1207/// to return N anymore.
1208bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1209 bool Erased = false;
1210 switch (N->getOpcode()) {
1211 case ISD::HANDLENODE: return false; // noop.
1212 case ISD::CONDCODE:
1213 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1214 "Cond code doesn't exist!");
1215 Erased = CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] != nullptr;
1216 CondCodeNodes[cast<CondCodeSDNode>(Val: N)->get()] = nullptr;
1217 break;
1218 case ISD::ExternalSymbol:
1219 Erased = ExternalSymbols.erase(Key: cast<ExternalSymbolSDNode>(Val: N)->getSymbol());
1220 break;
1221 case ISD::TargetExternalSymbol: {
1222 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(Val: N);
1223 Erased = TargetExternalSymbols.erase(x: std::pair<std::string, unsigned>(
1224 ESN->getSymbol(), ESN->getTargetFlags()));
1225 break;
1226 }
1227 case ISD::MCSymbol: {
1228 auto *MCSN = cast<MCSymbolSDNode>(Val: N);
1229 Erased = MCSymbols.erase(Val: MCSN->getMCSymbol());
1230 break;
1231 }
1232 case ISD::VALUETYPE: {
1233 EVT VT = cast<VTSDNode>(Val: N)->getVT();
1234 if (VT.isExtended()) {
1235 Erased = ExtendedValueTypeNodes.erase(x: VT);
1236 } else {
1237 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1238 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1239 }
1240 break;
1241 }
1242 default:
1243 // Remove it from the CSE Map.
1244 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1245 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1246 Erased = CSEMap.RemoveNode(N);
1247 break;
1248 }
1249#ifndef NDEBUG
1250 // Verify that the node was actually in one of the CSE maps, unless it has a
1251 // glue result (which cannot be CSE'd) or is one of the special cases that are
1252 // not subject to CSE.
1253 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1254 !N->isMachineOpcode() && !doNotCSE(N)) {
1255 N->dump(this);
1256 dbgs() << "\n";
1257 llvm_unreachable("Node is not in map!");
1258 }
1259#endif
1260 return Erased;
1261}
1262
1263/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1264/// maps and modified in place. Add it back to the CSE maps, unless an identical
1265/// node already exists, in which case transfer all its users to the existing
1266/// node. This transfer can potentially trigger recursive merging.
1267void
1268SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1269 // For node types that aren't CSE'd, just act as if no identical node
1270 // already exists.
1271 if (!doNotCSE(N)) {
1272 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1273 if (Existing != N) {
1274 // If there was already an existing matching node, use ReplaceAllUsesWith
1275 // to replace the dead one with the existing one. This can cause
1276 // recursive merging of other unrelated nodes down the line.
1277 Existing->intersectFlagsWith(Flags: N->getFlags());
1278 if (auto *MemNode = dyn_cast<MemSDNode>(Val: Existing))
1279 MemNode->refineRanges(NewMMO: cast<MemSDNode>(Val: N)->getMemOperand());
1280 ReplaceAllUsesWith(From: N, To: Existing);
1281
1282 // N is now dead. Inform the listeners and delete it.
1283 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1284 DUL->NodeDeleted(N, Existing);
1285 DeleteNodeNotInCSEMaps(N);
1286 return;
1287 }
1288 }
1289
1290 // If the node doesn't already exist, we updated it. Inform listeners.
1291 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1292 DUL->NodeUpdated(N);
1293}
1294
1295/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1296/// were replaced with those specified. If this node is never memoized,
1297/// return null, otherwise return a pointer to the slot it would take. If a
1298/// node already exists with these operands, the slot will be non-null.
1299SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1300 void *&InsertPos) {
1301 if (doNotCSE(N))
1302 return nullptr;
1303
1304 SDValue Ops[] = { Op };
1305 FoldingSetNodeID ID;
1306 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1307 AddNodeIDCustom(ID, N);
1308 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1309 if (Node)
1310 Node->intersectFlagsWith(Flags: N->getFlags());
1311 return Node;
1312}
1313
1314/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1315/// were replaced with those specified. If this node is never memoized,
1316/// return null, otherwise return a pointer to the slot it would take. If a
1317/// node already exists with these operands, the slot will be non-null.
1318SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1319 SDValue Op1, SDValue Op2,
1320 void *&InsertPos) {
1321 if (doNotCSE(N))
1322 return nullptr;
1323
1324 SDValue Ops[] = { Op1, Op2 };
1325 FoldingSetNodeID ID;
1326 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1327 AddNodeIDCustom(ID, N);
1328 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1329 if (Node)
1330 Node->intersectFlagsWith(Flags: N->getFlags());
1331 return Node;
1332}
1333
1334/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1335/// were replaced with those specified. If this node is never memoized,
1336/// return null, otherwise return a pointer to the slot it would take. If a
1337/// node already exists with these operands, the slot will be non-null.
1338SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1339 void *&InsertPos) {
1340 if (doNotCSE(N))
1341 return nullptr;
1342
1343 FoldingSetNodeID ID;
1344 AddNodeIDNode(ID, OpC: N->getOpcode(), VTList: N->getVTList(), OpList: Ops);
1345 AddNodeIDCustom(ID, N);
1346 SDNode *Node = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos);
1347 if (Node)
1348 Node->intersectFlagsWith(Flags: N->getFlags());
1349 return Node;
1350}
1351
1352Align SelectionDAG::getEVTAlign(EVT VT) const {
1353 Type *Ty = VT == MVT::iPTR ? PointerType::get(C&: *getContext(), AddressSpace: 0)
1354 : VT.getTypeForEVT(Context&: *getContext());
1355
1356 return getDataLayout().getABITypeAlign(Ty);
1357}
1358
1359// EntryNode could meaningfully have debug info if we can find it...
1360SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOptLevel OL)
1361 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1362 getVTList(VT1: MVT::Other, VT2: MVT::Glue)),
1363 Root(getEntryNode()) {
1364 InsertNode(N: &EntryNode);
1365 DbgInfo = new SDDbgInfo();
1366}
1367
1368void SelectionDAG::init(MachineFunction &NewMF,
1369 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1370 const TargetLibraryInfo *LibraryInfo,
1371 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1372 BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
1373 FunctionVarLocs const *VarLocs) {
1374 MF = &NewMF;
1375 SDAGISelPass = PassPtr;
1376 ORE = &NewORE;
1377 TLI = getSubtarget().getTargetLowering();
1378 TSI = getSubtarget().getSelectionDAGInfo();
1379 LibInfo = LibraryInfo;
1380 Context = &MF->getFunction().getContext();
1381 UA = NewUA;
1382 PSI = PSIin;
1383 BFI = BFIin;
1384 MMI = &MMIin;
1385 FnVarLocs = VarLocs;
1386}
1387
1388SelectionDAG::~SelectionDAG() {
1389 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1390 allnodes_clear();
1391 OperandRecycler.clear(OperandAllocator);
1392 delete DbgInfo;
1393}
1394
1395bool SelectionDAG::shouldOptForSize() const {
1396 return llvm::shouldOptimizeForSize(BB: FLI->MBB->getBasicBlock(), PSI, BFI);
1397}
1398
1399void SelectionDAG::allnodes_clear() {
1400 assert(&*AllNodes.begin() == &EntryNode);
1401 AllNodes.remove(IT: AllNodes.begin());
1402 while (!AllNodes.empty())
1403 DeallocateNode(N: &AllNodes.front());
1404#ifndef NDEBUG
1405 NextPersistentId = 0;
1406#endif
1407}
1408
1409SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1410 void *&InsertPos) {
1411 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1412 if (N) {
1413 switch (N->getOpcode()) {
1414 default: break;
1415 case ISD::Constant:
1416 case ISD::ConstantFP:
1417 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1418 "debug location. Use another overload.");
1419 }
1420 }
1421 return N;
1422}
1423
1424SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1425 const SDLoc &DL, void *&InsertPos) {
1426 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1427 if (N) {
1428 switch (N->getOpcode()) {
1429 case ISD::Constant:
1430 case ISD::ConstantFP:
1431 // Erase debug location from the node if the node is used at several
1432 // different places. Do not propagate one location to all uses as it
1433 // will cause a worse single stepping debugging experience.
1434 if (N->getDebugLoc() != DL.getDebugLoc())
1435 N->setDebugLoc(DebugLoc());
1436 break;
1437 default:
1438 // When the node's point of use is located earlier in the instruction
1439 // sequence than its prior point of use, update its debug info to the
1440 // earlier location.
1441 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1442 N->setDebugLoc(DL.getDebugLoc());
1443 break;
1444 }
1445 }
1446 return N;
1447}
1448
1449void SelectionDAG::clear() {
1450 allnodes_clear();
1451 OperandRecycler.clear(OperandAllocator);
1452 OperandAllocator.Reset();
1453 CSEMap.clear();
1454
1455 ExtendedValueTypeNodes.clear();
1456 ExternalSymbols.clear();
1457 TargetExternalSymbols.clear();
1458 MCSymbols.clear();
1459 SDEI.clear();
1460 llvm::fill(Range&: CondCodeNodes, Value: nullptr);
1461 llvm::fill(Range&: ValueTypeNodes, Value: nullptr);
1462
1463 EntryNode.UseList = nullptr;
1464 InsertNode(N: &EntryNode);
1465 Root = getEntryNode();
1466 DbgInfo->clear();
1467}
1468
1469SDValue SelectionDAG::getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT) {
1470 return VT.bitsGT(VT: Op.getValueType())
1471 ? getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Op)
1472 : getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: Op,
1473 N2: getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
1474}
1475
1476std::pair<SDValue, SDValue>
1477SelectionDAG::getStrictFPExtendOrRound(SDValue Op, SDValue Chain,
1478 const SDLoc &DL, EVT VT) {
1479 assert(!VT.bitsEq(Op.getValueType()) &&
1480 "Strict no-op FP extend/round not allowed.");
1481 SDValue Res =
1482 VT.bitsGT(VT: Op.getValueType())
1483 ? getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {VT, MVT::Other}, Ops: {Chain, Op})
1484 : getNode(Opcode: ISD::STRICT_FP_ROUND, DL, ResultTys: {VT, MVT::Other},
1485 Ops: {Chain, Op, getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
1486
1487 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1488}
1489
1490SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1491 return VT.bitsGT(VT: Op.getValueType()) ?
1492 getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Op) :
1493 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1494}
1495
1496SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1497 return VT.bitsGT(VT: Op.getValueType()) ?
1498 getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op) :
1499 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1500}
1501
1502SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1503 return VT.bitsGT(VT: Op.getValueType()) ?
1504 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Op) :
1505 getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op);
1506}
1507
1508SDValue SelectionDAG::getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL,
1509 EVT VT) {
1510 assert(!VT.isVector());
1511 auto Type = Op.getValueType();
1512 SDValue DestOp;
1513 if (Type == VT)
1514 return Op;
1515 auto Size = Op.getValueSizeInBits();
1516 DestOp = getBitcast(VT: EVT::getIntegerVT(Context&: *Context, BitWidth: Size), V: Op);
1517 if (DestOp.getValueType() == VT)
1518 return DestOp;
1519
1520 return getAnyExtOrTrunc(Op: DestOp, DL, VT);
1521}
1522
1523SDValue SelectionDAG::getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL,
1524 EVT VT) {
1525 assert(!VT.isVector());
1526 auto Type = Op.getValueType();
1527 SDValue DestOp;
1528 if (Type == VT)
1529 return Op;
1530 auto Size = Op.getValueSizeInBits();
1531 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1532 if (DestOp.getValueType() == VT)
1533 return DestOp;
1534
1535 return getSExtOrTrunc(Op: DestOp, DL, VT);
1536}
1537
1538SDValue SelectionDAG::getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL,
1539 EVT VT) {
1540 assert(!VT.isVector());
1541 auto Type = Op.getValueType();
1542 SDValue DestOp;
1543 if (Type == VT)
1544 return Op;
1545 auto Size = Op.getValueSizeInBits();
1546 DestOp = getBitcast(VT: MVT::getIntegerVT(BitWidth: Size), V: Op);
1547 if (DestOp.getValueType() == VT)
1548 return DestOp;
1549
1550 return getZExtOrTrunc(Op: DestOp, DL, VT);
1551}
1552
1553SDValue SelectionDAG::getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT,
1554 EVT OpVT) {
1555 if (VT.bitsLE(VT: Op.getValueType()))
1556 return getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: Op);
1557
1558 TargetLowering::BooleanContent BType = TLI->getBooleanContents(Type: OpVT);
1559 return getNode(Opcode: TLI->getExtendForContent(Content: BType), DL: SL, VT, Operand: Op);
1560}
1561
1562SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1563 EVT OpVT = Op.getValueType();
1564 assert(VT.isInteger() && OpVT.isInteger() &&
1565 "Cannot getZeroExtendInReg FP types");
1566 assert(VT.isVector() == OpVT.isVector() &&
1567 "getZeroExtendInReg type should be vector iff the operand "
1568 "type is vector!");
1569 assert((!VT.isVector() ||
1570 VT.getVectorElementCount() == OpVT.getVectorElementCount()) &&
1571 "Vector element counts must match in getZeroExtendInReg");
1572 assert(VT.bitsLE(OpVT) && "Not extending!");
1573 if (OpVT == VT)
1574 return Op;
1575 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1576 loBitsSet: VT.getScalarSizeInBits());
1577 return getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT));
1578}
1579
1580SDValue SelectionDAG::getVPZeroExtendInReg(SDValue Op, SDValue Mask,
1581 SDValue EVL, const SDLoc &DL,
1582 EVT VT) {
1583 EVT OpVT = Op.getValueType();
1584 assert(VT.isInteger() && OpVT.isInteger() &&
1585 "Cannot getVPZeroExtendInReg FP types");
1586 assert(VT.isVector() && OpVT.isVector() &&
1587 "getVPZeroExtendInReg type and operand type should be vector!");
1588 assert(VT.getVectorElementCount() == OpVT.getVectorElementCount() &&
1589 "Vector element counts must match in getZeroExtendInReg");
1590 assert(VT.bitsLE(OpVT) && "Not extending!");
1591 if (OpVT == VT)
1592 return Op;
1593 APInt Imm = APInt::getLowBitsSet(numBits: OpVT.getScalarSizeInBits(),
1594 loBitsSet: VT.getScalarSizeInBits());
1595 return getNode(Opcode: ISD::VP_AND, DL, VT: OpVT, N1: Op, N2: getConstant(Val: Imm, DL, VT: OpVT), N3: Mask,
1596 N4: EVL);
1597}
1598
1599SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) {
1600 // Only unsigned pointer semantics are supported right now. In the future this
1601 // might delegate to TLI to check pointer signedness.
1602 return getZExtOrTrunc(Op, DL, VT);
1603}
1604
1605SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) {
1606 // Only unsigned pointer semantics are supported right now. In the future this
1607 // might delegate to TLI to check pointer signedness.
1608 return getZeroExtendInReg(Op, DL, VT);
1609}
1610
1611SDValue SelectionDAG::getNegative(SDValue Val, const SDLoc &DL, EVT VT) {
1612 return getNode(Opcode: ISD::SUB, DL, VT, N1: getConstant(Val: 0, DL, VT), N2: Val);
1613}
1614
1615/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1616SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1617 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: getAllOnesConstant(DL, VT));
1618}
1619
1620SDValue SelectionDAG::getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT) {
1621 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1622 return getNode(Opcode: ISD::XOR, DL, VT, N1: Val, N2: TrueValue);
1623}
1624
1625SDValue SelectionDAG::getVPLogicalNOT(const SDLoc &DL, SDValue Val,
1626 SDValue Mask, SDValue EVL, EVT VT) {
1627 SDValue TrueValue = getBoolConstant(V: true, DL, VT, OpVT: VT);
1628 return getNode(Opcode: ISD::VP_XOR, DL, VT, N1: Val, N2: TrueValue, N3: Mask, N4: EVL);
1629}
1630
1631SDValue SelectionDAG::getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1632 SDValue Mask, SDValue EVL) {
1633 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1634}
1635
1636SDValue SelectionDAG::getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op,
1637 SDValue Mask, SDValue EVL) {
1638 if (VT.bitsGT(VT: Op.getValueType()))
1639 return getNode(Opcode: ISD::VP_ZERO_EXTEND, DL, VT, N1: Op, N2: Mask, N3: EVL);
1640 if (VT.bitsLT(VT: Op.getValueType()))
1641 return getNode(Opcode: ISD::VP_TRUNCATE, DL, VT, N1: Op, N2: Mask, N3: EVL);
1642 return Op;
1643}
1644
1645SDValue SelectionDAG::getBoolConstant(bool V, const SDLoc &DL, EVT VT,
1646 EVT OpVT) {
1647 if (!V)
1648 return getConstant(Val: 0, DL, VT);
1649
1650 switch (TLI->getBooleanContents(Type: OpVT)) {
1651 case TargetLowering::ZeroOrOneBooleanContent:
1652 case TargetLowering::UndefinedBooleanContent:
1653 return getConstant(Val: 1, DL, VT);
1654 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1655 return getAllOnesConstant(DL, VT);
1656 }
1657 llvm_unreachable("Unexpected boolean content enum!");
1658}
1659
1660SDValue SelectionDAG::getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
1661 bool isT, bool isO) {
1662 return getConstant(Val: APInt(VT.getScalarSizeInBits(), Val, /*isSigned=*/false),
1663 DL, VT, isTarget: isT, isOpaque: isO);
1664}
1665
1666SDValue SelectionDAG::getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
1667 bool isT, bool isO) {
1668 return getConstant(Val: *ConstantInt::get(Context&: *Context, V: Val), DL, VT, isTarget: isT, isOpaque: isO);
1669}
1670
1671SDValue SelectionDAG::getConstant(const ConstantInt &Val, const SDLoc &DL,
1672 EVT VT, bool isT, bool isO) {
1673 assert(VT.isInteger() && "Cannot create FP integer constant!");
1674
1675 EVT EltVT = VT.getScalarType();
1676 const ConstantInt *Elt = &Val;
1677
1678 // Vector splats are explicit within the DAG, with ConstantSDNode holding the
1679 // to-be-splatted scalar ConstantInt.
1680 if (isa<VectorType>(Val: Elt->getType()))
1681 Elt = ConstantInt::get(Context&: *getContext(), V: Elt->getValue());
1682
1683 // In some cases the vector type is legal but the element type is illegal and
1684 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1685 // inserted value (the type does not need to match the vector element type).
1686 // Any extra bits introduced will be truncated away.
1687 if (VT.isVector() && TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1688 TargetLowering::TypePromoteInteger) {
1689 EltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1690 APInt NewVal;
1691 if (TLI->isSExtCheaperThanZExt(FromTy: VT.getScalarType(), ToTy: EltVT))
1692 NewVal = Elt->getValue().sextOrTrunc(width: EltVT.getSizeInBits());
1693 else
1694 NewVal = Elt->getValue().zextOrTrunc(width: EltVT.getSizeInBits());
1695 Elt = ConstantInt::get(Context&: *getContext(), V: NewVal);
1696 }
1697 // In other cases the element type is illegal and needs to be expanded, for
1698 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1699 // the value into n parts and use a vector type with n-times the elements.
1700 // Then bitcast to the type requested.
1701 // Legalizing constants too early makes the DAGCombiner's job harder so we
1702 // only legalize if the DAG tells us we must produce legal types.
1703 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1704 TLI->getTypeAction(Context&: *getContext(), VT: EltVT) ==
1705 TargetLowering::TypeExpandInteger) {
1706 const APInt &NewVal = Elt->getValue();
1707 EVT ViaEltVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: EltVT);
1708 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1709
1710 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1711 if (VT.isScalableVector() ||
1712 TLI->isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)) {
1713 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1714 "Can only handle an even split!");
1715 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1716
1717 SmallVector<SDValue, 2> ScalarParts;
1718 for (unsigned i = 0; i != Parts; ++i)
1719 ScalarParts.push_back(Elt: getConstant(
1720 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1721 VT: ViaEltVT, isT, isO));
1722
1723 return getNode(Opcode: ISD::SPLAT_VECTOR_PARTS, DL, VT, Ops: ScalarParts);
1724 }
1725
1726 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1727 EVT ViaVecVT = EVT::getVectorVT(Context&: *getContext(), VT: ViaEltVT, NumElements: ViaVecNumElts);
1728
1729 // Check the temporary vector is the correct size. If this fails then
1730 // getTypeToTransformTo() probably returned a type whose size (in bits)
1731 // isn't a power-of-2 factor of the requested type size.
1732 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1733
1734 SmallVector<SDValue, 2> EltParts;
1735 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1736 EltParts.push_back(Elt: getConstant(
1737 Val: NewVal.extractBits(numBits: ViaEltSizeInBits, bitPosition: i * ViaEltSizeInBits), DL,
1738 VT: ViaEltVT, isT, isO));
1739
1740 // EltParts is currently in little endian order. If we actually want
1741 // big-endian order then reverse it now.
1742 if (getDataLayout().isBigEndian())
1743 std::reverse(first: EltParts.begin(), last: EltParts.end());
1744
1745 // The elements must be reversed when the element order is different
1746 // to the endianness of the elements (because the BITCAST is itself a
1747 // vector shuffle in this situation). However, we do not need any code to
1748 // perform this reversal because getConstant() is producing a vector
1749 // splat.
1750 // This situation occurs in MIPS MSA.
1751
1752 SmallVector<SDValue, 8> Ops;
1753 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1754 llvm::append_range(C&: Ops, R&: EltParts);
1755
1756 SDValue V =
1757 getNode(Opcode: ISD::BITCAST, DL, VT, Operand: getBuildVector(VT: ViaVecVT, DL, Ops));
1758 return V;
1759 }
1760
1761 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1762 "APInt size does not match type size!");
1763 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1764 SDVTList VTs = getVTList(VT: EltVT);
1765 FoldingSetNodeID ID;
1766 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1767 ID.AddPointer(Ptr: Elt);
1768 ID.AddBoolean(B: isO);
1769 void *IP = nullptr;
1770 SDNode *N = nullptr;
1771 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1772 if (!VT.isVector())
1773 return SDValue(N, 0);
1774
1775 if (!N) {
1776 N = newSDNode<ConstantSDNode>(Args&: isT, Args&: isO, Args&: Elt, Args&: VTs);
1777 CSEMap.InsertNode(N, InsertPos: IP);
1778 InsertNode(N);
1779 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating constant: ", G: this);
1780 }
1781
1782 SDValue Result(N, 0);
1783 if (VT.isVector())
1784 Result = getSplat(VT, DL, Op: Result);
1785 return Result;
1786}
1787
1788SDValue SelectionDAG::getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT,
1789 bool isT, bool isO) {
1790 unsigned Size = VT.getScalarSizeInBits();
1791 return getConstant(Val: APInt(Size, Val, /*isSigned=*/true), DL, VT, isT, isO);
1792}
1793
1794SDValue SelectionDAG::getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget,
1795 bool IsOpaque) {
1796 return getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT,
1797 isT: IsTarget, isO: IsOpaque);
1798}
1799
1800SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL,
1801 bool isTarget) {
1802 return getConstant(Val, DL, VT: TLI->getPointerTy(DL: getDataLayout()), isT: isTarget);
1803}
1804
1805SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT,
1806 const SDLoc &DL) {
1807 assert(VT.isInteger() && "Shift amount is not an integer type!");
1808 EVT ShiftVT = TLI->getShiftAmountTy(LHSTy: VT, DL: getDataLayout());
1809 return getConstant(Val, DL, VT: ShiftVT);
1810}
1811
1812SDValue SelectionDAG::getShiftAmountConstant(const APInt &Val, EVT VT,
1813 const SDLoc &DL) {
1814 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1815 return getShiftAmountConstant(Val: Val.getZExtValue(), VT, DL);
1816}
1817
1818SDValue SelectionDAG::getVectorIdxConstant(uint64_t Val, const SDLoc &DL,
1819 bool isTarget) {
1820 return getConstant(Val, DL, VT: TLI->getVectorIdxTy(DL: getDataLayout()), isT: isTarget);
1821}
1822
1823SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT,
1824 bool isTarget) {
1825 return getConstantFP(V: *ConstantFP::get(Context&: *getContext(), V), DL, VT, isTarget);
1826}
1827
1828SDValue SelectionDAG::getConstantFP(const ConstantFP &V, const SDLoc &DL,
1829 EVT VT, bool isTarget) {
1830 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1831
1832 EVT EltVT = VT.getScalarType();
1833 const ConstantFP *Elt = &V;
1834
1835 // Vector splats are explicit within the DAG, with ConstantFPSDNode holding
1836 // the to-be-splatted scalar ConstantFP.
1837 if (isa<VectorType>(Val: Elt->getType()))
1838 Elt = ConstantFP::get(Context&: *getContext(), V: Elt->getValue());
1839
1840 // Do the map lookup using the actual bit pattern for the floating point
1841 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1842 // we don't have issues with SNANs.
1843 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1844 SDVTList VTs = getVTList(VT: EltVT);
1845 FoldingSetNodeID ID;
1846 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1847 ID.AddPointer(Ptr: Elt);
1848 void *IP = nullptr;
1849 SDNode *N = nullptr;
1850 if ((N = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)))
1851 if (!VT.isVector())
1852 return SDValue(N, 0);
1853
1854 if (!N) {
1855 N = newSDNode<ConstantFPSDNode>(Args&: isTarget, Args&: Elt, Args&: VTs);
1856 CSEMap.InsertNode(N, InsertPos: IP);
1857 InsertNode(N);
1858 }
1859
1860 SDValue Result(N, 0);
1861 if (VT.isVector())
1862 Result = getSplat(VT, DL, Op: Result);
1863 NewSDValueDbgMsg(V: Result, Msg: "Creating fp constant: ", G: this);
1864 return Result;
1865}
1866
1867SDValue SelectionDAG::getConstantFP(double Val, const SDLoc &DL, EVT VT,
1868 bool isTarget) {
1869 EVT EltVT = VT.getScalarType();
1870 if (EltVT == MVT::f32)
1871 return getConstantFP(V: APFloat((float)Val), DL, VT, isTarget);
1872 if (EltVT == MVT::f64)
1873 return getConstantFP(V: APFloat(Val), DL, VT, isTarget);
1874 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1875 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1876 bool Ignored;
1877 APFloat APF = APFloat(Val);
1878 APF.convert(ToSemantics: EltVT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
1879 losesInfo: &Ignored);
1880 return getConstantFP(V: APF, DL, VT, isTarget);
1881 }
1882 llvm_unreachable("Unsupported type in getConstantFP");
1883}
1884
1885SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, const SDLoc &DL,
1886 EVT VT, int64_t Offset, bool isTargetGA,
1887 unsigned TargetFlags) {
1888 assert((TargetFlags == 0 || isTargetGA) &&
1889 "Cannot set target flags on target-independent globals");
1890
1891 // Truncate (with sign-extension) the offset value to the pointer size.
1892 unsigned BitWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
1893 if (BitWidth < 64)
1894 Offset = SignExtend64(X: Offset, B: BitWidth);
1895
1896 unsigned Opc;
1897 if (GV->isThreadLocal())
1898 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress;
1899 else
1900 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1901
1902 SDVTList VTs = getVTList(VT);
1903 FoldingSetNodeID ID;
1904 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1905 ID.AddPointer(Ptr: GV);
1906 ID.AddInteger(I: Offset);
1907 ID.AddInteger(I: TargetFlags);
1908 void *IP = nullptr;
1909 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
1910 return SDValue(E, 0);
1911
1912 auto *N = newSDNode<GlobalAddressSDNode>(
1913 Args&: Opc, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: GV, Args&: VTs, Args&: Offset, Args&: TargetFlags);
1914 CSEMap.InsertNode(N, InsertPos: IP);
1915 InsertNode(N);
1916 return SDValue(N, 0);
1917}
1918
1919SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1920 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1921 SDVTList VTs = getVTList(VT);
1922 FoldingSetNodeID ID;
1923 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1924 ID.AddInteger(I: FI);
1925 void *IP = nullptr;
1926 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1927 return SDValue(E, 0);
1928
1929 auto *N = newSDNode<FrameIndexSDNode>(Args&: FI, Args&: VTs, Args&: isTarget);
1930 CSEMap.InsertNode(N, InsertPos: IP);
1931 InsertNode(N);
1932 return SDValue(N, 0);
1933}
1934
1935SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1936 unsigned TargetFlags) {
1937 assert((TargetFlags == 0 || isTarget) &&
1938 "Cannot set target flags on target-independent jump tables");
1939 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1940 SDVTList VTs = getVTList(VT);
1941 FoldingSetNodeID ID;
1942 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1943 ID.AddInteger(I: JTI);
1944 ID.AddInteger(I: TargetFlags);
1945 void *IP = nullptr;
1946 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1947 return SDValue(E, 0);
1948
1949 auto *N = newSDNode<JumpTableSDNode>(Args&: JTI, Args&: VTs, Args&: isTarget, Args&: TargetFlags);
1950 CSEMap.InsertNode(N, InsertPos: IP);
1951 InsertNode(N);
1952 return SDValue(N, 0);
1953}
1954
1955SDValue SelectionDAG::getJumpTableDebugInfo(int JTI, SDValue Chain,
1956 const SDLoc &DL) {
1957 EVT PTy = getTargetLoweringInfo().getPointerTy(DL: getDataLayout());
1958 return getNode(Opcode: ISD::JUMP_TABLE_DEBUG_INFO, DL, VT: MVT::Glue, N1: Chain,
1959 N2: getTargetConstant(Val: static_cast<uint64_t>(JTI), DL, VT: PTy, isOpaque: true));
1960}
1961
1962SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
1963 MaybeAlign Alignment, int Offset,
1964 bool isTarget, unsigned TargetFlags) {
1965 assert((TargetFlags == 0 || isTarget) &&
1966 "Cannot set target flags on target-independent globals");
1967 if (!Alignment)
1968 Alignment = shouldOptForSize()
1969 ? getDataLayout().getABITypeAlign(Ty: C->getType())
1970 : getDataLayout().getPrefTypeAlign(Ty: C->getType());
1971 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1972 SDVTList VTs = getVTList(VT);
1973 FoldingSetNodeID ID;
1974 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
1975 ID.AddInteger(I: Alignment->value());
1976 ID.AddInteger(I: Offset);
1977 ID.AddPointer(Ptr: C);
1978 ID.AddInteger(I: TargetFlags);
1979 void *IP = nullptr;
1980 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
1981 return SDValue(E, 0);
1982
1983 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
1984 Args&: TargetFlags);
1985 CSEMap.InsertNode(N, InsertPos: IP);
1986 InsertNode(N);
1987 SDValue V = SDValue(N, 0);
1988 NewSDValueDbgMsg(V, Msg: "Creating new constant pool: ", G: this);
1989 return V;
1990}
1991
1992SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
1993 MaybeAlign Alignment, int Offset,
1994 bool isTarget, unsigned TargetFlags) {
1995 assert((TargetFlags == 0 || isTarget) &&
1996 "Cannot set target flags on target-independent globals");
1997 if (!Alignment)
1998 Alignment = getDataLayout().getPrefTypeAlign(Ty: C->getType());
1999 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
2000 SDVTList VTs = getVTList(VT);
2001 FoldingSetNodeID ID;
2002 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2003 ID.AddInteger(I: Alignment->value());
2004 ID.AddInteger(I: Offset);
2005 C->addSelectionDAGCSEId(ID);
2006 ID.AddInteger(I: TargetFlags);
2007 void *IP = nullptr;
2008 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2009 return SDValue(E, 0);
2010
2011 auto *N = newSDNode<ConstantPoolSDNode>(Args&: isTarget, Args&: C, Args&: VTs, Args&: Offset, Args&: *Alignment,
2012 Args&: TargetFlags);
2013 CSEMap.InsertNode(N, InsertPos: IP);
2014 InsertNode(N);
2015 return SDValue(N, 0);
2016}
2017
2018SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) {
2019 FoldingSetNodeID ID;
2020 AddNodeIDNode(ID, OpC: ISD::BasicBlock, VTList: getVTList(VT: MVT::Other), OpList: {});
2021 ID.AddPointer(Ptr: MBB);
2022 void *IP = nullptr;
2023 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2024 return SDValue(E, 0);
2025
2026 auto *N = newSDNode<BasicBlockSDNode>(Args&: MBB);
2027 CSEMap.InsertNode(N, InsertPos: IP);
2028 InsertNode(N);
2029 return SDValue(N, 0);
2030}
2031
2032SDValue SelectionDAG::getValueType(EVT VT) {
2033 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
2034 ValueTypeNodes.size())
2035 ValueTypeNodes.resize(new_size: VT.getSimpleVT().SimpleTy+1);
2036
2037 SDNode *&N = VT.isExtended() ?
2038 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
2039
2040 if (N) return SDValue(N, 0);
2041 N = newSDNode<VTSDNode>(Args&: VT);
2042 InsertNode(N);
2043 return SDValue(N, 0);
2044}
2045
2046SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) {
2047 SDNode *&N = ExternalSymbols[Sym];
2048 if (N) return SDValue(N, 0);
2049 N = newSDNode<ExternalSymbolSDNode>(Args: false, Args&: Sym, Args: 0, Args: getVTList(VT));
2050 InsertNode(N);
2051 return SDValue(N, 0);
2052}
2053
2054SDValue SelectionDAG::getMCSymbol(MCSymbol *Sym, EVT VT) {
2055 SDNode *&N = MCSymbols[Sym];
2056 if (N)
2057 return SDValue(N, 0);
2058 N = newSDNode<MCSymbolSDNode>(Args&: Sym, Args: getVTList(VT));
2059 InsertNode(N);
2060 return SDValue(N, 0);
2061}
2062
2063SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT,
2064 unsigned TargetFlags) {
2065 SDNode *&N =
2066 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
2067 if (N) return SDValue(N, 0);
2068 N = newSDNode<ExternalSymbolSDNode>(Args: true, Args&: Sym, Args&: TargetFlags, Args: getVTList(VT));
2069 InsertNode(N);
2070 return SDValue(N, 0);
2071}
2072
2073SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) {
2074 if ((unsigned)Cond >= CondCodeNodes.size())
2075 CondCodeNodes.resize(new_size: Cond+1);
2076
2077 if (!CondCodeNodes[Cond]) {
2078 auto *N = newSDNode<CondCodeSDNode>(Args&: Cond);
2079 CondCodeNodes[Cond] = N;
2080 InsertNode(N);
2081 }
2082
2083 return SDValue(CondCodeNodes[Cond], 0);
2084}
2085
2086SDValue SelectionDAG::getVScale(const SDLoc &DL, EVT VT, APInt MulImm,
2087 bool ConstantFold) {
2088 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2089 "APInt size does not match type size!");
2090
2091 if (MulImm == 0)
2092 return getConstant(Val: 0, DL, VT);
2093
2094 if (ConstantFold) {
2095 const MachineFunction &MF = getMachineFunction();
2096 const Function &F = MF.getFunction();
2097 ConstantRange CR = getVScaleRange(F: &F, BitWidth: 64);
2098 if (const APInt *C = CR.getSingleElement())
2099 return getConstant(Val: MulImm * C->getZExtValue(), DL, VT);
2100 }
2101
2102 return getNode(Opcode: ISD::VSCALE, DL, VT, Operand: getConstant(Val: MulImm, DL, VT));
2103}
2104
2105SDValue SelectionDAG::getElementCount(const SDLoc &DL, EVT VT, ElementCount EC,
2106 bool ConstantFold) {
2107 if (EC.isScalable())
2108 return getVScale(DL, VT,
2109 MulImm: APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2110
2111 return getConstant(Val: EC.getKnownMinValue(), DL, VT);
2112}
2113
2114SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT) {
2115 APInt One(ResVT.getScalarSizeInBits(), 1);
2116 return getStepVector(DL, ResVT, StepVal: One);
2117}
2118
2119SDValue SelectionDAG::getStepVector(const SDLoc &DL, EVT ResVT,
2120 const APInt &StepVal) {
2121 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2122 if (ResVT.isScalableVector())
2123 return getNode(
2124 Opcode: ISD::STEP_VECTOR, DL, VT: ResVT,
2125 Operand: getTargetConstant(Val: StepVal, DL, VT: ResVT.getVectorElementType()));
2126
2127 SmallVector<SDValue, 16> OpsStepConstants;
2128 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2129 OpsStepConstants.push_back(
2130 Elt: getConstant(Val: StepVal * i, DL, VT: ResVT.getVectorElementType()));
2131 return getBuildVector(VT: ResVT, DL, Ops: OpsStepConstants);
2132}
2133
2134/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2135/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2136static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef<int> M) {
2137 std::swap(a&: N1, b&: N2);
2138 ShuffleVectorSDNode::commuteMask(Mask: M);
2139}
2140
2141SDValue SelectionDAG::getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1,
2142 SDValue N2, ArrayRef<int> Mask) {
2143 assert(VT.getVectorNumElements() == Mask.size() &&
2144 "Must have the same number of vector elements as mask elements!");
2145 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2146 "Invalid VECTOR_SHUFFLE");
2147
2148 // Canonicalize shuffle undef, undef -> undef
2149 if (N1.isUndef() && N2.isUndef())
2150 return getUNDEF(VT);
2151
2152 // Validate that all indices in Mask are within the range of the elements
2153 // input to the shuffle.
2154 int NElts = Mask.size();
2155 assert(llvm::all_of(Mask,
2156 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2157 "Index out of range");
2158
2159 // Copy the mask so we can do any needed cleanup.
2160 SmallVector<int, 8> MaskVec(Mask);
2161
2162 // Canonicalize shuffle v, v -> v, undef
2163 if (N1 == N2) {
2164 N2 = getUNDEF(VT);
2165 for (int i = 0; i != NElts; ++i)
2166 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2167 }
2168
2169 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2170 if (N1.isUndef())
2171 commuteShuffle(N1, N2, M: MaskVec);
2172
2173 if (TLI->hasVectorBlend()) {
2174 // If shuffling a splat, try to blend the splat instead. We do this here so
2175 // that even when this arises during lowering we don't have to re-handle it.
2176 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2177 BitVector UndefElements;
2178 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2179 if (!Splat)
2180 return;
2181
2182 for (int i = 0; i < NElts; ++i) {
2183 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2184 continue;
2185
2186 // If this input comes from undef, mark it as such.
2187 if (UndefElements[MaskVec[i] - Offset]) {
2188 MaskVec[i] = -1;
2189 continue;
2190 }
2191
2192 // If we can blend a non-undef lane, use that instead.
2193 if (!UndefElements[i])
2194 MaskVec[i] = i + Offset;
2195 }
2196 };
2197 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(Val&: N1))
2198 BlendSplat(N1BV, 0);
2199 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(Val&: N2))
2200 BlendSplat(N2BV, NElts);
2201 }
2202
2203 // Canonicalize all index into lhs, -> shuffle lhs, undef
2204 // Canonicalize all index into rhs, -> shuffle rhs, undef
2205 bool AllLHS = true, AllRHS = true;
2206 bool N2Undef = N2.isUndef();
2207 for (int i = 0; i != NElts; ++i) {
2208 if (MaskVec[i] >= NElts) {
2209 if (N2Undef)
2210 MaskVec[i] = -1;
2211 else
2212 AllLHS = false;
2213 } else if (MaskVec[i] >= 0) {
2214 AllRHS = false;
2215 }
2216 }
2217 if (AllLHS && AllRHS)
2218 return getUNDEF(VT);
2219 if (AllLHS && !N2Undef)
2220 N2 = getUNDEF(VT);
2221 if (AllRHS) {
2222 N1 = getUNDEF(VT);
2223 commuteShuffle(N1, N2, M: MaskVec);
2224 }
2225 // Reset our undef status after accounting for the mask.
2226 N2Undef = N2.isUndef();
2227 // Re-check whether both sides ended up undef.
2228 if (N1.isUndef() && N2Undef)
2229 return getUNDEF(VT);
2230
2231 // If Identity shuffle return that node.
2232 bool Identity = true, AllSame = true;
2233 for (int i = 0; i != NElts; ++i) {
2234 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2235 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2236 }
2237 if (Identity && NElts)
2238 return N1;
2239
2240 // Shuffling a constant splat doesn't change the result.
2241 if (N2Undef) {
2242 SDValue V = N1;
2243
2244 // Look through any bitcasts. We check that these don't change the number
2245 // (and size) of elements and just changes their types.
2246 while (V.getOpcode() == ISD::BITCAST)
2247 V = V->getOperand(Num: 0);
2248
2249 // A splat should always show up as a build vector node.
2250 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
2251 BitVector UndefElements;
2252 SDValue Splat = BV->getSplatValue(UndefElements: &UndefElements);
2253 // If this is a splat of an undef, shuffling it is also undef.
2254 if (Splat && Splat.isUndef())
2255 return getUNDEF(VT);
2256
2257 bool SameNumElts =
2258 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2259
2260 // We only have a splat which can skip shuffles if there is a splatted
2261 // value and no undef lanes rearranged by the shuffle.
2262 if (Splat && UndefElements.none()) {
2263 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2264 // number of elements match or the value splatted is a zero constant.
2265 if (SameNumElts || isNullConstant(V: Splat))
2266 return N1;
2267 }
2268
2269 // If the shuffle itself creates a splat, build the vector directly.
2270 if (AllSame && SameNumElts) {
2271 EVT BuildVT = BV->getValueType(ResNo: 0);
2272 const SDValue &Splatted = BV->getOperand(Num: MaskVec[0]);
2273 SDValue NewBV = getSplatBuildVector(VT: BuildVT, DL: dl, Op: Splatted);
2274
2275 // We may have jumped through bitcasts, so the type of the
2276 // BUILD_VECTOR may not match the type of the shuffle.
2277 if (BuildVT != VT)
2278 NewBV = getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: NewBV);
2279 return NewBV;
2280 }
2281 }
2282 }
2283
2284 SDVTList VTs = getVTList(VT);
2285 FoldingSetNodeID ID;
2286 SDValue Ops[2] = { N1, N2 };
2287 AddNodeIDNode(ID, OpC: ISD::VECTOR_SHUFFLE, VTList: VTs, OpList: Ops);
2288 for (int i = 0; i != NElts; ++i)
2289 ID.AddInteger(I: MaskVec[i]);
2290
2291 void* IP = nullptr;
2292 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2293 return SDValue(E, 0);
2294
2295 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2296 // SDNode doesn't have access to it. This memory will be "leaked" when
2297 // the node is deallocated, but recovered when the NodeAllocator is released.
2298 int *MaskAlloc = OperandAllocator.Allocate<int>(Num: NElts);
2299 llvm::copy(Range&: MaskVec, Out: MaskAlloc);
2300
2301 auto *N = newSDNode<ShuffleVectorSDNode>(Args&: VTs, Args: dl.getIROrder(),
2302 Args: dl.getDebugLoc(), Args&: MaskAlloc);
2303 createOperands(Node: N, Vals: Ops);
2304
2305 CSEMap.InsertNode(N, InsertPos: IP);
2306 InsertNode(N);
2307 SDValue V = SDValue(N, 0);
2308 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
2309 return V;
2310}
2311
2312SDValue SelectionDAG::getCommutedVectorShuffle(const ShuffleVectorSDNode &SV) {
2313 EVT VT = SV.getValueType(ResNo: 0);
2314 SmallVector<int, 8> MaskVec(SV.getMask());
2315 ShuffleVectorSDNode::commuteMask(Mask: MaskVec);
2316
2317 SDValue Op0 = SV.getOperand(Num: 0);
2318 SDValue Op1 = SV.getOperand(Num: 1);
2319 return getVectorShuffle(VT, dl: SDLoc(&SV), N1: Op1, N2: Op0, Mask: MaskVec);
2320}
2321
2322SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
2323 SDVTList VTs = getVTList(VT);
2324 FoldingSetNodeID ID;
2325 AddNodeIDNode(ID, OpC: ISD::Register, VTList: VTs, OpList: {});
2326 ID.AddInteger(I: Reg.id());
2327 void *IP = nullptr;
2328 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2329 return SDValue(E, 0);
2330
2331 auto *N = newSDNode<RegisterSDNode>(Args&: Reg, Args&: VTs);
2332 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2333 CSEMap.InsertNode(N, InsertPos: IP);
2334 InsertNode(N);
2335 return SDValue(N, 0);
2336}
2337
2338SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) {
2339 FoldingSetNodeID ID;
2340 AddNodeIDNode(ID, OpC: ISD::RegisterMask, VTList: getVTList(VT: MVT::Untyped), OpList: {});
2341 ID.AddPointer(Ptr: RegMask);
2342 void *IP = nullptr;
2343 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2344 return SDValue(E, 0);
2345
2346 auto *N = newSDNode<RegisterMaskSDNode>(Args&: RegMask);
2347 CSEMap.InsertNode(N, InsertPos: IP);
2348 InsertNode(N);
2349 return SDValue(N, 0);
2350}
2351
2352SDValue SelectionDAG::getEHLabel(const SDLoc &dl, SDValue Root,
2353 MCSymbol *Label) {
2354 return getLabelNode(Opcode: ISD::EH_LABEL, dl, Root, Label);
2355}
2356
2357SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2358 SDValue Root, MCSymbol *Label) {
2359 FoldingSetNodeID ID;
2360 SDValue Ops[] = { Root };
2361 AddNodeIDNode(ID, OpC: Opcode, VTList: getVTList(VT: MVT::Other), OpList: Ops);
2362 ID.AddPointer(Ptr: Label);
2363 void *IP = nullptr;
2364 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2365 return SDValue(E, 0);
2366
2367 auto *N =
2368 newSDNode<LabelSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Label);
2369 createOperands(Node: N, Vals: Ops);
2370
2371 CSEMap.InsertNode(N, InsertPos: IP);
2372 InsertNode(N);
2373 return SDValue(N, 0);
2374}
2375
2376SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
2377 int64_t Offset, bool isTarget,
2378 unsigned TargetFlags) {
2379 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2380 SDVTList VTs = getVTList(VT);
2381
2382 FoldingSetNodeID ID;
2383 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: {});
2384 ID.AddPointer(Ptr: BA);
2385 ID.AddInteger(I: Offset);
2386 ID.AddInteger(I: TargetFlags);
2387 void *IP = nullptr;
2388 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2389 return SDValue(E, 0);
2390
2391 auto *N = newSDNode<BlockAddressSDNode>(Args&: Opc, Args&: VTs, Args&: BA, Args&: Offset, Args&: TargetFlags);
2392 CSEMap.InsertNode(N, InsertPos: IP);
2393 InsertNode(N);
2394 return SDValue(N, 0);
2395}
2396
2397SDValue SelectionDAG::getSrcValue(const Value *V) {
2398 FoldingSetNodeID ID;
2399 AddNodeIDNode(ID, OpC: ISD::SRCVALUE, VTList: getVTList(VT: MVT::Other), OpList: {});
2400 ID.AddPointer(Ptr: V);
2401
2402 void *IP = nullptr;
2403 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2404 return SDValue(E, 0);
2405
2406 auto *N = newSDNode<SrcValueSDNode>(Args&: V);
2407 CSEMap.InsertNode(N, InsertPos: IP);
2408 InsertNode(N);
2409 return SDValue(N, 0);
2410}
2411
2412SDValue SelectionDAG::getMDNode(const MDNode *MD) {
2413 FoldingSetNodeID ID;
2414 AddNodeIDNode(ID, OpC: ISD::MDNODE_SDNODE, VTList: getVTList(VT: MVT::Other), OpList: {});
2415 ID.AddPointer(Ptr: MD);
2416
2417 void *IP = nullptr;
2418 if (SDNode *E = FindNodeOrInsertPos(ID, InsertPos&: IP))
2419 return SDValue(E, 0);
2420
2421 auto *N = newSDNode<MDNodeSDNode>(Args&: MD);
2422 CSEMap.InsertNode(N, InsertPos: IP);
2423 InsertNode(N);
2424 return SDValue(N, 0);
2425}
2426
2427SDValue SelectionDAG::getBitcast(EVT VT, SDValue V) {
2428 if (VT == V.getValueType())
2429 return V;
2430
2431 return getNode(Opcode: ISD::BITCAST, DL: SDLoc(V), VT, Operand: V);
2432}
2433
2434SDValue SelectionDAG::getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr,
2435 unsigned SrcAS, unsigned DestAS) {
2436 SDVTList VTs = getVTList(VT);
2437 SDValue Ops[] = {Ptr};
2438 FoldingSetNodeID ID;
2439 AddNodeIDNode(ID, OpC: ISD::ADDRSPACECAST, VTList: VTs, OpList: Ops);
2440 ID.AddInteger(I: SrcAS);
2441 ID.AddInteger(I: DestAS);
2442
2443 void *IP = nullptr;
2444 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
2445 return SDValue(E, 0);
2446
2447 auto *N = newSDNode<AddrSpaceCastSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
2448 Args&: VTs, Args&: SrcAS, Args&: DestAS);
2449 createOperands(Node: N, Vals: Ops);
2450
2451 CSEMap.InsertNode(N, InsertPos: IP);
2452 InsertNode(N);
2453 return SDValue(N, 0);
2454}
2455
2456SDValue SelectionDAG::getFreeze(SDValue V) {
2457 return getNode(Opcode: ISD::FREEZE, DL: SDLoc(V), VT: V.getValueType(), Operand: V);
2458}
2459
2460/// getShiftAmountOperand - Return the specified value casted to
2461/// the target's desired shift amount type.
2462SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) {
2463 EVT OpTy = Op.getValueType();
2464 EVT ShTy = TLI->getShiftAmountTy(LHSTy, DL: getDataLayout());
2465 if (OpTy == ShTy || OpTy.isVector()) return Op;
2466
2467 return getZExtOrTrunc(Op, DL: SDLoc(Op), VT: ShTy);
2468}
2469
2470/// Given a store node \p StoreNode, return true if it is safe to fold that node
2471/// into \p FPNode, which expands to a library call with output pointers.
2472static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
2473 SDNode *FPNode) {
2474 SmallVector<const SDNode *, 8> Worklist;
2475 SmallVector<const SDNode *, 8> DeferredNodes;
2476 SmallPtrSet<const SDNode *, 16> Visited;
2477
2478 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
2479 for (SDValue Op : StoreNode->ops())
2480 if (Op.getNode() != FPNode)
2481 Worklist.push_back(Elt: Op.getNode());
2482
2483 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
2484 while (!Worklist.empty()) {
2485 const SDNode *Node = Worklist.pop_back_val();
2486 auto [_, Inserted] = Visited.insert(Ptr: Node);
2487 if (!Inserted)
2488 continue;
2489
2490 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
2491 return false;
2492
2493 // Reached the FPNode (would result in a cycle).
2494 // OR Reached CALLSEQ_START (would result in nested call sequences).
2495 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
2496 return false;
2497
2498 if (Node->getOpcode() == ISD::CALLSEQ_END) {
2499 // Defer looking into call sequences (so we can check we're outside one).
2500 // We still need to look through these for the predecessor check.
2501 DeferredNodes.push_back(Elt: Node);
2502 continue;
2503 }
2504
2505 for (SDValue Op : Node->ops())
2506 Worklist.push_back(Elt: Op.getNode());
2507 }
2508
2509 // True if we're outside a call sequence and don't have the FPNode as a
2510 // predecessor. No cycles or nested call sequences possible.
2511 return !SDNode::hasPredecessorHelper(N: FPNode, Visited, Worklist&: DeferredNodes,
2512 MaxSteps);
2513}
2514
2515bool SelectionDAG::expandMultipleResultFPLibCall(
2516 RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl<SDValue> &Results,
2517 std::optional<unsigned> CallRetResNo) {
2518 LLVMContext &Ctx = *getContext();
2519 EVT VT = Node->getValueType(ResNo: 0);
2520 unsigned NumResults = Node->getNumValues();
2521
2522 const char *LCName = TLI->getLibcallName(Call: LC);
2523 if (!LC || !LCName)
2524 return false;
2525
2526 auto getVecDesc = [&]() -> VecDesc const * {
2527 for (bool Masked : {false, true}) {
2528 if (VecDesc const *VD = getLibInfo().getVectorMappingInfo(
2529 F: LCName, VF: VT.getVectorElementCount(), Masked)) {
2530 return VD;
2531 }
2532 }
2533 return nullptr;
2534 };
2535
2536 // For vector types, we must find a vector mapping for the libcall.
2537 VecDesc const *VD = nullptr;
2538 if (VT.isVector() && !(VD = getVecDesc()))
2539 return false;
2540
2541 // Find users of the node that store the results (and share input chains). The
2542 // destination pointers can be used instead of creating stack allocations.
2543 SDValue StoresInChain;
2544 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
2545 for (SDNode *User : Node->users()) {
2546 if (!ISD::isNormalStore(N: User))
2547 continue;
2548 auto *ST = cast<StoreSDNode>(Val: User);
2549 SDValue StoreValue = ST->getValue();
2550 unsigned ResNo = StoreValue.getResNo();
2551 // Ensure the store corresponds to an output pointer.
2552 if (CallRetResNo == ResNo)
2553 continue;
2554 // Ensure the store to the default address space and not atomic or volatile.
2555 if (!ST->isSimple() || ST->getAddressSpace() != 0)
2556 continue;
2557 // Ensure all store chains are the same (so they don't alias).
2558 if (StoresInChain && ST->getChain() != StoresInChain)
2559 continue;
2560 // Ensure the store is properly aligned.
2561 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Context&: Ctx);
2562 if (ST->getAlign() <
2563 getDataLayout().getABITypeAlign(Ty: StoreType->getScalarType()))
2564 continue;
2565 // Avoid:
2566 // 1. Creating cyclic dependencies.
2567 // 2. Expanding the node to a call within a call sequence.
2568 if (!canFoldStoreIntoLibCallOutputPointers(StoreNode: ST, FPNode: Node))
2569 continue;
2570 ResultStores[ResNo] = ST;
2571 StoresInChain = ST->getChain();
2572 }
2573
2574 TargetLowering::ArgListTy Args;
2575 auto AddArgListEntry = [&](SDValue Node, Type *Ty) {
2576 TargetLowering::ArgListEntry Entry{};
2577 Entry.Ty = Ty;
2578 Entry.Node = Node;
2579 Args.push_back(x: Entry);
2580 };
2581
2582 // Pass the arguments.
2583 for (const SDValue &Op : Node->op_values()) {
2584 EVT ArgVT = Op.getValueType();
2585 Type *ArgTy = ArgVT.getTypeForEVT(Context&: Ctx);
2586 AddArgListEntry(Op, ArgTy);
2587 }
2588
2589 // Pass the output pointers.
2590 SmallVector<SDValue, 2> ResultPtrs(NumResults);
2591 Type *PointerTy = PointerType::getUnqual(C&: Ctx);
2592 for (auto [ResNo, ST] : llvm::enumerate(First&: ResultStores)) {
2593 if (ResNo == CallRetResNo)
2594 continue;
2595 EVT ResVT = Node->getValueType(ResNo);
2596 SDValue ResultPtr = ST ? ST->getBasePtr() : CreateStackTemporary(VT: ResVT);
2597 ResultPtrs[ResNo] = ResultPtr;
2598 AddArgListEntry(ResultPtr, PointerTy);
2599 }
2600
2601 SDLoc DL(Node);
2602
2603 // Pass the vector mask (if required).
2604 if (VD && VD->isMasked()) {
2605 EVT MaskVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: Ctx, VT);
2606 SDValue Mask = getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT);
2607 AddArgListEntry(Mask, MaskVT.getTypeForEVT(Context&: Ctx));
2608 }
2609
2610 Type *RetType = CallRetResNo.has_value()
2611 ? Node->getValueType(ResNo: *CallRetResNo).getTypeForEVT(Context&: Ctx)
2612 : Type::getVoidTy(C&: Ctx);
2613 SDValue InChain = StoresInChain ? StoresInChain : getEntryNode();
2614 SDValue Callee = getExternalSymbol(Sym: VD ? VD->getVectorFnName().data() : LCName,
2615 VT: TLI->getPointerTy(DL: getDataLayout()));
2616 TargetLowering::CallLoweringInfo CLI(*this);
2617 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
2618 CC: TLI->getLibcallCallingConv(Call: LC), ResultType: RetType, Target: Callee, ArgsList: std::move(Args));
2619
2620 auto [Call, CallChain] = TLI->LowerCallTo(CLI);
2621
2622 for (auto [ResNo, ResultPtr] : llvm::enumerate(First&: ResultPtrs)) {
2623 if (ResNo == CallRetResNo) {
2624 Results.push_back(Elt: Call);
2625 continue;
2626 }
2627 MachinePointerInfo PtrInfo;
2628 SDValue LoadResult =
2629 getLoad(VT: Node->getValueType(ResNo), dl: DL, Chain: CallChain, Ptr: ResultPtr, PtrInfo);
2630 SDValue OutChain = LoadResult.getValue(R: 1);
2631
2632 if (StoreSDNode *ST = ResultStores[ResNo]) {
2633 // Replace store with the library call.
2634 ReplaceAllUsesOfValueWith(From: SDValue(ST, 0), To: OutChain);
2635 PtrInfo = ST->getPointerInfo();
2636 } else {
2637 PtrInfo = MachinePointerInfo::getFixedStack(
2638 MF&: getMachineFunction(), FI: cast<FrameIndexSDNode>(Val&: ResultPtr)->getIndex());
2639 }
2640
2641 Results.push_back(Elt: LoadResult);
2642 }
2643
2644 return true;
2645}
2646
2647SDValue SelectionDAG::expandVAArg(SDNode *Node) {
2648 SDLoc dl(Node);
2649 const TargetLowering &TLI = getTargetLoweringInfo();
2650 const Value *V = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
2651 EVT VT = Node->getValueType(ResNo: 0);
2652 SDValue Tmp1 = Node->getOperand(Num: 0);
2653 SDValue Tmp2 = Node->getOperand(Num: 1);
2654 const MaybeAlign MA(Node->getConstantOperandVal(Num: 3));
2655
2656 SDValue VAListLoad = getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Tmp1,
2657 Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2658 SDValue VAList = VAListLoad;
2659
2660 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2661 VAList = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2662 N2: getConstant(Val: MA->value() - 1, DL: dl, VT: VAList.getValueType()));
2663
2664 VAList = getNode(
2665 Opcode: ISD::AND, DL: dl, VT: VAList.getValueType(), N1: VAList,
2666 N2: getSignedConstant(Val: -(int64_t)MA->value(), DL: dl, VT: VAList.getValueType()));
2667 }
2668
2669 // Increment the pointer, VAList, to the next vaarg
2670 Tmp1 = getNode(Opcode: ISD::ADD, DL: dl, VT: VAList.getValueType(), N1: VAList,
2671 N2: getConstant(Val: getDataLayout().getTypeAllocSize(
2672 Ty: VT.getTypeForEVT(Context&: *getContext())),
2673 DL: dl, VT: VAList.getValueType()));
2674 // Store the incremented VAList to the legalized pointer
2675 Tmp1 =
2676 getStore(Chain: VAListLoad.getValue(R: 1), dl, Val: Tmp1, Ptr: Tmp2, PtrInfo: MachinePointerInfo(V));
2677 // Load the actual argument out of the pointer VAList
2678 return getLoad(VT, dl, Chain: Tmp1, Ptr: VAList, PtrInfo: MachinePointerInfo());
2679}
2680
2681SDValue SelectionDAG::expandVACopy(SDNode *Node) {
2682 SDLoc dl(Node);
2683 const TargetLowering &TLI = getTargetLoweringInfo();
2684 // This defaults to loading a pointer from the input and storing it to the
2685 // output, returning the chain.
2686 const Value *VD = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 3))->getValue();
2687 const Value *VS = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 4))->getValue();
2688 SDValue Tmp1 =
2689 getLoad(VT: TLI.getPointerTy(DL: getDataLayout()), dl, Chain: Node->getOperand(Num: 0),
2690 Ptr: Node->getOperand(Num: 2), PtrInfo: MachinePointerInfo(VS));
2691 return getStore(Chain: Tmp1.getValue(R: 1), dl, Val: Tmp1, Ptr: Node->getOperand(Num: 1),
2692 PtrInfo: MachinePointerInfo(VD));
2693}
2694
2695Align SelectionDAG::getReducedAlign(EVT VT, bool UseABI) {
2696 const DataLayout &DL = getDataLayout();
2697 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2698 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2699
2700 if (TLI->isTypeLegal(VT) || !VT.isVector())
2701 return RedAlign;
2702
2703 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2704 const Align StackAlign = TFI->getStackAlign();
2705
2706 // See if we can choose a smaller ABI alignment in cases where it's an
2707 // illegal vector type that will get broken down.
2708 if (RedAlign > StackAlign) {
2709 EVT IntermediateVT;
2710 MVT RegisterVT;
2711 unsigned NumIntermediates;
2712 TLI->getVectorTypeBreakdown(Context&: *getContext(), VT, IntermediateVT,
2713 NumIntermediates, RegisterVT);
2714 Ty = IntermediateVT.getTypeForEVT(Context&: *getContext());
2715 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2716 if (RedAlign2 < RedAlign)
2717 RedAlign = RedAlign2;
2718
2719 if (!getMachineFunction().getFrameInfo().isStackRealignable())
2720 // If the stack is not realignable, the alignment should be limited to the
2721 // StackAlignment
2722 RedAlign = std::min(a: RedAlign, b: StackAlign);
2723 }
2724
2725 return RedAlign;
2726}
2727
2728SDValue SelectionDAG::CreateStackTemporary(TypeSize Bytes, Align Alignment) {
2729 MachineFrameInfo &MFI = MF->getFrameInfo();
2730 const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
2731 int StackID = 0;
2732 if (Bytes.isScalable())
2733 StackID = TFI->getStackIDForScalableVectors();
2734 // The stack id gives an indication of whether the object is scalable or
2735 // not, so it's safe to pass in the minimum size here.
2736 int FrameIdx = MFI.CreateStackObject(Size: Bytes.getKnownMinValue(), Alignment,
2737 isSpillSlot: false, Alloca: nullptr, ID: StackID);
2738 return getFrameIndex(FI: FrameIdx, VT: TLI->getFrameIndexTy(DL: getDataLayout()));
2739}
2740
2741SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
2742 Type *Ty = VT.getTypeForEVT(Context&: *getContext());
2743 Align StackAlign =
2744 std::max(a: getDataLayout().getPrefTypeAlign(Ty), b: Align(minAlign));
2745 return CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: StackAlign);
2746}
2747
2748SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
2749 TypeSize VT1Size = VT1.getStoreSize();
2750 TypeSize VT2Size = VT2.getStoreSize();
2751 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2752 "Don't know how to choose the maximum size when creating a stack "
2753 "temporary");
2754 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2755 ? VT1Size
2756 : VT2Size;
2757
2758 Type *Ty1 = VT1.getTypeForEVT(Context&: *getContext());
2759 Type *Ty2 = VT2.getTypeForEVT(Context&: *getContext());
2760 const DataLayout &DL = getDataLayout();
2761 Align Align = std::max(a: DL.getPrefTypeAlign(Ty: Ty1), b: DL.getPrefTypeAlign(Ty: Ty2));
2762 return CreateStackTemporary(Bytes, Alignment: Align);
2763}
2764
2765SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2,
2766 ISD::CondCode Cond, const SDLoc &dl) {
2767 EVT OpVT = N1.getValueType();
2768
2769 auto GetUndefBooleanConstant = [&]() {
2770 if (VT.getScalarType() == MVT::i1 ||
2771 TLI->getBooleanContents(Type: OpVT) ==
2772 TargetLowering::UndefinedBooleanContent)
2773 return getUNDEF(VT);
2774 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2775 // so we cannot use getUNDEF(). Return zero instead.
2776 return getConstant(Val: 0, DL: dl, VT);
2777 };
2778
2779 // These setcc operations always fold.
2780 switch (Cond) {
2781 default: break;
2782 case ISD::SETFALSE:
2783 case ISD::SETFALSE2: return getBoolConstant(V: false, DL: dl, VT, OpVT);
2784 case ISD::SETTRUE:
2785 case ISD::SETTRUE2: return getBoolConstant(V: true, DL: dl, VT, OpVT);
2786
2787 case ISD::SETOEQ:
2788 case ISD::SETOGT:
2789 case ISD::SETOGE:
2790 case ISD::SETOLT:
2791 case ISD::SETOLE:
2792 case ISD::SETONE:
2793 case ISD::SETO:
2794 case ISD::SETUO:
2795 case ISD::SETUEQ:
2796 case ISD::SETUNE:
2797 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2798 break;
2799 }
2800
2801 if (OpVT.isInteger()) {
2802 // For EQ and NE, we can always pick a value for the undef to make the
2803 // predicate pass or fail, so we can return undef.
2804 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2805 // icmp eq/ne X, undef -> undef.
2806 if ((N1.isUndef() || N2.isUndef()) &&
2807 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2808 return GetUndefBooleanConstant();
2809
2810 // If both operands are undef, we can return undef for int comparison.
2811 // icmp undef, undef -> undef.
2812 if (N1.isUndef() && N2.isUndef())
2813 return GetUndefBooleanConstant();
2814
2815 // icmp X, X -> true/false
2816 // icmp X, undef -> true/false because undef could be X.
2817 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2818 return getBoolConstant(V: ISD::isTrueWhenEqual(Cond), DL: dl, VT, OpVT);
2819 }
2820
2821 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Val&: N2)) {
2822 const APInt &C2 = N2C->getAPIntValue();
2823 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1)) {
2824 const APInt &C1 = N1C->getAPIntValue();
2825
2826 return getBoolConstant(V: ICmpInst::compare(LHS: C1, RHS: C2, Pred: getICmpCondCode(Pred: Cond)),
2827 DL: dl, VT, OpVT);
2828 }
2829 }
2830
2831 auto *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
2832 auto *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
2833
2834 if (N1CFP && N2CFP) {
2835 APFloat::cmpResult R = N1CFP->getValueAPF().compare(RHS: N2CFP->getValueAPF());
2836 switch (Cond) {
2837 default: break;
2838 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2839 return GetUndefBooleanConstant();
2840 [[fallthrough]];
2841 case ISD::SETOEQ: return getBoolConstant(V: R==APFloat::cmpEqual, DL: dl, VT,
2842 OpVT);
2843 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2844 return GetUndefBooleanConstant();
2845 [[fallthrough]];
2846 case ISD::SETONE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2847 R==APFloat::cmpLessThan, DL: dl, VT,
2848 OpVT);
2849 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2850 return GetUndefBooleanConstant();
2851 [[fallthrough]];
2852 case ISD::SETOLT: return getBoolConstant(V: R==APFloat::cmpLessThan, DL: dl, VT,
2853 OpVT);
2854 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2855 return GetUndefBooleanConstant();
2856 [[fallthrough]];
2857 case ISD::SETOGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan, DL: dl,
2858 VT, OpVT);
2859 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2860 return GetUndefBooleanConstant();
2861 [[fallthrough]];
2862 case ISD::SETOLE: return getBoolConstant(V: R==APFloat::cmpLessThan ||
2863 R==APFloat::cmpEqual, DL: dl, VT,
2864 OpVT);
2865 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2866 return GetUndefBooleanConstant();
2867 [[fallthrough]];
2868 case ISD::SETOGE: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2869 R==APFloat::cmpEqual, DL: dl, VT, OpVT);
2870 case ISD::SETO: return getBoolConstant(V: R!=APFloat::cmpUnordered, DL: dl, VT,
2871 OpVT);
2872 case ISD::SETUO: return getBoolConstant(V: R==APFloat::cmpUnordered, DL: dl, VT,
2873 OpVT);
2874 case ISD::SETUEQ: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2875 R==APFloat::cmpEqual, DL: dl, VT,
2876 OpVT);
2877 case ISD::SETUNE: return getBoolConstant(V: R!=APFloat::cmpEqual, DL: dl, VT,
2878 OpVT);
2879 case ISD::SETULT: return getBoolConstant(V: R==APFloat::cmpUnordered ||
2880 R==APFloat::cmpLessThan, DL: dl, VT,
2881 OpVT);
2882 case ISD::SETUGT: return getBoolConstant(V: R==APFloat::cmpGreaterThan ||
2883 R==APFloat::cmpUnordered, DL: dl, VT,
2884 OpVT);
2885 case ISD::SETULE: return getBoolConstant(V: R!=APFloat::cmpGreaterThan, DL: dl,
2886 VT, OpVT);
2887 case ISD::SETUGE: return getBoolConstant(V: R!=APFloat::cmpLessThan, DL: dl, VT,
2888 OpVT);
2889 }
2890 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2891 // Ensure that the constant occurs on the RHS.
2892 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Operation: Cond);
2893 if (!TLI->isCondCodeLegal(CC: SwappedCond, VT: OpVT.getSimpleVT()))
2894 return SDValue();
2895 return getSetCC(DL: dl, VT, LHS: N2, RHS: N1, Cond: SwappedCond);
2896 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2897 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2898 // If an operand is known to be a nan (or undef that could be a nan), we can
2899 // fold it.
2900 // Choosing NaN for the undef will always make unordered comparison succeed
2901 // and ordered comparison fails.
2902 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2903 switch (ISD::getUnorderedFlavor(Cond)) {
2904 default:
2905 llvm_unreachable("Unknown flavor!");
2906 case 0: // Known false.
2907 return getBoolConstant(V: false, DL: dl, VT, OpVT);
2908 case 1: // Known true.
2909 return getBoolConstant(V: true, DL: dl, VT, OpVT);
2910 case 2: // Undefined.
2911 return GetUndefBooleanConstant();
2912 }
2913 }
2914
2915 // Could not fold it.
2916 return SDValue();
2917}
2918
2919/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2920/// use this predicate to simplify operations downstream.
2921bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const {
2922 unsigned BitWidth = Op.getScalarValueSizeInBits();
2923 return MaskedValueIsZero(Op, Mask: APInt::getSignMask(BitWidth), Depth);
2924}
2925
2926/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2927/// this predicate to simplify operations downstream. Mask is known to be zero
2928/// for bits that V cannot have.
2929bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2930 unsigned Depth) const {
2931 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).Zero);
2932}
2933
2934/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2935/// DemandedElts. We use this predicate to simplify operations downstream.
2936/// Mask is known to be zero for bits that V cannot have.
2937bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask,
2938 const APInt &DemandedElts,
2939 unsigned Depth) const {
2940 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, DemandedElts, Depth).Zero);
2941}
2942
2943/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2944/// DemandedElts. We use this predicate to simplify operations downstream.
2945bool SelectionDAG::MaskedVectorIsZero(SDValue V, const APInt &DemandedElts,
2946 unsigned Depth /* = 0 */) const {
2947 return computeKnownBits(Op: V, DemandedElts, Depth).isZero();
2948}
2949
2950/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2951bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask,
2952 unsigned Depth) const {
2953 return Mask.isSubsetOf(RHS: computeKnownBits(Op: V, Depth).One);
2954}
2955
2956APInt SelectionDAG::computeVectorKnownZeroElements(SDValue Op,
2957 const APInt &DemandedElts,
2958 unsigned Depth) const {
2959 EVT VT = Op.getValueType();
2960 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2961
2962 unsigned NumElts = VT.getVectorNumElements();
2963 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2964
2965 APInt KnownZeroElements = APInt::getZero(numBits: NumElts);
2966 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2967 if (!DemandedElts[EltIdx])
2968 continue; // Don't query elements that are not demanded.
2969 APInt Mask = APInt::getOneBitSet(numBits: NumElts, BitNo: EltIdx);
2970 if (MaskedVectorIsZero(V: Op, DemandedElts: Mask, Depth))
2971 KnownZeroElements.setBit(EltIdx);
2972 }
2973 return KnownZeroElements;
2974}
2975
2976/// isSplatValue - Return true if the vector V has the same value
2977/// across all DemandedElts. For scalable vectors, we don't know the
2978/// number of lanes at compile time. Instead, we use a 1 bit APInt
2979/// to represent a conservative value for all lanes; that is, that
2980/// one bit value is implicitly splatted across all lanes.
2981bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2982 APInt &UndefElts, unsigned Depth) const {
2983 unsigned Opcode = V.getOpcode();
2984 EVT VT = V.getValueType();
2985 assert(VT.isVector() && "Vector type expected");
2986 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2987 "scalable demanded bits are ignored");
2988
2989 if (!DemandedElts)
2990 return false; // No demanded elts, better to assume we don't know anything.
2991
2992 if (Depth >= MaxRecursionDepth)
2993 return false; // Limit search depth.
2994
2995 // Deal with some common cases here that work for both fixed and scalable
2996 // vector types.
2997 switch (Opcode) {
2998 case ISD::SPLAT_VECTOR:
2999 UndefElts = V.getOperand(i: 0).isUndef()
3000 ? APInt::getAllOnes(numBits: DemandedElts.getBitWidth())
3001 : APInt(DemandedElts.getBitWidth(), 0);
3002 return true;
3003 case ISD::ADD:
3004 case ISD::SUB:
3005 case ISD::AND:
3006 case ISD::XOR:
3007 case ISD::OR: {
3008 APInt UndefLHS, UndefRHS;
3009 SDValue LHS = V.getOperand(i: 0);
3010 SDValue RHS = V.getOperand(i: 1);
3011 // Only recognize splats with the same demanded undef elements for both
3012 // operands, otherwise we might fail to handle binop-specific undef
3013 // handling.
3014 // e.g. (and undef, 0) -> 0 etc.
3015 if (isSplatValue(V: LHS, DemandedElts, UndefElts&: UndefLHS, Depth: Depth + 1) &&
3016 isSplatValue(V: RHS, DemandedElts, UndefElts&: UndefRHS, Depth: Depth + 1) &&
3017 (DemandedElts & UndefLHS) == (DemandedElts & UndefRHS)) {
3018 UndefElts = UndefLHS | UndefRHS;
3019 return true;
3020 }
3021 return false;
3022 }
3023 case ISD::ABS:
3024 case ISD::TRUNCATE:
3025 case ISD::SIGN_EXTEND:
3026 case ISD::ZERO_EXTEND:
3027 return isSplatValue(V: V.getOperand(i: 0), DemandedElts, UndefElts, Depth: Depth + 1);
3028 default:
3029 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
3030 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
3031 return TLI->isSplatValueForTargetNode(Op: V, DemandedElts, UndefElts, DAG: *this,
3032 Depth);
3033 break;
3034}
3035
3036 // We don't support other cases than those above for scalable vectors at
3037 // the moment.
3038 if (VT.isScalableVector())
3039 return false;
3040
3041 unsigned NumElts = VT.getVectorNumElements();
3042 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
3043 UndefElts = APInt::getZero(numBits: NumElts);
3044
3045 switch (Opcode) {
3046 case ISD::BUILD_VECTOR: {
3047 SDValue Scl;
3048 for (unsigned i = 0; i != NumElts; ++i) {
3049 SDValue Op = V.getOperand(i);
3050 if (Op.isUndef()) {
3051 UndefElts.setBit(i);
3052 continue;
3053 }
3054 if (!DemandedElts[i])
3055 continue;
3056 if (Scl && Scl != Op)
3057 return false;
3058 Scl = Op;
3059 }
3060 return true;
3061 }
3062 case ISD::VECTOR_SHUFFLE: {
3063 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
3064 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
3065 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
3066 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: V)->getMask();
3067 for (int i = 0; i != (int)NumElts; ++i) {
3068 int M = Mask[i];
3069 if (M < 0) {
3070 UndefElts.setBit(i);
3071 continue;
3072 }
3073 if (!DemandedElts[i])
3074 continue;
3075 if (M < (int)NumElts)
3076 DemandedLHS.setBit(M);
3077 else
3078 DemandedRHS.setBit(M - NumElts);
3079 }
3080
3081 // If we aren't demanding either op, assume there's no splat.
3082 // If we are demanding both ops, assume there's no splat.
3083 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
3084 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
3085 return false;
3086
3087 // See if the demanded elts of the source op is a splat or we only demand
3088 // one element, which should always be a splat.
3089 // TODO: Handle source ops splats with undefs.
3090 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
3091 APInt SrcUndefs;
3092 return (SrcElts.popcount() == 1) ||
3093 (isSplatValue(V: Src, DemandedElts: SrcElts, UndefElts&: SrcUndefs, Depth: Depth + 1) &&
3094 (SrcElts & SrcUndefs).isZero());
3095 };
3096 if (!DemandedLHS.isZero())
3097 return CheckSplatSrc(V.getOperand(i: 0), DemandedLHS);
3098 return CheckSplatSrc(V.getOperand(i: 1), DemandedRHS);
3099 }
3100 case ISD::EXTRACT_SUBVECTOR: {
3101 // Offset the demanded elts by the subvector index.
3102 SDValue Src = V.getOperand(i: 0);
3103 // We don't support scalable vectors at the moment.
3104 if (Src.getValueType().isScalableVector())
3105 return false;
3106 uint64_t Idx = V.getConstantOperandVal(i: 1);
3107 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3108 APInt UndefSrcElts;
3109 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3110 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3111 UndefElts = UndefSrcElts.extractBits(numBits: NumElts, bitPosition: Idx);
3112 return true;
3113 }
3114 break;
3115 }
3116 case ISD::ANY_EXTEND_VECTOR_INREG:
3117 case ISD::SIGN_EXTEND_VECTOR_INREG:
3118 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3119 // Widen the demanded elts by the src element count.
3120 SDValue Src = V.getOperand(i: 0);
3121 // We don't support scalable vectors at the moment.
3122 if (Src.getValueType().isScalableVector())
3123 return false;
3124 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3125 APInt UndefSrcElts;
3126 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3127 if (isSplatValue(V: Src, DemandedElts: DemandedSrcElts, UndefElts&: UndefSrcElts, Depth: Depth + 1)) {
3128 UndefElts = UndefSrcElts.trunc(width: NumElts);
3129 return true;
3130 }
3131 break;
3132 }
3133 case ISD::BITCAST: {
3134 SDValue Src = V.getOperand(i: 0);
3135 EVT SrcVT = Src.getValueType();
3136 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
3137 unsigned BitWidth = VT.getScalarSizeInBits();
3138
3139 // Ignore bitcasts from unsupported types.
3140 // TODO: Add fp support?
3141 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
3142 break;
3143
3144 // Bitcast 'small element' vector to 'large element' vector.
3145 if ((BitWidth % SrcBitWidth) == 0) {
3146 // See if each sub element is a splat.
3147 unsigned Scale = BitWidth / SrcBitWidth;
3148 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3149 APInt ScaledDemandedElts =
3150 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3151 for (unsigned I = 0; I != Scale; ++I) {
3152 APInt SubUndefElts;
3153 APInt SubDemandedElt = APInt::getOneBitSet(numBits: Scale, BitNo: I);
3154 APInt SubDemandedElts = APInt::getSplat(NewLen: NumSrcElts, V: SubDemandedElt);
3155 SubDemandedElts &= ScaledDemandedElts;
3156 if (!isSplatValue(V: Src, DemandedElts: SubDemandedElts, UndefElts&: SubUndefElts, Depth: Depth + 1))
3157 return false;
3158 // TODO: Add support for merging sub undef elements.
3159 if (!SubUndefElts.isZero())
3160 return false;
3161 }
3162 return true;
3163 }
3164 break;
3165 }
3166 }
3167
3168 return false;
3169}
3170
3171/// Helper wrapper to main isSplatValue function.
3172bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
3173 EVT VT = V.getValueType();
3174 assert(VT.isVector() && "Vector type expected");
3175
3176 APInt UndefElts;
3177 // Since the number of lanes in a scalable vector is unknown at compile time,
3178 // we track one bit which is implicitly broadcast to all lanes. This means
3179 // that all lanes in a scalable vector are considered demanded.
3180 APInt DemandedElts
3181 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3182 return isSplatValue(V, DemandedElts, UndefElts) &&
3183 (AllowUndefs || !UndefElts);
3184}
3185
3186SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) {
3187 V = peekThroughExtractSubvectors(V);
3188
3189 EVT VT = V.getValueType();
3190 unsigned Opcode = V.getOpcode();
3191 switch (Opcode) {
3192 default: {
3193 APInt UndefElts;
3194 // Since the number of lanes in a scalable vector is unknown at compile time,
3195 // we track one bit which is implicitly broadcast to all lanes. This means
3196 // that all lanes in a scalable vector are considered demanded.
3197 APInt DemandedElts
3198 = APInt::getAllOnes(numBits: VT.isScalableVector() ? 1 : VT.getVectorNumElements());
3199
3200 if (isSplatValue(V, DemandedElts, UndefElts)) {
3201 if (VT.isScalableVector()) {
3202 // DemandedElts and UndefElts are ignored for scalable vectors, since
3203 // the only supported cases are SPLAT_VECTOR nodes.
3204 SplatIdx = 0;
3205 } else {
3206 // Handle case where all demanded elements are UNDEF.
3207 if (DemandedElts.isSubsetOf(RHS: UndefElts)) {
3208 SplatIdx = 0;
3209 return getUNDEF(VT);
3210 }
3211 SplatIdx = (UndefElts & DemandedElts).countr_one();
3212 }
3213 return V;
3214 }
3215 break;
3216 }
3217 case ISD::SPLAT_VECTOR:
3218 SplatIdx = 0;
3219 return V;
3220 case ISD::VECTOR_SHUFFLE: {
3221 assert(!VT.isScalableVector());
3222 // Check if this is a shuffle node doing a splat.
3223 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
3224 // getTargetVShiftNode currently struggles without the splat source.
3225 auto *SVN = cast<ShuffleVectorSDNode>(Val&: V);
3226 if (!SVN->isSplat())
3227 break;
3228 int Idx = SVN->getSplatIndex();
3229 int NumElts = V.getValueType().getVectorNumElements();
3230 SplatIdx = Idx % NumElts;
3231 return V.getOperand(i: Idx / NumElts);
3232 }
3233 }
3234
3235 return SDValue();
3236}
3237
3238SDValue SelectionDAG::getSplatValue(SDValue V, bool LegalTypes) {
3239 int SplatIdx;
3240 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
3241 EVT SVT = SrcVector.getValueType().getScalarType();
3242 EVT LegalSVT = SVT;
3243 if (LegalTypes && !TLI->isTypeLegal(VT: SVT)) {
3244 if (!SVT.isInteger())
3245 return SDValue();
3246 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
3247 if (LegalSVT.bitsLT(VT: SVT))
3248 return SDValue();
3249 }
3250 return getExtractVectorElt(DL: SDLoc(V), VT: LegalSVT, Vec: SrcVector, Idx: SplatIdx);
3251 }
3252 return SDValue();
3253}
3254
3255std::optional<ConstantRange>
3256SelectionDAG::getValidShiftAmountRange(SDValue V, const APInt &DemandedElts,
3257 unsigned Depth) const {
3258 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3259 V.getOpcode() == ISD::SRA) &&
3260 "Unknown shift node");
3261 // Shifting more than the bitwidth is not valid.
3262 unsigned BitWidth = V.getScalarValueSizeInBits();
3263
3264 if (auto *Cst = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1))) {
3265 const APInt &ShAmt = Cst->getAPIntValue();
3266 if (ShAmt.uge(RHS: BitWidth))
3267 return std::nullopt;
3268 return ConstantRange(ShAmt);
3269 }
3270
3271 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val: V.getOperand(i: 1))) {
3272 const APInt *MinAmt = nullptr, *MaxAmt = nullptr;
3273 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3274 if (!DemandedElts[i])
3275 continue;
3276 auto *SA = dyn_cast<ConstantSDNode>(Val: BV->getOperand(Num: i));
3277 if (!SA) {
3278 MinAmt = MaxAmt = nullptr;
3279 break;
3280 }
3281 const APInt &ShAmt = SA->getAPIntValue();
3282 if (ShAmt.uge(RHS: BitWidth))
3283 return std::nullopt;
3284 if (!MinAmt || MinAmt->ugt(RHS: ShAmt))
3285 MinAmt = &ShAmt;
3286 if (!MaxAmt || MaxAmt->ult(RHS: ShAmt))
3287 MaxAmt = &ShAmt;
3288 }
3289 assert(((!MinAmt && !MaxAmt) || (MinAmt && MaxAmt)) &&
3290 "Failed to find matching min/max shift amounts");
3291 if (MinAmt && MaxAmt)
3292 return ConstantRange(*MinAmt, *MaxAmt + 1);
3293 }
3294
3295 // Use computeKnownBits to find a hidden constant/knownbits (usually type
3296 // legalized). e.g. Hidden behind multiple bitcasts/build_vector/casts etc.
3297 KnownBits KnownAmt = computeKnownBits(Op: V.getOperand(i: 1), DemandedElts, Depth);
3298 if (KnownAmt.getMaxValue().ult(RHS: BitWidth))
3299 return ConstantRange::fromKnownBits(Known: KnownAmt, /*IsSigned=*/false);
3300
3301 return std::nullopt;
3302}
3303
3304std::optional<uint64_t>
3305SelectionDAG::getValidShiftAmount(SDValue V, const APInt &DemandedElts,
3306 unsigned Depth) const {
3307 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3308 V.getOpcode() == ISD::SRA) &&
3309 "Unknown shift node");
3310 if (std::optional<ConstantRange> AmtRange =
3311 getValidShiftAmountRange(V, DemandedElts, Depth))
3312 if (const APInt *ShAmt = AmtRange->getSingleElement())
3313 return ShAmt->getZExtValue();
3314 return std::nullopt;
3315}
3316
3317std::optional<uint64_t>
3318SelectionDAG::getValidShiftAmount(SDValue V, unsigned Depth) const {
3319 EVT VT = V.getValueType();
3320 APInt DemandedElts = VT.isFixedLengthVector()
3321 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3322 : APInt(1, 1);
3323 return getValidShiftAmount(V, DemandedElts, Depth);
3324}
3325
3326std::optional<uint64_t>
3327SelectionDAG::getValidMinimumShiftAmount(SDValue V, const APInt &DemandedElts,
3328 unsigned Depth) const {
3329 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3330 V.getOpcode() == ISD::SRA) &&
3331 "Unknown shift node");
3332 if (std::optional<ConstantRange> AmtRange =
3333 getValidShiftAmountRange(V, DemandedElts, Depth))
3334 return AmtRange->getUnsignedMin().getZExtValue();
3335 return std::nullopt;
3336}
3337
3338std::optional<uint64_t>
3339SelectionDAG::getValidMinimumShiftAmount(SDValue V, unsigned Depth) const {
3340 EVT VT = V.getValueType();
3341 APInt DemandedElts = VT.isFixedLengthVector()
3342 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3343 : APInt(1, 1);
3344 return getValidMinimumShiftAmount(V, DemandedElts, Depth);
3345}
3346
3347std::optional<uint64_t>
3348SelectionDAG::getValidMaximumShiftAmount(SDValue V, const APInt &DemandedElts,
3349 unsigned Depth) const {
3350 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3351 V.getOpcode() == ISD::SRA) &&
3352 "Unknown shift node");
3353 if (std::optional<ConstantRange> AmtRange =
3354 getValidShiftAmountRange(V, DemandedElts, Depth))
3355 return AmtRange->getUnsignedMax().getZExtValue();
3356 return std::nullopt;
3357}
3358
3359std::optional<uint64_t>
3360SelectionDAG::getValidMaximumShiftAmount(SDValue V, unsigned Depth) const {
3361 EVT VT = V.getValueType();
3362 APInt DemandedElts = VT.isFixedLengthVector()
3363 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3364 : APInt(1, 1);
3365 return getValidMaximumShiftAmount(V, DemandedElts, Depth);
3366}
3367
3368/// Determine which bits of Op are known to be either zero or one and return
3369/// them in Known. For vectors, the known bits are those that are shared by
3370/// every vector element.
3371KnownBits SelectionDAG::computeKnownBits(SDValue Op, unsigned Depth) const {
3372 EVT VT = Op.getValueType();
3373
3374 // Since the number of lanes in a scalable vector is unknown at compile time,
3375 // we track one bit which is implicitly broadcast to all lanes. This means
3376 // that all lanes in a scalable vector are considered demanded.
3377 APInt DemandedElts = VT.isFixedLengthVector()
3378 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
3379 : APInt(1, 1);
3380 return computeKnownBits(Op, DemandedElts, Depth);
3381}
3382
3383/// Determine which bits of Op are known to be either zero or one and return
3384/// them in Known. The DemandedElts argument allows us to only collect the known
3385/// bits that are shared by the requested vector elements.
3386KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
3387 unsigned Depth) const {
3388 unsigned BitWidth = Op.getScalarValueSizeInBits();
3389
3390 KnownBits Known(BitWidth); // Don't know anything.
3391
3392 if (auto OptAPInt = Op->bitcastToAPInt()) {
3393 // We know all of the bits for a constant!
3394 return KnownBits::makeConstant(C: *std::move(OptAPInt));
3395 }
3396
3397 if (Depth >= MaxRecursionDepth)
3398 return Known; // Limit search depth.
3399
3400 KnownBits Known2;
3401 unsigned NumElts = DemandedElts.getBitWidth();
3402 assert((!Op.getValueType().isFixedLengthVector() ||
3403 NumElts == Op.getValueType().getVectorNumElements()) &&
3404 "Unexpected vector size");
3405
3406 if (!DemandedElts)
3407 return Known; // No demanded elts, better to assume we don't know anything.
3408
3409 unsigned Opcode = Op.getOpcode();
3410 switch (Opcode) {
3411 case ISD::MERGE_VALUES:
3412 return computeKnownBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
3413 Depth: Depth + 1);
3414 case ISD::SPLAT_VECTOR: {
3415 SDValue SrcOp = Op.getOperand(i: 0);
3416 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3417 "Expected SPLAT_VECTOR implicit truncation");
3418 // Implicitly truncate the bits to match the official semantics of
3419 // SPLAT_VECTOR.
3420 Known = computeKnownBits(Op: SrcOp, Depth: Depth + 1).trunc(BitWidth);
3421 break;
3422 }
3423 case ISD::SPLAT_VECTOR_PARTS: {
3424 unsigned ScalarSize = Op.getOperand(i: 0).getScalarValueSizeInBits();
3425 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3426 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3427 for (auto [I, SrcOp] : enumerate(First: Op->ops())) {
3428 Known.insertBits(SubBits: computeKnownBits(Op: SrcOp, Depth: Depth + 1), BitPosition: ScalarSize * I);
3429 }
3430 break;
3431 }
3432 case ISD::STEP_VECTOR: {
3433 const APInt &Step = Op.getConstantOperandAPInt(i: 0);
3434
3435 if (Step.isPowerOf2())
3436 Known.Zero.setLowBits(Step.logBase2());
3437
3438 const Function &F = getMachineFunction().getFunction();
3439
3440 if (!isUIntN(N: BitWidth, x: Op.getValueType().getVectorMinNumElements()))
3441 break;
3442 const APInt MinNumElts =
3443 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3444
3445 bool Overflow;
3446 const APInt MaxNumElts = getVScaleRange(F: &F, BitWidth)
3447 .getUnsignedMax()
3448 .umul_ov(RHS: MinNumElts, Overflow);
3449 if (Overflow)
3450 break;
3451
3452 const APInt MaxValue = (MaxNumElts - 1).umul_ov(RHS: Step, Overflow);
3453 if (Overflow)
3454 break;
3455
3456 Known.Zero.setHighBits(MaxValue.countl_zero());
3457 break;
3458 }
3459 case ISD::BUILD_VECTOR:
3460 assert(!Op.getValueType().isScalableVector());
3461 // Collect the known bits that are shared by every demanded vector element.
3462 Known.Zero.setAllBits(); Known.One.setAllBits();
3463 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3464 if (!DemandedElts[i])
3465 continue;
3466
3467 SDValue SrcOp = Op.getOperand(i);
3468 Known2 = computeKnownBits(Op: SrcOp, Depth: Depth + 1);
3469
3470 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3471 if (SrcOp.getValueSizeInBits() != BitWidth) {
3472 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3473 "Expected BUILD_VECTOR implicit truncation");
3474 Known2 = Known2.trunc(BitWidth);
3475 }
3476
3477 // Known bits are the values that are shared by every demanded element.
3478 Known = Known.intersectWith(RHS: Known2);
3479
3480 // If we don't know any bits, early out.
3481 if (Known.isUnknown())
3482 break;
3483 }
3484 break;
3485 case ISD::VECTOR_SHUFFLE: {
3486 assert(!Op.getValueType().isScalableVector());
3487 // Collect the known bits that are shared by every vector element referenced
3488 // by the shuffle.
3489 APInt DemandedLHS, DemandedRHS;
3490 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
3491 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3492 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
3493 DemandedLHS, DemandedRHS))
3494 break;
3495
3496 // Known bits are the values that are shared by every demanded element.
3497 Known.Zero.setAllBits(); Known.One.setAllBits();
3498 if (!!DemandedLHS) {
3499 SDValue LHS = Op.getOperand(i: 0);
3500 Known2 = computeKnownBits(Op: LHS, DemandedElts: DemandedLHS, Depth: Depth + 1);
3501 Known = Known.intersectWith(RHS: Known2);
3502 }
3503 // If we don't know any bits, early out.
3504 if (Known.isUnknown())
3505 break;
3506 if (!!DemandedRHS) {
3507 SDValue RHS = Op.getOperand(i: 1);
3508 Known2 = computeKnownBits(Op: RHS, DemandedElts: DemandedRHS, Depth: Depth + 1);
3509 Known = Known.intersectWith(RHS: Known2);
3510 }
3511 break;
3512 }
3513 case ISD::VSCALE: {
3514 const Function &F = getMachineFunction().getFunction();
3515 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
3516 Known = getVScaleRange(F: &F, BitWidth).multiply(Other: Multiplier).toKnownBits();
3517 break;
3518 }
3519 case ISD::CONCAT_VECTORS: {
3520 if (Op.getValueType().isScalableVector())
3521 break;
3522 // Split DemandedElts and test each of the demanded subvectors.
3523 Known.Zero.setAllBits(); Known.One.setAllBits();
3524 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
3525 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3526 unsigned NumSubVectors = Op.getNumOperands();
3527 for (unsigned i = 0; i != NumSubVectors; ++i) {
3528 APInt DemandedSub =
3529 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
3530 if (!!DemandedSub) {
3531 SDValue Sub = Op.getOperand(i);
3532 Known2 = computeKnownBits(Op: Sub, DemandedElts: DemandedSub, Depth: Depth + 1);
3533 Known = Known.intersectWith(RHS: Known2);
3534 }
3535 // If we don't know any bits, early out.
3536 if (Known.isUnknown())
3537 break;
3538 }
3539 break;
3540 }
3541 case ISD::INSERT_SUBVECTOR: {
3542 if (Op.getValueType().isScalableVector())
3543 break;
3544 // Demand any elements from the subvector and the remainder from the src its
3545 // inserted into.
3546 SDValue Src = Op.getOperand(i: 0);
3547 SDValue Sub = Op.getOperand(i: 1);
3548 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3549 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3550 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3551 APInt DemandedSrcElts = DemandedElts;
3552 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3553
3554 Known.One.setAllBits();
3555 Known.Zero.setAllBits();
3556 if (!!DemandedSubElts) {
3557 Known = computeKnownBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
3558 if (Known.isUnknown())
3559 break; // early-out.
3560 }
3561 if (!!DemandedSrcElts) {
3562 Known2 = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3563 Known = Known.intersectWith(RHS: Known2);
3564 }
3565 break;
3566 }
3567 case ISD::EXTRACT_SUBVECTOR: {
3568 // Offset the demanded elts by the subvector index.
3569 SDValue Src = Op.getOperand(i: 0);
3570 // Bail until we can represent demanded elements for scalable vectors.
3571 if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3572 break;
3573 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3574 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3575 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3576 Known = computeKnownBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
3577 break;
3578 }
3579 case ISD::SCALAR_TO_VECTOR: {
3580 if (Op.getValueType().isScalableVector())
3581 break;
3582 // We know about scalar_to_vector as much as we know about it source,
3583 // which becomes the first element of otherwise unknown vector.
3584 if (DemandedElts != 1)
3585 break;
3586
3587 SDValue N0 = Op.getOperand(i: 0);
3588 Known = computeKnownBits(Op: N0, Depth: Depth + 1);
3589 if (N0.getValueSizeInBits() != BitWidth)
3590 Known = Known.trunc(BitWidth);
3591
3592 break;
3593 }
3594 case ISD::BITCAST: {
3595 if (Op.getValueType().isScalableVector())
3596 break;
3597
3598 SDValue N0 = Op.getOperand(i: 0);
3599 EVT SubVT = N0.getValueType();
3600 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3601
3602 // Ignore bitcasts from unsupported types.
3603 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3604 break;
3605
3606 // Fast handling of 'identity' bitcasts.
3607 if (BitWidth == SubBitWidth) {
3608 Known = computeKnownBits(Op: N0, DemandedElts, Depth: Depth + 1);
3609 break;
3610 }
3611
3612 bool IsLE = getDataLayout().isLittleEndian();
3613
3614 // Bitcast 'small element' vector to 'large element' scalar/vector.
3615 if ((BitWidth % SubBitWidth) == 0) {
3616 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3617
3618 // Collect known bits for the (larger) output by collecting the known
3619 // bits from each set of sub elements and shift these into place.
3620 // We need to separately call computeKnownBits for each set of
3621 // sub elements as the knownbits for each is likely to be different.
3622 unsigned SubScale = BitWidth / SubBitWidth;
3623 APInt SubDemandedElts(NumElts * SubScale, 0);
3624 for (unsigned i = 0; i != NumElts; ++i)
3625 if (DemandedElts[i])
3626 SubDemandedElts.setBit(i * SubScale);
3627
3628 for (unsigned i = 0; i != SubScale; ++i) {
3629 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts.shl(shiftAmt: i),
3630 Depth: Depth + 1);
3631 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3632 Known.insertBits(SubBits: Known2, BitPosition: SubBitWidth * Shifts);
3633 }
3634 }
3635
3636 // Bitcast 'large element' scalar/vector to 'small element' vector.
3637 if ((SubBitWidth % BitWidth) == 0) {
3638 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3639
3640 // Collect known bits for the (smaller) output by collecting the known
3641 // bits from the overlapping larger input elements and extracting the
3642 // sub sections we actually care about.
3643 unsigned SubScale = SubBitWidth / BitWidth;
3644 APInt SubDemandedElts =
3645 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / SubScale);
3646 Known2 = computeKnownBits(Op: N0, DemandedElts: SubDemandedElts, Depth: Depth + 1);
3647
3648 Known.Zero.setAllBits(); Known.One.setAllBits();
3649 for (unsigned i = 0; i != NumElts; ++i)
3650 if (DemandedElts[i]) {
3651 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3652 unsigned Offset = (Shifts % SubScale) * BitWidth;
3653 Known = Known.intersectWith(RHS: Known2.extractBits(NumBits: BitWidth, BitPosition: Offset));
3654 // If we don't know any bits, early out.
3655 if (Known.isUnknown())
3656 break;
3657 }
3658 }
3659 break;
3660 }
3661 case ISD::AND:
3662 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3663 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3664
3665 Known &= Known2;
3666 break;
3667 case ISD::OR:
3668 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3669 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3670
3671 Known |= Known2;
3672 break;
3673 case ISD::XOR:
3674 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3675 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3676
3677 Known ^= Known2;
3678 break;
3679 case ISD::MUL: {
3680 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3681 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3682 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3683 // TODO: SelfMultiply can be poison, but not undef.
3684 if (SelfMultiply)
3685 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3686 Op: Op.getOperand(i: 0), DemandedElts, PoisonOnly: false, Depth: Depth + 1);
3687 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3688
3689 // If the multiplication is known not to overflow, the product of a number
3690 // with itself is non-negative. Only do this if we didn't already computed
3691 // the opposite value for the sign bit.
3692 if (Op->getFlags().hasNoSignedWrap() &&
3693 Op.getOperand(i: 0) == Op.getOperand(i: 1) &&
3694 !Known.isNegative())
3695 Known.makeNonNegative();
3696 break;
3697 }
3698 case ISD::MULHU: {
3699 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3700 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3701 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3702 break;
3703 }
3704 case ISD::MULHS: {
3705 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3706 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3707 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3708 break;
3709 }
3710 case ISD::ABDU: {
3711 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3712 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3713 Known = KnownBits::abdu(LHS: Known, RHS: Known2);
3714 break;
3715 }
3716 case ISD::ABDS: {
3717 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3718 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3719 Known = KnownBits::abds(LHS: Known, RHS: Known2);
3720 unsigned SignBits1 =
3721 ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3722 if (SignBits1 == 1)
3723 break;
3724 unsigned SignBits0 =
3725 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3726 Known.Zero.setHighBits(std::min(a: SignBits0, b: SignBits1) - 1);
3727 break;
3728 }
3729 case ISD::UMUL_LOHI: {
3730 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3731 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3732 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3733 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3734 if (Op.getResNo() == 0)
3735 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3736 else
3737 Known = KnownBits::mulhu(LHS: Known, RHS: Known2);
3738 break;
3739 }
3740 case ISD::SMUL_LOHI: {
3741 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3742 Known = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3743 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3744 bool SelfMultiply = Op.getOperand(i: 0) == Op.getOperand(i: 1);
3745 if (Op.getResNo() == 0)
3746 Known = KnownBits::mul(LHS: Known, RHS: Known2, NoUndefSelfMultiply: SelfMultiply);
3747 else
3748 Known = KnownBits::mulhs(LHS: Known, RHS: Known2);
3749 break;
3750 }
3751 case ISD::AVGFLOORU: {
3752 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3753 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3754 Known = KnownBits::avgFloorU(LHS: Known, RHS: Known2);
3755 break;
3756 }
3757 case ISD::AVGCEILU: {
3758 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3759 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3760 Known = KnownBits::avgCeilU(LHS: Known, RHS: Known2);
3761 break;
3762 }
3763 case ISD::AVGFLOORS: {
3764 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3765 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3766 Known = KnownBits::avgFloorS(LHS: Known, RHS: Known2);
3767 break;
3768 }
3769 case ISD::AVGCEILS: {
3770 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3771 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3772 Known = KnownBits::avgCeilS(LHS: Known, RHS: Known2);
3773 break;
3774 }
3775 case ISD::SELECT:
3776 case ISD::VSELECT:
3777 Known = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3778 // If we don't know any bits, early out.
3779 if (Known.isUnknown())
3780 break;
3781 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
3782
3783 // Only known if known in both the LHS and RHS.
3784 Known = Known.intersectWith(RHS: Known2);
3785 break;
3786 case ISD::SELECT_CC:
3787 Known = computeKnownBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
3788 // If we don't know any bits, early out.
3789 if (Known.isUnknown())
3790 break;
3791 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
3792
3793 // Only known if known in both the LHS and RHS.
3794 Known = Known.intersectWith(RHS: Known2);
3795 break;
3796 case ISD::SMULO:
3797 case ISD::UMULO:
3798 if (Op.getResNo() != 1)
3799 break;
3800 // The boolean result conforms to getBooleanContents.
3801 // If we know the result of a setcc has the top bits zero, use this info.
3802 // We know that we have an integer-based boolean since these operations
3803 // are only available for integer.
3804 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
3805 TargetLowering::ZeroOrOneBooleanContent &&
3806 BitWidth > 1)
3807 Known.Zero.setBitsFrom(1);
3808 break;
3809 case ISD::SETCC:
3810 case ISD::SETCCCARRY:
3811 case ISD::STRICT_FSETCC:
3812 case ISD::STRICT_FSETCCS: {
3813 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3814 // If we know the result of a setcc has the top bits zero, use this info.
3815 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
3816 TargetLowering::ZeroOrOneBooleanContent &&
3817 BitWidth > 1)
3818 Known.Zero.setBitsFrom(1);
3819 break;
3820 }
3821 case ISD::SHL: {
3822 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3823 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3824
3825 bool NUW = Op->getFlags().hasNoUnsignedWrap();
3826 bool NSW = Op->getFlags().hasNoSignedWrap();
3827
3828 bool ShAmtNonZero = Known2.isNonZero();
3829
3830 Known = KnownBits::shl(LHS: Known, RHS: Known2, NUW, NSW, ShAmtNonZero);
3831
3832 // Minimum shift low bits are known zero.
3833 if (std::optional<uint64_t> ShMinAmt =
3834 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3835 Known.Zero.setLowBits(*ShMinAmt);
3836 break;
3837 }
3838 case ISD::SRL:
3839 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3840 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3841 Known = KnownBits::lshr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3842 Exact: Op->getFlags().hasExact());
3843
3844 // Minimum shift high bits are known zero.
3845 if (std::optional<uint64_t> ShMinAmt =
3846 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
3847 Known.Zero.setHighBits(*ShMinAmt);
3848 break;
3849 case ISD::SRA:
3850 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3851 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3852 Known = KnownBits::ashr(LHS: Known, RHS: Known2, /*ShAmtNonZero=*/false,
3853 Exact: Op->getFlags().hasExact());
3854 break;
3855 case ISD::FSHL:
3856 case ISD::FSHR:
3857 if (ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 2), DemandedElts)) {
3858 unsigned Amt = C->getAPIntValue().urem(RHS: BitWidth);
3859
3860 // For fshl, 0-shift returns the 1st arg.
3861 // For fshr, 0-shift returns the 2nd arg.
3862 if (Amt == 0) {
3863 Known = computeKnownBits(Op: Op.getOperand(i: Opcode == ISD::FSHL ? 0 : 1),
3864 DemandedElts, Depth: Depth + 1);
3865 break;
3866 }
3867
3868 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3869 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3870 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3871 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3872 if (Opcode == ISD::FSHL) {
3873 Known.One <<= Amt;
3874 Known.Zero <<= Amt;
3875 Known2.One.lshrInPlace(ShiftAmt: BitWidth - Amt);
3876 Known2.Zero.lshrInPlace(ShiftAmt: BitWidth - Amt);
3877 } else {
3878 Known.One <<= BitWidth - Amt;
3879 Known.Zero <<= BitWidth - Amt;
3880 Known2.One.lshrInPlace(ShiftAmt: Amt);
3881 Known2.Zero.lshrInPlace(ShiftAmt: Amt);
3882 }
3883 Known = Known.unionWith(RHS: Known2);
3884 }
3885 break;
3886 case ISD::SHL_PARTS:
3887 case ISD::SRA_PARTS:
3888 case ISD::SRL_PARTS: {
3889 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3890
3891 // Collect lo/hi source values and concatenate.
3892 unsigned LoBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
3893 unsigned HiBits = Op.getOperand(i: 1).getScalarValueSizeInBits();
3894 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3895 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
3896 Known = Known2.concat(Lo: Known);
3897
3898 // Collect shift amount.
3899 Known2 = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
3900
3901 if (Opcode == ISD::SHL_PARTS)
3902 Known = KnownBits::shl(LHS: Known, RHS: Known2);
3903 else if (Opcode == ISD::SRA_PARTS)
3904 Known = KnownBits::ashr(LHS: Known, RHS: Known2);
3905 else // if (Opcode == ISD::SRL_PARTS)
3906 Known = KnownBits::lshr(LHS: Known, RHS: Known2);
3907
3908 // TODO: Minimum shift low/high bits are known zero.
3909
3910 if (Op.getResNo() == 0)
3911 Known = Known.extractBits(NumBits: LoBits, BitPosition: 0);
3912 else
3913 Known = Known.extractBits(NumBits: HiBits, BitPosition: LoBits);
3914 break;
3915 }
3916 case ISD::SIGN_EXTEND_INREG: {
3917 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3918 EVT EVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3919 Known = Known.sextInReg(SrcBitWidth: EVT.getScalarSizeInBits());
3920 break;
3921 }
3922 case ISD::CTTZ:
3923 case ISD::CTTZ_ZERO_UNDEF: {
3924 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3925 // If we have a known 1, its position is our upper bound.
3926 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3927 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
3928 Known.Zero.setBitsFrom(LowBits);
3929 break;
3930 }
3931 case ISD::CTLZ:
3932 case ISD::CTLZ_ZERO_UNDEF: {
3933 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3934 // If we have a known 1, its position is our upper bound.
3935 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3936 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
3937 Known.Zero.setBitsFrom(LowBits);
3938 break;
3939 }
3940 case ISD::CTPOP: {
3941 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
3942 // If we know some of the bits are zero, they can't be one.
3943 unsigned PossibleOnes = Known2.countMaxPopulation();
3944 Known.Zero.setBitsFrom(llvm::bit_width(Value: PossibleOnes));
3945 break;
3946 }
3947 case ISD::PARITY: {
3948 // Parity returns 0 everywhere but the LSB.
3949 Known.Zero.setBitsFrom(1);
3950 break;
3951 }
3952 case ISD::MGATHER:
3953 case ISD::MLOAD: {
3954 ISD::LoadExtType ETy =
3955 (Opcode == ISD::MGATHER)
3956 ? cast<MaskedGatherSDNode>(Val&: Op)->getExtensionType()
3957 : cast<MaskedLoadSDNode>(Val&: Op)->getExtensionType();
3958 if (ETy == ISD::ZEXTLOAD) {
3959 EVT MemVT = cast<MemSDNode>(Val&: Op)->getMemoryVT();
3960 KnownBits Known0(MemVT.getScalarSizeInBits());
3961 return Known0.zext(BitWidth);
3962 }
3963 break;
3964 }
3965 case ISD::LOAD: {
3966 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
3967 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3968 if (ISD::isNON_EXTLoad(N: LD) && Cst) {
3969 // Determine any common known bits from the loaded constant pool value.
3970 Type *CstTy = Cst->getType();
3971 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3972 !Op.getValueType().isScalableVector()) {
3973 // If its a vector splat, then we can (quickly) reuse the scalar path.
3974 // NOTE: We assume all elements match and none are UNDEF.
3975 if (CstTy->isVectorTy()) {
3976 if (const Constant *Splat = Cst->getSplatValue()) {
3977 Cst = Splat;
3978 CstTy = Cst->getType();
3979 }
3980 }
3981 // TODO - do we need to handle different bitwidths?
3982 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3983 // Iterate across all vector elements finding common known bits.
3984 Known.One.setAllBits();
3985 Known.Zero.setAllBits();
3986 for (unsigned i = 0; i != NumElts; ++i) {
3987 if (!DemandedElts[i])
3988 continue;
3989 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
3990 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
3991 const APInt &Value = CInt->getValue();
3992 Known.One &= Value;
3993 Known.Zero &= ~Value;
3994 continue;
3995 }
3996 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
3997 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3998 Known.One &= Value;
3999 Known.Zero &= ~Value;
4000 continue;
4001 }
4002 }
4003 Known.One.clearAllBits();
4004 Known.Zero.clearAllBits();
4005 break;
4006 }
4007 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
4008 if (auto *CInt = dyn_cast<ConstantInt>(Val: Cst)) {
4009 Known = KnownBits::makeConstant(C: CInt->getValue());
4010 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: Cst)) {
4011 Known =
4012 KnownBits::makeConstant(C: CFP->getValueAPF().bitcastToAPInt());
4013 }
4014 }
4015 }
4016 } else if (Op.getResNo() == 0) {
4017 unsigned ScalarMemorySize = LD->getMemoryVT().getScalarSizeInBits();
4018 KnownBits KnownScalarMemory(ScalarMemorySize);
4019 if (const MDNode *MD = LD->getRanges())
4020 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4021
4022 // Extend the Known bits from memory to the size of the scalar result.
4023 if (ISD::isZEXTLoad(N: Op.getNode()))
4024 Known = KnownScalarMemory.zext(BitWidth);
4025 else if (ISD::isSEXTLoad(N: Op.getNode()))
4026 Known = KnownScalarMemory.sext(BitWidth);
4027 else if (ISD::isEXTLoad(N: Op.getNode()))
4028 Known = KnownScalarMemory.anyext(BitWidth);
4029 else
4030 Known = KnownScalarMemory;
4031 assert(Known.getBitWidth() == BitWidth);
4032 return Known;
4033 }
4034 break;
4035 }
4036 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4037 if (Op.getValueType().isScalableVector())
4038 break;
4039 EVT InVT = Op.getOperand(i: 0).getValueType();
4040 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4041 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4042 Known = Known.zext(BitWidth);
4043 break;
4044 }
4045 case ISD::ZERO_EXTEND: {
4046 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4047 Known = Known.zext(BitWidth);
4048 break;
4049 }
4050 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4051 if (Op.getValueType().isScalableVector())
4052 break;
4053 EVT InVT = Op.getOperand(i: 0).getValueType();
4054 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4055 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4056 // If the sign bit is known to be zero or one, then sext will extend
4057 // it to the top bits, else it will just zext.
4058 Known = Known.sext(BitWidth);
4059 break;
4060 }
4061 case ISD::SIGN_EXTEND: {
4062 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4063 // If the sign bit is known to be zero or one, then sext will extend
4064 // it to the top bits, else it will just zext.
4065 Known = Known.sext(BitWidth);
4066 break;
4067 }
4068 case ISD::ANY_EXTEND_VECTOR_INREG: {
4069 if (Op.getValueType().isScalableVector())
4070 break;
4071 EVT InVT = Op.getOperand(i: 0).getValueType();
4072 APInt InDemandedElts = DemandedElts.zext(width: InVT.getVectorNumElements());
4073 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts: InDemandedElts, Depth: Depth + 1);
4074 Known = Known.anyext(BitWidth);
4075 break;
4076 }
4077 case ISD::ANY_EXTEND: {
4078 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4079 Known = Known.anyext(BitWidth);
4080 break;
4081 }
4082 case ISD::TRUNCATE: {
4083 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4084 Known = Known.trunc(BitWidth);
4085 break;
4086 }
4087 case ISD::AssertZext: {
4088 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4089 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: VT.getSizeInBits());
4090 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4091 Known.Zero |= (~InMask);
4092 Known.One &= (~Known.Zero);
4093 break;
4094 }
4095 case ISD::AssertAlign: {
4096 unsigned LogOfAlign = Log2(A: cast<AssertAlignSDNode>(Val&: Op)->getAlign());
4097 assert(LogOfAlign != 0);
4098
4099 // TODO: Should use maximum with source
4100 // If a node is guaranteed to be aligned, set low zero bits accordingly as
4101 // well as clearing one bits.
4102 Known.Zero.setLowBits(LogOfAlign);
4103 Known.One.clearLowBits(loBits: LogOfAlign);
4104 break;
4105 }
4106 case ISD::FGETSIGN:
4107 // All bits are zero except the low bit.
4108 Known.Zero.setBitsFrom(1);
4109 break;
4110 case ISD::ADD:
4111 case ISD::SUB: {
4112 SDNodeFlags Flags = Op.getNode()->getFlags();
4113 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4114 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4115 Known = KnownBits::computeForAddSub(
4116 Add: Op.getOpcode() == ISD::ADD, NSW: Flags.hasNoSignedWrap(),
4117 NUW: Flags.hasNoUnsignedWrap(), LHS: Known, RHS: Known2);
4118 break;
4119 }
4120 case ISD::USUBO:
4121 case ISD::SSUBO:
4122 case ISD::USUBO_CARRY:
4123 case ISD::SSUBO_CARRY:
4124 if (Op.getResNo() == 1) {
4125 // If we know the result of a setcc has the top bits zero, use this info.
4126 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4127 TargetLowering::ZeroOrOneBooleanContent &&
4128 BitWidth > 1)
4129 Known.Zero.setBitsFrom(1);
4130 break;
4131 }
4132 [[fallthrough]];
4133 case ISD::SUBC: {
4134 assert(Op.getResNo() == 0 &&
4135 "We only compute knownbits for the difference here.");
4136
4137 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
4138 KnownBits Borrow(1);
4139 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
4140 Borrow = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4141 // Borrow has bit width 1
4142 Borrow = Borrow.trunc(BitWidth: 1);
4143 } else {
4144 Borrow.setAllZero();
4145 }
4146
4147 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4148 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4149 Known = KnownBits::computeForSubBorrow(LHS: Known, RHS: Known2, Borrow);
4150 break;
4151 }
4152 case ISD::UADDO:
4153 case ISD::SADDO:
4154 case ISD::UADDO_CARRY:
4155 case ISD::SADDO_CARRY:
4156 if (Op.getResNo() == 1) {
4157 // If we know the result of a setcc has the top bits zero, use this info.
4158 if (TLI->getBooleanContents(Type: Op.getOperand(i: 0).getValueType()) ==
4159 TargetLowering::ZeroOrOneBooleanContent &&
4160 BitWidth > 1)
4161 Known.Zero.setBitsFrom(1);
4162 break;
4163 }
4164 [[fallthrough]];
4165 case ISD::ADDC:
4166 case ISD::ADDE: {
4167 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
4168
4169 // With ADDE and UADDO_CARRY, a carry bit may be added in.
4170 KnownBits Carry(1);
4171 if (Opcode == ISD::ADDE)
4172 // Can't track carry from glue, set carry to unknown.
4173 Carry.resetAll();
4174 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
4175 Carry = computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
4176 // Carry has bit width 1
4177 Carry = Carry.trunc(BitWidth: 1);
4178 } else {
4179 Carry.setAllZero();
4180 }
4181
4182 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4183 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4184 Known = KnownBits::computeForAddCarry(LHS: Known, RHS: Known2, Carry);
4185 break;
4186 }
4187 case ISD::UDIV: {
4188 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4189 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4190 Known = KnownBits::udiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4191 break;
4192 }
4193 case ISD::SDIV: {
4194 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4195 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4196 Known = KnownBits::sdiv(LHS: Known, RHS: Known2, Exact: Op->getFlags().hasExact());
4197 break;
4198 }
4199 case ISD::SREM: {
4200 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4201 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4202 Known = KnownBits::srem(LHS: Known, RHS: Known2);
4203 break;
4204 }
4205 case ISD::UREM: {
4206 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4207 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4208 Known = KnownBits::urem(LHS: Known, RHS: Known2);
4209 break;
4210 }
4211 case ISD::EXTRACT_ELEMENT: {
4212 Known = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
4213 const unsigned Index = Op.getConstantOperandVal(i: 1);
4214 const unsigned EltBitWidth = Op.getValueSizeInBits();
4215
4216 // Remove low part of known bits mask
4217 Known.Zero = Known.Zero.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4218 Known.One = Known.One.getHiBits(numBits: Known.getBitWidth() - Index * EltBitWidth);
4219
4220 // Remove high part of known bit mask
4221 Known = Known.trunc(BitWidth: EltBitWidth);
4222 break;
4223 }
4224 case ISD::EXTRACT_VECTOR_ELT: {
4225 SDValue InVec = Op.getOperand(i: 0);
4226 SDValue EltNo = Op.getOperand(i: 1);
4227 EVT VecVT = InVec.getValueType();
4228 // computeKnownBits not yet implemented for scalable vectors.
4229 if (VecVT.isScalableVector())
4230 break;
4231 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
4232 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4233
4234 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
4235 // anything about the extended bits.
4236 if (BitWidth > EltBitWidth)
4237 Known = Known.trunc(BitWidth: EltBitWidth);
4238
4239 // If we know the element index, just demand that vector element, else for
4240 // an unknown element index, ignore DemandedElts and demand them all.
4241 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
4242 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4243 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
4244 DemandedSrcElts =
4245 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
4246
4247 Known = computeKnownBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
4248 if (BitWidth > EltBitWidth)
4249 Known = Known.anyext(BitWidth);
4250 break;
4251 }
4252 case ISD::INSERT_VECTOR_ELT: {
4253 if (Op.getValueType().isScalableVector())
4254 break;
4255
4256 // If we know the element index, split the demand between the
4257 // source vector and the inserted element, otherwise assume we need
4258 // the original demanded vector elements and the value.
4259 SDValue InVec = Op.getOperand(i: 0);
4260 SDValue InVal = Op.getOperand(i: 1);
4261 SDValue EltNo = Op.getOperand(i: 2);
4262 bool DemandedVal = true;
4263 APInt DemandedVecElts = DemandedElts;
4264 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
4265 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
4266 unsigned EltIdx = CEltNo->getZExtValue();
4267 DemandedVal = !!DemandedElts[EltIdx];
4268 DemandedVecElts.clearBit(BitPosition: EltIdx);
4269 }
4270 Known.One.setAllBits();
4271 Known.Zero.setAllBits();
4272 if (DemandedVal) {
4273 Known2 = computeKnownBits(Op: InVal, Depth: Depth + 1);
4274 Known = Known.intersectWith(RHS: Known2.zextOrTrunc(BitWidth));
4275 }
4276 if (!!DemandedVecElts) {
4277 Known2 = computeKnownBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
4278 Known = Known.intersectWith(RHS: Known2);
4279 }
4280 break;
4281 }
4282 case ISD::BITREVERSE: {
4283 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4284 Known = Known2.reverseBits();
4285 break;
4286 }
4287 case ISD::BSWAP: {
4288 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4289 Known = Known2.byteSwap();
4290 break;
4291 }
4292 case ISD::ABS: {
4293 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4294 Known = Known2.abs();
4295 Known.Zero.setHighBits(
4296 ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1) - 1);
4297 break;
4298 }
4299 case ISD::USUBSAT: {
4300 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4301 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4302 Known = KnownBits::usub_sat(LHS: Known, RHS: Known2);
4303 break;
4304 }
4305 case ISD::UMIN: {
4306 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4307 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4308 Known = KnownBits::umin(LHS: Known, RHS: Known2);
4309 break;
4310 }
4311 case ISD::UMAX: {
4312 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4313 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4314 Known = KnownBits::umax(LHS: Known, RHS: Known2);
4315 break;
4316 }
4317 case ISD::SMIN:
4318 case ISD::SMAX: {
4319 // If we have a clamp pattern, we know that the number of sign bits will be
4320 // the minimum of the clamp min/max range.
4321 bool IsMax = (Opcode == ISD::SMAX);
4322 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4323 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4324 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4325 CstHigh =
4326 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4327 if (CstLow && CstHigh) {
4328 if (!IsMax)
4329 std::swap(a&: CstLow, b&: CstHigh);
4330
4331 const APInt &ValueLow = CstLow->getAPIntValue();
4332 const APInt &ValueHigh = CstHigh->getAPIntValue();
4333 if (ValueLow.sle(RHS: ValueHigh)) {
4334 unsigned LowSignBits = ValueLow.getNumSignBits();
4335 unsigned HighSignBits = ValueHigh.getNumSignBits();
4336 unsigned MinSignBits = std::min(a: LowSignBits, b: HighSignBits);
4337 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
4338 Known.One.setHighBits(MinSignBits);
4339 break;
4340 }
4341 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
4342 Known.Zero.setHighBits(MinSignBits);
4343 break;
4344 }
4345 }
4346 }
4347
4348 Known = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4349 Known2 = computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4350 if (IsMax)
4351 Known = KnownBits::smax(LHS: Known, RHS: Known2);
4352 else
4353 Known = KnownBits::smin(LHS: Known, RHS: Known2);
4354
4355 // For SMAX, if CstLow is non-negative we know the result will be
4356 // non-negative and thus all sign bits are 0.
4357 // TODO: There's an equivalent of this for smin with negative constant for
4358 // known ones.
4359 if (IsMax && CstLow) {
4360 const APInt &ValueLow = CstLow->getAPIntValue();
4361 if (ValueLow.isNonNegative()) {
4362 unsigned SignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4363 Known.Zero.setHighBits(std::min(a: SignBits, b: ValueLow.getNumSignBits()));
4364 }
4365 }
4366
4367 break;
4368 }
4369 case ISD::UINT_TO_FP: {
4370 Known.makeNonNegative();
4371 break;
4372 }
4373 case ISD::SINT_TO_FP: {
4374 Known2 = computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4375 if (Known2.isNonNegative())
4376 Known.makeNonNegative();
4377 else if (Known2.isNegative())
4378 Known.makeNegative();
4379 break;
4380 }
4381 case ISD::FP_TO_UINT_SAT: {
4382 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4383 EVT VT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
4384 Known.Zero |= APInt::getBitsSetFrom(numBits: BitWidth, loBit: VT.getScalarSizeInBits());
4385 break;
4386 }
4387 case ISD::ATOMIC_LOAD: {
4388 // If we are looking at the loaded value.
4389 if (Op.getResNo() == 0) {
4390 auto *AT = cast<AtomicSDNode>(Val&: Op);
4391 unsigned ScalarMemorySize = AT->getMemoryVT().getScalarSizeInBits();
4392 KnownBits KnownScalarMemory(ScalarMemorySize);
4393 if (const MDNode *MD = AT->getRanges())
4394 computeKnownBitsFromRangeMetadata(Ranges: *MD, Known&: KnownScalarMemory);
4395
4396 switch (AT->getExtensionType()) {
4397 case ISD::ZEXTLOAD:
4398 Known = KnownScalarMemory.zext(BitWidth);
4399 break;
4400 case ISD::SEXTLOAD:
4401 Known = KnownScalarMemory.sext(BitWidth);
4402 break;
4403 case ISD::EXTLOAD:
4404 switch (TLI->getExtendForAtomicOps()) {
4405 case ISD::ZERO_EXTEND:
4406 Known = KnownScalarMemory.zext(BitWidth);
4407 break;
4408 case ISD::SIGN_EXTEND:
4409 Known = KnownScalarMemory.sext(BitWidth);
4410 break;
4411 default:
4412 Known = KnownScalarMemory.anyext(BitWidth);
4413 break;
4414 }
4415 break;
4416 case ISD::NON_EXTLOAD:
4417 Known = KnownScalarMemory;
4418 break;
4419 }
4420 assert(Known.getBitWidth() == BitWidth);
4421 }
4422 break;
4423 }
4424 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
4425 if (Op.getResNo() == 1) {
4426 // The boolean result conforms to getBooleanContents.
4427 // If we know the result of a setcc has the top bits zero, use this info.
4428 // We know that we have an integer-based boolean since these operations
4429 // are only available for integer.
4430 if (TLI->getBooleanContents(isVec: Op.getValueType().isVector(), isFloat: false) ==
4431 TargetLowering::ZeroOrOneBooleanContent &&
4432 BitWidth > 1)
4433 Known.Zero.setBitsFrom(1);
4434 break;
4435 }
4436 [[fallthrough]];
4437 case ISD::ATOMIC_CMP_SWAP:
4438 case ISD::ATOMIC_SWAP:
4439 case ISD::ATOMIC_LOAD_ADD:
4440 case ISD::ATOMIC_LOAD_SUB:
4441 case ISD::ATOMIC_LOAD_AND:
4442 case ISD::ATOMIC_LOAD_CLR:
4443 case ISD::ATOMIC_LOAD_OR:
4444 case ISD::ATOMIC_LOAD_XOR:
4445 case ISD::ATOMIC_LOAD_NAND:
4446 case ISD::ATOMIC_LOAD_MIN:
4447 case ISD::ATOMIC_LOAD_MAX:
4448 case ISD::ATOMIC_LOAD_UMIN:
4449 case ISD::ATOMIC_LOAD_UMAX: {
4450 // If we are looking at the loaded value.
4451 if (Op.getResNo() == 0) {
4452 auto *AT = cast<AtomicSDNode>(Val&: Op);
4453 unsigned MemBits = AT->getMemoryVT().getScalarSizeInBits();
4454
4455 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
4456 Known.Zero.setBitsFrom(MemBits);
4457 }
4458 break;
4459 }
4460 case ISD::FrameIndex:
4461 case ISD::TargetFrameIndex:
4462 TLI->computeKnownBitsForFrameIndex(FIOp: cast<FrameIndexSDNode>(Val&: Op)->getIndex(),
4463 Known, MF: getMachineFunction());
4464 break;
4465
4466 default:
4467 if (Opcode < ISD::BUILTIN_OP_END)
4468 break;
4469 [[fallthrough]];
4470 case ISD::INTRINSIC_WO_CHAIN:
4471 case ISD::INTRINSIC_W_CHAIN:
4472 case ISD::INTRINSIC_VOID:
4473 // TODO: Probably okay to remove after audit; here to reduce change size
4474 // in initial enablement patch for scalable vectors
4475 if (Op.getValueType().isScalableVector())
4476 break;
4477
4478 // Allow the target to implement this method for its nodes.
4479 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: *this, Depth);
4480 break;
4481 }
4482
4483 return Known;
4484}
4485
4486/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4487static SelectionDAG::OverflowKind mapOverflowResult(ConstantRange::OverflowResult OR) {
4488 switch (OR) {
4489 case ConstantRange::OverflowResult::MayOverflow:
4490 return SelectionDAG::OFK_Sometime;
4491 case ConstantRange::OverflowResult::AlwaysOverflowsLow:
4492 case ConstantRange::OverflowResult::AlwaysOverflowsHigh:
4493 return SelectionDAG::OFK_Always;
4494 case ConstantRange::OverflowResult::NeverOverflows:
4495 return SelectionDAG::OFK_Never;
4496 }
4497 llvm_unreachable("Unknown OverflowResult");
4498}
4499
4500SelectionDAG::OverflowKind
4501SelectionDAG::computeOverflowForSignedAdd(SDValue N0, SDValue N1) const {
4502 // X + 0 never overflow
4503 if (isNullConstant(V: N1))
4504 return OFK_Never;
4505
4506 // If both operands each have at least two sign bits, the addition
4507 // cannot overflow.
4508 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4509 return OFK_Never;
4510
4511 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4512 return OFK_Sometime;
4513}
4514
4515SelectionDAG::OverflowKind
4516SelectionDAG::computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const {
4517 // X + 0 never overflow
4518 if (isNullConstant(V: N1))
4519 return OFK_Never;
4520
4521 // mulhi + 1 never overflow
4522 KnownBits N1Known = computeKnownBits(Op: N1);
4523 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4524 N1Known.getMaxValue().ult(RHS: 2))
4525 return OFK_Never;
4526
4527 KnownBits N0Known = computeKnownBits(Op: N0);
4528 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4529 N0Known.getMaxValue().ult(RHS: 2))
4530 return OFK_Never;
4531
4532 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4533 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4534 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4535 return mapOverflowResult(OR: N0Range.unsignedAddMayOverflow(Other: N1Range));
4536}
4537
4538SelectionDAG::OverflowKind
4539SelectionDAG::computeOverflowForSignedSub(SDValue N0, SDValue N1) const {
4540 // X - 0 never overflow
4541 if (isNullConstant(V: N1))
4542 return OFK_Never;
4543
4544 // If both operands each have at least two sign bits, the subtraction
4545 // cannot overflow.
4546 if (ComputeNumSignBits(Op: N0) > 1 && ComputeNumSignBits(Op: N1) > 1)
4547 return OFK_Never;
4548
4549 KnownBits N0Known = computeKnownBits(Op: N0);
4550 KnownBits N1Known = computeKnownBits(Op: N1);
4551 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: true);
4552 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: true);
4553 return mapOverflowResult(OR: N0Range.signedSubMayOverflow(Other: N1Range));
4554}
4555
4556SelectionDAG::OverflowKind
4557SelectionDAG::computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const {
4558 // X - 0 never overflow
4559 if (isNullConstant(V: N1))
4560 return OFK_Never;
4561
4562 KnownBits N0Known = computeKnownBits(Op: N0);
4563 KnownBits N1Known = computeKnownBits(Op: N1);
4564 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4565 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4566 return mapOverflowResult(OR: N0Range.unsignedSubMayOverflow(Other: N1Range));
4567}
4568
4569SelectionDAG::OverflowKind
4570SelectionDAG::computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const {
4571 // X * 0 and X * 1 never overflow.
4572 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4573 return OFK_Never;
4574
4575 KnownBits N0Known = computeKnownBits(Op: N0);
4576 KnownBits N1Known = computeKnownBits(Op: N1);
4577 ConstantRange N0Range = ConstantRange::fromKnownBits(Known: N0Known, IsSigned: false);
4578 ConstantRange N1Range = ConstantRange::fromKnownBits(Known: N1Known, IsSigned: false);
4579 return mapOverflowResult(OR: N0Range.unsignedMulMayOverflow(Other: N1Range));
4580}
4581
4582SelectionDAG::OverflowKind
4583SelectionDAG::computeOverflowForSignedMul(SDValue N0, SDValue N1) const {
4584 // X * 0 and X * 1 never overflow.
4585 if (isNullConstant(V: N1) || isOneConstant(V: N1))
4586 return OFK_Never;
4587
4588 // Get the size of the result.
4589 unsigned BitWidth = N0.getScalarValueSizeInBits();
4590
4591 // Sum of the sign bits.
4592 unsigned SignBits = ComputeNumSignBits(Op: N0) + ComputeNumSignBits(Op: N1);
4593
4594 // If we have enough sign bits, then there's no overflow.
4595 if (SignBits > BitWidth + 1)
4596 return OFK_Never;
4597
4598 if (SignBits == BitWidth + 1) {
4599 // The overflow occurs when the true multiplication of the
4600 // the operands is the minimum negative number.
4601 KnownBits N0Known = computeKnownBits(Op: N0);
4602 KnownBits N1Known = computeKnownBits(Op: N1);
4603 // If one of the operands is non-negative, then there's no
4604 // overflow.
4605 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4606 return OFK_Never;
4607 }
4608
4609 return OFK_Sometime;
4610}
4611
4612bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
4613 if (Depth >= MaxRecursionDepth)
4614 return false; // Limit search depth.
4615
4616 EVT OpVT = Val.getValueType();
4617 unsigned BitWidth = OpVT.getScalarSizeInBits();
4618
4619 // Is the constant a known power of 2?
4620 if (ISD::matchUnaryPredicate(Op: Val, Match: [BitWidth](ConstantSDNode *C) {
4621 return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4622 }))
4623 return true;
4624
4625 // A left-shift of a constant one will have exactly one bit set because
4626 // shifting the bit off the end is undefined.
4627 if (Val.getOpcode() == ISD::SHL) {
4628 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4629 if (C && C->getAPIntValue() == 1)
4630 return true;
4631 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4632 isKnownNeverZero(Op: Val, Depth);
4633 }
4634
4635 // Similarly, a logical right-shift of a constant sign-bit will have exactly
4636 // one bit set.
4637 if (Val.getOpcode() == ISD::SRL) {
4638 auto *C = isConstOrConstSplat(N: Val.getOperand(i: 0));
4639 if (C && C->getAPIntValue().isSignMask())
4640 return true;
4641 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1) &&
4642 isKnownNeverZero(Op: Val, Depth);
4643 }
4644
4645 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4646 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4647
4648 // Are all operands of a build vector constant powers of two?
4649 if (Val.getOpcode() == ISD::BUILD_VECTOR)
4650 if (llvm::all_of(Range: Val->ops(), P: [BitWidth](SDValue E) {
4651 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: E))
4652 return C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2();
4653 return false;
4654 }))
4655 return true;
4656
4657 // Is the operand of a splat vector a constant power of two?
4658 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4659 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Val->getOperand(Num: 0)))
4660 if (C->getAPIntValue().zextOrTrunc(width: BitWidth).isPowerOf2())
4661 return true;
4662
4663 // vscale(power-of-two) is a power-of-two for some targets
4664 if (Val.getOpcode() == ISD::VSCALE &&
4665 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4666 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1))
4667 return true;
4668
4669 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4670 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4671 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1) &&
4672 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4673
4674 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4675 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 2), Depth: Depth + 1) &&
4676 isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 1), Depth: Depth + 1);
4677
4678 // Looking for `x & -x` pattern:
4679 // If x == 0:
4680 // x & -x -> 0
4681 // If x != 0:
4682 // x & -x -> non-zero pow2
4683 // so if we find the pattern return whether we know `x` is non-zero.
4684 SDValue X;
4685 if (sd_match(N: Val, P: m_And(L: m_Value(N&: X), R: m_Neg(V: m_Deferred(V&: X)))))
4686 return isKnownNeverZero(Op: X, Depth);
4687
4688 if (Val.getOpcode() == ISD::ZERO_EXTEND)
4689 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4690
4691 // More could be done here, though the above checks are enough
4692 // to handle some common cases.
4693 return false;
4694}
4695
4696bool SelectionDAG::isKnownToBeAPowerOfTwoFP(SDValue Val, unsigned Depth) const {
4697 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Val, AllowUndefs: true))
4698 return C1->getValueAPF().getExactLog2Abs() >= 0;
4699
4700 if (Val.getOpcode() == ISD::UINT_TO_FP || Val.getOpcode() == ISD::SINT_TO_FP)
4701 return isKnownToBeAPowerOfTwo(Val: Val.getOperand(i: 0), Depth: Depth + 1);
4702
4703 return false;
4704}
4705
4706unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const {
4707 EVT VT = Op.getValueType();
4708
4709 // Since the number of lanes in a scalable vector is unknown at compile time,
4710 // we track one bit which is implicitly broadcast to all lanes. This means
4711 // that all lanes in a scalable vector are considered demanded.
4712 APInt DemandedElts = VT.isFixedLengthVector()
4713 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
4714 : APInt(1, 1);
4715 return ComputeNumSignBits(Op, DemandedElts, Depth);
4716}
4717
4718unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4719 unsigned Depth) const {
4720 EVT VT = Op.getValueType();
4721 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4722 unsigned VTBits = VT.getScalarSizeInBits();
4723 unsigned NumElts = DemandedElts.getBitWidth();
4724 unsigned Tmp, Tmp2;
4725 unsigned FirstAnswer = 1;
4726
4727 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
4728 const APInt &Val = C->getAPIntValue();
4729 return Val.getNumSignBits();
4730 }
4731
4732 if (Depth >= MaxRecursionDepth)
4733 return 1; // Limit search depth.
4734
4735 if (!DemandedElts)
4736 return 1; // No demanded elts, better to assume we don't know anything.
4737
4738 unsigned Opcode = Op.getOpcode();
4739 switch (Opcode) {
4740 default: break;
4741 case ISD::AssertSext:
4742 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4743 return VTBits-Tmp+1;
4744 case ISD::AssertZext:
4745 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getSizeInBits();
4746 return VTBits-Tmp;
4747 case ISD::MERGE_VALUES:
4748 return ComputeNumSignBits(Op: Op.getOperand(i: Op.getResNo()), DemandedElts,
4749 Depth: Depth + 1);
4750 case ISD::SPLAT_VECTOR: {
4751 // Check if the sign bits of source go down as far as the truncated value.
4752 unsigned NumSrcBits = Op.getOperand(i: 0).getValueSizeInBits();
4753 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
4754 if (NumSrcSignBits > (NumSrcBits - VTBits))
4755 return NumSrcSignBits - (NumSrcBits - VTBits);
4756 break;
4757 }
4758 case ISD::BUILD_VECTOR:
4759 assert(!VT.isScalableVector());
4760 Tmp = VTBits;
4761 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4762 if (!DemandedElts[i])
4763 continue;
4764
4765 SDValue SrcOp = Op.getOperand(i);
4766 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4767 // for constant nodes to ensure we only look at the sign bits.
4768 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: SrcOp)) {
4769 APInt T = C->getAPIntValue().trunc(width: VTBits);
4770 Tmp2 = T.getNumSignBits();
4771 } else {
4772 Tmp2 = ComputeNumSignBits(Op: SrcOp, Depth: Depth + 1);
4773
4774 if (SrcOp.getValueSizeInBits() != VTBits) {
4775 assert(SrcOp.getValueSizeInBits() > VTBits &&
4776 "Expected BUILD_VECTOR implicit truncation");
4777 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4778 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4779 }
4780 }
4781 Tmp = std::min(a: Tmp, b: Tmp2);
4782 }
4783 return Tmp;
4784
4785 case ISD::VECTOR_SHUFFLE: {
4786 // Collect the minimum number of sign bits that are shared by every vector
4787 // element referenced by the shuffle.
4788 APInt DemandedLHS, DemandedRHS;
4789 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
4790 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4791 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: SVN->getMask(), DemandedElts,
4792 DemandedLHS, DemandedRHS))
4793 return 1;
4794
4795 Tmp = std::numeric_limits<unsigned>::max();
4796 if (!!DemandedLHS)
4797 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS, Depth: Depth + 1);
4798 if (!!DemandedRHS) {
4799 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS, Depth: Depth + 1);
4800 Tmp = std::min(a: Tmp, b: Tmp2);
4801 }
4802 // If we don't know anything, early out and try computeKnownBits fall-back.
4803 if (Tmp == 1)
4804 break;
4805 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4806 return Tmp;
4807 }
4808
4809 case ISD::BITCAST: {
4810 if (VT.isScalableVector())
4811 break;
4812 SDValue N0 = Op.getOperand(i: 0);
4813 EVT SrcVT = N0.getValueType();
4814 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4815
4816 // Ignore bitcasts from unsupported types..
4817 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4818 break;
4819
4820 // Fast handling of 'identity' bitcasts.
4821 if (VTBits == SrcBits)
4822 return ComputeNumSignBits(Op: N0, DemandedElts, Depth: Depth + 1);
4823
4824 bool IsLE = getDataLayout().isLittleEndian();
4825
4826 // Bitcast 'large element' scalar/vector to 'small element' vector.
4827 if ((SrcBits % VTBits) == 0) {
4828 assert(VT.isVector() && "Expected bitcast to vector");
4829
4830 unsigned Scale = SrcBits / VTBits;
4831 APInt SrcDemandedElts =
4832 APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumElts / Scale);
4833
4834 // Fast case - sign splat can be simply split across the small elements.
4835 Tmp = ComputeNumSignBits(Op: N0, DemandedElts: SrcDemandedElts, Depth: Depth + 1);
4836 if (Tmp == SrcBits)
4837 return VTBits;
4838
4839 // Slow case - determine how far the sign extends into each sub-element.
4840 Tmp2 = VTBits;
4841 for (unsigned i = 0; i != NumElts; ++i)
4842 if (DemandedElts[i]) {
4843 unsigned SubOffset = i % Scale;
4844 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4845 SubOffset = SubOffset * VTBits;
4846 if (Tmp <= SubOffset)
4847 return 1;
4848 Tmp2 = std::min(a: Tmp2, b: Tmp - SubOffset);
4849 }
4850 return Tmp2;
4851 }
4852 break;
4853 }
4854
4855 case ISD::FP_TO_SINT_SAT:
4856 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4857 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4858 return VTBits - Tmp + 1;
4859 case ISD::SIGN_EXTEND:
4860 Tmp = VTBits - Op.getOperand(i: 0).getScalarValueSizeInBits();
4861 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1) + Tmp;
4862 case ISD::SIGN_EXTEND_INREG:
4863 // Max of the input and what this extends.
4864 Tmp = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT().getScalarSizeInBits();
4865 Tmp = VTBits-Tmp+1;
4866 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4867 return std::max(a: Tmp, b: Tmp2);
4868 case ISD::SIGN_EXTEND_VECTOR_INREG: {
4869 if (VT.isScalableVector())
4870 break;
4871 SDValue Src = Op.getOperand(i: 0);
4872 EVT SrcVT = Src.getValueType();
4873 APInt DemandedSrcElts = DemandedElts.zext(width: SrcVT.getVectorNumElements());
4874 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4875 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth+1) + Tmp;
4876 }
4877 case ISD::SRA:
4878 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4879 // SRA X, C -> adds C sign bits.
4880 if (std::optional<uint64_t> ShAmt =
4881 getValidMinimumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1))
4882 Tmp = std::min<uint64_t>(a: Tmp + *ShAmt, b: VTBits);
4883 return Tmp;
4884 case ISD::SHL:
4885 if (std::optional<ConstantRange> ShAmtRange =
4886 getValidShiftAmountRange(V: Op, DemandedElts, Depth: Depth + 1)) {
4887 uint64_t MaxShAmt = ShAmtRange->getUnsignedMax().getZExtValue();
4888 uint64_t MinShAmt = ShAmtRange->getUnsignedMin().getZExtValue();
4889 // Try to look through ZERO/SIGN/ANY_EXTEND. If all extended bits are
4890 // shifted out, then we can compute the number of sign bits for the
4891 // operand being extended. A future improvement could be to pass along the
4892 // "shifted left by" information in the recursive calls to
4893 // ComputeKnownSignBits. Allowing us to handle this more generically.
4894 if (ISD::isExtOpcode(Opcode: Op.getOperand(i: 0).getOpcode())) {
4895 SDValue Ext = Op.getOperand(i: 0);
4896 EVT ExtVT = Ext.getValueType();
4897 SDValue Extendee = Ext.getOperand(i: 0);
4898 EVT ExtendeeVT = Extendee.getValueType();
4899 uint64_t SizeDifference =
4900 ExtVT.getScalarSizeInBits() - ExtendeeVT.getScalarSizeInBits();
4901 if (SizeDifference <= MinShAmt) {
4902 Tmp = SizeDifference +
4903 ComputeNumSignBits(Op: Extendee, DemandedElts, Depth: Depth + 1);
4904 if (MaxShAmt < Tmp)
4905 return Tmp - MaxShAmt;
4906 }
4907 }
4908 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4909 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4910 if (MaxShAmt < Tmp)
4911 return Tmp - MaxShAmt;
4912 }
4913 break;
4914 case ISD::AND:
4915 case ISD::OR:
4916 case ISD::XOR: // NOT is handled here.
4917 // Logical binary ops preserve the number of sign bits at the worst.
4918 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth+1);
4919 if (Tmp != 1) {
4920 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4921 FirstAnswer = std::min(a: Tmp, b: Tmp2);
4922 // We computed what we know about the sign bits as our first
4923 // answer. Now proceed to the generic code that uses
4924 // computeKnownBits, and pick whichever answer is better.
4925 }
4926 break;
4927
4928 case ISD::SELECT:
4929 case ISD::VSELECT:
4930 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth+1);
4931 if (Tmp == 1) return 1; // Early out.
4932 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4933 return std::min(a: Tmp, b: Tmp2);
4934 case ISD::SELECT_CC:
4935 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth+1);
4936 if (Tmp == 1) return 1; // Early out.
4937 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth+1);
4938 return std::min(a: Tmp, b: Tmp2);
4939
4940 case ISD::SMIN:
4941 case ISD::SMAX: {
4942 // If we have a clamp pattern, we know that the number of sign bits will be
4943 // the minimum of the clamp min/max range.
4944 bool IsMax = (Opcode == ISD::SMAX);
4945 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4946 if ((CstLow = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)))
4947 if (Op.getOperand(i: 0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4948 CstHigh =
4949 isConstOrConstSplat(N: Op.getOperand(i: 0).getOperand(i: 1), DemandedElts);
4950 if (CstLow && CstHigh) {
4951 if (!IsMax)
4952 std::swap(a&: CstLow, b&: CstHigh);
4953 if (CstLow->getAPIntValue().sle(RHS: CstHigh->getAPIntValue())) {
4954 Tmp = CstLow->getAPIntValue().getNumSignBits();
4955 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4956 return std::min(a: Tmp, b: Tmp2);
4957 }
4958 }
4959
4960 // Fallback - just get the minimum number of sign bits of the operands.
4961 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4962 if (Tmp == 1)
4963 return 1; // Early out.
4964 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4965 return std::min(a: Tmp, b: Tmp2);
4966 }
4967 case ISD::UMIN:
4968 case ISD::UMAX:
4969 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
4970 if (Tmp == 1)
4971 return 1; // Early out.
4972 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
4973 return std::min(a: Tmp, b: Tmp2);
4974 case ISD::SSUBO_CARRY:
4975 case ISD::USUBO_CARRY:
4976 // sub_carry(x,x,c) -> 0/-1 (sext carry)
4977 if (Op.getResNo() == 0 && Op.getOperand(i: 0) == Op.getOperand(i: 1))
4978 return VTBits;
4979 [[fallthrough]];
4980 case ISD::SADDO:
4981 case ISD::UADDO:
4982 case ISD::SADDO_CARRY:
4983 case ISD::UADDO_CARRY:
4984 case ISD::SSUBO:
4985 case ISD::USUBO:
4986 case ISD::SMULO:
4987 case ISD::UMULO:
4988 if (Op.getResNo() != 1)
4989 break;
4990 // The boolean result conforms to getBooleanContents. Fall through.
4991 // If setcc returns 0/-1, all bits are sign bits.
4992 // We know that we have an integer-based boolean since these operations
4993 // are only available for integer.
4994 if (TLI->getBooleanContents(isVec: VT.isVector(), isFloat: false) ==
4995 TargetLowering::ZeroOrNegativeOneBooleanContent)
4996 return VTBits;
4997 break;
4998 case ISD::SETCC:
4999 case ISD::SETCCCARRY:
5000 case ISD::STRICT_FSETCC:
5001 case ISD::STRICT_FSETCCS: {
5002 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
5003 // If setcc returns 0/-1, all bits are sign bits.
5004 if (TLI->getBooleanContents(Type: Op.getOperand(i: OpNo).getValueType()) ==
5005 TargetLowering::ZeroOrNegativeOneBooleanContent)
5006 return VTBits;
5007 break;
5008 }
5009 case ISD::ROTL:
5010 case ISD::ROTR:
5011 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5012
5013 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
5014 if (Tmp == VTBits)
5015 return VTBits;
5016
5017 if (ConstantSDNode *C =
5018 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts)) {
5019 unsigned RotAmt = C->getAPIntValue().urem(RHS: VTBits);
5020
5021 // Handle rotate right by N like a rotate left by 32-N.
5022 if (Opcode == ISD::ROTR)
5023 RotAmt = (VTBits - RotAmt) % VTBits;
5024
5025 // If we aren't rotating out all of the known-in sign bits, return the
5026 // number that are left. This handles rotl(sext(x), 1) for example.
5027 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
5028 }
5029 break;
5030 case ISD::ADD:
5031 case ISD::ADDC:
5032 // Add can have at most one carry bit. Thus we know that the output
5033 // is, at worst, one more bit than the inputs.
5034 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5035 if (Tmp == 1) return 1; // Early out.
5036
5037 // Special case decrementing a value (ADD X, -1):
5038 if (ConstantSDNode *CRHS =
5039 isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts))
5040 if (CRHS->isAllOnes()) {
5041 KnownBits Known =
5042 computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5043
5044 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5045 // sign bits set.
5046 if ((Known.Zero | 1).isAllOnes())
5047 return VTBits;
5048
5049 // If we are subtracting one from a positive number, there is no carry
5050 // out of the result.
5051 if (Known.isNonNegative())
5052 return Tmp;
5053 }
5054
5055 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5056 if (Tmp2 == 1) return 1; // Early out.
5057 return std::min(a: Tmp, b: Tmp2) - 1;
5058 case ISD::SUB:
5059 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5060 if (Tmp2 == 1) return 1; // Early out.
5061
5062 // Handle NEG.
5063 if (ConstantSDNode *CLHS =
5064 isConstOrConstSplat(N: Op.getOperand(i: 0), DemandedElts))
5065 if (CLHS->isZero()) {
5066 KnownBits Known =
5067 computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5068 // If the input is known to be 0 or 1, the output is 0/-1, which is all
5069 // sign bits set.
5070 if ((Known.Zero | 1).isAllOnes())
5071 return VTBits;
5072
5073 // If the input is known to be positive (the sign bit is known clear),
5074 // the output of the NEG has the same number of sign bits as the input.
5075 if (Known.isNonNegative())
5076 return Tmp2;
5077
5078 // Otherwise, we treat this like a SUB.
5079 }
5080
5081 // Sub can have at most one carry bit. Thus we know that the output
5082 // is, at worst, one more bit than the inputs.
5083 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5084 if (Tmp == 1) return 1; // Early out.
5085 return std::min(a: Tmp, b: Tmp2) - 1;
5086 case ISD::MUL: {
5087 // The output of the Mul can be at most twice the valid bits in the inputs.
5088 unsigned SignBitsOp0 = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5089 if (SignBitsOp0 == 1)
5090 break;
5091 unsigned SignBitsOp1 = ComputeNumSignBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5092 if (SignBitsOp1 == 1)
5093 break;
5094 unsigned OutValidBits =
5095 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
5096 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
5097 }
5098 case ISD::AVGCEILS:
5099 case ISD::AVGFLOORS:
5100 Tmp = ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5101 if (Tmp == 1)
5102 return 1; // Early out.
5103 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
5104 return std::min(a: Tmp, b: Tmp2);
5105 case ISD::SREM:
5106 // The sign bit is the LHS's sign bit, except when the result of the
5107 // remainder is zero. The magnitude of the result should be less than or
5108 // equal to the magnitude of the LHS. Therefore, the result should have
5109 // at least as many sign bits as the left hand side.
5110 return ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
5111 case ISD::TRUNCATE: {
5112 // Check if the sign bits of source go down as far as the truncated value.
5113 unsigned NumSrcBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
5114 unsigned NumSrcSignBits = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5115 if (NumSrcSignBits > (NumSrcBits - VTBits))
5116 return NumSrcSignBits - (NumSrcBits - VTBits);
5117 break;
5118 }
5119 case ISD::EXTRACT_ELEMENT: {
5120 if (VT.isScalableVector())
5121 break;
5122 const int KnownSign = ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth+1);
5123 const int BitWidth = Op.getValueSizeInBits();
5124 const int Items = Op.getOperand(i: 0).getValueSizeInBits() / BitWidth;
5125
5126 // Get reverse index (starting from 1), Op1 value indexes elements from
5127 // little end. Sign starts at big end.
5128 const int rIndex = Items - 1 - Op.getConstantOperandVal(i: 1);
5129
5130 // If the sign portion ends in our element the subtraction gives correct
5131 // result. Otherwise it gives either negative or > bitwidth result
5132 return std::clamp(val: KnownSign - rIndex * BitWidth, lo: 0, hi: BitWidth);
5133 }
5134 case ISD::INSERT_VECTOR_ELT: {
5135 if (VT.isScalableVector())
5136 break;
5137 // If we know the element index, split the demand between the
5138 // source vector and the inserted element, otherwise assume we need
5139 // the original demanded vector elements and the value.
5140 SDValue InVec = Op.getOperand(i: 0);
5141 SDValue InVal = Op.getOperand(i: 1);
5142 SDValue EltNo = Op.getOperand(i: 2);
5143 bool DemandedVal = true;
5144 APInt DemandedVecElts = DemandedElts;
5145 auto *CEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5146 if (CEltNo && CEltNo->getAPIntValue().ult(RHS: NumElts)) {
5147 unsigned EltIdx = CEltNo->getZExtValue();
5148 DemandedVal = !!DemandedElts[EltIdx];
5149 DemandedVecElts.clearBit(BitPosition: EltIdx);
5150 }
5151 Tmp = std::numeric_limits<unsigned>::max();
5152 if (DemandedVal) {
5153 // TODO - handle implicit truncation of inserted elements.
5154 if (InVal.getScalarValueSizeInBits() != VTBits)
5155 break;
5156 Tmp2 = ComputeNumSignBits(Op: InVal, Depth: Depth + 1);
5157 Tmp = std::min(a: Tmp, b: Tmp2);
5158 }
5159 if (!!DemandedVecElts) {
5160 Tmp2 = ComputeNumSignBits(Op: InVec, DemandedElts: DemandedVecElts, Depth: Depth + 1);
5161 Tmp = std::min(a: Tmp, b: Tmp2);
5162 }
5163 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5164 return Tmp;
5165 }
5166 case ISD::EXTRACT_VECTOR_ELT: {
5167 assert(!VT.isScalableVector());
5168 SDValue InVec = Op.getOperand(i: 0);
5169 SDValue EltNo = Op.getOperand(i: 1);
5170 EVT VecVT = InVec.getValueType();
5171 // ComputeNumSignBits not yet implemented for scalable vectors.
5172 if (VecVT.isScalableVector())
5173 break;
5174 const unsigned BitWidth = Op.getValueSizeInBits();
5175 const unsigned EltBitWidth = Op.getOperand(i: 0).getScalarValueSizeInBits();
5176 const unsigned NumSrcElts = VecVT.getVectorNumElements();
5177
5178 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
5179 // anything about sign bits. But if the sizes match we can derive knowledge
5180 // about sign bits from the vector operand.
5181 if (BitWidth != EltBitWidth)
5182 break;
5183
5184 // If we know the element index, just demand that vector element, else for
5185 // an unknown element index, ignore DemandedElts and demand them all.
5186 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
5187 auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo);
5188 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(RHS: NumSrcElts))
5189 DemandedSrcElts =
5190 APInt::getOneBitSet(numBits: NumSrcElts, BitNo: ConstEltNo->getZExtValue());
5191
5192 return ComputeNumSignBits(Op: InVec, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5193 }
5194 case ISD::EXTRACT_SUBVECTOR: {
5195 // Offset the demanded elts by the subvector index.
5196 SDValue Src = Op.getOperand(i: 0);
5197 // Bail until we can represent demanded elements for scalable vectors.
5198 if (Src.getValueType().isScalableVector())
5199 break;
5200 uint64_t Idx = Op.getConstantOperandVal(i: 1);
5201 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5202 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5203 return ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5204 }
5205 case ISD::CONCAT_VECTORS: {
5206 if (VT.isScalableVector())
5207 break;
5208 // Determine the minimum number of sign bits across all demanded
5209 // elts of the input vectors. Early out if the result is already 1.
5210 Tmp = std::numeric_limits<unsigned>::max();
5211 EVT SubVectorVT = Op.getOperand(i: 0).getValueType();
5212 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
5213 unsigned NumSubVectors = Op.getNumOperands();
5214 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
5215 APInt DemandedSub =
5216 DemandedElts.extractBits(numBits: NumSubVectorElts, bitPosition: i * NumSubVectorElts);
5217 if (!DemandedSub)
5218 continue;
5219 Tmp2 = ComputeNumSignBits(Op: Op.getOperand(i), DemandedElts: DemandedSub, Depth: Depth + 1);
5220 Tmp = std::min(a: Tmp, b: Tmp2);
5221 }
5222 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5223 return Tmp;
5224 }
5225 case ISD::INSERT_SUBVECTOR: {
5226 if (VT.isScalableVector())
5227 break;
5228 // Demand any elements from the subvector and the remainder from the src its
5229 // inserted into.
5230 SDValue Src = Op.getOperand(i: 0);
5231 SDValue Sub = Op.getOperand(i: 1);
5232 uint64_t Idx = Op.getConstantOperandVal(i: 2);
5233 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
5234 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5235 APInt DemandedSrcElts = DemandedElts;
5236 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
5237
5238 Tmp = std::numeric_limits<unsigned>::max();
5239 if (!!DemandedSubElts) {
5240 Tmp = ComputeNumSignBits(Op: Sub, DemandedElts: DemandedSubElts, Depth: Depth + 1);
5241 if (Tmp == 1)
5242 return 1; // early-out
5243 }
5244 if (!!DemandedSrcElts) {
5245 Tmp2 = ComputeNumSignBits(Op: Src, DemandedElts: DemandedSrcElts, Depth: Depth + 1);
5246 Tmp = std::min(a: Tmp, b: Tmp2);
5247 }
5248 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
5249 return Tmp;
5250 }
5251 case ISD::LOAD: {
5252 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
5253 if (const MDNode *Ranges = LD->getRanges()) {
5254 if (DemandedElts != 1)
5255 break;
5256
5257 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *Ranges);
5258 if (VTBits > CR.getBitWidth()) {
5259 switch (LD->getExtensionType()) {
5260 case ISD::SEXTLOAD:
5261 CR = CR.signExtend(BitWidth: VTBits);
5262 break;
5263 case ISD::ZEXTLOAD:
5264 CR = CR.zeroExtend(BitWidth: VTBits);
5265 break;
5266 default:
5267 break;
5268 }
5269 }
5270
5271 if (VTBits != CR.getBitWidth())
5272 break;
5273 return std::min(a: CR.getSignedMin().getNumSignBits(),
5274 b: CR.getSignedMax().getNumSignBits());
5275 }
5276
5277 break;
5278 }
5279 case ISD::ATOMIC_CMP_SWAP:
5280 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
5281 case ISD::ATOMIC_SWAP:
5282 case ISD::ATOMIC_LOAD_ADD:
5283 case ISD::ATOMIC_LOAD_SUB:
5284 case ISD::ATOMIC_LOAD_AND:
5285 case ISD::ATOMIC_LOAD_CLR:
5286 case ISD::ATOMIC_LOAD_OR:
5287 case ISD::ATOMIC_LOAD_XOR:
5288 case ISD::ATOMIC_LOAD_NAND:
5289 case ISD::ATOMIC_LOAD_MIN:
5290 case ISD::ATOMIC_LOAD_MAX:
5291 case ISD::ATOMIC_LOAD_UMIN:
5292 case ISD::ATOMIC_LOAD_UMAX:
5293 case ISD::ATOMIC_LOAD: {
5294 auto *AT = cast<AtomicSDNode>(Val&: Op);
5295 // If we are looking at the loaded value.
5296 if (Op.getResNo() == 0) {
5297 Tmp = AT->getMemoryVT().getScalarSizeInBits();
5298 if (Tmp == VTBits)
5299 return 1; // early-out
5300
5301 // For atomic_load, prefer to use the extension type.
5302 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
5303 switch (AT->getExtensionType()) {
5304 default:
5305 break;
5306 case ISD::SEXTLOAD:
5307 return VTBits - Tmp + 1;
5308 case ISD::ZEXTLOAD:
5309 return VTBits - Tmp;
5310 }
5311 }
5312
5313 if (TLI->getExtendForAtomicOps() == ISD::SIGN_EXTEND)
5314 return VTBits - Tmp + 1;
5315 if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
5316 return VTBits - Tmp;
5317 }
5318 break;
5319 }
5320 }
5321
5322 // If we are looking at the loaded value of the SDNode.
5323 if (Op.getResNo() == 0) {
5324 // Handle LOADX separately here. EXTLOAD case will fallthrough.
5325 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op)) {
5326 unsigned ExtType = LD->getExtensionType();
5327 switch (ExtType) {
5328 default: break;
5329 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
5330 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5331 return VTBits - Tmp + 1;
5332 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
5333 Tmp = LD->getMemoryVT().getScalarSizeInBits();
5334 return VTBits - Tmp;
5335 case ISD::NON_EXTLOAD:
5336 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
5337 // We only need to handle vectors - computeKnownBits should handle
5338 // scalar cases.
5339 Type *CstTy = Cst->getType();
5340 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
5341 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
5342 VTBits == CstTy->getScalarSizeInBits()) {
5343 Tmp = VTBits;
5344 for (unsigned i = 0; i != NumElts; ++i) {
5345 if (!DemandedElts[i])
5346 continue;
5347 if (Constant *Elt = Cst->getAggregateElement(Elt: i)) {
5348 if (auto *CInt = dyn_cast<ConstantInt>(Val: Elt)) {
5349 const APInt &Value = CInt->getValue();
5350 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5351 continue;
5352 }
5353 if (auto *CFP = dyn_cast<ConstantFP>(Val: Elt)) {
5354 APInt Value = CFP->getValueAPF().bitcastToAPInt();
5355 Tmp = std::min(a: Tmp, b: Value.getNumSignBits());
5356 continue;
5357 }
5358 }
5359 // Unknown type. Conservatively assume no bits match sign bit.
5360 return 1;
5361 }
5362 return Tmp;
5363 }
5364 }
5365 break;
5366 }
5367 }
5368 }
5369
5370 // Allow the target to implement this method for its nodes.
5371 if (Opcode >= ISD::BUILTIN_OP_END ||
5372 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5373 Opcode == ISD::INTRINSIC_W_CHAIN ||
5374 Opcode == ISD::INTRINSIC_VOID) {
5375 // TODO: This can probably be removed once target code is audited. This
5376 // is here purely to reduce patch size and review complexity.
5377 if (!VT.isScalableVector()) {
5378 unsigned NumBits =
5379 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, DAG: *this, Depth);
5380 if (NumBits > 1)
5381 FirstAnswer = std::max(a: FirstAnswer, b: NumBits);
5382 }
5383 }
5384
5385 // Finally, if we can prove that the top bits of the result are 0's or 1's,
5386 // use this information.
5387 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
5388 return std::max(a: FirstAnswer, b: Known.countMinSignBits());
5389}
5390
5391unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5392 unsigned Depth) const {
5393 unsigned SignBits = ComputeNumSignBits(Op, Depth);
5394 return Op.getScalarValueSizeInBits() - SignBits + 1;
5395}
5396
5397unsigned SelectionDAG::ComputeMaxSignificantBits(SDValue Op,
5398 const APInt &DemandedElts,
5399 unsigned Depth) const {
5400 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
5401 return Op.getScalarValueSizeInBits() - SignBits + 1;
5402}
5403
5404bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly,
5405 unsigned Depth) const {
5406 // Early out for FREEZE.
5407 if (Op.getOpcode() == ISD::FREEZE)
5408 return true;
5409
5410 EVT VT = Op.getValueType();
5411 APInt DemandedElts = VT.isFixedLengthVector()
5412 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5413 : APInt(1, 1);
5414 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
5415}
5416
5417bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
5418 const APInt &DemandedElts,
5419 bool PoisonOnly,
5420 unsigned Depth) const {
5421 unsigned Opcode = Op.getOpcode();
5422
5423 // Early out for FREEZE.
5424 if (Opcode == ISD::FREEZE)
5425 return true;
5426
5427 if (Depth >= MaxRecursionDepth)
5428 return false; // Limit search depth.
5429
5430 if (isIntOrFPConstant(V: Op))
5431 return true;
5432
5433 switch (Opcode) {
5434 case ISD::CONDCODE:
5435 case ISD::VALUETYPE:
5436 case ISD::FrameIndex:
5437 case ISD::TargetFrameIndex:
5438 case ISD::CopyFromReg:
5439 return true;
5440
5441 case ISD::POISON:
5442 return false;
5443
5444 case ISD::UNDEF:
5445 return PoisonOnly;
5446
5447 case ISD::BUILD_VECTOR:
5448 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5449 // this shouldn't affect the result.
5450 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5451 if (!DemandedElts[i])
5452 continue;
5453 if (!isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i), PoisonOnly,
5454 Depth: Depth + 1))
5455 return false;
5456 }
5457 return true;
5458
5459 case ISD::SPLAT_VECTOR:
5460 return isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), PoisonOnly,
5461 Depth: Depth + 1);
5462
5463 case ISD::VECTOR_SHUFFLE: {
5464 APInt DemandedLHS, DemandedRHS;
5465 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5466 if (!getShuffleDemandedElts(SrcWidth: DemandedElts.getBitWidth(), Mask: SVN->getMask(),
5467 DemandedElts, DemandedLHS, DemandedRHS,
5468 /*AllowUndefElts=*/false))
5469 return false;
5470 if (!DemandedLHS.isZero() &&
5471 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 0), DemandedElts: DemandedLHS,
5472 PoisonOnly, Depth: Depth + 1))
5473 return false;
5474 if (!DemandedRHS.isZero() &&
5475 !isGuaranteedNotToBeUndefOrPoison(Op: Op.getOperand(i: 1), DemandedElts: DemandedRHS,
5476 PoisonOnly, Depth: Depth + 1))
5477 return false;
5478 return true;
5479 }
5480
5481 // TODO: Search for noundef attributes from library functions.
5482
5483 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5484
5485 default:
5486 // Allow the target to implement this method for its nodes.
5487 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5488 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5489 return TLI->isGuaranteedNotToBeUndefOrPoisonForTargetNode(
5490 Op, DemandedElts, DAG: *this, PoisonOnly, Depth);
5491 break;
5492 }
5493
5494 // If Op can't create undef/poison and none of its operands are undef/poison
5495 // then Op is never undef/poison.
5496 // NOTE: TargetNodes can handle this in themselves in
5497 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5498 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5499 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5500 Depth) &&
5501 all_of(Range: Op->ops(), P: [&](SDValue V) {
5502 return isGuaranteedNotToBeUndefOrPoison(Op: V, PoisonOnly, Depth: Depth + 1);
5503 });
5504}
5505
5506bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, bool PoisonOnly,
5507 bool ConsiderFlags,
5508 unsigned Depth) const {
5509 EVT VT = Op.getValueType();
5510 APInt DemandedElts = VT.isFixedLengthVector()
5511 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5512 : APInt(1, 1);
5513 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5514 Depth);
5515}
5516
5517bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
5518 bool PoisonOnly, bool ConsiderFlags,
5519 unsigned Depth) const {
5520 if (ConsiderFlags && Op->hasPoisonGeneratingFlags())
5521 return true;
5522
5523 unsigned Opcode = Op.getOpcode();
5524 switch (Opcode) {
5525 case ISD::AssertSext:
5526 case ISD::AssertZext:
5527 case ISD::AssertAlign:
5528 case ISD::AssertNoFPClass:
5529 // Assertion nodes can create poison if the assertion fails.
5530 return true;
5531
5532 case ISD::FREEZE:
5533 case ISD::CONCAT_VECTORS:
5534 case ISD::INSERT_SUBVECTOR:
5535 case ISD::EXTRACT_SUBVECTOR:
5536 case ISD::SADDSAT:
5537 case ISD::UADDSAT:
5538 case ISD::SSUBSAT:
5539 case ISD::USUBSAT:
5540 case ISD::MULHU:
5541 case ISD::MULHS:
5542 case ISD::SMIN:
5543 case ISD::SMAX:
5544 case ISD::UMIN:
5545 case ISD::UMAX:
5546 case ISD::AND:
5547 case ISD::XOR:
5548 case ISD::ROTL:
5549 case ISD::ROTR:
5550 case ISD::FSHL:
5551 case ISD::FSHR:
5552 case ISD::BSWAP:
5553 case ISD::CTTZ:
5554 case ISD::CTLZ:
5555 case ISD::CTPOP:
5556 case ISD::BITREVERSE:
5557 case ISD::PARITY:
5558 case ISD::SIGN_EXTEND:
5559 case ISD::TRUNCATE:
5560 case ISD::SIGN_EXTEND_INREG:
5561 case ISD::SIGN_EXTEND_VECTOR_INREG:
5562 case ISD::ZERO_EXTEND_VECTOR_INREG:
5563 case ISD::BITCAST:
5564 case ISD::BUILD_VECTOR:
5565 case ISD::BUILD_PAIR:
5566 case ISD::SPLAT_VECTOR:
5567 return false;
5568
5569 case ISD::ADDC:
5570 case ISD::SUBC:
5571 case ISD::ADDE:
5572 case ISD::SUBE:
5573 case ISD::SADDO:
5574 case ISD::SSUBO:
5575 case ISD::SMULO:
5576 case ISD::SADDO_CARRY:
5577 case ISD::SSUBO_CARRY:
5578 case ISD::UADDO:
5579 case ISD::USUBO:
5580 case ISD::UMULO:
5581 case ISD::UADDO_CARRY:
5582 case ISD::USUBO_CARRY:
5583 // No poison on result or overflow flags.
5584 return false;
5585
5586 case ISD::SELECT_CC:
5587 case ISD::SETCC: {
5588 // Integer setcc cannot create undef or poison.
5589 if (Op.getOperand(i: 0).getValueType().isInteger())
5590 return false;
5591
5592 // FP compares are more complicated. They can create poison for nan/infinity
5593 // based on options and flags. The options and flags also cause special
5594 // nonan condition codes to be used. Those condition codes may be preserved
5595 // even if the nonan flag is dropped somewhere.
5596 unsigned CCOp = Opcode == ISD::SETCC ? 2 : 4;
5597 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val: Op.getOperand(i: CCOp))->get();
5598 if (((unsigned)CCCode & 0x10U))
5599 return true;
5600
5601 const TargetOptions &Options = getTarget().Options;
5602 return Options.NoNaNsFPMath || Options.NoInfsFPMath;
5603 }
5604
5605 case ISD::OR:
5606 case ISD::ZERO_EXTEND:
5607 case ISD::SELECT:
5608 case ISD::VSELECT:
5609 case ISD::ADD:
5610 case ISD::SUB:
5611 case ISD::MUL:
5612 case ISD::FNEG:
5613 case ISD::FADD:
5614 case ISD::FSUB:
5615 case ISD::FMUL:
5616 case ISD::FDIV:
5617 case ISD::FREM:
5618 // No poison except from flags (which is handled above)
5619 return false;
5620
5621 case ISD::SHL:
5622 case ISD::SRL:
5623 case ISD::SRA:
5624 // If the max shift amount isn't in range, then the shift can
5625 // create poison.
5626 return !getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1);
5627
5628 case ISD::CTTZ_ZERO_UNDEF:
5629 case ISD::CTLZ_ZERO_UNDEF:
5630 // If the amount is zero then the result will be poison.
5631 // TODO: Add isKnownNeverZero DemandedElts handling.
5632 return !isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5633
5634 case ISD::SCALAR_TO_VECTOR:
5635 // Check if we demand any upper (undef) elements.
5636 return !PoisonOnly && DemandedElts.ugt(RHS: 1);
5637
5638 case ISD::INSERT_VECTOR_ELT:
5639 case ISD::EXTRACT_VECTOR_ELT: {
5640 // Ensure that the element index is in bounds.
5641 EVT VecVT = Op.getOperand(i: 0).getValueType();
5642 SDValue Idx = Op.getOperand(i: Opcode == ISD::INSERT_VECTOR_ELT ? 2 : 1);
5643 KnownBits KnownIdx = computeKnownBits(Op: Idx, Depth: Depth + 1);
5644 return KnownIdx.getMaxValue().uge(RHS: VecVT.getVectorMinNumElements());
5645 }
5646
5647 case ISD::VECTOR_SHUFFLE: {
5648 // Check for any demanded shuffle element that is undef.
5649 auto *SVN = cast<ShuffleVectorSDNode>(Val&: Op);
5650 for (auto [Idx, Elt] : enumerate(First: SVN->getMask()))
5651 if (Elt < 0 && DemandedElts[Idx])
5652 return true;
5653 return false;
5654 }
5655
5656 default:
5657 // Allow the target to implement this method for its nodes.
5658 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5659 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5660 return TLI->canCreateUndefOrPoisonForTargetNode(
5661 Op, DemandedElts, DAG: *this, PoisonOnly, ConsiderFlags, Depth);
5662 break;
5663 }
5664
5665 // Be conservative and return true.
5666 return true;
5667}
5668
5669bool SelectionDAG::isADDLike(SDValue Op, bool NoWrap) const {
5670 unsigned Opcode = Op.getOpcode();
5671 if (Opcode == ISD::OR)
5672 return Op->getFlags().hasDisjoint() ||
5673 haveNoCommonBitsSet(A: Op.getOperand(i: 0), B: Op.getOperand(i: 1));
5674 if (Opcode == ISD::XOR)
5675 return !NoWrap && isMinSignedConstant(V: Op.getOperand(i: 1));
5676 return false;
5677}
5678
5679bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
5680 return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
5681 (Op.isAnyAdd() || isADDLike(Op));
5682}
5683
5684bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
5685 unsigned Depth) const {
5686 EVT VT = Op.getValueType();
5687
5688 // Since the number of lanes in a scalable vector is unknown at compile time,
5689 // we track one bit which is implicitly broadcast to all lanes. This means
5690 // that all lanes in a scalable vector are considered demanded.
5691 APInt DemandedElts = VT.isFixedLengthVector()
5692 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
5693 : APInt(1, 1);
5694
5695 return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
5696}
5697
5698bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
5699 bool SNaN, unsigned Depth) const {
5700 assert(!DemandedElts.isZero() && "No demanded elements");
5701
5702 // If we're told that NaNs won't happen, assume they won't.
5703 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5704 return true;
5705
5706 if (Depth >= MaxRecursionDepth)
5707 return false; // Limit search depth.
5708
5709 // If the value is a constant, we can obviously see if it is a NaN or not.
5710 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
5711 return !C->getValueAPF().isNaN() ||
5712 (SNaN && !C->getValueAPF().isSignaling());
5713 }
5714
5715 unsigned Opcode = Op.getOpcode();
5716 switch (Opcode) {
5717 case ISD::FADD:
5718 case ISD::FSUB:
5719 case ISD::FMUL:
5720 case ISD::FDIV:
5721 case ISD::FREM:
5722 case ISD::FSIN:
5723 case ISD::FCOS:
5724 case ISD::FTAN:
5725 case ISD::FASIN:
5726 case ISD::FACOS:
5727 case ISD::FATAN:
5728 case ISD::FATAN2:
5729 case ISD::FSINH:
5730 case ISD::FCOSH:
5731 case ISD::FTANH:
5732 case ISD::FMA:
5733 case ISD::FMAD: {
5734 if (SNaN)
5735 return true;
5736 // TODO: Need isKnownNeverInfinity
5737 return false;
5738 }
5739 case ISD::FCANONICALIZE:
5740 case ISD::FEXP:
5741 case ISD::FEXP2:
5742 case ISD::FEXP10:
5743 case ISD::FTRUNC:
5744 case ISD::FFLOOR:
5745 case ISD::FCEIL:
5746 case ISD::FROUND:
5747 case ISD::FROUNDEVEN:
5748 case ISD::LROUND:
5749 case ISD::LLROUND:
5750 case ISD::FRINT:
5751 case ISD::LRINT:
5752 case ISD::LLRINT:
5753 case ISD::FNEARBYINT:
5754 case ISD::FLDEXP: {
5755 if (SNaN)
5756 return true;
5757 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
5758 }
5759 case ISD::FABS:
5760 case ISD::FNEG:
5761 case ISD::FCOPYSIGN: {
5762 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
5763 }
5764 case ISD::SELECT:
5765 return isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1) &&
5766 isKnownNeverNaN(Op: Op.getOperand(i: 2), DemandedElts, SNaN, Depth: Depth + 1);
5767 case ISD::FP_EXTEND:
5768 case ISD::FP_ROUND: {
5769 if (SNaN)
5770 return true;
5771 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
5772 }
5773 case ISD::SINT_TO_FP:
5774 case ISD::UINT_TO_FP:
5775 return true;
5776 case ISD::FSQRT: // Need is known positive
5777 case ISD::FLOG:
5778 case ISD::FLOG2:
5779 case ISD::FLOG10:
5780 case ISD::FPOWI:
5781 case ISD::FPOW: {
5782 if (SNaN)
5783 return true;
5784 // TODO: Refine on operand
5785 return false;
5786 }
5787 case ISD::FMINNUM:
5788 case ISD::FMAXNUM:
5789 case ISD::FMINIMUMNUM:
5790 case ISD::FMAXIMUMNUM: {
5791 // Only one needs to be known not-nan, since it will be returned if the
5792 // other ends up being one.
5793 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) ||
5794 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
5795 }
5796 case ISD::FMINNUM_IEEE:
5797 case ISD::FMAXNUM_IEEE: {
5798 if (SNaN)
5799 return true;
5800 // This can return a NaN if either operand is an sNaN, or if both operands
5801 // are NaN.
5802 return (isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN: false, Depth: Depth + 1) &&
5803 isKnownNeverSNaN(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1)) ||
5804 (isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN: false, Depth: Depth + 1) &&
5805 isKnownNeverSNaN(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1));
5806 }
5807 case ISD::FMINIMUM:
5808 case ISD::FMAXIMUM: {
5809 // TODO: Does this quiet or return the origina NaN as-is?
5810 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1) &&
5811 isKnownNeverNaN(Op: Op.getOperand(i: 1), DemandedElts, SNaN, Depth: Depth + 1);
5812 }
5813 case ISD::EXTRACT_VECTOR_ELT: {
5814 SDValue Src = Op.getOperand(i: 0);
5815 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
5816 EVT SrcVT = Src.getValueType();
5817 if (SrcVT.isFixedLengthVector() && Idx &&
5818 Idx->getAPIntValue().ult(RHS: SrcVT.getVectorNumElements())) {
5819 APInt DemandedSrcElts = APInt::getOneBitSet(numBits: SrcVT.getVectorNumElements(),
5820 BitNo: Idx->getZExtValue());
5821 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
5822 }
5823 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
5824 }
5825 case ISD::EXTRACT_SUBVECTOR: {
5826 SDValue Src = Op.getOperand(i: 0);
5827 if (Src.getValueType().isFixedLengthVector()) {
5828 unsigned Idx = Op.getConstantOperandVal(i: 1);
5829 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
5830 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
5831 return isKnownNeverNaN(Op: Src, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
5832 }
5833 return isKnownNeverNaN(Op: Src, SNaN, Depth: Depth + 1);
5834 }
5835 case ISD::INSERT_SUBVECTOR: {
5836 SDValue BaseVector = Op.getOperand(i: 0);
5837 SDValue SubVector = Op.getOperand(i: 1);
5838 EVT BaseVectorVT = BaseVector.getValueType();
5839 if (BaseVectorVT.isFixedLengthVector()) {
5840 unsigned Idx = Op.getConstantOperandVal(i: 2);
5841 unsigned NumBaseElts = BaseVectorVT.getVectorNumElements();
5842 unsigned NumSubElts = SubVector.getValueType().getVectorNumElements();
5843
5844 // Clear/Extract the bits at the position where the subvector will be
5845 // inserted.
5846 APInt DemandedMask =
5847 APInt::getBitsSet(numBits: NumBaseElts, loBit: Idx, hiBit: Idx + NumSubElts);
5848 APInt DemandedSrcElts = DemandedElts & ~DemandedMask;
5849 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
5850
5851 bool NeverNaN = true;
5852 if (!DemandedSrcElts.isZero())
5853 NeverNaN &=
5854 isKnownNeverNaN(Op: BaseVector, DemandedElts: DemandedSrcElts, SNaN, Depth: Depth + 1);
5855 if (NeverNaN && !DemandedSubElts.isZero())
5856 NeverNaN &=
5857 isKnownNeverNaN(Op: SubVector, DemandedElts: DemandedSubElts, SNaN, Depth: Depth + 1);
5858 return NeverNaN;
5859 }
5860 return isKnownNeverNaN(Op: BaseVector, SNaN, Depth: Depth + 1) &&
5861 isKnownNeverNaN(Op: SubVector, SNaN, Depth: Depth + 1);
5862 }
5863 case ISD::BUILD_VECTOR: {
5864 unsigned NumElts = Op.getNumOperands();
5865 for (unsigned I = 0; I != NumElts; ++I)
5866 if (DemandedElts[I] &&
5867 !isKnownNeverNaN(Op: Op.getOperand(i: I), SNaN, Depth: Depth + 1))
5868 return false;
5869 return true;
5870 }
5871 case ISD::AssertNoFPClass: {
5872 FPClassTest NoFPClass =
5873 static_cast<FPClassTest>(Op.getConstantOperandVal(i: 1));
5874 if ((NoFPClass & fcNan) == fcNan)
5875 return true;
5876 if (SNaN && (NoFPClass & fcSNan) == fcSNan)
5877 return true;
5878 return isKnownNeverNaN(Op: Op.getOperand(i: 0), DemandedElts, SNaN, Depth: Depth + 1);
5879 }
5880 default:
5881 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5882 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
5883 return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, DAG: *this, SNaN,
5884 Depth);
5885 }
5886
5887 return false;
5888 }
5889}
5890
5891bool SelectionDAG::isKnownNeverZeroFloat(SDValue Op) const {
5892 assert(Op.getValueType().isFloatingPoint() &&
5893 "Floating point type expected");
5894
5895 // If the value is a constant, we can obviously see if it is a zero or not.
5896 return ISD::matchUnaryFpPredicate(
5897 Op, Match: [](ConstantFPSDNode *C) { return !C->isZero(); });
5898}
5899
5900bool SelectionDAG::isKnownNeverZero(SDValue Op, unsigned Depth) const {
5901 if (Depth >= MaxRecursionDepth)
5902 return false; // Limit search depth.
5903
5904 assert(!Op.getValueType().isFloatingPoint() &&
5905 "Floating point types unsupported - use isKnownNeverZeroFloat");
5906
5907 // If the value is a constant, we can obviously see if it is a zero or not.
5908 if (ISD::matchUnaryPredicate(Op,
5909 Match: [](ConstantSDNode *C) { return !C->isZero(); }))
5910 return true;
5911
5912 // TODO: Recognize more cases here. Most of the cases are also incomplete to
5913 // some degree.
5914 switch (Op.getOpcode()) {
5915 default:
5916 break;
5917
5918 case ISD::OR:
5919 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5920 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5921
5922 case ISD::VSELECT:
5923 case ISD::SELECT:
5924 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5925 isKnownNeverZero(Op: Op.getOperand(i: 2), Depth: Depth + 1);
5926
5927 case ISD::SHL: {
5928 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5929 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5930 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5931 // 1 << X is never zero.
5932 if (ValKnown.One[0])
5933 return true;
5934 // If max shift cnt of known ones is non-zero, result is non-zero.
5935 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
5936 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
5937 !ValKnown.One.shl(ShiftAmt: MaxCnt).isZero())
5938 return true;
5939 break;
5940 }
5941 case ISD::UADDSAT:
5942 case ISD::UMAX:
5943 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
5944 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5945
5946 // For smin/smax: If either operand is known negative/positive
5947 // respectively we don't need the other to be known at all.
5948 case ISD::SMAX: {
5949 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5950 if (Op1.isStrictlyPositive())
5951 return true;
5952
5953 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5954 if (Op0.isStrictlyPositive())
5955 return true;
5956
5957 if (Op1.isNonZero() && Op0.isNonZero())
5958 return true;
5959
5960 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5961 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5962 }
5963 case ISD::SMIN: {
5964 KnownBits Op1 = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1);
5965 if (Op1.isNegative())
5966 return true;
5967
5968 KnownBits Op0 = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5969 if (Op0.isNegative())
5970 return true;
5971
5972 if (Op1.isNonZero() && Op0.isNonZero())
5973 return true;
5974
5975 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5976 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5977 }
5978 case ISD::UMIN:
5979 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
5980 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5981
5982 case ISD::ROTL:
5983 case ISD::ROTR:
5984 case ISD::BITREVERSE:
5985 case ISD::BSWAP:
5986 case ISD::CTPOP:
5987 case ISD::ABS:
5988 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5989
5990 case ISD::SRA:
5991 case ISD::SRL: {
5992 if (Op->getFlags().hasExact())
5993 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5994 KnownBits ValKnown = computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
5995 if (ValKnown.isNegative())
5996 return true;
5997 // If max shift cnt of known ones is non-zero, result is non-zero.
5998 APInt MaxCnt = computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1).getMaxValue();
5999 if (MaxCnt.ult(RHS: ValKnown.getBitWidth()) &&
6000 !ValKnown.One.lshr(ShiftAmt: MaxCnt).isZero())
6001 return true;
6002 break;
6003 }
6004 case ISD::UDIV:
6005 case ISD::SDIV:
6006 // div exact can only produce a zero if the dividend is zero.
6007 // TODO: For udiv this is also true if Op1 u<= Op0
6008 if (Op->getFlags().hasExact())
6009 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6010 break;
6011
6012 case ISD::ADD:
6013 if (Op->getFlags().hasNoUnsignedWrap())
6014 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) ||
6015 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6016 return true;
6017 // TODO: There are a lot more cases we can prove for add.
6018 break;
6019
6020 case ISD::SUB: {
6021 if (isNullConstant(V: Op.getOperand(i: 0)))
6022 return isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1);
6023
6024 std::optional<bool> ne =
6025 KnownBits::ne(LHS: computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1),
6026 RHS: computeKnownBits(Op: Op.getOperand(i: 1), Depth: Depth + 1));
6027 return ne && *ne;
6028 }
6029
6030 case ISD::MUL:
6031 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
6032 if (isKnownNeverZero(Op: Op.getOperand(i: 1), Depth: Depth + 1) &&
6033 isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1))
6034 return true;
6035 break;
6036
6037 case ISD::ZERO_EXTEND:
6038 case ISD::SIGN_EXTEND:
6039 return isKnownNeverZero(Op: Op.getOperand(i: 0), Depth: Depth + 1);
6040 case ISD::VSCALE: {
6041 const Function &F = getMachineFunction().getFunction();
6042 const APInt &Multiplier = Op.getConstantOperandAPInt(i: 0);
6043 ConstantRange CR =
6044 getVScaleRange(F: &F, BitWidth: Op.getScalarValueSizeInBits()).multiply(Other: Multiplier);
6045 if (!CR.contains(Val: APInt(CR.getBitWidth(), 0)))
6046 return true;
6047 break;
6048 }
6049 }
6050
6051 return computeKnownBits(Op, Depth).isNonZero();
6052}
6053
6054bool SelectionDAG::cannotBeOrderedNegativeFP(SDValue Op) const {
6055 if (ConstantFPSDNode *C1 = isConstOrConstSplatFP(N: Op, AllowUndefs: true))
6056 return !C1->isNegative();
6057
6058 return Op.getOpcode() == ISD::FABS;
6059}
6060
6061bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const {
6062 // Check the obvious case.
6063 if (A == B) return true;
6064
6065 // For negative and positive zero.
6066 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(Val&: A))
6067 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(Val&: B))
6068 if (CA->isZero() && CB->isZero()) return true;
6069
6070 // Otherwise they may not be equal.
6071 return false;
6072}
6073
6074// Only bits set in Mask must be negated, other bits may be arbitrary.
6075SDValue llvm::getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs) {
6076 if (isBitwiseNot(V, AllowUndefs))
6077 return V.getOperand(i: 0);
6078
6079 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
6080 // bits in the non-extended part.
6081 ConstantSDNode *MaskC = isConstOrConstSplat(N: Mask);
6082 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
6083 return SDValue();
6084 SDValue ExtArg = V.getOperand(i: 0);
6085 if (ExtArg.getScalarValueSizeInBits() >=
6086 MaskC->getAPIntValue().getActiveBits() &&
6087 isBitwiseNot(V: ExtArg, AllowUndefs) &&
6088 ExtArg.getOperand(i: 0).getOpcode() == ISD::TRUNCATE &&
6089 ExtArg.getOperand(i: 0).getOperand(i: 0).getValueType() == V.getValueType())
6090 return ExtArg.getOperand(i: 0).getOperand(i: 0);
6091 return SDValue();
6092}
6093
6094static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B) {
6095 // Match masked merge pattern (X & ~M) op (Y & M)
6096 // Including degenerate case (X & ~M) op M
6097 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
6098 SDValue Other) {
6099 if (SDValue NotOperand =
6100 getBitwiseNotOperand(V: Not, Mask, /* AllowUndefs */ true)) {
6101 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
6102 NotOperand->getOpcode() == ISD::TRUNCATE)
6103 NotOperand = NotOperand->getOperand(Num: 0);
6104
6105 if (Other == NotOperand)
6106 return true;
6107 if (Other->getOpcode() == ISD::AND)
6108 return NotOperand == Other->getOperand(Num: 0) ||
6109 NotOperand == Other->getOperand(Num: 1);
6110 }
6111 return false;
6112 };
6113
6114 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
6115 A = A->getOperand(Num: 0);
6116
6117 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
6118 B = B->getOperand(Num: 0);
6119
6120 if (A->getOpcode() == ISD::AND)
6121 return MatchNoCommonBitsPattern(A->getOperand(Num: 0), A->getOperand(Num: 1), B) ||
6122 MatchNoCommonBitsPattern(A->getOperand(Num: 1), A->getOperand(Num: 0), B);
6123 return false;
6124}
6125
6126// FIXME: unify with llvm::haveNoCommonBitsSet.
6127bool SelectionDAG::haveNoCommonBitsSet(SDValue A, SDValue B) const {
6128 assert(A.getValueType() == B.getValueType() &&
6129 "Values must have the same type");
6130 if (haveNoCommonBitsSetCommutative(A, B) ||
6131 haveNoCommonBitsSetCommutative(A: B, B: A))
6132 return true;
6133 return KnownBits::haveNoCommonBitsSet(LHS: computeKnownBits(Op: A),
6134 RHS: computeKnownBits(Op: B));
6135}
6136
6137static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
6138 SelectionDAG &DAG) {
6139 if (cast<ConstantSDNode>(Val&: Step)->isZero())
6140 return DAG.getConstant(Val: 0, DL, VT);
6141
6142 return SDValue();
6143}
6144
6145static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT,
6146 ArrayRef<SDValue> Ops,
6147 SelectionDAG &DAG) {
6148 int NumOps = Ops.size();
6149 assert(NumOps != 0 && "Can't build an empty vector!");
6150 assert(!VT.isScalableVector() &&
6151 "BUILD_VECTOR cannot be used with scalable types");
6152 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
6153 "Incorrect element count in BUILD_VECTOR!");
6154
6155 // BUILD_VECTOR of UNDEFs is UNDEF.
6156 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6157 return DAG.getUNDEF(VT);
6158
6159 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
6160 SDValue IdentitySrc;
6161 bool IsIdentity = true;
6162 for (int i = 0; i != NumOps; ++i) {
6163 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
6164 Ops[i].getOperand(i: 0).getValueType() != VT ||
6165 (IdentitySrc && Ops[i].getOperand(i: 0) != IdentitySrc) ||
6166 !isa<ConstantSDNode>(Val: Ops[i].getOperand(i: 1)) ||
6167 Ops[i].getConstantOperandAPInt(i: 1) != i) {
6168 IsIdentity = false;
6169 break;
6170 }
6171 IdentitySrc = Ops[i].getOperand(i: 0);
6172 }
6173 if (IsIdentity)
6174 return IdentitySrc;
6175
6176 return SDValue();
6177}
6178
6179/// Try to simplify vector concatenation to an input value, undef, or build
6180/// vector.
6181static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT,
6182 ArrayRef<SDValue> Ops,
6183 SelectionDAG &DAG) {
6184 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
6185 assert(llvm::all_of(Ops,
6186 [Ops](SDValue Op) {
6187 return Ops[0].getValueType() == Op.getValueType();
6188 }) &&
6189 "Concatenation of vectors with inconsistent value types!");
6190 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
6191 VT.getVectorElementCount() &&
6192 "Incorrect element count in vector concatenation!");
6193
6194 if (Ops.size() == 1)
6195 return Ops[0];
6196
6197 // Concat of UNDEFs is UNDEF.
6198 if (llvm::all_of(Range&: Ops, P: [](SDValue Op) { return Op.isUndef(); }))
6199 return DAG.getUNDEF(VT);
6200
6201 // Scan the operands and look for extract operations from a single source
6202 // that correspond to insertion at the same location via this concatenation:
6203 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
6204 SDValue IdentitySrc;
6205 bool IsIdentity = true;
6206 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
6207 SDValue Op = Ops[i];
6208 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
6209 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
6210 Op.getOperand(i: 0).getValueType() != VT ||
6211 (IdentitySrc && Op.getOperand(i: 0) != IdentitySrc) ||
6212 Op.getConstantOperandVal(i: 1) != IdentityIndex) {
6213 IsIdentity = false;
6214 break;
6215 }
6216 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
6217 "Unexpected identity source vector for concat of extracts");
6218 IdentitySrc = Op.getOperand(i: 0);
6219 }
6220 if (IsIdentity) {
6221 assert(IdentitySrc && "Failed to set source vector of extracts");
6222 return IdentitySrc;
6223 }
6224
6225 // The code below this point is only designed to work for fixed width
6226 // vectors, so we bail out for now.
6227 if (VT.isScalableVector())
6228 return SDValue();
6229
6230 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
6231 // simplified to one big BUILD_VECTOR.
6232 // FIXME: Add support for SCALAR_TO_VECTOR as well.
6233 EVT SVT = VT.getScalarType();
6234 SmallVector<SDValue, 16> Elts;
6235 for (SDValue Op : Ops) {
6236 EVT OpVT = Op.getValueType();
6237 if (Op.isUndef())
6238 Elts.append(NumInputs: OpVT.getVectorNumElements(), Elt: DAG.getUNDEF(VT: SVT));
6239 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
6240 Elts.append(in_start: Op->op_begin(), in_end: Op->op_end());
6241 else
6242 return SDValue();
6243 }
6244
6245 // BUILD_VECTOR requires all inputs to be of the same type, find the
6246 // maximum type and extend them all.
6247 for (SDValue Op : Elts)
6248 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
6249
6250 if (SVT.bitsGT(VT: VT.getScalarType())) {
6251 for (SDValue &Op : Elts) {
6252 if (Op.isUndef())
6253 Op = DAG.getUNDEF(VT: SVT);
6254 else
6255 Op = DAG.getTargetLoweringInfo().isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
6256 ? DAG.getZExtOrTrunc(Op, DL, VT: SVT)
6257 : DAG.getSExtOrTrunc(Op, DL, VT: SVT);
6258 }
6259 }
6260
6261 SDValue V = DAG.getBuildVector(VT, DL, Ops: Elts);
6262 NewSDValueDbgMsg(V, Msg: "New node fold concat vectors: ", G: &DAG);
6263 return V;
6264}
6265
6266/// Gets or creates the specified node.
6267SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
6268 SDVTList VTs = getVTList(VT);
6269 FoldingSetNodeID ID;
6270 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: {});
6271 void *IP = nullptr;
6272 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
6273 return SDValue(E, 0);
6274
6275 auto *N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6276 CSEMap.InsertNode(N, InsertPos: IP);
6277
6278 InsertNode(N);
6279 SDValue V = SDValue(N, 0);
6280 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6281 return V;
6282}
6283
6284SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6285 SDValue N1) {
6286 SDNodeFlags Flags;
6287 if (Inserter)
6288 Flags = Inserter->getFlags();
6289 return getNode(Opcode, DL, VT, Operand: N1, Flags);
6290}
6291
6292SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6293 SDValue N1, const SDNodeFlags Flags) {
6294 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
6295
6296 // Constant fold unary operations with a vector integer or float operand.
6297 switch (Opcode) {
6298 default:
6299 // FIXME: Entirely reasonable to perform folding of other unary
6300 // operations here as the need arises.
6301 break;
6302 case ISD::FNEG:
6303 case ISD::FABS:
6304 case ISD::FCEIL:
6305 case ISD::FTRUNC:
6306 case ISD::FFLOOR:
6307 case ISD::FP_EXTEND:
6308 case ISD::FP_TO_SINT:
6309 case ISD::FP_TO_UINT:
6310 case ISD::FP_TO_FP16:
6311 case ISD::FP_TO_BF16:
6312 case ISD::TRUNCATE:
6313 case ISD::ANY_EXTEND:
6314 case ISD::ZERO_EXTEND:
6315 case ISD::SIGN_EXTEND:
6316 case ISD::UINT_TO_FP:
6317 case ISD::SINT_TO_FP:
6318 case ISD::FP16_TO_FP:
6319 case ISD::BF16_TO_FP:
6320 case ISD::BITCAST:
6321 case ISD::ABS:
6322 case ISD::BITREVERSE:
6323 case ISD::BSWAP:
6324 case ISD::CTLZ:
6325 case ISD::CTLZ_ZERO_UNDEF:
6326 case ISD::CTTZ:
6327 case ISD::CTTZ_ZERO_UNDEF:
6328 case ISD::CTPOP:
6329 case ISD::STEP_VECTOR: {
6330 SDValue Ops = {N1};
6331 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
6332 return Fold;
6333 }
6334 }
6335
6336 unsigned OpOpcode = N1.getNode()->getOpcode();
6337 switch (Opcode) {
6338 case ISD::STEP_VECTOR:
6339 assert(VT.isScalableVector() &&
6340 "STEP_VECTOR can only be used with scalable types");
6341 assert(OpOpcode == ISD::TargetConstant &&
6342 VT.getVectorElementType() == N1.getValueType() &&
6343 "Unexpected step operand");
6344 break;
6345 case ISD::FREEZE:
6346 assert(VT == N1.getValueType() && "Unexpected VT!");
6347 if (isGuaranteedNotToBeUndefOrPoison(Op: N1, /*PoisonOnly*/ false,
6348 /*Depth*/ 1))
6349 return N1;
6350 break;
6351 case ISD::TokenFactor:
6352 case ISD::MERGE_VALUES:
6353 case ISD::CONCAT_VECTORS:
6354 return N1; // Factor, merge or concat of one node? No need.
6355 case ISD::BUILD_VECTOR: {
6356 // Attempt to simplify BUILD_VECTOR.
6357 SDValue Ops[] = {N1};
6358 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
6359 return V;
6360 break;
6361 }
6362 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
6363 case ISD::FP_EXTEND:
6364 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
6365 "Invalid FP cast!");
6366 if (N1.getValueType() == VT) return N1; // noop conversion.
6367 assert((!VT.isVector() || VT.getVectorElementCount() ==
6368 N1.getValueType().getVectorElementCount()) &&
6369 "Vector element count mismatch!");
6370 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
6371 if (N1.isUndef())
6372 return getUNDEF(VT);
6373 break;
6374 case ISD::FP_TO_SINT:
6375 case ISD::FP_TO_UINT:
6376 if (N1.isUndef())
6377 return getUNDEF(VT);
6378 break;
6379 case ISD::SINT_TO_FP:
6380 case ISD::UINT_TO_FP:
6381 // [us]itofp(undef) = 0, because the result value is bounded.
6382 if (N1.isUndef())
6383 return getConstantFP(Val: 0.0, DL, VT);
6384 break;
6385 case ISD::SIGN_EXTEND:
6386 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6387 "Invalid SIGN_EXTEND!");
6388 assert(VT.isVector() == N1.getValueType().isVector() &&
6389 "SIGN_EXTEND result type type should be vector iff the operand "
6390 "type is vector!");
6391 if (N1.getValueType() == VT) return N1; // noop extension
6392 assert((!VT.isVector() || VT.getVectorElementCount() ==
6393 N1.getValueType().getVectorElementCount()) &&
6394 "Vector element count mismatch!");
6395 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
6396 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
6397 SDNodeFlags Flags;
6398 if (OpOpcode == ISD::ZERO_EXTEND)
6399 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6400 SDValue NewVal = getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6401 transferDbgValues(From: N1, To: NewVal);
6402 return NewVal;
6403 }
6404
6405 if (OpOpcode == ISD::POISON)
6406 return getPOISON(VT);
6407
6408 if (N1.isUndef())
6409 // sext(undef) = 0, because the top bits will all be the same.
6410 return getConstant(Val: 0, DL, VT);
6411 break;
6412 case ISD::ZERO_EXTEND:
6413 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6414 "Invalid ZERO_EXTEND!");
6415 assert(VT.isVector() == N1.getValueType().isVector() &&
6416 "ZERO_EXTEND result type type should be vector iff the operand "
6417 "type is vector!");
6418 if (N1.getValueType() == VT) return N1; // noop extension
6419 assert((!VT.isVector() || VT.getVectorElementCount() ==
6420 N1.getValueType().getVectorElementCount()) &&
6421 "Vector element count mismatch!");
6422 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
6423 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
6424 SDNodeFlags Flags;
6425 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6426 SDValue NewVal =
6427 getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, N1: N1.getOperand(i: 0), Flags);
6428 transferDbgValues(From: N1, To: NewVal);
6429 return NewVal;
6430 }
6431
6432 if (OpOpcode == ISD::POISON)
6433 return getPOISON(VT);
6434
6435 if (N1.isUndef())
6436 // zext(undef) = 0, because the top bits will be zero.
6437 return getConstant(Val: 0, DL, VT);
6438
6439 // Skip unnecessary zext_inreg pattern:
6440 // (zext (trunc x)) -> x iff the upper bits are known zero.
6441 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
6442 // use to recognise zext_inreg patterns.
6443 if (OpOpcode == ISD::TRUNCATE) {
6444 SDValue OpOp = N1.getOperand(i: 0);
6445 if (OpOp.getValueType() == VT) {
6446 if (OpOp.getOpcode() != ISD::AND) {
6447 APInt HiBits = APInt::getBitsSetFrom(numBits: VT.getScalarSizeInBits(),
6448 loBit: N1.getScalarValueSizeInBits());
6449 if (MaskedValueIsZero(V: OpOp, Mask: HiBits)) {
6450 transferDbgValues(From: N1, To: OpOp);
6451 return OpOp;
6452 }
6453 }
6454 }
6455 }
6456 break;
6457 case ISD::ANY_EXTEND:
6458 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6459 "Invalid ANY_EXTEND!");
6460 assert(VT.isVector() == N1.getValueType().isVector() &&
6461 "ANY_EXTEND result type type should be vector iff the operand "
6462 "type is vector!");
6463 if (N1.getValueType() == VT) return N1; // noop extension
6464 assert((!VT.isVector() || VT.getVectorElementCount() ==
6465 N1.getValueType().getVectorElementCount()) &&
6466 "Vector element count mismatch!");
6467 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
6468
6469 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6470 OpOpcode == ISD::ANY_EXTEND) {
6471 SDNodeFlags Flags;
6472 if (OpOpcode == ISD::ZERO_EXTEND)
6473 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6474 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
6475 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6476 }
6477 if (N1.isUndef())
6478 return getUNDEF(VT);
6479
6480 // (ext (trunc x)) -> x
6481 if (OpOpcode == ISD::TRUNCATE) {
6482 SDValue OpOp = N1.getOperand(i: 0);
6483 if (OpOp.getValueType() == VT) {
6484 transferDbgValues(From: N1, To: OpOp);
6485 return OpOp;
6486 }
6487 }
6488 break;
6489 case ISD::TRUNCATE:
6490 assert(VT.isInteger() && N1.getValueType().isInteger() &&
6491 "Invalid TRUNCATE!");
6492 assert(VT.isVector() == N1.getValueType().isVector() &&
6493 "TRUNCATE result type type should be vector iff the operand "
6494 "type is vector!");
6495 if (N1.getValueType() == VT) return N1; // noop truncate
6496 assert((!VT.isVector() || VT.getVectorElementCount() ==
6497 N1.getValueType().getVectorElementCount()) &&
6498 "Vector element count mismatch!");
6499 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
6500 if (OpOpcode == ISD::TRUNCATE)
6501 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6502 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
6503 OpOpcode == ISD::ANY_EXTEND) {
6504 // If the source is smaller than the dest, we still need an extend.
6505 if (N1.getOperand(i: 0).getValueType().getScalarType().bitsLT(
6506 VT: VT.getScalarType())) {
6507 SDNodeFlags Flags;
6508 if (OpOpcode == ISD::ZERO_EXTEND)
6509 Flags.setNonNeg(N1->getFlags().hasNonNeg());
6510 return getNode(Opcode: OpOpcode, DL, VT, N1: N1.getOperand(i: 0), Flags);
6511 }
6512 if (N1.getOperand(i: 0).getValueType().bitsGT(VT))
6513 return getNode(Opcode: ISD::TRUNCATE, DL, VT, N1: N1.getOperand(i: 0));
6514 return N1.getOperand(i: 0);
6515 }
6516 if (N1.isUndef())
6517 return getUNDEF(VT);
6518 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
6519 return getVScale(DL, VT,
6520 MulImm: N1.getConstantOperandAPInt(i: 0).trunc(width: VT.getSizeInBits()));
6521 break;
6522 case ISD::ANY_EXTEND_VECTOR_INREG:
6523 case ISD::ZERO_EXTEND_VECTOR_INREG:
6524 case ISD::SIGN_EXTEND_VECTOR_INREG:
6525 assert(VT.isVector() && "This DAG node is restricted to vector types.");
6526 assert(N1.getValueType().bitsLE(VT) &&
6527 "The input must be the same size or smaller than the result.");
6528 assert(VT.getVectorMinNumElements() <
6529 N1.getValueType().getVectorMinNumElements() &&
6530 "The destination vector type must have fewer lanes than the input.");
6531 break;
6532 case ISD::ABS:
6533 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
6534 if (N1.isUndef())
6535 return getConstant(Val: 0, DL, VT);
6536 break;
6537 case ISD::BSWAP:
6538 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
6539 assert((VT.getScalarSizeInBits() % 16 == 0) &&
6540 "BSWAP types must be a multiple of 16 bits!");
6541 if (N1.isUndef())
6542 return getUNDEF(VT);
6543 // bswap(bswap(X)) -> X.
6544 if (OpOpcode == ISD::BSWAP)
6545 return N1.getOperand(i: 0);
6546 break;
6547 case ISD::BITREVERSE:
6548 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
6549 if (N1.isUndef())
6550 return getUNDEF(VT);
6551 break;
6552 case ISD::BITCAST:
6553 assert(VT.getSizeInBits() == N1.getValueSizeInBits() &&
6554 "Cannot BITCAST between types of different sizes!");
6555 if (VT == N1.getValueType()) return N1; // noop conversion.
6556 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
6557 return getNode(Opcode: ISD::BITCAST, DL, VT, N1: N1.getOperand(i: 0));
6558 if (N1.isUndef())
6559 return getUNDEF(VT);
6560 break;
6561 case ISD::SCALAR_TO_VECTOR:
6562 assert(VT.isVector() && !N1.getValueType().isVector() &&
6563 (VT.getVectorElementType() == N1.getValueType() ||
6564 (VT.getVectorElementType().isInteger() &&
6565 N1.getValueType().isInteger() &&
6566 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
6567 "Illegal SCALAR_TO_VECTOR node!");
6568 if (N1.isUndef())
6569 return getUNDEF(VT);
6570 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
6571 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
6572 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)) &&
6573 N1.getConstantOperandVal(i: 1) == 0 &&
6574 N1.getOperand(i: 0).getValueType() == VT)
6575 return N1.getOperand(i: 0);
6576 break;
6577 case ISD::FNEG:
6578 // Negation of an unknown bag of bits is still completely undefined.
6579 if (N1.isUndef())
6580 return getUNDEF(VT);
6581
6582 if (OpOpcode == ISD::FNEG) // --X -> X
6583 return N1.getOperand(i: 0);
6584 break;
6585 case ISD::FABS:
6586 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
6587 return getNode(Opcode: ISD::FABS, DL, VT, N1: N1.getOperand(i: 0));
6588 break;
6589 case ISD::VSCALE:
6590 assert(VT == N1.getValueType() && "Unexpected VT!");
6591 break;
6592 case ISD::CTPOP:
6593 if (N1.getValueType().getScalarType() == MVT::i1)
6594 return N1;
6595 break;
6596 case ISD::CTLZ:
6597 case ISD::CTTZ:
6598 if (N1.getValueType().getScalarType() == MVT::i1)
6599 return getNOT(DL, Val: N1, VT: N1.getValueType());
6600 break;
6601 case ISD::VECREDUCE_ADD:
6602 if (N1.getValueType().getScalarType() == MVT::i1)
6603 return getNode(Opcode: ISD::VECREDUCE_XOR, DL, VT, N1);
6604 break;
6605 case ISD::VECREDUCE_SMIN:
6606 case ISD::VECREDUCE_UMAX:
6607 if (N1.getValueType().getScalarType() == MVT::i1)
6608 return getNode(Opcode: ISD::VECREDUCE_OR, DL, VT, N1);
6609 break;
6610 case ISD::VECREDUCE_SMAX:
6611 case ISD::VECREDUCE_UMIN:
6612 if (N1.getValueType().getScalarType() == MVT::i1)
6613 return getNode(Opcode: ISD::VECREDUCE_AND, DL, VT, N1);
6614 break;
6615 case ISD::SPLAT_VECTOR:
6616 assert(VT.isVector() && "Wrong return type!");
6617 // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
6618 // that for now.
6619 assert((VT.getVectorElementType() == N1.getValueType() ||
6620 (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
6621 (VT.getVectorElementType().isInteger() &&
6622 N1.getValueType().isInteger() &&
6623 VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
6624 "Wrong operand type!");
6625 break;
6626 }
6627
6628 SDNode *N;
6629 SDVTList VTs = getVTList(VT);
6630 SDValue Ops[] = {N1};
6631 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
6632 FoldingSetNodeID ID;
6633 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
6634 void *IP = nullptr;
6635 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
6636 E->intersectFlagsWith(Flags);
6637 return SDValue(E, 0);
6638 }
6639
6640 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6641 N->setFlags(Flags);
6642 createOperands(Node: N, Vals: Ops);
6643 CSEMap.InsertNode(N, InsertPos: IP);
6644 } else {
6645 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
6646 createOperands(Node: N, Vals: Ops);
6647 }
6648
6649 InsertNode(N);
6650 SDValue V = SDValue(N, 0);
6651 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
6652 return V;
6653}
6654
6655static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
6656 const APInt &C2) {
6657 switch (Opcode) {
6658 case ISD::ADD: return C1 + C2;
6659 case ISD::SUB: return C1 - C2;
6660 case ISD::MUL: return C1 * C2;
6661 case ISD::AND: return C1 & C2;
6662 case ISD::OR: return C1 | C2;
6663 case ISD::XOR: return C1 ^ C2;
6664 case ISD::SHL: return C1 << C2;
6665 case ISD::SRL: return C1.lshr(ShiftAmt: C2);
6666 case ISD::SRA: return C1.ashr(ShiftAmt: C2);
6667 case ISD::ROTL: return C1.rotl(rotateAmt: C2);
6668 case ISD::ROTR: return C1.rotr(rotateAmt: C2);
6669 case ISD::SMIN: return C1.sle(RHS: C2) ? C1 : C2;
6670 case ISD::SMAX: return C1.sge(RHS: C2) ? C1 : C2;
6671 case ISD::UMIN: return C1.ule(RHS: C2) ? C1 : C2;
6672 case ISD::UMAX: return C1.uge(RHS: C2) ? C1 : C2;
6673 case ISD::SADDSAT: return C1.sadd_sat(RHS: C2);
6674 case ISD::UADDSAT: return C1.uadd_sat(RHS: C2);
6675 case ISD::SSUBSAT: return C1.ssub_sat(RHS: C2);
6676 case ISD::USUBSAT: return C1.usub_sat(RHS: C2);
6677 case ISD::SSHLSAT: return C1.sshl_sat(RHS: C2);
6678 case ISD::USHLSAT: return C1.ushl_sat(RHS: C2);
6679 case ISD::UDIV:
6680 if (!C2.getBoolValue())
6681 break;
6682 return C1.udiv(RHS: C2);
6683 case ISD::UREM:
6684 if (!C2.getBoolValue())
6685 break;
6686 return C1.urem(RHS: C2);
6687 case ISD::SDIV:
6688 if (!C2.getBoolValue())
6689 break;
6690 return C1.sdiv(RHS: C2);
6691 case ISD::SREM:
6692 if (!C2.getBoolValue())
6693 break;
6694 return C1.srem(RHS: C2);
6695 case ISD::AVGFLOORS:
6696 return APIntOps::avgFloorS(C1, C2);
6697 case ISD::AVGFLOORU:
6698 return APIntOps::avgFloorU(C1, C2);
6699 case ISD::AVGCEILS:
6700 return APIntOps::avgCeilS(C1, C2);
6701 case ISD::AVGCEILU:
6702 return APIntOps::avgCeilU(C1, C2);
6703 case ISD::ABDS:
6704 return APIntOps::abds(A: C1, B: C2);
6705 case ISD::ABDU:
6706 return APIntOps::abdu(A: C1, B: C2);
6707 case ISD::MULHS:
6708 return APIntOps::mulhs(C1, C2);
6709 case ISD::MULHU:
6710 return APIntOps::mulhu(C1, C2);
6711 }
6712 return std::nullopt;
6713}
6714// Handle constant folding with UNDEF.
6715// TODO: Handle more cases.
6716static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
6717 bool IsUndef1, const APInt &C2,
6718 bool IsUndef2) {
6719 if (!(IsUndef1 || IsUndef2))
6720 return FoldValue(Opcode, C1, C2);
6721
6722 // Fold and(x, undef) -> 0
6723 // Fold mul(x, undef) -> 0
6724 if (Opcode == ISD::AND || Opcode == ISD::MUL)
6725 return APInt::getZero(numBits: C1.getBitWidth());
6726
6727 return std::nullopt;
6728}
6729
6730SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
6731 const GlobalAddressSDNode *GA,
6732 const SDNode *N2) {
6733 if (GA->getOpcode() != ISD::GlobalAddress)
6734 return SDValue();
6735 if (!TLI->isOffsetFoldingLegal(GA))
6736 return SDValue();
6737 auto *C2 = dyn_cast<ConstantSDNode>(Val: N2);
6738 if (!C2)
6739 return SDValue();
6740 int64_t Offset = C2->getSExtValue();
6741 switch (Opcode) {
6742 case ISD::ADD: break;
6743 case ISD::SUB: Offset = -uint64_t(Offset); break;
6744 default: return SDValue();
6745 }
6746 return getGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(C2), VT,
6747 Offset: GA->getOffset() + uint64_t(Offset));
6748}
6749
6750bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6751 switch (Opcode) {
6752 case ISD::SDIV:
6753 case ISD::UDIV:
6754 case ISD::SREM:
6755 case ISD::UREM: {
6756 // If a divisor is zero/undef or any element of a divisor vector is
6757 // zero/undef, the whole op is undef.
6758 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6759 SDValue Divisor = Ops[1];
6760 if (Divisor.isUndef() || isNullConstant(V: Divisor))
6761 return true;
6762
6763 return ISD::isBuildVectorOfConstantSDNodes(N: Divisor.getNode()) &&
6764 llvm::any_of(Range: Divisor->op_values(),
6765 P: [](SDValue V) { return V.isUndef() ||
6766 isNullConstant(V); });
6767 // TODO: Handle signed overflow.
6768 }
6769 // TODO: Handle oversized shifts.
6770 default:
6771 return false;
6772 }
6773}
6774
6775SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
6776 EVT VT, ArrayRef<SDValue> Ops,
6777 SDNodeFlags Flags) {
6778 // If the opcode is a target-specific ISD node, there's nothing we can
6779 // do here and the operand rules may not line up with the below, so
6780 // bail early.
6781 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
6782 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6783 // foldCONCAT_VECTORS in getNode before this is called.
6784 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6785 return SDValue();
6786
6787 unsigned NumOps = Ops.size();
6788 if (NumOps == 0)
6789 return SDValue();
6790
6791 if (isUndef(Opcode, Ops))
6792 return getUNDEF(VT);
6793
6794 // Handle unary special cases.
6795 if (NumOps == 1) {
6796 SDValue N1 = Ops[0];
6797
6798 // Constant fold unary operations with an integer constant operand. Even
6799 // opaque constant will be folded, because the folding of unary operations
6800 // doesn't create new constants with different values. Nevertheless, the
6801 // opaque flag is preserved during folding to prevent future folding with
6802 // other constants.
6803 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N1)) {
6804 const APInt &Val = C->getAPIntValue();
6805 switch (Opcode) {
6806 case ISD::SIGN_EXTEND:
6807 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6808 isT: C->isTargetOpcode(), isO: C->isOpaque());
6809 case ISD::TRUNCATE:
6810 if (C->isOpaque())
6811 break;
6812 [[fallthrough]];
6813 case ISD::ZERO_EXTEND:
6814 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6815 isT: C->isTargetOpcode(), isO: C->isOpaque());
6816 case ISD::ANY_EXTEND:
6817 // Some targets like RISCV prefer to sign extend some types.
6818 if (TLI->isSExtCheaperThanZExt(FromTy: N1.getValueType(), ToTy: VT))
6819 return getConstant(Val: Val.sextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6820 isT: C->isTargetOpcode(), isO: C->isOpaque());
6821 return getConstant(Val: Val.zextOrTrunc(width: VT.getSizeInBits()), DL, VT,
6822 isT: C->isTargetOpcode(), isO: C->isOpaque());
6823 case ISD::ABS:
6824 return getConstant(Val: Val.abs(), DL, VT, isT: C->isTargetOpcode(),
6825 isO: C->isOpaque());
6826 case ISD::BITREVERSE:
6827 return getConstant(Val: Val.reverseBits(), DL, VT, isT: C->isTargetOpcode(),
6828 isO: C->isOpaque());
6829 case ISD::BSWAP:
6830 return getConstant(Val: Val.byteSwap(), DL, VT, isT: C->isTargetOpcode(),
6831 isO: C->isOpaque());
6832 case ISD::CTPOP:
6833 return getConstant(Val: Val.popcount(), DL, VT, isT: C->isTargetOpcode(),
6834 isO: C->isOpaque());
6835 case ISD::CTLZ:
6836 case ISD::CTLZ_ZERO_UNDEF:
6837 return getConstant(Val: Val.countl_zero(), DL, VT, isT: C->isTargetOpcode(),
6838 isO: C->isOpaque());
6839 case ISD::CTTZ:
6840 case ISD::CTTZ_ZERO_UNDEF:
6841 return getConstant(Val: Val.countr_zero(), DL, VT, isT: C->isTargetOpcode(),
6842 isO: C->isOpaque());
6843 case ISD::UINT_TO_FP:
6844 case ISD::SINT_TO_FP: {
6845 APFloat FPV(VT.getFltSemantics(), APInt::getZero(numBits: VT.getSizeInBits()));
6846 (void)FPV.convertFromAPInt(Input: Val, IsSigned: Opcode == ISD::SINT_TO_FP,
6847 RM: APFloat::rmNearestTiesToEven);
6848 return getConstantFP(V: FPV, DL, VT);
6849 }
6850 case ISD::FP16_TO_FP:
6851 case ISD::BF16_TO_FP: {
6852 bool Ignored;
6853 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6854 : APFloat::BFloat(),
6855 (Val.getBitWidth() == 16) ? Val : Val.trunc(width: 16));
6856
6857 // This can return overflow, underflow, or inexact; we don't care.
6858 // FIXME need to be more flexible about rounding mode.
6859 (void)FPV.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
6860 losesInfo: &Ignored);
6861 return getConstantFP(V: FPV, DL, VT);
6862 }
6863 case ISD::STEP_VECTOR:
6864 if (SDValue V = FoldSTEP_VECTOR(DL, VT, Step: N1, DAG&: *this))
6865 return V;
6866 break;
6867 case ISD::BITCAST:
6868 if (VT == MVT::f16 && C->getValueType(ResNo: 0) == MVT::i16)
6869 return getConstantFP(V: APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6870 if (VT == MVT::f32 && C->getValueType(ResNo: 0) == MVT::i32)
6871 return getConstantFP(V: APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6872 if (VT == MVT::f64 && C->getValueType(ResNo: 0) == MVT::i64)
6873 return getConstantFP(V: APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6874 if (VT == MVT::f128 && C->getValueType(ResNo: 0) == MVT::i128)
6875 return getConstantFP(V: APFloat(APFloat::IEEEquad(), Val), DL, VT);
6876 break;
6877 }
6878 }
6879
6880 // Constant fold unary operations with a floating point constant operand.
6881 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: N1)) {
6882 APFloat V = C->getValueAPF(); // make copy
6883 switch (Opcode) {
6884 case ISD::FNEG:
6885 V.changeSign();
6886 return getConstantFP(V, DL, VT);
6887 case ISD::FABS:
6888 V.clearSign();
6889 return getConstantFP(V, DL, VT);
6890 case ISD::FCEIL: {
6891 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardPositive);
6892 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6893 return getConstantFP(V, DL, VT);
6894 return SDValue();
6895 }
6896 case ISD::FTRUNC: {
6897 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardZero);
6898 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6899 return getConstantFP(V, DL, VT);
6900 return SDValue();
6901 }
6902 case ISD::FFLOOR: {
6903 APFloat::opStatus fs = V.roundToIntegral(RM: APFloat::rmTowardNegative);
6904 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6905 return getConstantFP(V, DL, VT);
6906 return SDValue();
6907 }
6908 case ISD::FP_EXTEND: {
6909 bool ignored;
6910 // This can return overflow, underflow, or inexact; we don't care.
6911 // FIXME need to be more flexible about rounding mode.
6912 (void)V.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
6913 losesInfo: &ignored);
6914 return getConstantFP(V, DL, VT);
6915 }
6916 case ISD::FP_TO_SINT:
6917 case ISD::FP_TO_UINT: {
6918 bool ignored;
6919 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6920 // FIXME need to be more flexible about rounding mode.
6921 APFloat::opStatus s =
6922 V.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &ignored);
6923 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6924 break;
6925 return getConstant(Val: IntVal, DL, VT);
6926 }
6927 case ISD::FP_TO_FP16:
6928 case ISD::FP_TO_BF16: {
6929 bool Ignored;
6930 // This can return overflow, underflow, or inexact; we don't care.
6931 // FIXME need to be more flexible about rounding mode.
6932 (void)V.convert(ToSemantics: Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6933 : APFloat::BFloat(),
6934 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
6935 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
6936 }
6937 case ISD::BITCAST:
6938 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::f16)
6939 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6940 VT);
6941 if (VT == MVT::i16 && C->getValueType(ResNo: 0) == MVT::bf16)
6942 return getConstant(Val: (uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6943 VT);
6944 if (VT == MVT::i32 && C->getValueType(ResNo: 0) == MVT::f32)
6945 return getConstant(Val: (uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6946 VT);
6947 if (VT == MVT::i64 && C->getValueType(ResNo: 0) == MVT::f64)
6948 return getConstant(Val: V.bitcastToAPInt().getZExtValue(), DL, VT);
6949 break;
6950 }
6951 }
6952
6953 // Early-out if we failed to constant fold a bitcast.
6954 if (Opcode == ISD::BITCAST)
6955 return SDValue();
6956 }
6957
6958 // Handle binops special cases.
6959 if (NumOps == 2) {
6960 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6961 return CFP;
6962
6963 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
6964 if (auto *C2 = dyn_cast<ConstantSDNode>(Val: Ops[1])) {
6965 if (C1->isOpaque() || C2->isOpaque())
6966 return SDValue();
6967
6968 std::optional<APInt> FoldAttempt =
6969 FoldValue(Opcode, C1: C1->getAPIntValue(), C2: C2->getAPIntValue());
6970 if (!FoldAttempt)
6971 return SDValue();
6972
6973 SDValue Folded = getConstant(Val: *FoldAttempt, DL, VT);
6974 assert((!Folded || !VT.isVector()) &&
6975 "Can't fold vectors ops with scalar operands");
6976 return Folded;
6977 }
6978 }
6979
6980 // fold (add Sym, c) -> Sym+c
6981 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[0]))
6982 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[1].getNode());
6983 if (TLI->isCommutativeBinOp(Opcode))
6984 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val: Ops[1]))
6985 return FoldSymbolOffset(Opcode, VT, GA, N2: Ops[0].getNode());
6986
6987 // fold (sext_in_reg c1) -> c2
6988 if (Opcode == ISD::SIGN_EXTEND_INREG) {
6989 EVT EVT = cast<VTSDNode>(Val: Ops[1])->getVT();
6990
6991 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6992 unsigned FromBits = EVT.getScalarSizeInBits();
6993 Val <<= Val.getBitWidth() - FromBits;
6994 Val.ashrInPlace(ShiftAmt: Val.getBitWidth() - FromBits);
6995 return getConstant(Val, DL, VT: ConstantVT);
6996 };
6997
6998 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ops[0])) {
6999 const APInt &Val = C1->getAPIntValue();
7000 return SignExtendInReg(Val, VT);
7001 }
7002
7003 if (ISD::isBuildVectorOfConstantSDNodes(N: Ops[0].getNode())) {
7004 SmallVector<SDValue, 8> ScalarOps;
7005 llvm::EVT OpVT = Ops[0].getOperand(i: 0).getValueType();
7006 for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I) {
7007 SDValue Op = Ops[0].getOperand(i: I);
7008 if (Op.isUndef()) {
7009 ScalarOps.push_back(Elt: getUNDEF(VT: OpVT));
7010 continue;
7011 }
7012 const APInt &Val = cast<ConstantSDNode>(Val&: Op)->getAPIntValue();
7013 ScalarOps.push_back(Elt: SignExtendInReg(Val, OpVT));
7014 }
7015 return getBuildVector(VT, DL, Ops: ScalarOps);
7016 }
7017
7018 if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR &&
7019 isa<ConstantSDNode>(Val: Ops[0].getOperand(i: 0)))
7020 return getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT,
7021 N1: SignExtendInReg(Ops[0].getConstantOperandAPInt(i: 0),
7022 Ops[0].getOperand(i: 0).getValueType()));
7023 }
7024 }
7025
7026 // This is for vector folding only from here on.
7027 if (!VT.isVector())
7028 return SDValue();
7029
7030 ElementCount NumElts = VT.getVectorElementCount();
7031
7032 // See if we can fold through any bitcasted integer ops.
7033 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
7034 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
7035 (Ops[0].getOpcode() == ISD::BITCAST ||
7036 Ops[1].getOpcode() == ISD::BITCAST)) {
7037 SDValue N1 = peekThroughBitcasts(V: Ops[0]);
7038 SDValue N2 = peekThroughBitcasts(V: Ops[1]);
7039 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
7040 auto *BV2 = dyn_cast<BuildVectorSDNode>(Val&: N2);
7041 if (BV1 && BV2 && N1.getValueType().isInteger() &&
7042 N2.getValueType().isInteger()) {
7043 bool IsLE = getDataLayout().isLittleEndian();
7044 unsigned EltBits = VT.getScalarSizeInBits();
7045 SmallVector<APInt> RawBits1, RawBits2;
7046 BitVector UndefElts1, UndefElts2;
7047 if (BV1->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits1, UndefElements&: UndefElts1) &&
7048 BV2->getConstantRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: EltBits, RawBitElements&: RawBits2, UndefElements&: UndefElts2)) {
7049 SmallVector<APInt> RawBits;
7050 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
7051 std::optional<APInt> Fold = FoldValueWithUndef(
7052 Opcode, C1: RawBits1[I], IsUndef1: UndefElts1[I], C2: RawBits2[I], IsUndef2: UndefElts2[I]);
7053 if (!Fold)
7054 break;
7055 RawBits.push_back(Elt: *Fold);
7056 }
7057 if (RawBits.size() == NumElts.getFixedValue()) {
7058 // We have constant folded, but we might need to cast this again back
7059 // to the original (possibly legalized) type.
7060 EVT BVVT, BVEltVT;
7061 if (N1.getValueType() == VT) {
7062 BVVT = N1.getValueType();
7063 BVEltVT = BV1->getOperand(Num: 0).getValueType();
7064 } else {
7065 BVVT = N2.getValueType();
7066 BVEltVT = BV2->getOperand(Num: 0).getValueType();
7067 }
7068 unsigned BVEltBits = BVEltVT.getSizeInBits();
7069 SmallVector<APInt> DstBits;
7070 BitVector DstUndefs;
7071 BuildVectorSDNode::recastRawBits(IsLittleEndian: IsLE, DstEltSizeInBits: BVVT.getScalarSizeInBits(),
7072 DstBitElements&: DstBits, SrcBitElements: RawBits, DstUndefElements&: DstUndefs,
7073 SrcUndefElements: BitVector(RawBits.size(), false));
7074 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(VT: BVEltVT));
7075 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
7076 if (DstUndefs[I])
7077 continue;
7078 Ops[I] = getConstant(Val: DstBits[I].sext(width: BVEltBits), DL, VT: BVEltVT);
7079 }
7080 return getBitcast(VT, V: getBuildVector(VT: BVVT, DL, Ops));
7081 }
7082 }
7083 }
7084 }
7085
7086 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
7087 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
7088 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
7089 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
7090 APInt RHSVal;
7091 if (ISD::isConstantSplatVector(N: Ops[1].getNode(), SplatVal&: RHSVal)) {
7092 APInt NewStep = Opcode == ISD::MUL
7093 ? Ops[0].getConstantOperandAPInt(i: 0) * RHSVal
7094 : Ops[0].getConstantOperandAPInt(i: 0) << RHSVal;
7095 return getStepVector(DL, ResVT: VT, StepVal: NewStep);
7096 }
7097 }
7098
7099 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
7100 return !Op.getValueType().isVector() ||
7101 Op.getValueType().getVectorElementCount() == NumElts;
7102 };
7103
7104 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
7105 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
7106 Op.getOpcode() == ISD::BUILD_VECTOR ||
7107 Op.getOpcode() == ISD::SPLAT_VECTOR;
7108 };
7109
7110 // All operands must be vector types with the same number of elements as
7111 // the result type and must be either UNDEF or a build/splat vector
7112 // or UNDEF scalars.
7113 if (!llvm::all_of(Range&: Ops, P: IsBuildVectorSplatVectorOrUndef) ||
7114 !llvm::all_of(Range&: Ops, P: IsScalarOrSameVectorSize))
7115 return SDValue();
7116
7117 // If we are comparing vectors, then the result needs to be a i1 boolean that
7118 // is then extended back to the legal result type depending on how booleans
7119 // are represented.
7120 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
7121 ISD::NodeType ExtendCode =
7122 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
7123 ? TargetLowering::getExtendForContent(Content: TLI->getBooleanContents(Type: VT))
7124 : ISD::SIGN_EXTEND;
7125
7126 // Find legal integer scalar type for constant promotion and
7127 // ensure that its scalar size is at least as large as source.
7128 EVT LegalSVT = VT.getScalarType();
7129 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
7130 LegalSVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT: LegalSVT);
7131 if (LegalSVT.bitsLT(VT: VT.getScalarType()))
7132 return SDValue();
7133 }
7134
7135 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
7136 // only have one operand to check. For fixed-length vector types we may have
7137 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
7138 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
7139
7140 // Constant fold each scalar lane separately.
7141 SmallVector<SDValue, 4> ScalarResults;
7142 for (unsigned I = 0; I != NumVectorElts; I++) {
7143 SmallVector<SDValue, 4> ScalarOps;
7144 for (SDValue Op : Ops) {
7145 EVT InSVT = Op.getValueType().getScalarType();
7146 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
7147 Op.getOpcode() != ISD::SPLAT_VECTOR) {
7148 if (Op.isUndef())
7149 ScalarOps.push_back(Elt: getUNDEF(VT: InSVT));
7150 else
7151 ScalarOps.push_back(Elt: Op);
7152 continue;
7153 }
7154
7155 SDValue ScalarOp =
7156 Op.getOperand(i: Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
7157 EVT ScalarVT = ScalarOp.getValueType();
7158
7159 // Build vector (integer) scalar operands may need implicit
7160 // truncation - do this before constant folding.
7161 if (ScalarVT.isInteger() && ScalarVT.bitsGT(VT: InSVT)) {
7162 // Don't create illegally-typed nodes unless they're constants or undef
7163 // - if we fail to constant fold we can't guarantee the (dead) nodes
7164 // we're creating will be cleaned up before being visited for
7165 // legalization.
7166 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
7167 !isa<ConstantSDNode>(Val: ScalarOp) &&
7168 TLI->getTypeAction(Context&: *getContext(), VT: InSVT) !=
7169 TargetLowering::TypeLegal)
7170 return SDValue();
7171 ScalarOp = getNode(Opcode: ISD::TRUNCATE, DL, VT: InSVT, N1: ScalarOp);
7172 }
7173
7174 ScalarOps.push_back(Elt: ScalarOp);
7175 }
7176
7177 // Constant fold the scalar operands.
7178 SDValue ScalarResult = getNode(Opcode, DL, VT: SVT, Ops: ScalarOps, Flags);
7179
7180 // Scalar folding only succeeded if the result is a constant or UNDEF.
7181 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
7182 ScalarResult.getOpcode() != ISD::ConstantFP)
7183 return SDValue();
7184
7185 // Legalize the (integer) scalar constant if necessary. We only do
7186 // this once we know the folding succeeded, since otherwise we would
7187 // get a node with illegal type which has a user.
7188 if (LegalSVT != SVT)
7189 ScalarResult = getNode(Opcode: ExtendCode, DL, VT: LegalSVT, N1: ScalarResult);
7190
7191 ScalarResults.push_back(Elt: ScalarResult);
7192 }
7193
7194 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, Op: ScalarResults[0])
7195 : getBuildVector(VT, DL, Ops: ScalarResults);
7196 NewSDValueDbgMsg(V, Msg: "New node fold constant vector: ", G: this);
7197 return V;
7198}
7199
7200SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL,
7201 EVT VT, ArrayRef<SDValue> Ops) {
7202 // TODO: Add support for unary/ternary fp opcodes.
7203 if (Ops.size() != 2)
7204 return SDValue();
7205
7206 // TODO: We don't do any constant folding for strict FP opcodes here, but we
7207 // should. That will require dealing with a potentially non-default
7208 // rounding mode, checking the "opStatus" return value from the APFloat
7209 // math calculations, and possibly other variations.
7210 SDValue N1 = Ops[0];
7211 SDValue N2 = Ops[1];
7212 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ false);
7213 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N: N2, /*AllowUndefs*/ false);
7214 if (N1CFP && N2CFP) {
7215 APFloat C1 = N1CFP->getValueAPF(); // make copy
7216 const APFloat &C2 = N2CFP->getValueAPF();
7217 switch (Opcode) {
7218 case ISD::FADD:
7219 C1.add(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7220 return getConstantFP(V: C1, DL, VT);
7221 case ISD::FSUB:
7222 C1.subtract(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7223 return getConstantFP(V: C1, DL, VT);
7224 case ISD::FMUL:
7225 C1.multiply(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7226 return getConstantFP(V: C1, DL, VT);
7227 case ISD::FDIV:
7228 C1.divide(RHS: C2, RM: APFloat::rmNearestTiesToEven);
7229 return getConstantFP(V: C1, DL, VT);
7230 case ISD::FREM:
7231 C1.mod(RHS: C2);
7232 return getConstantFP(V: C1, DL, VT);
7233 case ISD::FCOPYSIGN:
7234 C1.copySign(RHS: C2);
7235 return getConstantFP(V: C1, DL, VT);
7236 case ISD::FMINNUM:
7237 return getConstantFP(V: minnum(A: C1, B: C2), DL, VT);
7238 case ISD::FMAXNUM:
7239 return getConstantFP(V: maxnum(A: C1, B: C2), DL, VT);
7240 case ISD::FMINIMUM:
7241 return getConstantFP(V: minimum(A: C1, B: C2), DL, VT);
7242 case ISD::FMAXIMUM:
7243 return getConstantFP(V: maximum(A: C1, B: C2), DL, VT);
7244 case ISD::FMINIMUMNUM:
7245 return getConstantFP(V: minimumnum(A: C1, B: C2), DL, VT);
7246 case ISD::FMAXIMUMNUM:
7247 return getConstantFP(V: maximumnum(A: C1, B: C2), DL, VT);
7248 default: break;
7249 }
7250 }
7251 if (N1CFP && Opcode == ISD::FP_ROUND) {
7252 APFloat C1 = N1CFP->getValueAPF(); // make copy
7253 bool Unused;
7254 // This can return overflow, underflow, or inexact; we don't care.
7255 // FIXME need to be more flexible about rounding mode.
7256 (void)C1.convert(ToSemantics: VT.getFltSemantics(), RM: APFloat::rmNearestTiesToEven,
7257 losesInfo: &Unused);
7258 return getConstantFP(V: C1, DL, VT);
7259 }
7260
7261 switch (Opcode) {
7262 case ISD::FSUB:
7263 // -0.0 - undef --> undef (consistent with "fneg undef")
7264 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, /*AllowUndefs*/ true))
7265 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
7266 return getUNDEF(VT);
7267 [[fallthrough]];
7268
7269 case ISD::FADD:
7270 case ISD::FMUL:
7271 case ISD::FDIV:
7272 case ISD::FREM:
7273 // If both operands are undef, the result is undef. If 1 operand is undef,
7274 // the result is NaN. This should match the behavior of the IR optimizer.
7275 if (N1.isUndef() && N2.isUndef())
7276 return getUNDEF(VT);
7277 if (N1.isUndef() || N2.isUndef())
7278 return getConstantFP(V: APFloat::getNaN(Sem: VT.getFltSemantics()), DL, VT);
7279 }
7280 return SDValue();
7281}
7282
7283SDValue SelectionDAG::getAssertAlign(const SDLoc &DL, SDValue Val, Align A) {
7284 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
7285
7286 // There's no need to assert on a byte-aligned pointer. All pointers are at
7287 // least byte aligned.
7288 if (A == Align(1))
7289 return Val;
7290
7291 SDVTList VTs = getVTList(VT: Val.getValueType());
7292 FoldingSetNodeID ID;
7293 AddNodeIDNode(ID, OpC: ISD::AssertAlign, VTList: VTs, OpList: {Val});
7294 ID.AddInteger(I: A.value());
7295
7296 void *IP = nullptr;
7297 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP))
7298 return SDValue(E, 0);
7299
7300 auto *N =
7301 newSDNode<AssertAlignSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: A);
7302 createOperands(Node: N, Vals: {Val});
7303
7304 CSEMap.InsertNode(N, InsertPos: IP);
7305 InsertNode(N);
7306
7307 SDValue V(N, 0);
7308 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7309 return V;
7310}
7311
7312SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7313 SDValue N1, SDValue N2) {
7314 SDNodeFlags Flags;
7315 if (Inserter)
7316 Flags = Inserter->getFlags();
7317 return getNode(Opcode, DL, VT, N1, N2, Flags);
7318}
7319
7320void SelectionDAG::canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1,
7321 SDValue &N2) const {
7322 if (!TLI->isCommutativeBinOp(Opcode))
7323 return;
7324
7325 // Canonicalize:
7326 // binop(const, nonconst) -> binop(nonconst, const)
7327 bool N1C = isConstantIntBuildVectorOrConstantInt(N: N1);
7328 bool N2C = isConstantIntBuildVectorOrConstantInt(N: N2);
7329 bool N1CFP = isConstantFPBuildVectorOrConstantFP(N: N1);
7330 bool N2CFP = isConstantFPBuildVectorOrConstantFP(N: N2);
7331 if ((N1C && !N2C) || (N1CFP && !N2CFP))
7332 std::swap(a&: N1, b&: N2);
7333
7334 // Canonicalize:
7335 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
7336 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
7337 N2.getOpcode() == ISD::STEP_VECTOR)
7338 std::swap(a&: N1, b&: N2);
7339}
7340
7341SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7342 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
7343 assert(N1.getOpcode() != ISD::DELETED_NODE &&
7344 N2.getOpcode() != ISD::DELETED_NODE &&
7345 "Operand is DELETED_NODE!");
7346
7347 canonicalizeCommutativeBinop(Opcode, N1, N2);
7348
7349 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
7350 auto *N2C = dyn_cast<ConstantSDNode>(Val&: N2);
7351
7352 // Don't allow undefs in vector splats - we might be returning N2 when folding
7353 // to zero etc.
7354 ConstantSDNode *N2CV =
7355 isConstOrConstSplat(N: N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
7356
7357 switch (Opcode) {
7358 default: break;
7359 case ISD::TokenFactor:
7360 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
7361 N2.getValueType() == MVT::Other && "Invalid token factor!");
7362 // Fold trivial token factors.
7363 if (N1.getOpcode() == ISD::EntryToken) return N2;
7364 if (N2.getOpcode() == ISD::EntryToken) return N1;
7365 if (N1 == N2) return N1;
7366 break;
7367 case ISD::BUILD_VECTOR: {
7368 // Attempt to simplify BUILD_VECTOR.
7369 SDValue Ops[] = {N1, N2};
7370 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7371 return V;
7372 break;
7373 }
7374 case ISD::CONCAT_VECTORS: {
7375 SDValue Ops[] = {N1, N2};
7376 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
7377 return V;
7378 break;
7379 }
7380 case ISD::AND:
7381 assert(VT.isInteger() && "This operator does not apply to FP types!");
7382 assert(N1.getValueType() == N2.getValueType() &&
7383 N1.getValueType() == VT && "Binary operator types must match!");
7384 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
7385 // worth handling here.
7386 if (N2CV && N2CV->isZero())
7387 return N2;
7388 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
7389 return N1;
7390 break;
7391 case ISD::OR:
7392 case ISD::XOR:
7393 case ISD::ADD:
7394 case ISD::PTRADD:
7395 case ISD::SUB:
7396 assert(VT.isInteger() && "This operator does not apply to FP types!");
7397 assert(N1.getValueType() == N2.getValueType() &&
7398 N1.getValueType() == VT && "Binary operator types must match!");
7399 // The equal operand types requirement is unnecessarily strong for PTRADD.
7400 // However, the SelectionDAGBuilder does not generate PTRADDs with different
7401 // operand types, and we'd need to re-implement GEP's non-standard wrapping
7402 // logic everywhere where PTRADDs may be folded or combined to properly
7403 // support them. If/when we introduce pointer types to the SDAG, we will
7404 // need to relax this constraint.
7405
7406 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
7407 // it's worth handling here.
7408 if (N2CV && N2CV->isZero())
7409 return N1;
7410 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
7411 VT.getScalarType() == MVT::i1)
7412 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
7413 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
7414 if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
7415 N2.getOpcode() == ISD::VSCALE) {
7416 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
7417 const APInt &C2 = N2->getConstantOperandAPInt(Num: 0);
7418 return getVScale(DL, VT, MulImm: C1 + C2);
7419 }
7420 break;
7421 case ISD::MUL:
7422 assert(VT.isInteger() && "This operator does not apply to FP types!");
7423 assert(N1.getValueType() == N2.getValueType() &&
7424 N1.getValueType() == VT && "Binary operator types must match!");
7425 if (VT.getScalarType() == MVT::i1)
7426 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
7427 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7428 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
7429 const APInt &N2CImm = N2C->getAPIntValue();
7430 return getVScale(DL, VT, MulImm: MulImm * N2CImm);
7431 }
7432 break;
7433 case ISD::UDIV:
7434 case ISD::UREM:
7435 case ISD::MULHU:
7436 case ISD::MULHS:
7437 case ISD::SDIV:
7438 case ISD::SREM:
7439 case ISD::SADDSAT:
7440 case ISD::SSUBSAT:
7441 case ISD::UADDSAT:
7442 case ISD::USUBSAT:
7443 assert(VT.isInteger() && "This operator does not apply to FP types!");
7444 assert(N1.getValueType() == N2.getValueType() &&
7445 N1.getValueType() == VT && "Binary operator types must match!");
7446 if (VT.getScalarType() == MVT::i1) {
7447 // fold (add_sat x, y) -> (or x, y) for bool types.
7448 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
7449 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
7450 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
7451 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
7452 return getNode(Opcode: ISD::AND, DL, VT, N1, N2: getNOT(DL, Val: N2, VT));
7453 }
7454 break;
7455 case ISD::SCMP:
7456 case ISD::UCMP:
7457 assert(N1.getValueType() == N2.getValueType() &&
7458 "Types of operands of UCMP/SCMP must match");
7459 assert(N1.getValueType().isVector() == VT.isVector() &&
7460 "Operands and return type of must both be scalars or vectors");
7461 if (VT.isVector())
7462 assert(VT.getVectorElementCount() ==
7463 N1.getValueType().getVectorElementCount() &&
7464 "Result and operands must have the same number of elements");
7465 break;
7466 case ISD::AVGFLOORS:
7467 case ISD::AVGFLOORU:
7468 case ISD::AVGCEILS:
7469 case ISD::AVGCEILU:
7470 assert(VT.isInteger() && "This operator does not apply to FP types!");
7471 assert(N1.getValueType() == N2.getValueType() &&
7472 N1.getValueType() == VT && "Binary operator types must match!");
7473 break;
7474 case ISD::ABDS:
7475 case ISD::ABDU:
7476 assert(VT.isInteger() && "This operator does not apply to FP types!");
7477 assert(N1.getValueType() == N2.getValueType() &&
7478 N1.getValueType() == VT && "Binary operator types must match!");
7479 if (VT.getScalarType() == MVT::i1)
7480 return getNode(Opcode: ISD::XOR, DL, VT, N1, N2);
7481 break;
7482 case ISD::SMIN:
7483 case ISD::UMAX:
7484 assert(VT.isInteger() && "This operator does not apply to FP types!");
7485 assert(N1.getValueType() == N2.getValueType() &&
7486 N1.getValueType() == VT && "Binary operator types must match!");
7487 if (VT.getScalarType() == MVT::i1)
7488 return getNode(Opcode: ISD::OR, DL, VT, N1, N2);
7489 break;
7490 case ISD::SMAX:
7491 case ISD::UMIN:
7492 assert(VT.isInteger() && "This operator does not apply to FP types!");
7493 assert(N1.getValueType() == N2.getValueType() &&
7494 N1.getValueType() == VT && "Binary operator types must match!");
7495 if (VT.getScalarType() == MVT::i1)
7496 return getNode(Opcode: ISD::AND, DL, VT, N1, N2);
7497 break;
7498 case ISD::FADD:
7499 case ISD::FSUB:
7500 case ISD::FMUL:
7501 case ISD::FDIV:
7502 case ISD::FREM:
7503 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7504 assert(N1.getValueType() == N2.getValueType() &&
7505 N1.getValueType() == VT && "Binary operator types must match!");
7506 if (SDValue V = simplifyFPBinop(Opcode, X: N1, Y: N2, Flags))
7507 return V;
7508 break;
7509 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
7510 assert(N1.getValueType() == VT &&
7511 N1.getValueType().isFloatingPoint() &&
7512 N2.getValueType().isFloatingPoint() &&
7513 "Invalid FCOPYSIGN!");
7514 break;
7515 case ISD::SHL:
7516 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
7517 const APInt &MulImm = N1->getConstantOperandAPInt(Num: 0);
7518 const APInt &ShiftImm = N2C->getAPIntValue();
7519 return getVScale(DL, VT, MulImm: MulImm << ShiftImm);
7520 }
7521 [[fallthrough]];
7522 case ISD::SRA:
7523 case ISD::SRL:
7524 if (SDValue V = simplifyShift(X: N1, Y: N2))
7525 return V;
7526 [[fallthrough]];
7527 case ISD::ROTL:
7528 case ISD::ROTR:
7529 assert(VT == N1.getValueType() &&
7530 "Shift operators return type must be the same as their first arg");
7531 assert(VT.isInteger() && N2.getValueType().isInteger() &&
7532 "Shifts only work on integers");
7533 assert((!VT.isVector() || VT == N2.getValueType()) &&
7534 "Vector shift amounts must be in the same as their first arg");
7535 // Verify that the shift amount VT is big enough to hold valid shift
7536 // amounts. This catches things like trying to shift an i1024 value by an
7537 // i8, which is easy to fall into in generic code that uses
7538 // TLI.getShiftAmount().
7539 assert(N2.getValueType().getScalarSizeInBits() >=
7540 Log2_32_Ceil(VT.getScalarSizeInBits()) &&
7541 "Invalid use of small shift amount with oversized value!");
7542
7543 // Always fold shifts of i1 values so the code generator doesn't need to
7544 // handle them. Since we know the size of the shift has to be less than the
7545 // size of the value, the shift/rotate count is guaranteed to be zero.
7546 if (VT == MVT::i1)
7547 return N1;
7548 if (N2CV && N2CV->isZero())
7549 return N1;
7550 break;
7551 case ISD::FP_ROUND:
7552 assert(VT.isFloatingPoint() && N1.getValueType().isFloatingPoint() &&
7553 VT.bitsLE(N1.getValueType()) && N2C &&
7554 (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
7555 N2.getOpcode() == ISD::TargetConstant && "Invalid FP_ROUND!");
7556 if (N1.getValueType() == VT) return N1; // noop conversion.
7557 break;
7558 case ISD::AssertNoFPClass: {
7559 assert(N1.getValueType().isFloatingPoint() &&
7560 "AssertNoFPClass is used for a non-floating type");
7561 assert(isa<ConstantSDNode>(N2) && "NoFPClass is not Constant");
7562 FPClassTest NoFPClass = static_cast<FPClassTest>(N2->getAsZExtVal());
7563 assert(llvm::to_underlying(NoFPClass) <=
7564 BitmaskEnumDetail::Mask<FPClassTest>() &&
7565 "FPClassTest value too large");
7566 (void)NoFPClass;
7567 break;
7568 }
7569 case ISD::AssertSext:
7570 case ISD::AssertZext: {
7571 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
7572 assert(VT == N1.getValueType() && "Not an inreg extend!");
7573 assert(VT.isInteger() && EVT.isInteger() &&
7574 "Cannot *_EXTEND_INREG FP types");
7575 assert(!EVT.isVector() &&
7576 "AssertSExt/AssertZExt type should be the vector element type "
7577 "rather than the vector type!");
7578 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
7579 if (VT.getScalarType() == EVT) return N1; // noop assertion.
7580 break;
7581 }
7582 case ISD::SIGN_EXTEND_INREG: {
7583 EVT EVT = cast<VTSDNode>(Val&: N2)->getVT();
7584 assert(VT == N1.getValueType() && "Not an inreg extend!");
7585 assert(VT.isInteger() && EVT.isInteger() &&
7586 "Cannot *_EXTEND_INREG FP types");
7587 assert(EVT.isVector() == VT.isVector() &&
7588 "SIGN_EXTEND_INREG type should be vector iff the operand "
7589 "type is vector!");
7590 assert((!EVT.isVector() ||
7591 EVT.getVectorElementCount() == VT.getVectorElementCount()) &&
7592 "Vector element counts must match in SIGN_EXTEND_INREG");
7593 assert(EVT.bitsLE(VT) && "Not extending!");
7594 if (EVT == VT) return N1; // Not actually extending
7595 break;
7596 }
7597 case ISD::FP_TO_SINT_SAT:
7598 case ISD::FP_TO_UINT_SAT: {
7599 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
7600 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
7601 assert(N1.getValueType().isVector() == VT.isVector() &&
7602 "FP_TO_*INT_SAT type should be vector iff the operand type is "
7603 "vector!");
7604 assert((!VT.isVector() || VT.getVectorElementCount() ==
7605 N1.getValueType().getVectorElementCount()) &&
7606 "Vector element counts must match in FP_TO_*INT_SAT");
7607 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
7608 "Type to saturate to must be a scalar.");
7609 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
7610 "Not extending!");
7611 break;
7612 }
7613 case ISD::EXTRACT_VECTOR_ELT:
7614 assert(VT.getSizeInBits() >= N1.getValueType().getScalarSizeInBits() &&
7615 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
7616 element type of the vector.");
7617
7618 // Extract from an undefined value or using an undefined index is undefined.
7619 if (N1.isUndef() || N2.isUndef())
7620 return getUNDEF(VT);
7621
7622 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
7623 // vectors. For scalable vectors we will provide appropriate support for
7624 // dealing with arbitrary indices.
7625 if (N2C && N1.getValueType().isFixedLengthVector() &&
7626 N2C->getAPIntValue().uge(RHS: N1.getValueType().getVectorNumElements()))
7627 return getUNDEF(VT);
7628
7629 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
7630 // expanding copies of large vectors from registers. This only works for
7631 // fixed length vectors, since we need to know the exact number of
7632 // elements.
7633 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
7634 N1.getOperand(i: 0).getValueType().isFixedLengthVector()) {
7635 unsigned Factor = N1.getOperand(i: 0).getValueType().getVectorNumElements();
7636 return getExtractVectorElt(DL, VT,
7637 Vec: N1.getOperand(i: N2C->getZExtValue() / Factor),
7638 Idx: N2C->getZExtValue() % Factor);
7639 }
7640
7641 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
7642 // lowering is expanding large vector constants.
7643 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
7644 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
7645 assert((N1.getOpcode() != ISD::BUILD_VECTOR ||
7646 N1.getValueType().isFixedLengthVector()) &&
7647 "BUILD_VECTOR used for scalable vectors");
7648 unsigned Index =
7649 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
7650 SDValue Elt = N1.getOperand(i: Index);
7651
7652 if (VT != Elt.getValueType())
7653 // If the vector element type is not legal, the BUILD_VECTOR operands
7654 // are promoted and implicitly truncated, and the result implicitly
7655 // extended. Make that explicit here.
7656 Elt = getAnyExtOrTrunc(Op: Elt, DL, VT);
7657
7658 return Elt;
7659 }
7660
7661 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
7662 // operations are lowered to scalars.
7663 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
7664 // If the indices are the same, return the inserted element else
7665 // if the indices are known different, extract the element from
7666 // the original vector.
7667 SDValue N1Op2 = N1.getOperand(i: 2);
7668 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(Val&: N1Op2);
7669
7670 if (N1Op2C && N2C) {
7671 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
7672 if (VT == N1.getOperand(i: 1).getValueType())
7673 return N1.getOperand(i: 1);
7674 if (VT.isFloatingPoint()) {
7675 assert(VT.getSizeInBits() > N1.getOperand(1).getValueType().getSizeInBits());
7676 return getFPExtendOrRound(Op: N1.getOperand(i: 1), DL, VT);
7677 }
7678 return getSExtOrTrunc(Op: N1.getOperand(i: 1), DL, VT);
7679 }
7680 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0), N2);
7681 }
7682 }
7683
7684 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
7685 // when vector types are scalarized and v1iX is legal.
7686 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
7687 // Here we are completely ignoring the extract element index (N2),
7688 // which is fine for fixed width vectors, since any index other than 0
7689 // is undefined anyway. However, this cannot be ignored for scalable
7690 // vectors - in theory we could support this, but we don't want to do this
7691 // without a profitability check.
7692 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7693 N1.getValueType().isFixedLengthVector() &&
7694 N1.getValueType().getVectorNumElements() == 1) {
7695 return getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: N1.getOperand(i: 0),
7696 N2: N1.getOperand(i: 1));
7697 }
7698 break;
7699 case ISD::EXTRACT_ELEMENT:
7700 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
7701 assert(!N1.getValueType().isVector() && !VT.isVector() &&
7702 (N1.getValueType().isInteger() == VT.isInteger()) &&
7703 N1.getValueType() != VT &&
7704 "Wrong types for EXTRACT_ELEMENT!");
7705
7706 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
7707 // 64-bit integers into 32-bit parts. Instead of building the extract of
7708 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
7709 if (N1.getOpcode() == ISD::BUILD_PAIR)
7710 return N1.getOperand(i: N2C->getZExtValue());
7711
7712 // EXTRACT_ELEMENT of a constant int is also very common.
7713 if (N1C) {
7714 unsigned ElementSize = VT.getSizeInBits();
7715 unsigned Shift = ElementSize * N2C->getZExtValue();
7716 const APInt &Val = N1C->getAPIntValue();
7717 return getConstant(Val: Val.extractBits(numBits: ElementSize, bitPosition: Shift), DL, VT);
7718 }
7719 break;
7720 case ISD::EXTRACT_SUBVECTOR: {
7721 EVT N1VT = N1.getValueType();
7722 assert(VT.isVector() && N1VT.isVector() &&
7723 "Extract subvector VTs must be vectors!");
7724 assert(VT.getVectorElementType() == N1VT.getVectorElementType() &&
7725 "Extract subvector VTs must have the same element type!");
7726 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
7727 "Cannot extract a scalable vector from a fixed length vector!");
7728 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7729 VT.getVectorMinNumElements() <= N1VT.getVectorMinNumElements()) &&
7730 "Extract subvector must be from larger vector to smaller vector!");
7731 assert(N2C && "Extract subvector index must be a constant");
7732 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7733 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
7734 N1VT.getVectorMinNumElements()) &&
7735 "Extract subvector overflow!");
7736 assert(N2C->getAPIntValue().getBitWidth() ==
7737 TLI->getVectorIdxWidth(getDataLayout()) &&
7738 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
7739
7740 // Trivial extraction.
7741 if (VT == N1VT)
7742 return N1;
7743
7744 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
7745 if (N1.isUndef())
7746 return getUNDEF(VT);
7747
7748 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
7749 // the concat have the same type as the extract.
7750 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
7751 VT == N1.getOperand(i: 0).getValueType()) {
7752 unsigned Factor = VT.getVectorMinNumElements();
7753 return N1.getOperand(i: N2C->getZExtValue() / Factor);
7754 }
7755
7756 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
7757 // during shuffle legalization.
7758 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(i: 2) &&
7759 VT == N1.getOperand(i: 1).getValueType())
7760 return N1.getOperand(i: 1);
7761 break;
7762 }
7763 }
7764
7765 // Perform trivial constant folding.
7766 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, Ops: {N1, N2}, Flags))
7767 return SV;
7768
7769 // Canonicalize an UNDEF to the RHS, even over a constant.
7770 if (N1.isUndef()) {
7771 if (TLI->isCommutativeBinOp(Opcode)) {
7772 std::swap(a&: N1, b&: N2);
7773 } else {
7774 switch (Opcode) {
7775 case ISD::PTRADD:
7776 case ISD::SUB:
7777 // fold op(undef, arg2) -> undef, fold op(poison, arg2) ->poison.
7778 return N1.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
7779 case ISD::SIGN_EXTEND_INREG:
7780 case ISD::UDIV:
7781 case ISD::SDIV:
7782 case ISD::UREM:
7783 case ISD::SREM:
7784 case ISD::SSUBSAT:
7785 case ISD::USUBSAT:
7786 // fold op(undef, arg2) -> 0, fold op(poison, arg2) -> poison.
7787 return N1.getOpcode() == ISD::POISON ? getPOISON(VT)
7788 : getConstant(Val: 0, DL, VT);
7789 }
7790 }
7791 }
7792
7793 // Fold a bunch of operators when the RHS is undef.
7794 if (N2.isUndef()) {
7795 switch (Opcode) {
7796 case ISD::XOR:
7797 if (N1.isUndef())
7798 // Handle undef ^ undef -> 0 special case. This is a common
7799 // idiom (misuse).
7800 return getConstant(Val: 0, DL, VT);
7801 [[fallthrough]];
7802 case ISD::ADD:
7803 case ISD::PTRADD:
7804 case ISD::SUB:
7805 case ISD::UDIV:
7806 case ISD::SDIV:
7807 case ISD::UREM:
7808 case ISD::SREM:
7809 // fold op(arg1, undef) -> undef, fold op(arg1, poison) -> poison.
7810 return N2.getOpcode() == ISD::POISON ? getPOISON(VT) : getUNDEF(VT);
7811 case ISD::MUL:
7812 case ISD::AND:
7813 case ISD::SSUBSAT:
7814 case ISD::USUBSAT:
7815 // fold op(arg1, undef) -> 0, fold op(arg1, poison) -> poison.
7816 return N2.getOpcode() == ISD::POISON ? getPOISON(VT)
7817 : getConstant(Val: 0, DL, VT);
7818 case ISD::OR:
7819 case ISD::SADDSAT:
7820 case ISD::UADDSAT:
7821 // fold op(arg1, undef) -> an all-ones constant, fold op(arg1, poison) ->
7822 // poison.
7823 return N2.getOpcode() == ISD::POISON ? getPOISON(VT)
7824 : getAllOnesConstant(DL, VT);
7825 }
7826 }
7827
7828 // Memoize this node if possible.
7829 SDNode *N;
7830 SDVTList VTs = getVTList(VT);
7831 SDValue Ops[] = {N1, N2};
7832 if (VT != MVT::Glue) {
7833 FoldingSetNodeID ID;
7834 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
7835 void *IP = nullptr;
7836 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
7837 E->intersectFlagsWith(Flags);
7838 return SDValue(E, 0);
7839 }
7840
7841 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7842 N->setFlags(Flags);
7843 createOperands(Node: N, Vals: Ops);
7844 CSEMap.InsertNode(N, InsertPos: IP);
7845 } else {
7846 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
7847 createOperands(Node: N, Vals: Ops);
7848 }
7849
7850 InsertNode(N);
7851 SDValue V = SDValue(N, 0);
7852 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
7853 return V;
7854}
7855
7856SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7857 SDValue N1, SDValue N2, SDValue N3) {
7858 SDNodeFlags Flags;
7859 if (Inserter)
7860 Flags = Inserter->getFlags();
7861 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7862}
7863
7864SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7865 SDValue N1, SDValue N2, SDValue N3,
7866 const SDNodeFlags Flags) {
7867 assert(N1.getOpcode() != ISD::DELETED_NODE &&
7868 N2.getOpcode() != ISD::DELETED_NODE &&
7869 N3.getOpcode() != ISD::DELETED_NODE &&
7870 "Operand is DELETED_NODE!");
7871 // Perform various simplifications.
7872 switch (Opcode) {
7873 case ISD::FMA:
7874 case ISD::FMAD: {
7875 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7876 assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7877 N3.getValueType() == VT && "FMA types must match!");
7878 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
7879 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
7880 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(Val&: N3);
7881 if (N1CFP && N2CFP && N3CFP) {
7882 APFloat V1 = N1CFP->getValueAPF();
7883 const APFloat &V2 = N2CFP->getValueAPF();
7884 const APFloat &V3 = N3CFP->getValueAPF();
7885 if (Opcode == ISD::FMAD) {
7886 V1.multiply(RHS: V2, RM: APFloat::rmNearestTiesToEven);
7887 V1.add(RHS: V3, RM: APFloat::rmNearestTiesToEven);
7888 } else
7889 V1.fusedMultiplyAdd(Multiplicand: V2, Addend: V3, RM: APFloat::rmNearestTiesToEven);
7890 return getConstantFP(V: V1, DL, VT);
7891 }
7892 break;
7893 }
7894 case ISD::BUILD_VECTOR: {
7895 // Attempt to simplify BUILD_VECTOR.
7896 SDValue Ops[] = {N1, N2, N3};
7897 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
7898 return V;
7899 break;
7900 }
7901 case ISD::CONCAT_VECTORS: {
7902 SDValue Ops[] = {N1, N2, N3};
7903 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
7904 return V;
7905 break;
7906 }
7907 case ISD::SETCC: {
7908 assert(VT.isInteger() && "SETCC result type must be an integer!");
7909 assert(N1.getValueType() == N2.getValueType() &&
7910 "SETCC operands must have the same type!");
7911 assert(VT.isVector() == N1.getValueType().isVector() &&
7912 "SETCC type should be vector iff the operand type is vector!");
7913 assert((!VT.isVector() || VT.getVectorElementCount() ==
7914 N1.getValueType().getVectorElementCount()) &&
7915 "SETCC vector element counts must match!");
7916 // Use FoldSetCC to simplify SETCC's.
7917 if (SDValue V = FoldSetCC(VT, N1, N2, Cond: cast<CondCodeSDNode>(Val&: N3)->get(), dl: DL))
7918 return V;
7919 // Vector constant folding.
7920 SDValue Ops[] = {N1, N2, N3};
7921 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7922 NewSDValueDbgMsg(V, Msg: "New node vector constant folding: ", G: this);
7923 return V;
7924 }
7925 break;
7926 }
7927 case ISD::SELECT:
7928 case ISD::VSELECT:
7929 if (SDValue V = simplifySelect(Cond: N1, TVal: N2, FVal: N3))
7930 return V;
7931 break;
7932 case ISD::VECTOR_SHUFFLE:
7933 llvm_unreachable("should use getVectorShuffle constructor!");
7934 case ISD::VECTOR_SPLICE: {
7935 if (cast<ConstantSDNode>(Val&: N3)->isZero())
7936 return N1;
7937 break;
7938 }
7939 case ISD::INSERT_VECTOR_ELT: {
7940 assert(VT.isVector() && VT == N1.getValueType() &&
7941 "INSERT_VECTOR_ELT vector type mismatch");
7942 assert(VT.isFloatingPoint() == N2.getValueType().isFloatingPoint() &&
7943 "INSERT_VECTOR_ELT scalar fp/int mismatch");
7944 assert((!VT.isFloatingPoint() ||
7945 VT.getVectorElementType() == N2.getValueType()) &&
7946 "INSERT_VECTOR_ELT fp scalar type mismatch");
7947 assert((!VT.isInteger() ||
7948 VT.getScalarSizeInBits() <= N2.getScalarValueSizeInBits()) &&
7949 "INSERT_VECTOR_ELT int scalar size mismatch");
7950
7951 auto *N3C = dyn_cast<ConstantSDNode>(Val&: N3);
7952 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7953 // for scalable vectors where we will generate appropriate code to
7954 // deal with out-of-bounds cases correctly.
7955 if (N3C && N1.getValueType().isFixedLengthVector() &&
7956 N3C->getZExtValue() >= N1.getValueType().getVectorNumElements())
7957 return getUNDEF(VT);
7958
7959 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7960 if (N3.isUndef())
7961 return getUNDEF(VT);
7962
7963 // If the inserted element is an UNDEF, just use the input vector.
7964 if (N2.isUndef())
7965 return N1;
7966
7967 break;
7968 }
7969 case ISD::INSERT_SUBVECTOR: {
7970 // Inserting undef into undef is still undef.
7971 if (N1.isUndef() && N2.isUndef())
7972 return getUNDEF(VT);
7973
7974 EVT N2VT = N2.getValueType();
7975 assert(VT == N1.getValueType() &&
7976 "Dest and insert subvector source types must match!");
7977 assert(VT.isVector() && N2VT.isVector() &&
7978 "Insert subvector VTs must be vectors!");
7979 assert(VT.getVectorElementType() == N2VT.getVectorElementType() &&
7980 "Insert subvector VTs must have the same element type!");
7981 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7982 "Cannot insert a scalable vector into a fixed length vector!");
7983 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7984 VT.getVectorMinNumElements() >= N2VT.getVectorMinNumElements()) &&
7985 "Insert subvector must be from smaller vector to larger vector!");
7986 assert(isa<ConstantSDNode>(N3) &&
7987 "Insert subvector index must be constant");
7988 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7989 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7990 VT.getVectorMinNumElements()) &&
7991 "Insert subvector overflow!");
7992 assert(N3->getAsAPIntVal().getBitWidth() ==
7993 TLI->getVectorIdxWidth(getDataLayout()) &&
7994 "Constant index for INSERT_SUBVECTOR has an invalid size");
7995
7996 // Trivial insertion.
7997 if (VT == N2VT)
7998 return N2;
7999
8000 // If this is an insert of an extracted vector into an undef vector, we
8001 // can just use the input to the extract.
8002 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
8003 N2.getOperand(i: 1) == N3 && N2.getOperand(i: 0).getValueType() == VT)
8004 return N2.getOperand(i: 0);
8005 break;
8006 }
8007 case ISD::BITCAST:
8008 // Fold bit_convert nodes from a type to themselves.
8009 if (N1.getValueType() == VT)
8010 return N1;
8011 break;
8012 case ISD::VP_TRUNCATE:
8013 case ISD::VP_SIGN_EXTEND:
8014 case ISD::VP_ZERO_EXTEND:
8015 // Don't create noop casts.
8016 if (N1.getValueType() == VT)
8017 return N1;
8018 break;
8019 case ISD::VECTOR_COMPRESS: {
8020 [[maybe_unused]] EVT VecVT = N1.getValueType();
8021 [[maybe_unused]] EVT MaskVT = N2.getValueType();
8022 [[maybe_unused]] EVT PassthruVT = N3.getValueType();
8023 assert(VT == VecVT && "Vector and result type don't match.");
8024 assert(VecVT.isVector() && MaskVT.isVector() && PassthruVT.isVector() &&
8025 "All inputs must be vectors.");
8026 assert(VecVT == PassthruVT && "Vector and passthru types don't match.");
8027 assert(VecVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
8028 "Vector and mask must have same number of elements.");
8029
8030 if (N1.isUndef() || N2.isUndef())
8031 return N3;
8032
8033 break;
8034 }
8035 case ISD::PARTIAL_REDUCE_UMLA:
8036 case ISD::PARTIAL_REDUCE_SMLA:
8037 case ISD::PARTIAL_REDUCE_SUMLA: {
8038 [[maybe_unused]] EVT AccVT = N1.getValueType();
8039 [[maybe_unused]] EVT Input1VT = N2.getValueType();
8040 [[maybe_unused]] EVT Input2VT = N3.getValueType();
8041 assert(Input1VT.isVector() && Input1VT == Input2VT &&
8042 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8043 "node to have the same type!");
8044 assert(VT.isVector() && VT == AccVT &&
8045 "Expected the first operand of the PARTIAL_REDUCE_MLA node to have "
8046 "the same type as its result!");
8047 assert(Input1VT.getVectorElementCount().hasKnownScalarFactor(
8048 AccVT.getVectorElementCount()) &&
8049 "Expected the element count of the second and third operands of the "
8050 "PARTIAL_REDUCE_MLA node to be a positive integer multiple of the "
8051 "element count of the first operand and the result!");
8052 assert(N2.getScalarValueSizeInBits() <= N1.getScalarValueSizeInBits() &&
8053 "Expected the second and third operands of the PARTIAL_REDUCE_MLA "
8054 "node to have an element type which is the same as or smaller than "
8055 "the element type of the first operand and result!");
8056 break;
8057 }
8058 }
8059
8060 // Memoize node if it doesn't produce a glue result.
8061 SDNode *N;
8062 SDVTList VTs = getVTList(VT);
8063 SDValue Ops[] = {N1, N2, N3};
8064 if (VT != MVT::Glue) {
8065 FoldingSetNodeID ID;
8066 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
8067 void *IP = nullptr;
8068 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
8069 E->intersectFlagsWith(Flags);
8070 return SDValue(E, 0);
8071 }
8072
8073 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8074 N->setFlags(Flags);
8075 createOperands(Node: N, Vals: Ops);
8076 CSEMap.InsertNode(N, InsertPos: IP);
8077 } else {
8078 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
8079 createOperands(Node: N, Vals: Ops);
8080 }
8081
8082 InsertNode(N);
8083 SDValue V = SDValue(N, 0);
8084 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
8085 return V;
8086}
8087
8088SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8089 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8090 const SDNodeFlags Flags) {
8091 SDValue Ops[] = { N1, N2, N3, N4 };
8092 return getNode(Opcode, DL, VT, Ops, Flags);
8093}
8094
8095SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8096 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
8097 SDNodeFlags Flags;
8098 if (Inserter)
8099 Flags = Inserter->getFlags();
8100 return getNode(Opcode, DL, VT, N1, N2, N3, N4, Flags);
8101}
8102
8103SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8104 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8105 SDValue N5, const SDNodeFlags Flags) {
8106 SDValue Ops[] = { N1, N2, N3, N4, N5 };
8107 return getNode(Opcode, DL, VT, Ops, Flags);
8108}
8109
8110SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
8111 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
8112 SDValue N5) {
8113 SDNodeFlags Flags;
8114 if (Inserter)
8115 Flags = Inserter->getFlags();
8116 return getNode(Opcode, DL, VT, N1, N2, N3, N4, N5, Flags);
8117}
8118
8119/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
8120/// the incoming stack arguments to be loaded from the stack.
8121SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) {
8122 SmallVector<SDValue, 8> ArgChains;
8123
8124 // Include the original chain at the beginning of the list. When this is
8125 // used by target LowerCall hooks, this helps legalize find the
8126 // CALLSEQ_BEGIN node.
8127 ArgChains.push_back(Elt: Chain);
8128
8129 // Add a chain value for each stack argument.
8130 for (SDNode *U : getEntryNode().getNode()->users())
8131 if (LoadSDNode *L = dyn_cast<LoadSDNode>(Val: U))
8132 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: L->getBasePtr()))
8133 if (FI->getIndex() < 0)
8134 ArgChains.push_back(Elt: SDValue(L, 1));
8135
8136 // Build a tokenfactor for all the chains.
8137 return getNode(Opcode: ISD::TokenFactor, DL: SDLoc(Chain), VT: MVT::Other, Ops: ArgChains);
8138}
8139
8140/// getMemsetValue - Vectorized representation of the memset value
8141/// operand.
8142static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG,
8143 const SDLoc &dl) {
8144 assert(!Value.isUndef());
8145
8146 unsigned NumBits = VT.getScalarSizeInBits();
8147 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Value)) {
8148 assert(C->getAPIntValue().getBitWidth() == 8);
8149 APInt Val = APInt::getSplat(NewLen: NumBits, V: C->getAPIntValue());
8150 if (VT.isInteger()) {
8151 bool IsOpaque = VT.getSizeInBits() > 64 ||
8152 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(Value: C->getSExtValue());
8153 return DAG.getConstant(Val, DL: dl, VT, isT: false, isO: IsOpaque);
8154 }
8155 return DAG.getConstantFP(V: APFloat(VT.getFltSemantics(), Val), DL: dl, VT);
8156 }
8157
8158 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
8159 EVT IntVT = VT.getScalarType();
8160 if (!IntVT.isInteger())
8161 IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: IntVT.getSizeInBits());
8162
8163 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, N1: Value);
8164 if (NumBits > 8) {
8165 // Use a multiplication with 0x010101... to extend the input to the
8166 // required length.
8167 APInt Magic = APInt::getSplat(NewLen: NumBits, V: APInt(8, 0x01));
8168 Value = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IntVT, N1: Value,
8169 N2: DAG.getConstant(Val: Magic, DL: dl, VT: IntVT));
8170 }
8171
8172 if (VT != Value.getValueType() && !VT.isInteger())
8173 Value = DAG.getBitcast(VT: VT.getScalarType(), V: Value);
8174 if (VT != Value.getValueType())
8175 Value = DAG.getSplatBuildVector(VT, DL: dl, Op: Value);
8176
8177 return Value;
8178}
8179
8180/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
8181/// used when a memcpy is turned into a memset when the source is a constant
8182/// string ptr.
8183static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG,
8184 const TargetLowering &TLI,
8185 const ConstantDataArraySlice &Slice) {
8186 // Handle vector with all elements zero.
8187 if (Slice.Array == nullptr) {
8188 if (VT.isInteger())
8189 return DAG.getConstant(Val: 0, DL: dl, VT);
8190 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT,
8191 N1: DAG.getConstant(Val: 0, DL: dl, VT: VT.changeTypeToInteger()));
8192 }
8193
8194 assert(!VT.isVector() && "Can't handle vector type here!");
8195 unsigned NumVTBits = VT.getSizeInBits();
8196 unsigned NumVTBytes = NumVTBits / 8;
8197 unsigned NumBytes = std::min(a: NumVTBytes, b: unsigned(Slice.Length));
8198
8199 APInt Val(NumVTBits, 0);
8200 if (DAG.getDataLayout().isLittleEndian()) {
8201 for (unsigned i = 0; i != NumBytes; ++i)
8202 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
8203 } else {
8204 for (unsigned i = 0; i != NumBytes; ++i)
8205 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
8206 }
8207
8208 // If the "cost" of materializing the integer immediate is less than the cost
8209 // of a load, then it is cost effective to turn the load into the immediate.
8210 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
8211 if (TLI.shouldConvertConstantLoadToIntImm(Imm: Val, Ty))
8212 return DAG.getConstant(Val, DL: dl, VT);
8213 return SDValue();
8214}
8215
8216SDValue SelectionDAG::getMemBasePlusOffset(SDValue Base, TypeSize Offset,
8217 const SDLoc &DL,
8218 const SDNodeFlags Flags) {
8219 EVT VT = Base.getValueType();
8220 SDValue Index;
8221
8222 if (Offset.isScalable())
8223 Index = getVScale(DL, VT: Base.getValueType(),
8224 MulImm: APInt(Base.getValueSizeInBits().getFixedValue(),
8225 Offset.getKnownMinValue()));
8226 else
8227 Index = getConstant(Val: Offset.getFixedValue(), DL, VT);
8228
8229 return getMemBasePlusOffset(Base, Offset: Index, DL, Flags);
8230}
8231
8232SDValue SelectionDAG::getMemBasePlusOffset(SDValue Ptr, SDValue Offset,
8233 const SDLoc &DL,
8234 const SDNodeFlags Flags) {
8235 assert(Offset.getValueType().isInteger());
8236 EVT BasePtrVT = Ptr.getValueType();
8237 if (TLI->shouldPreservePtrArith(F: this->getMachineFunction().getFunction(),
8238 PtrVT: BasePtrVT))
8239 return getNode(Opcode: ISD::PTRADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
8240 return getNode(Opcode: ISD::ADD, DL, VT: BasePtrVT, N1: Ptr, N2: Offset, Flags);
8241}
8242
8243/// Returns true if memcpy source is constant data.
8244static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) {
8245 uint64_t SrcDelta = 0;
8246 GlobalAddressSDNode *G = nullptr;
8247 if (Src.getOpcode() == ISD::GlobalAddress)
8248 G = cast<GlobalAddressSDNode>(Val&: Src);
8249 else if (Src.getOpcode() == ISD::ADD &&
8250 Src.getOperand(i: 0).getOpcode() == ISD::GlobalAddress &&
8251 Src.getOperand(i: 1).getOpcode() == ISD::Constant) {
8252 G = cast<GlobalAddressSDNode>(Val: Src.getOperand(i: 0));
8253 SrcDelta = Src.getConstantOperandVal(i: 1);
8254 }
8255 if (!G)
8256 return false;
8257
8258 return getConstantDataArrayInfo(V: G->getGlobal(), Slice, ElementSize: 8,
8259 Offset: SrcDelta + G->getOffset());
8260}
8261
8262static bool shouldLowerMemFuncForSize(const MachineFunction &MF,
8263 SelectionDAG &DAG) {
8264 // On Darwin, -Os means optimize for size without hurting performance, so
8265 // only really optimize for size when -Oz (MinSize) is used.
8266 if (MF.getTarget().getTargetTriple().isOSDarwin())
8267 return MF.getFunction().hasMinSize();
8268 return DAG.shouldOptForSize();
8269}
8270
8271static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
8272 SmallVector<SDValue, 32> &OutChains, unsigned From,
8273 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
8274 SmallVector<SDValue, 16> &OutStoreChains) {
8275 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
8276 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
8277 SmallVector<SDValue, 16> GluedLoadChains;
8278 for (unsigned i = From; i < To; ++i) {
8279 OutChains.push_back(Elt: OutLoadChains[i]);
8280 GluedLoadChains.push_back(Elt: OutLoadChains[i]);
8281 }
8282
8283 // Chain for all loads.
8284 SDValue LoadToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
8285 Ops: GluedLoadChains);
8286
8287 for (unsigned i = From; i < To; ++i) {
8288 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: OutStoreChains[i]);
8289 SDValue NewStore = DAG.getTruncStore(Chain: LoadToken, dl, Val: ST->getValue(),
8290 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
8291 MMO: ST->getMemOperand());
8292 OutChains.push_back(Elt: NewStore);
8293 }
8294}
8295
8296static SDValue getMemcpyLoadsAndStores(
8297 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
8298 uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline,
8299 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo,
8300 const AAMDNodes &AAInfo, BatchAAResults *BatchAA) {
8301 // Turn a memcpy of undef to nop.
8302 // FIXME: We need to honor volatile even is Src is undef.
8303 if (Src.isUndef())
8304 return Chain;
8305
8306 // Expand memcpy to a series of load and store ops if the size operand falls
8307 // below a certain threshold.
8308 // TODO: In the AlwaysInline case, if the size is big then generate a loop
8309 // rather than maybe a humongous number of loads and stores.
8310 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8311 const DataLayout &DL = DAG.getDataLayout();
8312 LLVMContext &C = *DAG.getContext();
8313 std::vector<EVT> MemOps;
8314 bool DstAlignCanChange = false;
8315 MachineFunction &MF = DAG.getMachineFunction();
8316 MachineFrameInfo &MFI = MF.getFrameInfo();
8317 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8318 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
8319 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
8320 DstAlignCanChange = true;
8321 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
8322 if (!SrcAlign || Alignment > *SrcAlign)
8323 SrcAlign = Alignment;
8324 assert(SrcAlign && "SrcAlign must be set");
8325 ConstantDataArraySlice Slice;
8326 // If marked as volatile, perform a copy even when marked as constant.
8327 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
8328 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
8329 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
8330 const MemOp Op = isZeroConstant
8331 ? MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment,
8332 /*IsZeroMemset*/ true, IsVolatile: isVol)
8333 : MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment,
8334 SrcAlign: *SrcAlign, IsVolatile: isVol, MemcpyStrSrc: CopyFromConstant);
8335 if (!TLI.findOptimalMemOpLowering(
8336 MemOps, Limit, Op, DstAS: DstPtrInfo.getAddrSpace(),
8337 SrcAS: SrcPtrInfo.getAddrSpace(), FuncAttributes: MF.getFunction().getAttributes()))
8338 return SDValue();
8339
8340 if (DstAlignCanChange) {
8341 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
8342 Align NewAlign = DL.getABITypeAlign(Ty);
8343
8344 // Don't promote to an alignment that would require dynamic stack
8345 // realignment which may conflict with optimizations such as tail call
8346 // optimization.
8347 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
8348 if (!TRI->hasStackRealignment(MF))
8349 if (MaybeAlign StackAlign = DL.getStackAlignment())
8350 NewAlign = std::min(a: NewAlign, b: *StackAlign);
8351
8352 if (NewAlign > Alignment) {
8353 // Give the stack frame object a larger alignment if needed.
8354 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
8355 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
8356 Alignment = NewAlign;
8357 }
8358 }
8359
8360 // Prepare AAInfo for loads/stores after lowering this memcpy.
8361 AAMDNodes NewAAInfo = AAInfo;
8362 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8363
8364 const Value *SrcVal = dyn_cast_if_present<const Value *>(Val&: SrcPtrInfo.V);
8365 bool isConstant =
8366 BatchAA && SrcVal &&
8367 BatchAA->pointsToConstantMemory(Loc: MemoryLocation(SrcVal, Size, AAInfo));
8368
8369 MachineMemOperand::Flags MMOFlags =
8370 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
8371 SmallVector<SDValue, 16> OutLoadChains;
8372 SmallVector<SDValue, 16> OutStoreChains;
8373 SmallVector<SDValue, 32> OutChains;
8374 unsigned NumMemOps = MemOps.size();
8375 uint64_t SrcOff = 0, DstOff = 0;
8376 for (unsigned i = 0; i != NumMemOps; ++i) {
8377 EVT VT = MemOps[i];
8378 unsigned VTSize = VT.getSizeInBits() / 8;
8379 SDValue Value, Store;
8380
8381 if (VTSize > Size) {
8382 // Issuing an unaligned load / store pair that overlaps with the previous
8383 // pair. Adjust the offset accordingly.
8384 assert(i == NumMemOps-1 && i != 0);
8385 SrcOff -= VTSize - Size;
8386 DstOff -= VTSize - Size;
8387 }
8388
8389 if (CopyFromConstant &&
8390 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
8391 // It's unlikely a store of a vector immediate can be done in a single
8392 // instruction. It would require a load from a constantpool first.
8393 // We only handle zero vectors here.
8394 // FIXME: Handle other cases where store of vector immediate is done in
8395 // a single instruction.
8396 ConstantDataArraySlice SubSlice;
8397 if (SrcOff < Slice.Length) {
8398 SubSlice = Slice;
8399 SubSlice.move(Delta: SrcOff);
8400 } else {
8401 // This is an out-of-bounds access and hence UB. Pretend we read zero.
8402 SubSlice.Array = nullptr;
8403 SubSlice.Offset = 0;
8404 SubSlice.Length = VTSize;
8405 }
8406 Value = getMemsetStringVal(VT, dl, DAG, TLI, Slice: SubSlice);
8407 if (Value.getNode()) {
8408 Store = DAG.getStore(
8409 Chain, dl, Val: Value,
8410 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
8411 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
8412 OutChains.push_back(Elt: Store);
8413 }
8414 }
8415
8416 if (!Store.getNode()) {
8417 // The type might not be legal for the target. This should only happen
8418 // if the type is smaller than a legal type, as on PPC, so the right
8419 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
8420 // to Load/Store if NVT==VT.
8421 // FIXME does the case above also need this?
8422 EVT NVT = TLI.getTypeToTransformTo(Context&: C, VT);
8423 assert(NVT.bitsGE(VT));
8424
8425 bool isDereferenceable =
8426 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
8427 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
8428 if (isDereferenceable)
8429 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
8430 if (isConstant)
8431 SrcMMOFlags |= MachineMemOperand::MOInvariant;
8432
8433 Value = DAG.getExtLoad(
8434 ExtType: ISD::EXTLOAD, dl, VT: NVT, Chain,
8435 Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl),
8436 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), MemVT: VT,
8437 Alignment: commonAlignment(A: *SrcAlign, Offset: SrcOff), MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
8438 OutLoadChains.push_back(Elt: Value.getValue(R: 1));
8439
8440 Store = DAG.getTruncStore(
8441 Chain, dl, Val: Value,
8442 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
8443 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), SVT: VT, Alignment, MMOFlags, AAInfo: NewAAInfo);
8444 OutStoreChains.push_back(Elt: Store);
8445 }
8446 SrcOff += VTSize;
8447 DstOff += VTSize;
8448 Size -= VTSize;
8449 }
8450
8451 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
8452 TLI.getMaxGluedStoresPerMemcpy() : MaxLdStGlue;
8453 unsigned NumLdStInMemcpy = OutStoreChains.size();
8454
8455 if (NumLdStInMemcpy) {
8456 // It may be that memcpy might be converted to memset if it's memcpy
8457 // of constants. In such a case, we won't have loads and stores, but
8458 // just stores. In the absence of loads, there is nothing to gang up.
8459 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
8460 // If target does not care, just leave as it.
8461 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
8462 OutChains.push_back(Elt: OutLoadChains[i]);
8463 OutChains.push_back(Elt: OutStoreChains[i]);
8464 }
8465 } else {
8466 // Ld/St less than/equal limit set by target.
8467 if (NumLdStInMemcpy <= GluedLdStLimit) {
8468 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
8469 To: NumLdStInMemcpy, OutLoadChains,
8470 OutStoreChains);
8471 } else {
8472 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
8473 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
8474 unsigned GlueIter = 0;
8475
8476 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
8477 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
8478 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
8479
8480 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: IndexFrom, To: IndexTo,
8481 OutLoadChains, OutStoreChains);
8482 GlueIter += GluedLdStLimit;
8483 }
8484
8485 // Residual ld/st.
8486 if (RemainingLdStInMemcpy) {
8487 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, From: 0,
8488 To: RemainingLdStInMemcpy, OutLoadChains,
8489 OutStoreChains);
8490 }
8491 }
8492 }
8493 }
8494 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
8495}
8496
8497static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
8498 SDValue Chain, SDValue Dst, SDValue Src,
8499 uint64_t Size, Align Alignment,
8500 bool isVol, bool AlwaysInline,
8501 MachinePointerInfo DstPtrInfo,
8502 MachinePointerInfo SrcPtrInfo,
8503 const AAMDNodes &AAInfo) {
8504 // Turn a memmove of undef to nop.
8505 // FIXME: We need to honor volatile even is Src is undef.
8506 if (Src.isUndef())
8507 return Chain;
8508
8509 // Expand memmove to a series of load and store ops if the size operand falls
8510 // below a certain threshold.
8511 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8512 const DataLayout &DL = DAG.getDataLayout();
8513 LLVMContext &C = *DAG.getContext();
8514 std::vector<EVT> MemOps;
8515 bool DstAlignCanChange = false;
8516 MachineFunction &MF = DAG.getMachineFunction();
8517 MachineFrameInfo &MFI = MF.getFrameInfo();
8518 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8519 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
8520 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
8521 DstAlignCanChange = true;
8522 MaybeAlign SrcAlign = DAG.InferPtrAlign(Ptr: Src);
8523 if (!SrcAlign || Alignment > *SrcAlign)
8524 SrcAlign = Alignment;
8525 assert(SrcAlign && "SrcAlign must be set");
8526 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
8527 if (!TLI.findOptimalMemOpLowering(
8528 MemOps, Limit,
8529 Op: MemOp::Copy(Size, DstAlignCanChange, DstAlign: Alignment, SrcAlign: *SrcAlign,
8530 /*IsVolatile*/ true),
8531 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: SrcPtrInfo.getAddrSpace(),
8532 FuncAttributes: MF.getFunction().getAttributes()))
8533 return SDValue();
8534
8535 if (DstAlignCanChange) {
8536 Type *Ty = MemOps[0].getTypeForEVT(Context&: C);
8537 Align NewAlign = DL.getABITypeAlign(Ty);
8538
8539 // Don't promote to an alignment that would require dynamic stack
8540 // realignment which may conflict with optimizations such as tail call
8541 // optimization.
8542 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
8543 if (!TRI->hasStackRealignment(MF))
8544 if (MaybeAlign StackAlign = DL.getStackAlignment())
8545 NewAlign = std::min(a: NewAlign, b: *StackAlign);
8546
8547 if (NewAlign > Alignment) {
8548 // Give the stack frame object a larger alignment if needed.
8549 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
8550 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
8551 Alignment = NewAlign;
8552 }
8553 }
8554
8555 // Prepare AAInfo for loads/stores after lowering this memmove.
8556 AAMDNodes NewAAInfo = AAInfo;
8557 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8558
8559 MachineMemOperand::Flags MMOFlags =
8560 isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
8561 uint64_t SrcOff = 0, DstOff = 0;
8562 SmallVector<SDValue, 8> LoadValues;
8563 SmallVector<SDValue, 8> LoadChains;
8564 SmallVector<SDValue, 8> OutChains;
8565 unsigned NumMemOps = MemOps.size();
8566 for (unsigned i = 0; i < NumMemOps; i++) {
8567 EVT VT = MemOps[i];
8568 unsigned VTSize = VT.getSizeInBits() / 8;
8569 SDValue Value;
8570
8571 bool isDereferenceable =
8572 SrcPtrInfo.getWithOffset(O: SrcOff).isDereferenceable(Size: VTSize, C, DL);
8573 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
8574 if (isDereferenceable)
8575 SrcMMOFlags |= MachineMemOperand::MODereferenceable;
8576
8577 Value = DAG.getLoad(
8578 VT, dl, Chain,
8579 Ptr: DAG.getMemBasePlusOffset(Base: Src, Offset: TypeSize::getFixed(ExactSize: SrcOff), DL: dl),
8580 PtrInfo: SrcPtrInfo.getWithOffset(O: SrcOff), Alignment: *SrcAlign, MMOFlags: SrcMMOFlags, AAInfo: NewAAInfo);
8581 LoadValues.push_back(Elt: Value);
8582 LoadChains.push_back(Elt: Value.getValue(R: 1));
8583 SrcOff += VTSize;
8584 }
8585 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
8586 OutChains.clear();
8587 for (unsigned i = 0; i < NumMemOps; i++) {
8588 EVT VT = MemOps[i];
8589 unsigned VTSize = VT.getSizeInBits() / 8;
8590 SDValue Store;
8591
8592 Store = DAG.getStore(
8593 Chain, dl, Val: LoadValues[i],
8594 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
8595 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment, MMOFlags, AAInfo: NewAAInfo);
8596 OutChains.push_back(Elt: Store);
8597 DstOff += VTSize;
8598 }
8599
8600 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
8601}
8602
8603/// Lower the call to 'memset' intrinsic function into a series of store
8604/// operations.
8605///
8606/// \param DAG Selection DAG where lowered code is placed.
8607/// \param dl Link to corresponding IR location.
8608/// \param Chain Control flow dependency.
8609/// \param Dst Pointer to destination memory location.
8610/// \param Src Value of byte to write into the memory.
8611/// \param Size Number of bytes to write.
8612/// \param Alignment Alignment of the destination in bytes.
8613/// \param isVol True if destination is volatile.
8614/// \param AlwaysInline Makes sure no function call is generated.
8615/// \param DstPtrInfo IR information on the memory pointer.
8616/// \returns New head in the control flow, if lowering was successful, empty
8617/// SDValue otherwise.
8618///
8619/// The function tries to replace 'llvm.memset' intrinsic with several store
8620/// operations and value calculation code. This is usually profitable for small
8621/// memory size or when the semantic requires inlining.
8622static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl,
8623 SDValue Chain, SDValue Dst, SDValue Src,
8624 uint64_t Size, Align Alignment, bool isVol,
8625 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
8626 const AAMDNodes &AAInfo) {
8627 // Turn a memset of undef to nop.
8628 // FIXME: We need to honor volatile even is Src is undef.
8629 if (Src.isUndef())
8630 return Chain;
8631
8632 // Expand memset to a series of load/store ops if the size operand
8633 // falls below a certain threshold.
8634 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8635 std::vector<EVT> MemOps;
8636 bool DstAlignCanChange = false;
8637 MachineFunction &MF = DAG.getMachineFunction();
8638 MachineFrameInfo &MFI = MF.getFrameInfo();
8639 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
8640 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Dst);
8641 if (FI && !MFI.isFixedObjectIndex(ObjectIdx: FI->getIndex()))
8642 DstAlignCanChange = true;
8643 bool IsZeroVal = isNullConstant(V: Src);
8644 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
8645
8646 if (!TLI.findOptimalMemOpLowering(
8647 MemOps, Limit,
8648 Op: MemOp::Set(Size, DstAlignCanChange, DstAlign: Alignment, IsZeroMemset: IsZeroVal, IsVolatile: isVol),
8649 DstAS: DstPtrInfo.getAddrSpace(), SrcAS: ~0u, FuncAttributes: MF.getFunction().getAttributes()))
8650 return SDValue();
8651
8652 if (DstAlignCanChange) {
8653 Type *Ty = MemOps[0].getTypeForEVT(Context&: *DAG.getContext());
8654 const DataLayout &DL = DAG.getDataLayout();
8655 Align NewAlign = DL.getABITypeAlign(Ty);
8656
8657 // Don't promote to an alignment that would require dynamic stack
8658 // realignment which may conflict with optimizations such as tail call
8659 // optimization.
8660 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
8661 if (!TRI->hasStackRealignment(MF))
8662 if (MaybeAlign StackAlign = DL.getStackAlignment())
8663 NewAlign = std::min(a: NewAlign, b: *StackAlign);
8664
8665 if (NewAlign > Alignment) {
8666 // Give the stack frame object a larger alignment if needed.
8667 if (MFI.getObjectAlign(ObjectIdx: FI->getIndex()) < NewAlign)
8668 MFI.setObjectAlignment(ObjectIdx: FI->getIndex(), Alignment: NewAlign);
8669 Alignment = NewAlign;
8670 }
8671 }
8672
8673 SmallVector<SDValue, 8> OutChains;
8674 uint64_t DstOff = 0;
8675 unsigned NumMemOps = MemOps.size();
8676
8677 // Find the largest store and generate the bit pattern for it.
8678 EVT LargestVT = MemOps[0];
8679 for (unsigned i = 1; i < NumMemOps; i++)
8680 if (MemOps[i].bitsGT(VT: LargestVT))
8681 LargestVT = MemOps[i];
8682 SDValue MemSetValue = getMemsetValue(Value: Src, VT: LargestVT, DAG, dl);
8683
8684 // Prepare AAInfo for loads/stores after lowering this memset.
8685 AAMDNodes NewAAInfo = AAInfo;
8686 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
8687
8688 for (unsigned i = 0; i < NumMemOps; i++) {
8689 EVT VT = MemOps[i];
8690 unsigned VTSize = VT.getSizeInBits() / 8;
8691 if (VTSize > Size) {
8692 // Issuing an unaligned load / store pair that overlaps with the previous
8693 // pair. Adjust the offset accordingly.
8694 assert(i == NumMemOps-1 && i != 0);
8695 DstOff -= VTSize - Size;
8696 }
8697
8698 // If this store is smaller than the largest store see whether we can get
8699 // the smaller value for free with a truncate or extract vector element and
8700 // then store.
8701 SDValue Value = MemSetValue;
8702 if (VT.bitsLT(VT: LargestVT)) {
8703 unsigned Index;
8704 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
8705 EVT SVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(), NumElements: NElts);
8706 if (!LargestVT.isVector() && !VT.isVector() &&
8707 TLI.isTruncateFree(FromVT: LargestVT, ToVT: VT))
8708 Value = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, N1: MemSetValue);
8709 else if (LargestVT.isVector() && !VT.isVector() &&
8710 TLI.shallExtractConstSplatVectorElementToStore(
8711 VectorTy: LargestVT.getTypeForEVT(Context&: *DAG.getContext()),
8712 ElemSizeInBits: VT.getSizeInBits(), Index) &&
8713 TLI.isTypeLegal(VT: SVT) &&
8714 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
8715 // Target which can combine store(extractelement VectorTy, Idx) can get
8716 // the smaller value for free.
8717 SDValue TailValue = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: SVT, N1: MemSetValue);
8718 Value = DAG.getExtractVectorElt(DL: dl, VT, Vec: TailValue, Idx: Index);
8719 } else
8720 Value = getMemsetValue(Value: Src, VT, DAG, dl);
8721 }
8722 assert(Value.getValueType() == VT && "Value with wrong type.");
8723 SDValue Store = DAG.getStore(
8724 Chain, dl, Val: Value,
8725 Ptr: DAG.getMemBasePlusOffset(Base: Dst, Offset: TypeSize::getFixed(ExactSize: DstOff), DL: dl),
8726 PtrInfo: DstPtrInfo.getWithOffset(O: DstOff), Alignment,
8727 MMOFlags: isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone,
8728 AAInfo: NewAAInfo);
8729 OutChains.push_back(Elt: Store);
8730 DstOff += VT.getSizeInBits() / 8;
8731 Size -= VTSize;
8732 }
8733
8734 return DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
8735}
8736
8737static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
8738 unsigned AS) {
8739 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
8740 // pointer operands can be losslessly bitcasted to pointers of address space 0
8741 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(SrcAS: AS, DestAS: 0)) {
8742 report_fatal_error(reason: "cannot lower memory intrinsic in address space " +
8743 Twine(AS));
8744 }
8745}
8746
8747SDValue SelectionDAG::getMemcpy(
8748 SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
8749 Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
8750 std::optional<bool> OverrideTailCall, MachinePointerInfo DstPtrInfo,
8751 MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo,
8752 BatchAAResults *BatchAA) {
8753 // Check to see if we should lower the memcpy to loads and stores first.
8754 // For cases within the target-specified limits, this is the best choice.
8755 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
8756 if (ConstantSize) {
8757 // Memcpy with size zero? Just return the original chain.
8758 if (ConstantSize->isZero())
8759 return Chain;
8760
8761 SDValue Result = getMemcpyLoadsAndStores(
8762 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
8763 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
8764 if (Result.getNode())
8765 return Result;
8766 }
8767
8768 // Then check to see if we should lower the memcpy with target-specific
8769 // code. If the target chooses to do this, this is the next best.
8770 if (TSI) {
8771 SDValue Result = TSI->EmitTargetCodeForMemcpy(
8772 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline,
8773 DstPtrInfo, SrcPtrInfo);
8774 if (Result.getNode())
8775 return Result;
8776 }
8777
8778 // If we really need inline code and the target declined to provide it,
8779 // use a (potentially long) sequence of loads and stores.
8780 if (AlwaysInline) {
8781 assert(ConstantSize && "AlwaysInline requires a constant size!");
8782 return getMemcpyLoadsAndStores(
8783 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
8784 isVol, AlwaysInline: true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA);
8785 }
8786
8787 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
8788 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
8789
8790 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
8791 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
8792 // respect volatile, so they may do things like read or write memory
8793 // beyond the given memory regions. But fixing this isn't easy, and most
8794 // people don't care.
8795
8796 // Emit a library call.
8797 TargetLowering::ArgListTy Args;
8798 TargetLowering::ArgListEntry Entry;
8799 Entry.Ty = PointerType::getUnqual(C&: *getContext());
8800 Entry.Node = Dst; Args.push_back(x: Entry);
8801 Entry.Node = Src; Args.push_back(x: Entry);
8802
8803 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8804 Entry.Node = Size; Args.push_back(x: Entry);
8805 // FIXME: pass in SDLoc
8806 TargetLowering::CallLoweringInfo CLI(*this);
8807 bool IsTailCall = false;
8808 const char *MemCpyName = TLI->getMemcpyName();
8809
8810 if (OverrideTailCall.has_value()) {
8811 IsTailCall = *OverrideTailCall;
8812 } else {
8813 bool LowersToMemcpy = StringRef(MemCpyName) == StringRef("memcpy");
8814 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI);
8815 IsTailCall = CI && CI->isTailCall() &&
8816 isInTailCallPosition(Call: *CI, TM: getTarget(),
8817 ReturnsFirstArg: ReturnsFirstArg && LowersToMemcpy);
8818 }
8819
8820 CLI.setDebugLoc(dl)
8821 .setChain(Chain)
8822 .setLibCallee(
8823 CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMCPY),
8824 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
8825 Target: getExternalSymbol(Sym: MemCpyName, VT: TLI->getPointerTy(DL: getDataLayout())),
8826 ArgsList: std::move(Args))
8827 .setDiscardResult()
8828 .setTailCall(IsTailCall);
8829
8830 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8831 return CallResult.second;
8832}
8833
8834SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
8835 SDValue Dst, SDValue Src, SDValue Size,
8836 Type *SizeTy, unsigned ElemSz,
8837 bool isTailCall,
8838 MachinePointerInfo DstPtrInfo,
8839 MachinePointerInfo SrcPtrInfo) {
8840 // Emit a library call.
8841 TargetLowering::ArgListTy Args;
8842 TargetLowering::ArgListEntry Entry;
8843 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8844 Entry.Node = Dst;
8845 Args.push_back(x: Entry);
8846
8847 Entry.Node = Src;
8848 Args.push_back(x: Entry);
8849
8850 Entry.Ty = SizeTy;
8851 Entry.Node = Size;
8852 Args.push_back(x: Entry);
8853
8854 RTLIB::Libcall LibraryCall =
8855 RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8856 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8857 report_fatal_error(reason: "Unsupported element size");
8858
8859 TargetLowering::CallLoweringInfo CLI(*this);
8860 CLI.setDebugLoc(dl)
8861 .setChain(Chain)
8862 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8863 ResultType: Type::getVoidTy(C&: *getContext()),
8864 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8865 VT: TLI->getPointerTy(DL: getDataLayout())),
8866 ArgsList: std::move(Args))
8867 .setDiscardResult()
8868 .setTailCall(isTailCall);
8869
8870 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8871 return CallResult.second;
8872}
8873
8874SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
8875 SDValue Src, SDValue Size, Align Alignment,
8876 bool isVol, const CallInst *CI,
8877 std::optional<bool> OverrideTailCall,
8878 MachinePointerInfo DstPtrInfo,
8879 MachinePointerInfo SrcPtrInfo,
8880 const AAMDNodes &AAInfo,
8881 BatchAAResults *BatchAA) {
8882 // Check to see if we should lower the memmove to loads and stores first.
8883 // For cases within the target-specified limits, this is the best choice.
8884 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
8885 if (ConstantSize) {
8886 // Memmove with size zero? Just return the original chain.
8887 if (ConstantSize->isZero())
8888 return Chain;
8889
8890 SDValue Result = getMemmoveLoadsAndStores(
8891 DAG&: *this, dl, Chain, Dst, Src, Size: ConstantSize->getZExtValue(), Alignment,
8892 isVol, AlwaysInline: false, DstPtrInfo, SrcPtrInfo, AAInfo);
8893 if (Result.getNode())
8894 return Result;
8895 }
8896
8897 // Then check to see if we should lower the memmove with target-specific
8898 // code. If the target chooses to do this, this is the next best.
8899 if (TSI) {
8900 SDValue Result =
8901 TSI->EmitTargetCodeForMemmove(DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size,
8902 Alignment, isVolatile: isVol, DstPtrInfo, SrcPtrInfo);
8903 if (Result.getNode())
8904 return Result;
8905 }
8906
8907 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
8908 checkAddrSpaceIsValidForLibcall(TLI, AS: SrcPtrInfo.getAddrSpace());
8909
8910 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8911 // not be safe. See memcpy above for more details.
8912
8913 // Emit a library call.
8914 TargetLowering::ArgListTy Args;
8915 TargetLowering::ArgListEntry Entry;
8916 Entry.Ty = PointerType::getUnqual(C&: *getContext());
8917 Entry.Node = Dst; Args.push_back(x: Entry);
8918 Entry.Node = Src; Args.push_back(x: Entry);
8919
8920 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8921 Entry.Node = Size; Args.push_back(x: Entry);
8922 // FIXME: pass in SDLoc
8923 TargetLowering::CallLoweringInfo CLI(*this);
8924
8925 bool IsTailCall = false;
8926 if (OverrideTailCall.has_value()) {
8927 IsTailCall = *OverrideTailCall;
8928 } else {
8929 bool LowersToMemmove =
8930 TLI->getLibcallName(Call: RTLIB::MEMMOVE) == StringRef("memmove");
8931 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI);
8932 IsTailCall = CI && CI->isTailCall() &&
8933 isInTailCallPosition(Call: *CI, TM: getTarget(),
8934 ReturnsFirstArg: ReturnsFirstArg && LowersToMemmove);
8935 }
8936
8937 CLI.setDebugLoc(dl)
8938 .setChain(Chain)
8939 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMMOVE),
8940 ResultType: Dst.getValueType().getTypeForEVT(Context&: *getContext()),
8941 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMMOVE),
8942 VT: TLI->getPointerTy(DL: getDataLayout())),
8943 ArgsList: std::move(Args))
8944 .setDiscardResult()
8945 .setTailCall(IsTailCall);
8946
8947 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8948 return CallResult.second;
8949}
8950
8951SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
8952 SDValue Dst, SDValue Src, SDValue Size,
8953 Type *SizeTy, unsigned ElemSz,
8954 bool isTailCall,
8955 MachinePointerInfo DstPtrInfo,
8956 MachinePointerInfo SrcPtrInfo) {
8957 // Emit a library call.
8958 TargetLowering::ArgListTy Args;
8959 TargetLowering::ArgListEntry Entry;
8960 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
8961 Entry.Node = Dst;
8962 Args.push_back(x: Entry);
8963
8964 Entry.Node = Src;
8965 Args.push_back(x: Entry);
8966
8967 Entry.Ty = SizeTy;
8968 Entry.Node = Size;
8969 Args.push_back(x: Entry);
8970
8971 RTLIB::Libcall LibraryCall =
8972 RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
8973 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8974 report_fatal_error(reason: "Unsupported element size");
8975
8976 TargetLowering::CallLoweringInfo CLI(*this);
8977 CLI.setDebugLoc(dl)
8978 .setChain(Chain)
8979 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
8980 ResultType: Type::getVoidTy(C&: *getContext()),
8981 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
8982 VT: TLI->getPointerTy(DL: getDataLayout())),
8983 ArgsList: std::move(Args))
8984 .setDiscardResult()
8985 .setTailCall(isTailCall);
8986
8987 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8988 return CallResult.second;
8989}
8990
8991SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
8992 SDValue Src, SDValue Size, Align Alignment,
8993 bool isVol, bool AlwaysInline,
8994 const CallInst *CI,
8995 MachinePointerInfo DstPtrInfo,
8996 const AAMDNodes &AAInfo) {
8997 // Check to see if we should lower the memset to stores first.
8998 // For cases within the target-specified limits, this is the best choice.
8999 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Val&: Size);
9000 if (ConstantSize) {
9001 // Memset with size zero? Just return the original chain.
9002 if (ConstantSize->isZero())
9003 return Chain;
9004
9005 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9006 Size: ConstantSize->getZExtValue(), Alignment,
9007 isVol, AlwaysInline: false, DstPtrInfo, AAInfo);
9008
9009 if (Result.getNode())
9010 return Result;
9011 }
9012
9013 // Then check to see if we should lower the memset with target-specific
9014 // code. If the target chooses to do this, this is the next best.
9015 if (TSI) {
9016 SDValue Result = TSI->EmitTargetCodeForMemset(
9017 DAG&: *this, dl, Chain, Op1: Dst, Op2: Src, Op3: Size, Alignment, isVolatile: isVol, AlwaysInline, DstPtrInfo);
9018 if (Result.getNode())
9019 return Result;
9020 }
9021
9022 // If we really need inline code and the target declined to provide it,
9023 // use a (potentially long) sequence of loads and stores.
9024 if (AlwaysInline) {
9025 assert(ConstantSize && "AlwaysInline requires a constant size!");
9026 SDValue Result = getMemsetStores(DAG&: *this, dl, Chain, Dst, Src,
9027 Size: ConstantSize->getZExtValue(), Alignment,
9028 isVol, AlwaysInline: true, DstPtrInfo, AAInfo);
9029 assert(Result &&
9030 "getMemsetStores must return a valid sequence when AlwaysInline");
9031 return Result;
9032 }
9033
9034 checkAddrSpaceIsValidForLibcall(TLI, AS: DstPtrInfo.getAddrSpace());
9035
9036 // Emit a library call.
9037 auto &Ctx = *getContext();
9038 const auto& DL = getDataLayout();
9039
9040 TargetLowering::CallLoweringInfo CLI(*this);
9041 // FIXME: pass in SDLoc
9042 CLI.setDebugLoc(dl).setChain(Chain);
9043
9044 const char *BzeroName = getTargetLoweringInfo().getLibcallName(Call: RTLIB::BZERO);
9045
9046 // Helper function to create an Entry from Node and Type.
9047 const auto CreateEntry = [](SDValue Node, Type *Ty) {
9048 TargetLowering::ArgListEntry Entry;
9049 Entry.Node = Node;
9050 Entry.Ty = Ty;
9051 return Entry;
9052 };
9053
9054 bool UseBZero = isNullConstant(V: Src) && BzeroName;
9055 // If zeroing out and bzero is present, use it.
9056 if (UseBZero) {
9057 TargetLowering::ArgListTy Args;
9058 Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx)));
9059 Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx)));
9060 CLI.setLibCallee(
9061 CC: TLI->getLibcallCallingConv(Call: RTLIB::BZERO), ResultType: Type::getVoidTy(C&: Ctx),
9062 Target: getExternalSymbol(Sym: BzeroName, VT: TLI->getPointerTy(DL)), ArgsList: std::move(Args));
9063 } else {
9064 TargetLowering::ArgListTy Args;
9065 Args.push_back(x: CreateEntry(Dst, PointerType::getUnqual(C&: Ctx)));
9066 Args.push_back(x: CreateEntry(Src, Src.getValueType().getTypeForEVT(Context&: Ctx)));
9067 Args.push_back(x: CreateEntry(Size, DL.getIntPtrType(C&: Ctx)));
9068 CLI.setLibCallee(CC: TLI->getLibcallCallingConv(Call: RTLIB::MEMSET),
9069 ResultType: Dst.getValueType().getTypeForEVT(Context&: Ctx),
9070 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: RTLIB::MEMSET),
9071 VT: TLI->getPointerTy(DL)),
9072 ArgsList: std::move(Args));
9073 }
9074 bool LowersToMemset =
9075 TLI->getLibcallName(Call: RTLIB::MEMSET) == StringRef("memset");
9076 // If we're going to use bzero, make sure not to tail call unless the
9077 // subsequent return doesn't need a value, as bzero doesn't return the first
9078 // arg unlike memset.
9079 bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(CI: *CI) && !UseBZero;
9080 bool IsTailCall =
9081 CI && CI->isTailCall() &&
9082 isInTailCallPosition(Call: *CI, TM: getTarget(), ReturnsFirstArg: ReturnsFirstArg && LowersToMemset);
9083 CLI.setDiscardResult().setTailCall(IsTailCall);
9084
9085 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9086 return CallResult.second;
9087}
9088
9089SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
9090 SDValue Dst, SDValue Value, SDValue Size,
9091 Type *SizeTy, unsigned ElemSz,
9092 bool isTailCall,
9093 MachinePointerInfo DstPtrInfo) {
9094 // Emit a library call.
9095 TargetLowering::ArgListTy Args;
9096 TargetLowering::ArgListEntry Entry;
9097 Entry.Ty = getDataLayout().getIntPtrType(C&: *getContext());
9098 Entry.Node = Dst;
9099 Args.push_back(x: Entry);
9100
9101 Entry.Ty = Type::getInt8Ty(C&: *getContext());
9102 Entry.Node = Value;
9103 Args.push_back(x: Entry);
9104
9105 Entry.Ty = SizeTy;
9106 Entry.Node = Size;
9107 Args.push_back(x: Entry);
9108
9109 RTLIB::Libcall LibraryCall =
9110 RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElementSize: ElemSz);
9111 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
9112 report_fatal_error(reason: "Unsupported element size");
9113
9114 TargetLowering::CallLoweringInfo CLI(*this);
9115 CLI.setDebugLoc(dl)
9116 .setChain(Chain)
9117 .setLibCallee(CC: TLI->getLibcallCallingConv(Call: LibraryCall),
9118 ResultType: Type::getVoidTy(C&: *getContext()),
9119 Target: getExternalSymbol(Sym: TLI->getLibcallName(Call: LibraryCall),
9120 VT: TLI->getPointerTy(DL: getDataLayout())),
9121 ArgsList: std::move(Args))
9122 .setDiscardResult()
9123 .setTailCall(isTailCall);
9124
9125 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
9126 return CallResult.second;
9127}
9128
9129SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9130 SDVTList VTList, ArrayRef<SDValue> Ops,
9131 MachineMemOperand *MMO,
9132 ISD::LoadExtType ExtType) {
9133 FoldingSetNodeID ID;
9134 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
9135 ID.AddInteger(I: MemVT.getRawBits());
9136 ID.AddInteger(I: getSyntheticNodeSubclassData<AtomicSDNode>(
9137 IROrder: dl.getIROrder(), Args&: Opcode, Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType));
9138 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9139 ID.AddInteger(I: MMO->getFlags());
9140 void* IP = nullptr;
9141 if (auto *E = cast_or_null<AtomicSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
9142 E->refineAlignment(NewMMO: MMO);
9143 E->refineRanges(NewMMO: MMO);
9144 return SDValue(E, 0);
9145 }
9146
9147 auto *N = newSDNode<AtomicSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: Opcode,
9148 Args&: VTList, Args&: MemVT, Args&: MMO, Args&: ExtType);
9149 createOperands(Node: N, Vals: Ops);
9150
9151 CSEMap.InsertNode(N, InsertPos: IP);
9152 InsertNode(N);
9153 SDValue V(N, 0);
9154 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9155 return V;
9156}
9157
9158SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
9159 EVT MemVT, SDVTList VTs, SDValue Chain,
9160 SDValue Ptr, SDValue Cmp, SDValue Swp,
9161 MachineMemOperand *MMO) {
9162 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
9163 Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
9164 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
9165
9166 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
9167 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9168}
9169
9170SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
9171 SDValue Chain, SDValue Ptr, SDValue Val,
9172 MachineMemOperand *MMO) {
9173 assert((Opcode == ISD::ATOMIC_LOAD_ADD || Opcode == ISD::ATOMIC_LOAD_SUB ||
9174 Opcode == ISD::ATOMIC_LOAD_AND || Opcode == ISD::ATOMIC_LOAD_CLR ||
9175 Opcode == ISD::ATOMIC_LOAD_OR || Opcode == ISD::ATOMIC_LOAD_XOR ||
9176 Opcode == ISD::ATOMIC_LOAD_NAND || Opcode == ISD::ATOMIC_LOAD_MIN ||
9177 Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN ||
9178 Opcode == ISD::ATOMIC_LOAD_UMAX || Opcode == ISD::ATOMIC_LOAD_FADD ||
9179 Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_LOAD_FMAX ||
9180 Opcode == ISD::ATOMIC_LOAD_FMIN ||
9181 Opcode == ISD::ATOMIC_LOAD_FMINIMUM ||
9182 Opcode == ISD::ATOMIC_LOAD_FMAXIMUM ||
9183 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
9184 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
9185 Opcode == ISD::ATOMIC_LOAD_USUB_COND ||
9186 Opcode == ISD::ATOMIC_LOAD_USUB_SAT || Opcode == ISD::ATOMIC_SWAP ||
9187 Opcode == ISD::ATOMIC_STORE) &&
9188 "Invalid Atomic Op");
9189
9190 EVT VT = Val.getValueType();
9191
9192 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(VT: MVT::Other) :
9193 getVTList(VT1: VT, VT2: MVT::Other);
9194 SDValue Ops[] = {Chain, Ptr, Val};
9195 return getAtomic(Opcode, dl, MemVT, VTList: VTs, Ops, MMO);
9196}
9197
9198SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
9199 EVT MemVT, EVT VT, SDValue Chain,
9200 SDValue Ptr, MachineMemOperand *MMO) {
9201 SDVTList VTs = getVTList(VT1: VT, VT2: MVT::Other);
9202 SDValue Ops[] = {Chain, Ptr};
9203 return getAtomic(Opcode: ISD::ATOMIC_LOAD, dl, MemVT, VTList: VTs, Ops, MMO, ExtType);
9204}
9205
9206/// getMergeValues - Create a MERGE_VALUES node from the given operands.
9207SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
9208 if (Ops.size() == 1)
9209 return Ops[0];
9210
9211 SmallVector<EVT, 4> VTs;
9212 VTs.reserve(N: Ops.size());
9213 for (const SDValue &Op : Ops)
9214 VTs.push_back(Elt: Op.getValueType());
9215 return getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: getVTList(VTs), Ops);
9216}
9217
9218SDValue SelectionDAG::getMemIntrinsicNode(
9219 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
9220 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
9221 MachineMemOperand::Flags Flags, LocationSize Size,
9222 const AAMDNodes &AAInfo) {
9223 if (Size.hasValue() && !Size.getValue())
9224 Size = LocationSize::precise(Value: MemVT.getStoreSize());
9225
9226 MachineFunction &MF = getMachineFunction();
9227 MachineMemOperand *MMO =
9228 MF.getMachineMemOperand(PtrInfo, F: Flags, Size, BaseAlignment: Alignment, AAInfo);
9229
9230 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
9231}
9232
9233SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl,
9234 SDVTList VTList,
9235 ArrayRef<SDValue> Ops, EVT MemVT,
9236 MachineMemOperand *MMO) {
9237 assert(
9238 (Opcode == ISD::INTRINSIC_VOID || Opcode == ISD::INTRINSIC_W_CHAIN ||
9239 Opcode == ISD::PREFETCH ||
9240 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
9241 Opcode >= ISD::BUILTIN_OP_END && TSI->isTargetMemoryOpcode(Opcode))) &&
9242 "Opcode is not a memory-accessing opcode!");
9243
9244 // Memoize the node unless it returns a glue result.
9245 MemIntrinsicSDNode *N;
9246 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9247 FoldingSetNodeID ID;
9248 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
9249 ID.AddInteger(I: getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
9250 Opc: Opcode, Order: dl.getIROrder(), VTs: VTList, MemoryVT: MemVT, MMO));
9251 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9252 ID.AddInteger(I: MMO->getFlags());
9253 ID.AddInteger(I: MemVT.getRawBits());
9254 void *IP = nullptr;
9255 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9256 cast<MemIntrinsicSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9257 return SDValue(E, 0);
9258 }
9259
9260 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9261 Args&: VTList, Args&: MemVT, Args&: MMO);
9262 createOperands(Node: N, Vals: Ops);
9263
9264 CSEMap.InsertNode(N, InsertPos: IP);
9265 } else {
9266 N = newSDNode<MemIntrinsicSDNode>(Args&: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(),
9267 Args&: VTList, Args&: MemVT, Args&: MMO);
9268 createOperands(Node: N, Vals: Ops);
9269 }
9270 InsertNode(N);
9271 SDValue V(N, 0);
9272 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9273 return V;
9274}
9275
9276SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl,
9277 SDValue Chain, int FrameIndex,
9278 int64_t Size, int64_t Offset) {
9279 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
9280 const auto VTs = getVTList(VT: MVT::Other);
9281 SDValue Ops[2] = {
9282 Chain,
9283 getFrameIndex(FI: FrameIndex,
9284 VT: getTargetLoweringInfo().getFrameIndexTy(DL: getDataLayout()),
9285 isTarget: true)};
9286
9287 FoldingSetNodeID ID;
9288 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9289 ID.AddInteger(I: FrameIndex);
9290 ID.AddInteger(I: Size);
9291 ID.AddInteger(I: Offset);
9292 void *IP = nullptr;
9293 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9294 return SDValue(E, 0);
9295
9296 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
9297 Args: Opcode, Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args: VTs, Args&: Size, Args&: Offset);
9298 createOperands(Node: N, Vals: Ops);
9299 CSEMap.InsertNode(N, InsertPos: IP);
9300 InsertNode(N);
9301 SDValue V(N, 0);
9302 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9303 return V;
9304}
9305
9306SDValue SelectionDAG::getPseudoProbeNode(const SDLoc &Dl, SDValue Chain,
9307 uint64_t Guid, uint64_t Index,
9308 uint32_t Attr) {
9309 const unsigned Opcode = ISD::PSEUDO_PROBE;
9310 const auto VTs = getVTList(VT: MVT::Other);
9311 SDValue Ops[] = {Chain};
9312 FoldingSetNodeID ID;
9313 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
9314 ID.AddInteger(I: Guid);
9315 ID.AddInteger(I: Index);
9316 void *IP = nullptr;
9317 if (SDNode *E = FindNodeOrInsertPos(ID, DL: Dl, InsertPos&: IP))
9318 return SDValue(E, 0);
9319
9320 auto *N = newSDNode<PseudoProbeSDNode>(
9321 Args: Opcode, Args: Dl.getIROrder(), Args: Dl.getDebugLoc(), Args: VTs, Args&: Guid, Args&: Index, Args&: Attr);
9322 createOperands(Node: N, Vals: Ops);
9323 CSEMap.InsertNode(N, InsertPos: IP);
9324 InsertNode(N);
9325 SDValue V(N, 0);
9326 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9327 return V;
9328}
9329
9330/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
9331/// MachinePointerInfo record from it. This is particularly useful because the
9332/// code generator has many cases where it doesn't bother passing in a
9333/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
9334static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
9335 SelectionDAG &DAG, SDValue Ptr,
9336 int64_t Offset = 0) {
9337 // If this is FI+Offset, we can model it.
9338 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr))
9339 return MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(),
9340 FI: FI->getIndex(), Offset);
9341
9342 // If this is (FI+Offset1)+Offset2, we can model it.
9343 if (Ptr.getOpcode() != ISD::ADD ||
9344 !isa<ConstantSDNode>(Val: Ptr.getOperand(i: 1)) ||
9345 !isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0)))
9346 return Info;
9347
9348 int FI = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
9349 return MachinePointerInfo::getFixedStack(
9350 MF&: DAG.getMachineFunction(), FI,
9351 Offset: Offset + cast<ConstantSDNode>(Val: Ptr.getOperand(i: 1))->getSExtValue());
9352}
9353
9354/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
9355/// MachinePointerInfo record from it. This is particularly useful because the
9356/// code generator has many cases where it doesn't bother passing in a
9357/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
9358static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info,
9359 SelectionDAG &DAG, SDValue Ptr,
9360 SDValue OffsetOp) {
9361 // If the 'Offset' value isn't a constant, we can't handle this.
9362 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(Val&: OffsetOp))
9363 return InferPointerInfo(Info, DAG, Ptr, Offset: OffsetNode->getSExtValue());
9364 if (OffsetOp.isUndef())
9365 return InferPointerInfo(Info, DAG, Ptr);
9366 return Info;
9367}
9368
9369SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
9370 EVT VT, const SDLoc &dl, SDValue Chain,
9371 SDValue Ptr, SDValue Offset,
9372 MachinePointerInfo PtrInfo, EVT MemVT,
9373 Align Alignment,
9374 MachineMemOperand::Flags MMOFlags,
9375 const AAMDNodes &AAInfo, const MDNode *Ranges) {
9376 assert(Chain.getValueType() == MVT::Other &&
9377 "Invalid chain type");
9378
9379 MMOFlags |= MachineMemOperand::MOLoad;
9380 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
9381 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
9382 // clients.
9383 if (PtrInfo.V.isNull())
9384 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
9385
9386 TypeSize Size = MemVT.getStoreSize();
9387 MachineFunction &MF = getMachineFunction();
9388 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
9389 BaseAlignment: Alignment, AAInfo, Ranges);
9390 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
9391}
9392
9393SDValue SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
9394 EVT VT, const SDLoc &dl, SDValue Chain,
9395 SDValue Ptr, SDValue Offset, EVT MemVT,
9396 MachineMemOperand *MMO) {
9397 if (VT == MemVT) {
9398 ExtType = ISD::NON_EXTLOAD;
9399 } else if (ExtType == ISD::NON_EXTLOAD) {
9400 assert(VT == MemVT && "Non-extending load from different memory type!");
9401 } else {
9402 // Extending load.
9403 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
9404 "Should only be an extending load, not truncating!");
9405 assert(VT.isInteger() == MemVT.isInteger() &&
9406 "Cannot convert from FP to Int or Int -> FP!");
9407 assert(VT.isVector() == MemVT.isVector() &&
9408 "Cannot use an ext load to convert to or from a vector!");
9409 assert((!VT.isVector() ||
9410 VT.getVectorElementCount() == MemVT.getVectorElementCount()) &&
9411 "Cannot use an ext load to change the number of vector elements!");
9412 }
9413
9414 assert((!MMO->getRanges() ||
9415 (mdconst::extract<ConstantInt>(MMO->getRanges()->getOperand(0))
9416 ->getBitWidth() == MemVT.getScalarSizeInBits() &&
9417 MemVT.isInteger())) &&
9418 "Range metadata and load type must match!");
9419
9420 bool Indexed = AM != ISD::UNINDEXED;
9421 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9422
9423 SDVTList VTs = Indexed ?
9424 getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other) : getVTList(VT1: VT, VT2: MVT::Other);
9425 SDValue Ops[] = { Chain, Ptr, Offset };
9426 FoldingSetNodeID ID;
9427 AddNodeIDNode(ID, OpC: ISD::LOAD, VTList: VTs, OpList: Ops);
9428 ID.AddInteger(I: MemVT.getRawBits());
9429 ID.AddInteger(I: getSyntheticNodeSubclassData<LoadSDNode>(
9430 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: MemVT, Args&: MMO));
9431 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9432 ID.AddInteger(I: MMO->getFlags());
9433 void *IP = nullptr;
9434 if (auto *E = cast_or_null<LoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
9435 E->refineAlignment(NewMMO: MMO);
9436 E->refineRanges(NewMMO: MMO);
9437 return SDValue(E, 0);
9438 }
9439 auto *N = newSDNode<LoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
9440 Args&: ExtType, Args&: MemVT, Args&: MMO);
9441 createOperands(Node: N, Vals: Ops);
9442
9443 CSEMap.InsertNode(N, InsertPos: IP);
9444 InsertNode(N);
9445 SDValue V(N, 0);
9446 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9447 return V;
9448}
9449
9450SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
9451 SDValue Ptr, MachinePointerInfo PtrInfo,
9452 MaybeAlign Alignment,
9453 MachineMemOperand::Flags MMOFlags,
9454 const AAMDNodes &AAInfo, const MDNode *Ranges) {
9455 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9456 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
9457 PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges);
9458}
9459
9460SDValue SelectionDAG::getLoad(EVT VT, const SDLoc &dl, SDValue Chain,
9461 SDValue Ptr, MachineMemOperand *MMO) {
9462 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9463 return getLoad(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
9464 MemVT: VT, MMO);
9465}
9466
9467SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
9468 EVT VT, SDValue Chain, SDValue Ptr,
9469 MachinePointerInfo PtrInfo, EVT MemVT,
9470 MaybeAlign Alignment,
9471 MachineMemOperand::Flags MMOFlags,
9472 const AAMDNodes &AAInfo) {
9473 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9474 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, PtrInfo,
9475 MemVT, Alignment, MMOFlags, AAInfo);
9476}
9477
9478SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
9479 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
9480 MachineMemOperand *MMO) {
9481 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9482 return getLoad(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef,
9483 MemVT, MMO);
9484}
9485
9486SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl,
9487 SDValue Base, SDValue Offset,
9488 ISD::MemIndexedMode AM) {
9489 LoadSDNode *LD = cast<LoadSDNode>(Val&: OrigLoad);
9490 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
9491 // Don't propagate the invariant or dereferenceable flags.
9492 auto MMOFlags =
9493 LD->getMemOperand()->getFlags() &
9494 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
9495 return getLoad(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
9496 Chain: LD->getChain(), Ptr: Base, Offset, PtrInfo: LD->getPointerInfo(),
9497 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo());
9498}
9499
9500SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
9501 SDValue Ptr, MachinePointerInfo PtrInfo,
9502 Align Alignment,
9503 MachineMemOperand::Flags MMOFlags,
9504 const AAMDNodes &AAInfo) {
9505 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9506
9507 MMOFlags |= MachineMemOperand::MOStore;
9508 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9509
9510 if (PtrInfo.V.isNull())
9511 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
9512
9513 MachineFunction &MF = getMachineFunction();
9514 TypeSize Size = Val.getValueType().getStoreSize();
9515 MachineMemOperand *MMO =
9516 MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size, BaseAlignment: Alignment, AAInfo);
9517 return getStore(Chain, dl, Val, Ptr, MMO);
9518}
9519
9520SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
9521 SDValue Ptr, MachineMemOperand *MMO) {
9522 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9523 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT: Val.getValueType(), MMO,
9524 AM: ISD::UNINDEXED);
9525}
9526
9527SDValue SelectionDAG::getStore(SDValue Chain, const SDLoc &dl, SDValue Val,
9528 SDValue Ptr, SDValue Offset, EVT SVT,
9529 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
9530 bool IsTruncating) {
9531 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9532 EVT VT = Val.getValueType();
9533 if (VT == SVT) {
9534 IsTruncating = false;
9535 } else if (!IsTruncating) {
9536 assert(VT == SVT && "No-truncating store from different memory type!");
9537 } else {
9538 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9539 "Should only be a truncating store, not extending!");
9540 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9541 assert(VT.isVector() == SVT.isVector() &&
9542 "Cannot use trunc store to convert to or from a vector!");
9543 assert((!VT.isVector() ||
9544 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9545 "Cannot use trunc store to change the number of vector elements!");
9546 }
9547
9548 bool Indexed = AM != ISD::UNINDEXED;
9549 assert((Indexed || Offset.isUndef()) && "Unindexed store with an offset!");
9550 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
9551 : getVTList(VT: MVT::Other);
9552 SDValue Ops[] = {Chain, Val, Ptr, Offset};
9553 FoldingSetNodeID ID;
9554 AddNodeIDNode(ID, OpC: ISD::STORE, VTList: VTs, OpList: Ops);
9555 ID.AddInteger(I: SVT.getRawBits());
9556 ID.AddInteger(I: getSyntheticNodeSubclassData<StoreSDNode>(
9557 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: SVT, Args&: MMO));
9558 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9559 ID.AddInteger(I: MMO->getFlags());
9560 void *IP = nullptr;
9561 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9562 cast<StoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9563 return SDValue(E, 0);
9564 }
9565 auto *N = newSDNode<StoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
9566 Args&: IsTruncating, Args&: SVT, Args&: MMO);
9567 createOperands(Node: N, Vals: Ops);
9568
9569 CSEMap.InsertNode(N, InsertPos: IP);
9570 InsertNode(N);
9571 SDValue V(N, 0);
9572 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9573 return V;
9574}
9575
9576SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
9577 SDValue Ptr, MachinePointerInfo PtrInfo,
9578 EVT SVT, Align Alignment,
9579 MachineMemOperand::Flags MMOFlags,
9580 const AAMDNodes &AAInfo) {
9581 assert(Chain.getValueType() == MVT::Other &&
9582 "Invalid chain type");
9583
9584 MMOFlags |= MachineMemOperand::MOStore;
9585 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9586
9587 if (PtrInfo.V.isNull())
9588 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
9589
9590 MachineFunction &MF = getMachineFunction();
9591 MachineMemOperand *MMO = MF.getMachineMemOperand(
9592 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
9593 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
9594}
9595
9596SDValue SelectionDAG::getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
9597 SDValue Ptr, EVT SVT,
9598 MachineMemOperand *MMO) {
9599 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9600 return getStore(Chain, dl, Val, Ptr, Offset: Undef, SVT, MMO, AM: ISD::UNINDEXED, IsTruncating: true);
9601}
9602
9603SDValue SelectionDAG::getIndexedStore(SDValue OrigStore, const SDLoc &dl,
9604 SDValue Base, SDValue Offset,
9605 ISD::MemIndexedMode AM) {
9606 StoreSDNode *ST = cast<StoreSDNode>(Val&: OrigStore);
9607 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
9608 return getStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Ptr: Base, Offset,
9609 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand(), AM,
9610 IsTruncating: ST->isTruncatingStore());
9611}
9612
9613SDValue SelectionDAG::getLoadVP(
9614 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
9615 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
9616 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
9617 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
9618 const MDNode *Ranges, bool IsExpanding) {
9619 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9620
9621 MMOFlags |= MachineMemOperand::MOLoad;
9622 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
9623 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
9624 // clients.
9625 if (PtrInfo.V.isNull())
9626 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr, OffsetOp: Offset);
9627
9628 TypeSize Size = MemVT.getStoreSize();
9629 MachineFunction &MF = getMachineFunction();
9630 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, F: MMOFlags, Size,
9631 BaseAlignment: Alignment, AAInfo, Ranges);
9632 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
9633 MMO, IsExpanding);
9634}
9635
9636SDValue SelectionDAG::getLoadVP(ISD::MemIndexedMode AM,
9637 ISD::LoadExtType ExtType, EVT VT,
9638 const SDLoc &dl, SDValue Chain, SDValue Ptr,
9639 SDValue Offset, SDValue Mask, SDValue EVL,
9640 EVT MemVT, MachineMemOperand *MMO,
9641 bool IsExpanding) {
9642 bool Indexed = AM != ISD::UNINDEXED;
9643 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9644
9645 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
9646 : getVTList(VT1: VT, VT2: MVT::Other);
9647 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
9648 FoldingSetNodeID ID;
9649 AddNodeIDNode(ID, OpC: ISD::VP_LOAD, VTList: VTs, OpList: Ops);
9650 ID.AddInteger(I: MemVT.getRawBits());
9651 ID.AddInteger(I: getSyntheticNodeSubclassData<VPLoadSDNode>(
9652 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
9653 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9654 ID.AddInteger(I: MMO->getFlags());
9655 void *IP = nullptr;
9656 if (auto *E = cast_or_null<VPLoadSDNode>(Val: FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))) {
9657 E->refineAlignment(NewMMO: MMO);
9658 E->refineRanges(NewMMO: MMO);
9659 return SDValue(E, 0);
9660 }
9661 auto *N = newSDNode<VPLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
9662 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
9663 createOperands(Node: N, Vals: Ops);
9664
9665 CSEMap.InsertNode(N, InsertPos: IP);
9666 InsertNode(N);
9667 SDValue V(N, 0);
9668 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9669 return V;
9670}
9671
9672SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
9673 SDValue Ptr, SDValue Mask, SDValue EVL,
9674 MachinePointerInfo PtrInfo,
9675 MaybeAlign Alignment,
9676 MachineMemOperand::Flags MMOFlags,
9677 const AAMDNodes &AAInfo, const MDNode *Ranges,
9678 bool IsExpanding) {
9679 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9680 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
9681 Mask, EVL, PtrInfo, MemVT: VT, Alignment, MMOFlags, AAInfo, Ranges,
9682 IsExpanding);
9683}
9684
9685SDValue SelectionDAG::getLoadVP(EVT VT, const SDLoc &dl, SDValue Chain,
9686 SDValue Ptr, SDValue Mask, SDValue EVL,
9687 MachineMemOperand *MMO, bool IsExpanding) {
9688 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9689 return getLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Offset: Undef,
9690 Mask, EVL, MemVT: VT, MMO, IsExpanding);
9691}
9692
9693SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
9694 EVT VT, SDValue Chain, SDValue Ptr,
9695 SDValue Mask, SDValue EVL,
9696 MachinePointerInfo PtrInfo, EVT MemVT,
9697 MaybeAlign Alignment,
9698 MachineMemOperand::Flags MMOFlags,
9699 const AAMDNodes &AAInfo, bool IsExpanding) {
9700 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9701 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
9702 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, Ranges: nullptr,
9703 IsExpanding);
9704}
9705
9706SDValue SelectionDAG::getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl,
9707 EVT VT, SDValue Chain, SDValue Ptr,
9708 SDValue Mask, SDValue EVL, EVT MemVT,
9709 MachineMemOperand *MMO, bool IsExpanding) {
9710 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9711 return getLoadVP(AM: ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Offset: Undef, Mask,
9712 EVL, MemVT, MMO, IsExpanding);
9713}
9714
9715SDValue SelectionDAG::getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl,
9716 SDValue Base, SDValue Offset,
9717 ISD::MemIndexedMode AM) {
9718 auto *LD = cast<VPLoadSDNode>(Val&: OrigLoad);
9719 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
9720 // Don't propagate the invariant or dereferenceable flags.
9721 auto MMOFlags =
9722 LD->getMemOperand()->getFlags() &
9723 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
9724 return getLoadVP(AM, ExtType: LD->getExtensionType(), VT: OrigLoad.getValueType(), dl,
9725 Chain: LD->getChain(), Ptr: Base, Offset, Mask: LD->getMask(),
9726 EVL: LD->getVectorLength(), PtrInfo: LD->getPointerInfo(),
9727 MemVT: LD->getMemoryVT(), Alignment: LD->getAlign(), MMOFlags, AAInfo: LD->getAAInfo(),
9728 Ranges: nullptr, IsExpanding: LD->isExpandingLoad());
9729}
9730
9731SDValue SelectionDAG::getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val,
9732 SDValue Ptr, SDValue Offset, SDValue Mask,
9733 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
9734 ISD::MemIndexedMode AM, bool IsTruncating,
9735 bool IsCompressing) {
9736 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9737 bool Indexed = AM != ISD::UNINDEXED;
9738 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9739 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
9740 : getVTList(VT: MVT::Other);
9741 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
9742 FoldingSetNodeID ID;
9743 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
9744 ID.AddInteger(I: MemVT.getRawBits());
9745 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
9746 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
9747 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9748 ID.AddInteger(I: MMO->getFlags());
9749 void *IP = nullptr;
9750 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9751 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9752 return SDValue(E, 0);
9753 }
9754 auto *N = newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
9755 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
9756 createOperands(Node: N, Vals: Ops);
9757
9758 CSEMap.InsertNode(N, InsertPos: IP);
9759 InsertNode(N);
9760 SDValue V(N, 0);
9761 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9762 return V;
9763}
9764
9765SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
9766 SDValue Val, SDValue Ptr, SDValue Mask,
9767 SDValue EVL, MachinePointerInfo PtrInfo,
9768 EVT SVT, Align Alignment,
9769 MachineMemOperand::Flags MMOFlags,
9770 const AAMDNodes &AAInfo,
9771 bool IsCompressing) {
9772 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9773
9774 MMOFlags |= MachineMemOperand::MOStore;
9775 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9776
9777 if (PtrInfo.V.isNull())
9778 PtrInfo = InferPointerInfo(Info: PtrInfo, DAG&: *this, Ptr);
9779
9780 MachineFunction &MF = getMachineFunction();
9781 MachineMemOperand *MMO = MF.getMachineMemOperand(
9782 PtrInfo, F: MMOFlags, Size: SVT.getStoreSize(), BaseAlignment: Alignment, AAInfo);
9783 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
9784 IsCompressing);
9785}
9786
9787SDValue SelectionDAG::getTruncStoreVP(SDValue Chain, const SDLoc &dl,
9788 SDValue Val, SDValue Ptr, SDValue Mask,
9789 SDValue EVL, EVT SVT,
9790 MachineMemOperand *MMO,
9791 bool IsCompressing) {
9792 EVT VT = Val.getValueType();
9793
9794 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9795 if (VT == SVT)
9796 return getStoreVP(Chain, dl, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()), Mask,
9797 EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
9798 /*IsTruncating*/ false, IsCompressing);
9799
9800 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9801 "Should only be a truncating store, not extending!");
9802 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9803 assert(VT.isVector() == SVT.isVector() &&
9804 "Cannot use trunc store to convert to or from a vector!");
9805 assert((!VT.isVector() ||
9806 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9807 "Cannot use trunc store to change the number of vector elements!");
9808
9809 SDVTList VTs = getVTList(VT: MVT::Other);
9810 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9811 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
9812 FoldingSetNodeID ID;
9813 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
9814 ID.AddInteger(I: SVT.getRawBits());
9815 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStoreSDNode>(
9816 IROrder: dl.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
9817 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9818 ID.AddInteger(I: MMO->getFlags());
9819 void *IP = nullptr;
9820 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
9821 cast<VPStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9822 return SDValue(E, 0);
9823 }
9824 auto *N =
9825 newSDNode<VPStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
9826 Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO);
9827 createOperands(Node: N, Vals: Ops);
9828
9829 CSEMap.InsertNode(N, InsertPos: IP);
9830 InsertNode(N);
9831 SDValue V(N, 0);
9832 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9833 return V;
9834}
9835
9836SDValue SelectionDAG::getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl,
9837 SDValue Base, SDValue Offset,
9838 ISD::MemIndexedMode AM) {
9839 auto *ST = cast<VPStoreSDNode>(Val&: OrigStore);
9840 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
9841 SDVTList VTs = getVTList(VT1: Base.getValueType(), VT2: MVT::Other);
9842 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
9843 Offset, ST->getMask(), ST->getVectorLength()};
9844 FoldingSetNodeID ID;
9845 AddNodeIDNode(ID, OpC: ISD::VP_STORE, VTList: VTs, OpList: Ops);
9846 ID.AddInteger(I: ST->getMemoryVT().getRawBits());
9847 ID.AddInteger(I: ST->getRawSubclassData());
9848 ID.AddInteger(I: ST->getPointerInfo().getAddrSpace());
9849 ID.AddInteger(I: ST->getMemOperand()->getFlags());
9850 void *IP = nullptr;
9851 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
9852 return SDValue(E, 0);
9853
9854 auto *N = newSDNode<VPStoreSDNode>(
9855 Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM, Args: ST->isTruncatingStore(),
9856 Args: ST->isCompressingStore(), Args: ST->getMemoryVT(), Args: ST->getMemOperand());
9857 createOperands(Node: N, Vals: Ops);
9858
9859 CSEMap.InsertNode(N, InsertPos: IP);
9860 InsertNode(N);
9861 SDValue V(N, 0);
9862 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9863 return V;
9864}
9865
9866SDValue SelectionDAG::getStridedLoadVP(
9867 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9868 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9869 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9870 bool Indexed = AM != ISD::UNINDEXED;
9871 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9872
9873 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9874 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Ptr.getValueType(), VT3: MVT::Other)
9875 : getVTList(VT1: VT, VT2: MVT::Other);
9876 FoldingSetNodeID ID;
9877 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTList: VTs, OpList: Ops);
9878 ID.AddInteger(I: VT.getRawBits());
9879 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9880 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO));
9881 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9882
9883 void *IP = nullptr;
9884 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9885 cast<VPStridedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9886 return SDValue(E, 0);
9887 }
9888
9889 auto *N =
9890 newSDNode<VPStridedLoadSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs, Args&: AM,
9891 Args&: ExtType, Args&: IsExpanding, Args&: MemVT, Args&: MMO);
9892 createOperands(Node: N, Vals: Ops);
9893 CSEMap.InsertNode(N, InsertPos: IP);
9894 InsertNode(N);
9895 SDValue V(N, 0);
9896 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9897 return V;
9898}
9899
9900SDValue SelectionDAG::getStridedLoadVP(EVT VT, const SDLoc &DL, SDValue Chain,
9901 SDValue Ptr, SDValue Stride,
9902 SDValue Mask, SDValue EVL,
9903 MachineMemOperand *MMO,
9904 bool IsExpanding) {
9905 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9906 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType: ISD::NON_EXTLOAD, VT, DL, Chain, Ptr,
9907 Offset: Undef, Stride, Mask, EVL, MemVT: VT, MMO, IsExpanding);
9908}
9909
9910SDValue SelectionDAG::getExtStridedLoadVP(
9911 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9912 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9913 MachineMemOperand *MMO, bool IsExpanding) {
9914 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9915 return getStridedLoadVP(AM: ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Offset: Undef,
9916 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9917}
9918
9919SDValue SelectionDAG::getStridedStoreVP(SDValue Chain, const SDLoc &DL,
9920 SDValue Val, SDValue Ptr,
9921 SDValue Offset, SDValue Stride,
9922 SDValue Mask, SDValue EVL, EVT MemVT,
9923 MachineMemOperand *MMO,
9924 ISD::MemIndexedMode AM,
9925 bool IsTruncating, bool IsCompressing) {
9926 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9927 bool Indexed = AM != ISD::UNINDEXED;
9928 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9929 SDVTList VTs = Indexed ? getVTList(VT1: Ptr.getValueType(), VT2: MVT::Other)
9930 : getVTList(VT: MVT::Other);
9931 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9932 FoldingSetNodeID ID;
9933 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
9934 ID.AddInteger(I: MemVT.getRawBits());
9935 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9936 IROrder: DL.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
9937 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9938 void *IP = nullptr;
9939 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9940 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9941 return SDValue(E, 0);
9942 }
9943 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
9944 Args&: VTs, Args&: AM, Args&: IsTruncating,
9945 Args&: IsCompressing, Args&: MemVT, Args&: MMO);
9946 createOperands(Node: N, Vals: Ops);
9947
9948 CSEMap.InsertNode(N, InsertPos: IP);
9949 InsertNode(N);
9950 SDValue V(N, 0);
9951 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
9952 return V;
9953}
9954
9955SDValue SelectionDAG::getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL,
9956 SDValue Val, SDValue Ptr,
9957 SDValue Stride, SDValue Mask,
9958 SDValue EVL, EVT SVT,
9959 MachineMemOperand *MMO,
9960 bool IsCompressing) {
9961 EVT VT = Val.getValueType();
9962
9963 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9964 if (VT == SVT)
9965 return getStridedStoreVP(Chain, DL, Val, Ptr, Offset: getUNDEF(VT: Ptr.getValueType()),
9966 Stride, Mask, EVL, MemVT: VT, MMO, AM: ISD::UNINDEXED,
9967 /*IsTruncating*/ false, IsCompressing);
9968
9969 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) &&
9970 "Should only be a truncating store, not extending!");
9971 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9972 assert(VT.isVector() == SVT.isVector() &&
9973 "Cannot use trunc store to convert to or from a vector!");
9974 assert((!VT.isVector() ||
9975 VT.getVectorElementCount() == SVT.getVectorElementCount()) &&
9976 "Cannot use trunc store to change the number of vector elements!");
9977
9978 SDVTList VTs = getVTList(VT: MVT::Other);
9979 SDValue Undef = getUNDEF(VT: Ptr.getValueType());
9980 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9981 FoldingSetNodeID ID;
9982 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTList: VTs, OpList: Ops);
9983 ID.AddInteger(I: SVT.getRawBits());
9984 ID.AddInteger(I: getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9985 IROrder: DL.getIROrder(), Args&: VTs, Args: ISD::UNINDEXED, Args: true, Args&: IsCompressing, Args&: SVT, Args&: MMO));
9986 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
9987 void *IP = nullptr;
9988 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
9989 cast<VPStridedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
9990 return SDValue(E, 0);
9991 }
9992 auto *N = newSDNode<VPStridedStoreSDNode>(Args: DL.getIROrder(), Args: DL.getDebugLoc(),
9993 Args&: VTs, Args: ISD::UNINDEXED, Args: true,
9994 Args&: IsCompressing, Args&: SVT, Args&: MMO);
9995 createOperands(Node: N, Vals: Ops);
9996
9997 CSEMap.InsertNode(N, InsertPos: IP);
9998 InsertNode(N);
9999 SDValue V(N, 0);
10000 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10001 return V;
10002}
10003
10004SDValue SelectionDAG::getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10005 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
10006 ISD::MemIndexType IndexType) {
10007 assert(Ops.size() == 6 && "Incompatible number of operands");
10008
10009 FoldingSetNodeID ID;
10010 AddNodeIDNode(ID, OpC: ISD::VP_GATHER, VTList: VTs, OpList: Ops);
10011 ID.AddInteger(I: VT.getRawBits());
10012 ID.AddInteger(I: getSyntheticNodeSubclassData<VPGatherSDNode>(
10013 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10014 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10015 ID.AddInteger(I: MMO->getFlags());
10016 void *IP = nullptr;
10017 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10018 cast<VPGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10019 return SDValue(E, 0);
10020 }
10021
10022 auto *N = newSDNode<VPGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10023 Args&: VT, Args&: MMO, Args&: IndexType);
10024 createOperands(Node: N, Vals: Ops);
10025
10026 assert(N->getMask().getValueType().getVectorElementCount() ==
10027 N->getValueType(0).getVectorElementCount() &&
10028 "Vector width mismatch between mask and data");
10029 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10030 N->getValueType(0).getVectorElementCount().isScalable() &&
10031 "Scalable flags of index and data do not match");
10032 assert(ElementCount::isKnownGE(
10033 N->getIndex().getValueType().getVectorElementCount(),
10034 N->getValueType(0).getVectorElementCount()) &&
10035 "Vector width mismatch between index and data");
10036 assert(isa<ConstantSDNode>(N->getScale()) &&
10037 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10038 "Scale should be a constant power of 2");
10039
10040 CSEMap.InsertNode(N, InsertPos: IP);
10041 InsertNode(N);
10042 SDValue V(N, 0);
10043 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10044 return V;
10045}
10046
10047SDValue SelectionDAG::getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl,
10048 ArrayRef<SDValue> Ops,
10049 MachineMemOperand *MMO,
10050 ISD::MemIndexType IndexType) {
10051 assert(Ops.size() == 7 && "Incompatible number of operands");
10052
10053 FoldingSetNodeID ID;
10054 AddNodeIDNode(ID, OpC: ISD::VP_SCATTER, VTList: VTs, OpList: Ops);
10055 ID.AddInteger(I: VT.getRawBits());
10056 ID.AddInteger(I: getSyntheticNodeSubclassData<VPScatterSDNode>(
10057 IROrder: dl.getIROrder(), Args&: VTs, Args&: VT, Args&: MMO, Args&: IndexType));
10058 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10059 ID.AddInteger(I: MMO->getFlags());
10060 void *IP = nullptr;
10061 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10062 cast<VPScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10063 return SDValue(E, 0);
10064 }
10065 auto *N = newSDNode<VPScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10066 Args&: VT, Args&: MMO, Args&: IndexType);
10067 createOperands(Node: N, Vals: Ops);
10068
10069 assert(N->getMask().getValueType().getVectorElementCount() ==
10070 N->getValue().getValueType().getVectorElementCount() &&
10071 "Vector width mismatch between mask and data");
10072 assert(
10073 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10074 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10075 "Scalable flags of index and data do not match");
10076 assert(ElementCount::isKnownGE(
10077 N->getIndex().getValueType().getVectorElementCount(),
10078 N->getValue().getValueType().getVectorElementCount()) &&
10079 "Vector width mismatch between index and data");
10080 assert(isa<ConstantSDNode>(N->getScale()) &&
10081 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10082 "Scale should be a constant power of 2");
10083
10084 CSEMap.InsertNode(N, InsertPos: IP);
10085 InsertNode(N);
10086 SDValue V(N, 0);
10087 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10088 return V;
10089}
10090
10091SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain,
10092 SDValue Base, SDValue Offset, SDValue Mask,
10093 SDValue PassThru, EVT MemVT,
10094 MachineMemOperand *MMO,
10095 ISD::MemIndexedMode AM,
10096 ISD::LoadExtType ExtTy, bool isExpanding) {
10097 bool Indexed = AM != ISD::UNINDEXED;
10098 assert((Indexed || Offset.isUndef()) &&
10099 "Unindexed masked load with an offset!");
10100 SDVTList VTs = Indexed ? getVTList(VT1: VT, VT2: Base.getValueType(), VT3: MVT::Other)
10101 : getVTList(VT1: VT, VT2: MVT::Other);
10102 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
10103 FoldingSetNodeID ID;
10104 AddNodeIDNode(ID, OpC: ISD::MLOAD, VTList: VTs, OpList: Ops);
10105 ID.AddInteger(I: MemVT.getRawBits());
10106 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedLoadSDNode>(
10107 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO));
10108 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10109 ID.AddInteger(I: MMO->getFlags());
10110 void *IP = nullptr;
10111 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10112 cast<MaskedLoadSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10113 return SDValue(E, 0);
10114 }
10115 auto *N = newSDNode<MaskedLoadSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs,
10116 Args&: AM, Args&: ExtTy, Args&: isExpanding, Args&: MemVT, Args&: MMO);
10117 createOperands(Node: N, Vals: Ops);
10118
10119 CSEMap.InsertNode(N, InsertPos: IP);
10120 InsertNode(N);
10121 SDValue V(N, 0);
10122 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10123 return V;
10124}
10125
10126SDValue SelectionDAG::getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl,
10127 SDValue Base, SDValue Offset,
10128 ISD::MemIndexedMode AM) {
10129 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(Val&: OrigLoad);
10130 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
10131 return getMaskedLoad(VT: OrigLoad.getValueType(), dl, Chain: LD->getChain(), Base,
10132 Offset, Mask: LD->getMask(), PassThru: LD->getPassThru(),
10133 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand(), AM,
10134 ExtTy: LD->getExtensionType(), isExpanding: LD->isExpandingLoad());
10135}
10136
10137SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl,
10138 SDValue Val, SDValue Base, SDValue Offset,
10139 SDValue Mask, EVT MemVT,
10140 MachineMemOperand *MMO,
10141 ISD::MemIndexedMode AM, bool IsTruncating,
10142 bool IsCompressing) {
10143 assert(Chain.getValueType() == MVT::Other &&
10144 "Invalid chain type");
10145 bool Indexed = AM != ISD::UNINDEXED;
10146 assert((Indexed || Offset.isUndef()) &&
10147 "Unindexed masked store with an offset!");
10148 SDVTList VTs = Indexed ? getVTList(VT1: Base.getValueType(), VT2: MVT::Other)
10149 : getVTList(VT: MVT::Other);
10150 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
10151 FoldingSetNodeID ID;
10152 AddNodeIDNode(ID, OpC: ISD::MSTORE, VTList: VTs, OpList: Ops);
10153 ID.AddInteger(I: MemVT.getRawBits());
10154 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedStoreSDNode>(
10155 IROrder: dl.getIROrder(), Args&: VTs, Args&: AM, Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO));
10156 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10157 ID.AddInteger(I: MMO->getFlags());
10158 void *IP = nullptr;
10159 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10160 cast<MaskedStoreSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10161 return SDValue(E, 0);
10162 }
10163 auto *N =
10164 newSDNode<MaskedStoreSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(), Args&: VTs, Args&: AM,
10165 Args&: IsTruncating, Args&: IsCompressing, Args&: MemVT, Args&: MMO);
10166 createOperands(Node: N, Vals: Ops);
10167
10168 CSEMap.InsertNode(N, InsertPos: IP);
10169 InsertNode(N);
10170 SDValue V(N, 0);
10171 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10172 return V;
10173}
10174
10175SDValue SelectionDAG::getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
10176 SDValue Base, SDValue Offset,
10177 ISD::MemIndexedMode AM) {
10178 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(Val&: OrigStore);
10179 assert(ST->getOffset().isUndef() &&
10180 "Masked store is already a indexed store!");
10181 return getMaskedStore(Chain: ST->getChain(), dl, Val: ST->getValue(), Base, Offset,
10182 Mask: ST->getMask(), MemVT: ST->getMemoryVT(), MMO: ST->getMemOperand(),
10183 AM, IsTruncating: ST->isTruncatingStore(), IsCompressing: ST->isCompressingStore());
10184}
10185
10186SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10187 ArrayRef<SDValue> Ops,
10188 MachineMemOperand *MMO,
10189 ISD::MemIndexType IndexType,
10190 ISD::LoadExtType ExtTy) {
10191 assert(Ops.size() == 6 && "Incompatible number of operands");
10192
10193 FoldingSetNodeID ID;
10194 AddNodeIDNode(ID, OpC: ISD::MGATHER, VTList: VTs, OpList: Ops);
10195 ID.AddInteger(I: MemVT.getRawBits());
10196 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedGatherSDNode>(
10197 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy));
10198 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10199 ID.AddInteger(I: MMO->getFlags());
10200 void *IP = nullptr;
10201 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10202 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10203 return SDValue(E, 0);
10204 }
10205
10206 auto *N = newSDNode<MaskedGatherSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10207 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: ExtTy);
10208 createOperands(Node: N, Vals: Ops);
10209
10210 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
10211 "Incompatible type of the PassThru value in MaskedGatherSDNode");
10212 assert(N->getMask().getValueType().getVectorElementCount() ==
10213 N->getValueType(0).getVectorElementCount() &&
10214 "Vector width mismatch between mask and data");
10215 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10216 N->getValueType(0).getVectorElementCount().isScalable() &&
10217 "Scalable flags of index and data do not match");
10218 assert(ElementCount::isKnownGE(
10219 N->getIndex().getValueType().getVectorElementCount(),
10220 N->getValueType(0).getVectorElementCount()) &&
10221 "Vector width mismatch between index and data");
10222 assert(isa<ConstantSDNode>(N->getScale()) &&
10223 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10224 "Scale should be a constant power of 2");
10225
10226 CSEMap.InsertNode(N, InsertPos: IP);
10227 InsertNode(N);
10228 SDValue V(N, 0);
10229 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10230 return V;
10231}
10232
10233SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl,
10234 ArrayRef<SDValue> Ops,
10235 MachineMemOperand *MMO,
10236 ISD::MemIndexType IndexType,
10237 bool IsTrunc) {
10238 assert(Ops.size() == 6 && "Incompatible number of operands");
10239
10240 FoldingSetNodeID ID;
10241 AddNodeIDNode(ID, OpC: ISD::MSCATTER, VTList: VTs, OpList: Ops);
10242 ID.AddInteger(I: MemVT.getRawBits());
10243 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedScatterSDNode>(
10244 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc));
10245 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10246 ID.AddInteger(I: MMO->getFlags());
10247 void *IP = nullptr;
10248 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10249 cast<MaskedScatterSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10250 return SDValue(E, 0);
10251 }
10252
10253 auto *N = newSDNode<MaskedScatterSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10254 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType, Args&: IsTrunc);
10255 createOperands(Node: N, Vals: Ops);
10256
10257 assert(N->getMask().getValueType().getVectorElementCount() ==
10258 N->getValue().getValueType().getVectorElementCount() &&
10259 "Vector width mismatch between mask and data");
10260 assert(
10261 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
10262 N->getValue().getValueType().getVectorElementCount().isScalable() &&
10263 "Scalable flags of index and data do not match");
10264 assert(ElementCount::isKnownGE(
10265 N->getIndex().getValueType().getVectorElementCount(),
10266 N->getValue().getValueType().getVectorElementCount()) &&
10267 "Vector width mismatch between index and data");
10268 assert(isa<ConstantSDNode>(N->getScale()) &&
10269 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10270 "Scale should be a constant power of 2");
10271
10272 CSEMap.InsertNode(N, InsertPos: IP);
10273 InsertNode(N);
10274 SDValue V(N, 0);
10275 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10276 return V;
10277}
10278
10279SDValue SelectionDAG::getMaskedHistogram(SDVTList VTs, EVT MemVT,
10280 const SDLoc &dl, ArrayRef<SDValue> Ops,
10281 MachineMemOperand *MMO,
10282 ISD::MemIndexType IndexType) {
10283 assert(Ops.size() == 7 && "Incompatible number of operands");
10284
10285 FoldingSetNodeID ID;
10286 AddNodeIDNode(ID, OpC: ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, VTList: VTs, OpList: Ops);
10287 ID.AddInteger(I: MemVT.getRawBits());
10288 ID.AddInteger(I: getSyntheticNodeSubclassData<MaskedHistogramSDNode>(
10289 IROrder: dl.getIROrder(), Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType));
10290 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10291 ID.AddInteger(I: MMO->getFlags());
10292 void *IP = nullptr;
10293 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP)) {
10294 cast<MaskedGatherSDNode>(Val: E)->refineAlignment(NewMMO: MMO);
10295 return SDValue(E, 0);
10296 }
10297
10298 auto *N = newSDNode<MaskedHistogramSDNode>(Args: dl.getIROrder(), Args: dl.getDebugLoc(),
10299 Args&: VTs, Args&: MemVT, Args&: MMO, Args&: IndexType);
10300 createOperands(Node: N, Vals: Ops);
10301
10302 assert(N->getMask().getValueType().getVectorElementCount() ==
10303 N->getIndex().getValueType().getVectorElementCount() &&
10304 "Vector width mismatch between mask and data");
10305 assert(isa<ConstantSDNode>(N->getScale()) &&
10306 N->getScale()->getAsAPIntVal().isPowerOf2() &&
10307 "Scale should be a constant power of 2");
10308 assert(N->getInc().getValueType().isInteger() && "Non integer update value");
10309
10310 CSEMap.InsertNode(N, InsertPos: IP);
10311 InsertNode(N);
10312 SDValue V(N, 0);
10313 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10314 return V;
10315}
10316
10317SDValue SelectionDAG::getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
10318 EVT MemVT, MachineMemOperand *MMO) {
10319 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10320 SDVTList VTs = getVTList(VT: MVT::Other);
10321 SDValue Ops[] = {Chain, Ptr};
10322 FoldingSetNodeID ID;
10323 AddNodeIDNode(ID, OpC: ISD::GET_FPENV_MEM, VTList: VTs, OpList: Ops);
10324 ID.AddInteger(I: MemVT.getRawBits());
10325 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
10326 Opc: ISD::GET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
10327 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10328 ID.AddInteger(I: MMO->getFlags());
10329 void *IP = nullptr;
10330 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10331 return SDValue(E, 0);
10332
10333 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::GET_FPENV_MEM, Args: dl.getIROrder(),
10334 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
10335 createOperands(Node: N, Vals: Ops);
10336
10337 CSEMap.InsertNode(N, InsertPos: IP);
10338 InsertNode(N);
10339 SDValue V(N, 0);
10340 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10341 return V;
10342}
10343
10344SDValue SelectionDAG::getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr,
10345 EVT MemVT, MachineMemOperand *MMO) {
10346 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
10347 SDVTList VTs = getVTList(VT: MVT::Other);
10348 SDValue Ops[] = {Chain, Ptr};
10349 FoldingSetNodeID ID;
10350 AddNodeIDNode(ID, OpC: ISD::SET_FPENV_MEM, VTList: VTs, OpList: Ops);
10351 ID.AddInteger(I: MemVT.getRawBits());
10352 ID.AddInteger(I: getSyntheticNodeSubclassData<FPStateAccessSDNode>(
10353 Opc: ISD::SET_FPENV_MEM, Order: dl.getIROrder(), VTs, MemoryVT: MemVT, MMO));
10354 ID.AddInteger(I: MMO->getPointerInfo().getAddrSpace());
10355 ID.AddInteger(I: MMO->getFlags());
10356 void *IP = nullptr;
10357 if (SDNode *E = FindNodeOrInsertPos(ID, DL: dl, InsertPos&: IP))
10358 return SDValue(E, 0);
10359
10360 auto *N = newSDNode<FPStateAccessSDNode>(Args: ISD::SET_FPENV_MEM, Args: dl.getIROrder(),
10361 Args: dl.getDebugLoc(), Args&: VTs, Args&: MemVT, Args&: MMO);
10362 createOperands(Node: N, Vals: Ops);
10363
10364 CSEMap.InsertNode(N, InsertPos: IP);
10365 InsertNode(N);
10366 SDValue V(N, 0);
10367 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10368 return V;
10369}
10370
10371SDValue SelectionDAG::simplifySelect(SDValue Cond, SDValue T, SDValue F) {
10372 // select undef, T, F --> T (if T is a constant), otherwise F
10373 // select, ?, undef, F --> F
10374 // select, ?, T, undef --> T
10375 if (Cond.isUndef())
10376 return isConstantValueOfAnyType(N: T) ? T : F;
10377 if (T.isUndef())
10378 return F;
10379 if (F.isUndef())
10380 return T;
10381
10382 // select true, T, F --> T
10383 // select false, T, F --> F
10384 if (auto C = isBoolConstant(N: Cond, /*AllowTruncation=*/true))
10385 return *C ? T : F;
10386
10387 // select ?, T, T --> T
10388 if (T == F)
10389 return T;
10390
10391 return SDValue();
10392}
10393
10394SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
10395 // shift undef, Y --> 0 (can always assume that the undef value is 0)
10396 if (X.isUndef())
10397 return getConstant(Val: 0, DL: SDLoc(X.getNode()), VT: X.getValueType());
10398 // shift X, undef --> undef (because it may shift by the bitwidth)
10399 if (Y.isUndef())
10400 return getUNDEF(VT: X.getValueType());
10401
10402 // shift 0, Y --> 0
10403 // shift X, 0 --> X
10404 if (isNullOrNullSplat(V: X) || isNullOrNullSplat(V: Y))
10405 return X;
10406
10407 // shift X, C >= bitwidth(X) --> undef
10408 // All vector elements must be too big (or undef) to avoid partial undefs.
10409 auto isShiftTooBig = [X](ConstantSDNode *Val) {
10410 return !Val || Val->getAPIntValue().uge(RHS: X.getScalarValueSizeInBits());
10411 };
10412 if (ISD::matchUnaryPredicate(Op: Y, Match: isShiftTooBig, AllowUndefs: true))
10413 return getUNDEF(VT: X.getValueType());
10414
10415 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
10416 if (X.getValueType().getScalarType() == MVT::i1)
10417 return X;
10418
10419 return SDValue();
10420}
10421
10422SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y,
10423 SDNodeFlags Flags) {
10424 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
10425 // (an undef operand can be chosen to be Nan/Inf), then the result of this
10426 // operation is poison. That result can be relaxed to undef.
10427 ConstantFPSDNode *XC = isConstOrConstSplatFP(N: X, /* AllowUndefs */ true);
10428 ConstantFPSDNode *YC = isConstOrConstSplatFP(N: Y, /* AllowUndefs */ true);
10429 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
10430 (YC && YC->getValueAPF().isNaN());
10431 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
10432 (YC && YC->getValueAPF().isInfinity());
10433
10434 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
10435 return getUNDEF(VT: X.getValueType());
10436
10437 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
10438 return getUNDEF(VT: X.getValueType());
10439
10440 if (!YC)
10441 return SDValue();
10442
10443 // X + -0.0 --> X
10444 if (Opcode == ISD::FADD)
10445 if (YC->getValueAPF().isNegZero())
10446 return X;
10447
10448 // X - +0.0 --> X
10449 if (Opcode == ISD::FSUB)
10450 if (YC->getValueAPF().isPosZero())
10451 return X;
10452
10453 // X * 1.0 --> X
10454 // X / 1.0 --> X
10455 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
10456 if (YC->getValueAPF().isExactlyValue(V: 1.0))
10457 return X;
10458
10459 // X * 0.0 --> 0.0
10460 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
10461 if (YC->getValueAPF().isZero())
10462 return getConstantFP(Val: 0.0, DL: SDLoc(Y), VT: Y.getValueType());
10463
10464 return SDValue();
10465}
10466
10467SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain,
10468 SDValue Ptr, SDValue SV, unsigned Align) {
10469 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Val: Align, DL: dl, VT: MVT::i32) };
10470 return getNode(Opcode: ISD::VAARG, DL: dl, VTList: getVTList(VT1: VT, VT2: MVT::Other), Ops);
10471}
10472
10473SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
10474 ArrayRef<SDUse> Ops) {
10475 switch (Ops.size()) {
10476 case 0: return getNode(Opcode, DL, VT);
10477 case 1: return getNode(Opcode, DL, VT, N1: static_cast<const SDValue>(Ops[0]));
10478 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1]);
10479 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2]);
10480 default: break;
10481 }
10482
10483 // Copy from an SDUse array into an SDValue array for use with
10484 // the regular getNode logic.
10485 SmallVector<SDValue, 8> NewOps(Ops);
10486 return getNode(Opcode, DL, VT, Ops: NewOps);
10487}
10488
10489SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
10490 ArrayRef<SDValue> Ops) {
10491 SDNodeFlags Flags;
10492 if (Inserter)
10493 Flags = Inserter->getFlags();
10494 return getNode(Opcode, DL, VT, Ops, Flags);
10495}
10496
10497SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
10498 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
10499 unsigned NumOps = Ops.size();
10500 switch (NumOps) {
10501 case 0: return getNode(Opcode, DL, VT);
10502 case 1: return getNode(Opcode, DL, VT, N1: Ops[0], Flags);
10503 case 2: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], Flags);
10504 case 3: return getNode(Opcode, DL, VT, N1: Ops[0], N2: Ops[1], N3: Ops[2], Flags);
10505 default: break;
10506 }
10507
10508#ifndef NDEBUG
10509 for (const auto &Op : Ops)
10510 assert(Op.getOpcode() != ISD::DELETED_NODE &&
10511 "Operand is DELETED_NODE!");
10512#endif
10513
10514 switch (Opcode) {
10515 default: break;
10516 case ISD::BUILD_VECTOR:
10517 // Attempt to simplify BUILD_VECTOR.
10518 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, DAG&: *this))
10519 return V;
10520 break;
10521 case ISD::CONCAT_VECTORS:
10522 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, DAG&: *this))
10523 return V;
10524 break;
10525 case ISD::SELECT_CC:
10526 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
10527 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
10528 "LHS and RHS of condition must have same type!");
10529 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
10530 "True and False arms of SelectCC must have same type!");
10531 assert(Ops[2].getValueType() == VT &&
10532 "select_cc node must be of same type as true and false value!");
10533 assert((!Ops[0].getValueType().isVector() ||
10534 Ops[0].getValueType().getVectorElementCount() ==
10535 VT.getVectorElementCount()) &&
10536 "Expected select_cc with vector result to have the same sized "
10537 "comparison type!");
10538 break;
10539 case ISD::BR_CC:
10540 assert(NumOps == 5 && "BR_CC takes 5 operands!");
10541 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
10542 "LHS/RHS of comparison should match types!");
10543 break;
10544 case ISD::VP_ADD:
10545 case ISD::VP_SUB:
10546 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
10547 if (VT.getScalarType() == MVT::i1)
10548 Opcode = ISD::VP_XOR;
10549 break;
10550 case ISD::VP_MUL:
10551 // If it is VP_MUL mask operation then turn it to VP_AND
10552 if (VT.getScalarType() == MVT::i1)
10553 Opcode = ISD::VP_AND;
10554 break;
10555 case ISD::VP_REDUCE_MUL:
10556 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
10557 if (VT == MVT::i1)
10558 Opcode = ISD::VP_REDUCE_AND;
10559 break;
10560 case ISD::VP_REDUCE_ADD:
10561 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
10562 if (VT == MVT::i1)
10563 Opcode = ISD::VP_REDUCE_XOR;
10564 break;
10565 case ISD::VP_REDUCE_SMAX:
10566 case ISD::VP_REDUCE_UMIN:
10567 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
10568 // VP_REDUCE_AND.
10569 if (VT == MVT::i1)
10570 Opcode = ISD::VP_REDUCE_AND;
10571 break;
10572 case ISD::VP_REDUCE_SMIN:
10573 case ISD::VP_REDUCE_UMAX:
10574 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
10575 // VP_REDUCE_OR.
10576 if (VT == MVT::i1)
10577 Opcode = ISD::VP_REDUCE_OR;
10578 break;
10579 }
10580
10581 // Memoize nodes.
10582 SDNode *N;
10583 SDVTList VTs = getVTList(VT);
10584
10585 if (VT != MVT::Glue) {
10586 FoldingSetNodeID ID;
10587 AddNodeIDNode(ID, OpC: Opcode, VTList: VTs, OpList: Ops);
10588 void *IP = nullptr;
10589
10590 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10591 E->intersectFlagsWith(Flags);
10592 return SDValue(E, 0);
10593 }
10594
10595 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
10596 createOperands(Node: N, Vals: Ops);
10597
10598 CSEMap.InsertNode(N, InsertPos: IP);
10599 } else {
10600 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
10601 createOperands(Node: N, Vals: Ops);
10602 }
10603
10604 N->setFlags(Flags);
10605 InsertNode(N);
10606 SDValue V(N, 0);
10607 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10608 return V;
10609}
10610
10611SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10612 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
10613 return getNode(Opcode, DL, VTList: getVTList(VTs: ResultTys), Ops);
10614}
10615
10616SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10617 ArrayRef<SDValue> Ops) {
10618 SDNodeFlags Flags;
10619 if (Inserter)
10620 Flags = Inserter->getFlags();
10621 return getNode(Opcode, DL, VTList, Ops, Flags);
10622}
10623
10624SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10625 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
10626 if (VTList.NumVTs == 1)
10627 return getNode(Opcode, DL, VT: VTList.VTs[0], Ops, Flags);
10628
10629#ifndef NDEBUG
10630 for (const auto &Op : Ops)
10631 assert(Op.getOpcode() != ISD::DELETED_NODE &&
10632 "Operand is DELETED_NODE!");
10633#endif
10634
10635 switch (Opcode) {
10636 case ISD::SADDO:
10637 case ISD::UADDO:
10638 case ISD::SSUBO:
10639 case ISD::USUBO: {
10640 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
10641 "Invalid add/sub overflow op!");
10642 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
10643 Ops[0].getValueType() == Ops[1].getValueType() &&
10644 Ops[0].getValueType() == VTList.VTs[0] &&
10645 "Binary operator types must match!");
10646 SDValue N1 = Ops[0], N2 = Ops[1];
10647 canonicalizeCommutativeBinop(Opcode, N1, N2);
10648
10649 // (X +- 0) -> X with zero-overflow.
10650 ConstantSDNode *N2CV = isConstOrConstSplat(N: N2, /*AllowUndefs*/ false,
10651 /*AllowTruncation*/ true);
10652 if (N2CV && N2CV->isZero()) {
10653 SDValue ZeroOverFlow = getConstant(Val: 0, DL, VT: VTList.VTs[1]);
10654 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {N1, ZeroOverFlow}, Flags);
10655 }
10656
10657 if (VTList.VTs[0].getScalarType() == MVT::i1 &&
10658 VTList.VTs[1].getScalarType() == MVT::i1) {
10659 SDValue F1 = getFreeze(V: N1);
10660 SDValue F2 = getFreeze(V: N2);
10661 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
10662 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
10663 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
10664 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
10665 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: F1, N2: F2)},
10666 Flags);
10667 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
10668 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
10669 SDValue NotF1 = getNOT(DL, Val: F1, VT: VTList.VTs[0]);
10670 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList,
10671 Ops: {getNode(Opcode: ISD::XOR, DL, VT: VTList.VTs[0], N1: F1, N2: F2),
10672 getNode(Opcode: ISD::AND, DL, VT: VTList.VTs[1], N1: NotF1, N2: F2)},
10673 Flags);
10674 }
10675 }
10676 break;
10677 }
10678 case ISD::SADDO_CARRY:
10679 case ISD::UADDO_CARRY:
10680 case ISD::SSUBO_CARRY:
10681 case ISD::USUBO_CARRY:
10682 assert(VTList.NumVTs == 2 && Ops.size() == 3 &&
10683 "Invalid add/sub overflow op!");
10684 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
10685 Ops[0].getValueType() == Ops[1].getValueType() &&
10686 Ops[0].getValueType() == VTList.VTs[0] &&
10687 Ops[2].getValueType() == VTList.VTs[1] &&
10688 "Binary operator types must match!");
10689 break;
10690 case ISD::SMUL_LOHI:
10691 case ISD::UMUL_LOHI: {
10692 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
10693 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
10694 VTList.VTs[0] == Ops[0].getValueType() &&
10695 VTList.VTs[0] == Ops[1].getValueType() &&
10696 "Binary operator types must match!");
10697 // Constant fold.
10698 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Val: Ops[0]);
10699 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Ops[1]);
10700 if (LHS && RHS) {
10701 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
10702 unsigned OutWidth = Width * 2;
10703 APInt Val = LHS->getAPIntValue();
10704 APInt Mul = RHS->getAPIntValue();
10705 if (Opcode == ISD::SMUL_LOHI) {
10706 Val = Val.sext(width: OutWidth);
10707 Mul = Mul.sext(width: OutWidth);
10708 } else {
10709 Val = Val.zext(width: OutWidth);
10710 Mul = Mul.zext(width: OutWidth);
10711 }
10712 Val *= Mul;
10713
10714 SDValue Hi =
10715 getConstant(Val: Val.extractBits(numBits: Width, bitPosition: Width), DL, VT: VTList.VTs[0]);
10716 SDValue Lo = getConstant(Val: Val.trunc(width: Width), DL, VT: VTList.VTs[0]);
10717 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Lo, Hi}, Flags);
10718 }
10719 break;
10720 }
10721 case ISD::FFREXP: {
10722 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
10723 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
10724 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
10725
10726 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: Ops[0])) {
10727 int FrexpExp;
10728 APFloat FrexpMant =
10729 frexp(X: C->getValueAPF(), Exp&: FrexpExp, RM: APFloat::rmNearestTiesToEven);
10730 SDValue Result0 = getConstantFP(V: FrexpMant, DL, VT: VTList.VTs[0]);
10731 SDValue Result1 =
10732 getConstant(Val: FrexpMant.isFinite() ? FrexpExp : 0, DL, VT: VTList.VTs[1]);
10733 return getNode(Opcode: ISD::MERGE_VALUES, DL, VTList, Ops: {Result0, Result1}, Flags);
10734 }
10735
10736 break;
10737 }
10738 case ISD::STRICT_FP_EXTEND:
10739 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
10740 "Invalid STRICT_FP_EXTEND!");
10741 assert(VTList.VTs[0].isFloatingPoint() &&
10742 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
10743 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
10744 "STRICT_FP_EXTEND result type should be vector iff the operand "
10745 "type is vector!");
10746 assert((!VTList.VTs[0].isVector() ||
10747 VTList.VTs[0].getVectorElementCount() ==
10748 Ops[1].getValueType().getVectorElementCount()) &&
10749 "Vector element count mismatch!");
10750 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
10751 "Invalid fpext node, dst <= src!");
10752 break;
10753 case ISD::STRICT_FP_ROUND:
10754 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
10755 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
10756 "STRICT_FP_ROUND result type should be vector iff the operand "
10757 "type is vector!");
10758 assert((!VTList.VTs[0].isVector() ||
10759 VTList.VTs[0].getVectorElementCount() ==
10760 Ops[1].getValueType().getVectorElementCount()) &&
10761 "Vector element count mismatch!");
10762 assert(VTList.VTs[0].isFloatingPoint() &&
10763 Ops[1].getValueType().isFloatingPoint() &&
10764 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
10765 Ops[2].getOpcode() == ISD::TargetConstant &&
10766 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
10767 "Invalid STRICT_FP_ROUND!");
10768 break;
10769#if 0
10770 // FIXME: figure out how to safely handle things like
10771 // int foo(int x) { return 1 << (x & 255); }
10772 // int bar() { return foo(256); }
10773 case ISD::SRA_PARTS:
10774 case ISD::SRL_PARTS:
10775 case ISD::SHL_PARTS:
10776 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
10777 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
10778 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10779 else if (N3.getOpcode() == ISD::AND)
10780 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
10781 // If the and is only masking out bits that cannot effect the shift,
10782 // eliminate the and.
10783 unsigned NumBits = VT.getScalarSizeInBits()*2;
10784 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
10785 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
10786 }
10787 break;
10788#endif
10789 }
10790
10791 // Memoize the node unless it returns a glue result.
10792 SDNode *N;
10793 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
10794 FoldingSetNodeID ID;
10795 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
10796 void *IP = nullptr;
10797 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
10798 E->intersectFlagsWith(Flags);
10799 return SDValue(E, 0);
10800 }
10801
10802 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
10803 createOperands(Node: N, Vals: Ops);
10804 CSEMap.InsertNode(N, InsertPos: IP);
10805 } else {
10806 N = newSDNode<SDNode>(Args&: Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTList);
10807 createOperands(Node: N, Vals: Ops);
10808 }
10809
10810 N->setFlags(Flags);
10811 InsertNode(N);
10812 SDValue V(N, 0);
10813 NewSDValueDbgMsg(V, Msg: "Creating new node: ", G: this);
10814 return V;
10815}
10816
10817SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10818 SDVTList VTList) {
10819 return getNode(Opcode, DL, VTList, Ops: ArrayRef<SDValue>());
10820}
10821
10822SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10823 SDValue N1) {
10824 SDValue Ops[] = { N1 };
10825 return getNode(Opcode, DL, VTList, Ops);
10826}
10827
10828SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10829 SDValue N1, SDValue N2) {
10830 SDValue Ops[] = { N1, N2 };
10831 return getNode(Opcode, DL, VTList, Ops);
10832}
10833
10834SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10835 SDValue N1, SDValue N2, SDValue N3) {
10836 SDValue Ops[] = { N1, N2, N3 };
10837 return getNode(Opcode, DL, VTList, Ops);
10838}
10839
10840SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10841 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
10842 SDValue Ops[] = { N1, N2, N3, N4 };
10843 return getNode(Opcode, DL, VTList, Ops);
10844}
10845
10846SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10847 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10848 SDValue N5) {
10849 SDValue Ops[] = { N1, N2, N3, N4, N5 };
10850 return getNode(Opcode, DL, VTList, Ops);
10851}
10852
10853SDVTList SelectionDAG::getVTList(EVT VT) {
10854 if (!VT.isExtended())
10855 return makeVTList(VTs: SDNode::getValueTypeList(VT: VT.getSimpleVT()), NumVTs: 1);
10856
10857 return makeVTList(VTs: &(*EVTs.insert(x: VT).first), NumVTs: 1);
10858}
10859
10860SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) {
10861 FoldingSetNodeID ID;
10862 ID.AddInteger(I: 2U);
10863 ID.AddInteger(I: VT1.getRawBits());
10864 ID.AddInteger(I: VT2.getRawBits());
10865
10866 void *IP = nullptr;
10867 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10868 if (!Result) {
10869 EVT *Array = Allocator.Allocate<EVT>(Num: 2);
10870 Array[0] = VT1;
10871 Array[1] = VT2;
10872 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10873 VTListMap.InsertNode(N: Result, InsertPos: IP);
10874 }
10875 return Result->getSDVTList();
10876}
10877
10878SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) {
10879 FoldingSetNodeID ID;
10880 ID.AddInteger(I: 3U);
10881 ID.AddInteger(I: VT1.getRawBits());
10882 ID.AddInteger(I: VT2.getRawBits());
10883 ID.AddInteger(I: VT3.getRawBits());
10884
10885 void *IP = nullptr;
10886 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10887 if (!Result) {
10888 EVT *Array = Allocator.Allocate<EVT>(Num: 3);
10889 Array[0] = VT1;
10890 Array[1] = VT2;
10891 Array[2] = VT3;
10892 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10893 VTListMap.InsertNode(N: Result, InsertPos: IP);
10894 }
10895 return Result->getSDVTList();
10896}
10897
10898SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) {
10899 FoldingSetNodeID ID;
10900 ID.AddInteger(I: 4U);
10901 ID.AddInteger(I: VT1.getRawBits());
10902 ID.AddInteger(I: VT2.getRawBits());
10903 ID.AddInteger(I: VT3.getRawBits());
10904 ID.AddInteger(I: VT4.getRawBits());
10905
10906 void *IP = nullptr;
10907 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10908 if (!Result) {
10909 EVT *Array = Allocator.Allocate<EVT>(Num: 4);
10910 Array[0] = VT1;
10911 Array[1] = VT2;
10912 Array[2] = VT3;
10913 Array[3] = VT4;
10914 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10915 VTListMap.InsertNode(N: Result, InsertPos: IP);
10916 }
10917 return Result->getSDVTList();
10918}
10919
10920SDVTList SelectionDAG::getVTList(ArrayRef<EVT> VTs) {
10921 unsigned NumVTs = VTs.size();
10922 FoldingSetNodeID ID;
10923 ID.AddInteger(I: NumVTs);
10924 for (unsigned index = 0; index < NumVTs; index++) {
10925 ID.AddInteger(I: VTs[index].getRawBits());
10926 }
10927
10928 void *IP = nullptr;
10929 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, InsertPos&: IP);
10930 if (!Result) {
10931 EVT *Array = Allocator.Allocate<EVT>(Num: NumVTs);
10932 llvm::copy(Range&: VTs, Out: Array);
10933 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10934 VTListMap.InsertNode(N: Result, InsertPos: IP);
10935 }
10936 return Result->getSDVTList();
10937}
10938
10939
10940/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10941/// specified operands. If the resultant node already exists in the DAG,
10942/// this does not modify the specified node, instead it returns the node that
10943/// already exists. If the resultant node does not exist in the DAG, the
10944/// input node is returned. As a degenerate case, if you specify the same
10945/// input operands as the node already has, the input node is returned.
10946SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) {
10947 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10948
10949 // Check to see if there is no change.
10950 if (Op == N->getOperand(Num: 0)) return N;
10951
10952 // See if the modified node already exists.
10953 void *InsertPos = nullptr;
10954 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10955 return Existing;
10956
10957 // Nope it doesn't. Remove the node from its current place in the maps.
10958 if (InsertPos)
10959 if (!RemoveNodeFromCSEMaps(N))
10960 InsertPos = nullptr;
10961
10962 // Now we update the operands.
10963 N->OperandList[0].set(Op);
10964
10965 updateDivergence(N);
10966 // If this gets put into a CSE map, add it.
10967 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10968 return N;
10969}
10970
10971SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) {
10972 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10973
10974 // Check to see if there is no change.
10975 if (Op1 == N->getOperand(Num: 0) && Op2 == N->getOperand(Num: 1))
10976 return N; // No operands changed, just return the input node.
10977
10978 // See if the modified node already exists.
10979 void *InsertPos = nullptr;
10980 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10981 return Existing;
10982
10983 // Nope it doesn't. Remove the node from its current place in the maps.
10984 if (InsertPos)
10985 if (!RemoveNodeFromCSEMaps(N))
10986 InsertPos = nullptr;
10987
10988 // Now we update the operands.
10989 if (N->OperandList[0] != Op1)
10990 N->OperandList[0].set(Op1);
10991 if (N->OperandList[1] != Op2)
10992 N->OperandList[1].set(Op2);
10993
10994 updateDivergence(N);
10995 // If this gets put into a CSE map, add it.
10996 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10997 return N;
10998}
10999
11000SDNode *SelectionDAG::
11001UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) {
11002 SDValue Ops[] = { Op1, Op2, Op3 };
11003 return UpdateNodeOperands(N, Ops);
11004}
11005
11006SDNode *SelectionDAG::
11007UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11008 SDValue Op3, SDValue Op4) {
11009 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
11010 return UpdateNodeOperands(N, Ops);
11011}
11012
11013SDNode *SelectionDAG::
11014UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
11015 SDValue Op3, SDValue Op4, SDValue Op5) {
11016 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
11017 return UpdateNodeOperands(N, Ops);
11018}
11019
11020SDNode *SelectionDAG::
11021UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops) {
11022 unsigned NumOps = Ops.size();
11023 assert(N->getNumOperands() == NumOps &&
11024 "Update with wrong number of operands");
11025
11026 // If no operands changed just return the input node.
11027 if (std::equal(first1: Ops.begin(), last1: Ops.end(), first2: N->op_begin()))
11028 return N;
11029
11030 // See if the modified node already exists.
11031 void *InsertPos = nullptr;
11032 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
11033 return Existing;
11034
11035 // Nope it doesn't. Remove the node from its current place in the maps.
11036 if (InsertPos)
11037 if (!RemoveNodeFromCSEMaps(N))
11038 InsertPos = nullptr;
11039
11040 // Now we update the operands.
11041 for (unsigned i = 0; i != NumOps; ++i)
11042 if (N->OperandList[i] != Ops[i])
11043 N->OperandList[i].set(Ops[i]);
11044
11045 updateDivergence(N);
11046 // If this gets put into a CSE map, add it.
11047 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
11048 return N;
11049}
11050
11051/// DropOperands - Release the operands and set this node to have
11052/// zero operands.
11053void SDNode::DropOperands() {
11054 // Unlike the code in MorphNodeTo that does this, we don't need to
11055 // watch for dead nodes here.
11056 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
11057 SDUse &Use = *I++;
11058 Use.set(SDValue());
11059 }
11060}
11061
11062void SelectionDAG::setNodeMemRefs(MachineSDNode *N,
11063 ArrayRef<MachineMemOperand *> NewMemRefs) {
11064 if (NewMemRefs.empty()) {
11065 N->clearMemRefs();
11066 return;
11067 }
11068
11069 // Check if we can avoid allocating by storing a single reference directly.
11070 if (NewMemRefs.size() == 1) {
11071 N->MemRefs = NewMemRefs[0];
11072 N->NumMemRefs = 1;
11073 return;
11074 }
11075
11076 MachineMemOperand **MemRefsBuffer =
11077 Allocator.template Allocate<MachineMemOperand *>(Num: NewMemRefs.size());
11078 llvm::copy(Range&: NewMemRefs, Out: MemRefsBuffer);
11079 N->MemRefs = MemRefsBuffer;
11080 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
11081}
11082
11083/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
11084/// machine opcode.
11085///
11086SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11087 EVT VT) {
11088 SDVTList VTs = getVTList(VT);
11089 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11090}
11091
11092SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11093 EVT VT, SDValue Op1) {
11094 SDVTList VTs = getVTList(VT);
11095 SDValue Ops[] = { Op1 };
11096 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11097}
11098
11099SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11100 EVT VT, SDValue Op1,
11101 SDValue Op2) {
11102 SDVTList VTs = getVTList(VT);
11103 SDValue Ops[] = { Op1, Op2 };
11104 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11105}
11106
11107SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11108 EVT VT, SDValue Op1,
11109 SDValue Op2, SDValue Op3) {
11110 SDVTList VTs = getVTList(VT);
11111 SDValue Ops[] = { Op1, Op2, Op3 };
11112 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11113}
11114
11115SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11116 EVT VT, ArrayRef<SDValue> Ops) {
11117 SDVTList VTs = getVTList(VT);
11118 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11119}
11120
11121SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11122 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
11123 SDVTList VTs = getVTList(VT1, VT2);
11124 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11125}
11126
11127SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11128 EVT VT1, EVT VT2) {
11129 SDVTList VTs = getVTList(VT1, VT2);
11130 return SelectNodeTo(N, MachineOpc, VTs, Ops: {});
11131}
11132
11133SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11134 EVT VT1, EVT VT2, EVT VT3,
11135 ArrayRef<SDValue> Ops) {
11136 SDVTList VTs = getVTList(VT1, VT2, VT3);
11137 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11138}
11139
11140SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11141 EVT VT1, EVT VT2,
11142 SDValue Op1, SDValue Op2) {
11143 SDVTList VTs = getVTList(VT1, VT2);
11144 SDValue Ops[] = { Op1, Op2 };
11145 return SelectNodeTo(N, MachineOpc, VTs, Ops);
11146}
11147
11148SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc,
11149 SDVTList VTs,ArrayRef<SDValue> Ops) {
11150 SDNode *New = MorphNodeTo(N, Opc: ~MachineOpc, VTs, Ops);
11151 // Reset the NodeID to -1.
11152 New->setNodeId(-1);
11153 if (New != N) {
11154 ReplaceAllUsesWith(From: N, To: New);
11155 RemoveDeadNode(N);
11156 }
11157 return New;
11158}
11159
11160/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
11161/// the line number information on the merged node since it is not possible to
11162/// preserve the information that operation is associated with multiple lines.
11163/// This will make the debugger working better at -O0, were there is a higher
11164/// probability having other instructions associated with that line.
11165///
11166/// For IROrder, we keep the smaller of the two
11167SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
11168 DebugLoc NLoc = N->getDebugLoc();
11169 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
11170 N->setDebugLoc(DebugLoc());
11171 }
11172 unsigned Order = std::min(a: N->getIROrder(), b: OLoc.getIROrder());
11173 N->setIROrder(Order);
11174 return N;
11175}
11176
11177/// MorphNodeTo - This *mutates* the specified node to have the specified
11178/// return type, opcode, and operands.
11179///
11180/// Note that MorphNodeTo returns the resultant node. If there is already a
11181/// node of the specified opcode and operands, it returns that node instead of
11182/// the current one. Note that the SDLoc need not be the same.
11183///
11184/// Using MorphNodeTo is faster than creating a new node and swapping it in
11185/// with ReplaceAllUsesWith both because it often avoids allocating a new
11186/// node, and because it doesn't require CSE recalculation for any of
11187/// the node's users.
11188///
11189/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
11190/// As a consequence it isn't appropriate to use from within the DAG combiner or
11191/// the legalizer which maintain worklists that would need to be updated when
11192/// deleting things.
11193SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
11194 SDVTList VTs, ArrayRef<SDValue> Ops) {
11195 // If an identical node already exists, use it.
11196 void *IP = nullptr;
11197 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
11198 FoldingSetNodeID ID;
11199 AddNodeIDNode(ID, OpC: Opc, VTList: VTs, OpList: Ops);
11200 if (SDNode *ON = FindNodeOrInsertPos(ID, DL: SDLoc(N), InsertPos&: IP))
11201 return UpdateSDLocOnMergeSDNode(N: ON, OLoc: SDLoc(N));
11202 }
11203
11204 if (!RemoveNodeFromCSEMaps(N))
11205 IP = nullptr;
11206
11207 // Start the morphing.
11208 N->NodeType = Opc;
11209 N->ValueList = VTs.VTs;
11210 N->NumValues = VTs.NumVTs;
11211
11212 // Clear the operands list, updating used nodes to remove this from their
11213 // use list. Keep track of any operands that become dead as a result.
11214 SmallPtrSet<SDNode*, 16> DeadNodeSet;
11215 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
11216 SDUse &Use = *I++;
11217 SDNode *Used = Use.getNode();
11218 Use.set(SDValue());
11219 if (Used->use_empty())
11220 DeadNodeSet.insert(Ptr: Used);
11221 }
11222
11223 // For MachineNode, initialize the memory references information.
11224 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N))
11225 MN->clearMemRefs();
11226
11227 // Swap for an appropriately sized array from the recycler.
11228 removeOperands(Node: N);
11229 createOperands(Node: N, Vals: Ops);
11230
11231 // Delete any nodes that are still dead after adding the uses for the
11232 // new operands.
11233 if (!DeadNodeSet.empty()) {
11234 SmallVector<SDNode *, 16> DeadNodes;
11235 for (SDNode *N : DeadNodeSet)
11236 if (N->use_empty())
11237 DeadNodes.push_back(Elt: N);
11238 RemoveDeadNodes(DeadNodes);
11239 }
11240
11241 if (IP)
11242 CSEMap.InsertNode(N, InsertPos: IP); // Memoize the new node.
11243 return N;
11244}
11245
11246SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) {
11247 unsigned OrigOpc = Node->getOpcode();
11248 unsigned NewOpc;
11249 switch (OrigOpc) {
11250 default:
11251 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
11252#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11253 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
11254#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
11255 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
11256#include "llvm/IR/ConstrainedOps.def"
11257 }
11258
11259 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
11260
11261 // We're taking this node out of the chain, so we need to re-link things.
11262 SDValue InputChain = Node->getOperand(Num: 0);
11263 SDValue OutputChain = SDValue(Node, 1);
11264 ReplaceAllUsesOfValueWith(From: OutputChain, To: InputChain);
11265
11266 SmallVector<SDValue, 3> Ops;
11267 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
11268 Ops.push_back(Elt: Node->getOperand(Num: i));
11269
11270 SDVTList VTs = getVTList(VT: Node->getValueType(ResNo: 0));
11271 SDNode *Res = MorphNodeTo(N: Node, Opc: NewOpc, VTs, Ops);
11272
11273 // MorphNodeTo can operate in two ways: if an existing node with the
11274 // specified operands exists, it can just return it. Otherwise, it
11275 // updates the node in place to have the requested operands.
11276 if (Res == Node) {
11277 // If we updated the node in place, reset the node ID. To the isel,
11278 // this should be just like a newly allocated machine node.
11279 Res->setNodeId(-1);
11280 } else {
11281 ReplaceAllUsesWith(From: Node, To: Res);
11282 RemoveDeadNode(N: Node);
11283 }
11284
11285 return Res;
11286}
11287
11288/// getMachineNode - These are used for target selectors to create a new node
11289/// with specified return type(s), MachineInstr opcode, and operands.
11290///
11291/// Note that getMachineNode returns the resultant node. If there is already a
11292/// node of the specified opcode and operands, it returns that node instead of
11293/// the current one.
11294MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11295 EVT VT) {
11296 SDVTList VTs = getVTList(VT);
11297 return getMachineNode(Opcode, dl, VTs, Ops: {});
11298}
11299
11300MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11301 EVT VT, SDValue Op1) {
11302 SDVTList VTs = getVTList(VT);
11303 SDValue Ops[] = { Op1 };
11304 return getMachineNode(Opcode, dl, VTs, Ops);
11305}
11306
11307MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11308 EVT VT, SDValue Op1, SDValue Op2) {
11309 SDVTList VTs = getVTList(VT);
11310 SDValue Ops[] = { Op1, Op2 };
11311 return getMachineNode(Opcode, dl, VTs, Ops);
11312}
11313
11314MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11315 EVT VT, SDValue Op1, SDValue Op2,
11316 SDValue Op3) {
11317 SDVTList VTs = getVTList(VT);
11318 SDValue Ops[] = { Op1, Op2, Op3 };
11319 return getMachineNode(Opcode, dl, VTs, Ops);
11320}
11321
11322MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11323 EVT VT, ArrayRef<SDValue> Ops) {
11324 SDVTList VTs = getVTList(VT);
11325 return getMachineNode(Opcode, dl, VTs, Ops);
11326}
11327
11328MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11329 EVT VT1, EVT VT2, SDValue Op1,
11330 SDValue Op2) {
11331 SDVTList VTs = getVTList(VT1, VT2);
11332 SDValue Ops[] = { Op1, Op2 };
11333 return getMachineNode(Opcode, dl, VTs, Ops);
11334}
11335
11336MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11337 EVT VT1, EVT VT2, SDValue Op1,
11338 SDValue Op2, SDValue Op3) {
11339 SDVTList VTs = getVTList(VT1, VT2);
11340 SDValue Ops[] = { Op1, Op2, Op3 };
11341 return getMachineNode(Opcode, dl, VTs, Ops);
11342}
11343
11344MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11345 EVT VT1, EVT VT2,
11346 ArrayRef<SDValue> Ops) {
11347 SDVTList VTs = getVTList(VT1, VT2);
11348 return getMachineNode(Opcode, dl, VTs, Ops);
11349}
11350
11351MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11352 EVT VT1, EVT VT2, EVT VT3,
11353 SDValue Op1, SDValue Op2) {
11354 SDVTList VTs = getVTList(VT1, VT2, VT3);
11355 SDValue Ops[] = { Op1, Op2 };
11356 return getMachineNode(Opcode, dl, VTs, Ops);
11357}
11358
11359MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11360 EVT VT1, EVT VT2, EVT VT3,
11361 SDValue Op1, SDValue Op2,
11362 SDValue Op3) {
11363 SDVTList VTs = getVTList(VT1, VT2, VT3);
11364 SDValue Ops[] = { Op1, Op2, Op3 };
11365 return getMachineNode(Opcode, dl, VTs, Ops);
11366}
11367
11368MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11369 EVT VT1, EVT VT2, EVT VT3,
11370 ArrayRef<SDValue> Ops) {
11371 SDVTList VTs = getVTList(VT1, VT2, VT3);
11372 return getMachineNode(Opcode, dl, VTs, Ops);
11373}
11374
11375MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &dl,
11376 ArrayRef<EVT> ResultTys,
11377 ArrayRef<SDValue> Ops) {
11378 SDVTList VTs = getVTList(VTs: ResultTys);
11379 return getMachineNode(Opcode, dl, VTs, Ops);
11380}
11381
11382MachineSDNode *SelectionDAG::getMachineNode(unsigned Opcode, const SDLoc &DL,
11383 SDVTList VTs,
11384 ArrayRef<SDValue> Ops) {
11385 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
11386 MachineSDNode *N;
11387 void *IP = nullptr;
11388
11389 if (DoCSE) {
11390 FoldingSetNodeID ID;
11391 AddNodeIDNode(ID, OpC: ~Opcode, VTList: VTs, OpList: Ops);
11392 IP = nullptr;
11393 if (SDNode *E = FindNodeOrInsertPos(ID, DL, InsertPos&: IP)) {
11394 return cast<MachineSDNode>(Val: UpdateSDLocOnMergeSDNode(N: E, OLoc: DL));
11395 }
11396 }
11397
11398 // Allocate a new MachineSDNode.
11399 N = newSDNode<MachineSDNode>(Args: ~Opcode, Args: DL.getIROrder(), Args: DL.getDebugLoc(), Args&: VTs);
11400 createOperands(Node: N, Vals: Ops);
11401
11402 if (DoCSE)
11403 CSEMap.InsertNode(N, InsertPos: IP);
11404
11405 InsertNode(N);
11406 NewSDValueDbgMsg(V: SDValue(N, 0), Msg: "Creating new machine node: ", G: this);
11407 return N;
11408}
11409
11410/// getTargetExtractSubreg - A convenience function for creating
11411/// TargetOpcode::EXTRACT_SUBREG nodes.
11412SDValue SelectionDAG::getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
11413 SDValue Operand) {
11414 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
11415 SDNode *Subreg = getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL,
11416 VT, Op1: Operand, Op2: SRIdxVal);
11417 return SDValue(Subreg, 0);
11418}
11419
11420/// getTargetInsertSubreg - A convenience function for creating
11421/// TargetOpcode::INSERT_SUBREG nodes.
11422SDValue SelectionDAG::getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
11423 SDValue Operand, SDValue Subreg) {
11424 SDValue SRIdxVal = getTargetConstant(Val: SRIdx, DL, VT: MVT::i32);
11425 SDNode *Result = getMachineNode(Opcode: TargetOpcode::INSERT_SUBREG, dl: DL,
11426 VT, Op1: Operand, Op2: Subreg, Op3: SRIdxVal);
11427 return SDValue(Result, 0);
11428}
11429
11430/// getNodeIfExists - Get the specified node if it's already available, or
11431/// else return NULL.
11432SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
11433 ArrayRef<SDValue> Ops) {
11434 SDNodeFlags Flags;
11435 if (Inserter)
11436 Flags = Inserter->getFlags();
11437 return getNodeIfExists(Opcode, VTList, Ops, Flags);
11438}
11439
11440SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList,
11441 ArrayRef<SDValue> Ops,
11442 const SDNodeFlags Flags) {
11443 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
11444 FoldingSetNodeID ID;
11445 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
11446 void *IP = nullptr;
11447 if (SDNode *E = FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP)) {
11448 E->intersectFlagsWith(Flags);
11449 return E;
11450 }
11451 }
11452 return nullptr;
11453}
11454
11455/// doesNodeExist - Check if a node exists without modifying its flags.
11456bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
11457 ArrayRef<SDValue> Ops) {
11458 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
11459 FoldingSetNodeID ID;
11460 AddNodeIDNode(ID, OpC: Opcode, VTList, OpList: Ops);
11461 void *IP = nullptr;
11462 if (FindNodeOrInsertPos(ID, DL: SDLoc(), InsertPos&: IP))
11463 return true;
11464 }
11465 return false;
11466}
11467
11468/// getDbgValue - Creates a SDDbgValue node.
11469///
11470/// SDNode
11471SDDbgValue *SelectionDAG::getDbgValue(DIVariable *Var, DIExpression *Expr,
11472 SDNode *N, unsigned R, bool IsIndirect,
11473 const DebugLoc &DL, unsigned O) {
11474 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11475 "Expected inlined-at fields to agree");
11476 return new (DbgInfo->getAlloc())
11477 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(Node: N, ResNo: R),
11478 {}, IsIndirect, DL, O,
11479 /*IsVariadic=*/false);
11480}
11481
11482/// Constant
11483SDDbgValue *SelectionDAG::getConstantDbgValue(DIVariable *Var,
11484 DIExpression *Expr,
11485 const Value *C,
11486 const DebugLoc &DL, unsigned O) {
11487 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11488 "Expected inlined-at fields to agree");
11489 return new (DbgInfo->getAlloc())
11490 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(Const: C), {},
11491 /*IsIndirect=*/false, DL, O,
11492 /*IsVariadic=*/false);
11493}
11494
11495/// FrameIndex
11496SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
11497 DIExpression *Expr, unsigned FI,
11498 bool IsIndirect,
11499 const DebugLoc &DL,
11500 unsigned O) {
11501 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11502 "Expected inlined-at fields to agree");
11503 return getFrameIndexDbgValue(Var, Expr, FI, Dependencies: {}, IsIndirect, DL, O);
11504}
11505
11506/// FrameIndex with dependencies
11507SDDbgValue *SelectionDAG::getFrameIndexDbgValue(DIVariable *Var,
11508 DIExpression *Expr, unsigned FI,
11509 ArrayRef<SDNode *> Dependencies,
11510 bool IsIndirect,
11511 const DebugLoc &DL,
11512 unsigned O) {
11513 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11514 "Expected inlined-at fields to agree");
11515 return new (DbgInfo->getAlloc())
11516 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FrameIdx: FI),
11517 Dependencies, IsIndirect, DL, O,
11518 /*IsVariadic=*/false);
11519}
11520
11521/// VReg
11522SDDbgValue *SelectionDAG::getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
11523 Register VReg, bool IsIndirect,
11524 const DebugLoc &DL, unsigned O) {
11525 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11526 "Expected inlined-at fields to agree");
11527 return new (DbgInfo->getAlloc())
11528 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
11529 {}, IsIndirect, DL, O,
11530 /*IsVariadic=*/false);
11531}
11532
11533SDDbgValue *SelectionDAG::getDbgValueList(DIVariable *Var, DIExpression *Expr,
11534 ArrayRef<SDDbgOperand> Locs,
11535 ArrayRef<SDNode *> Dependencies,
11536 bool IsIndirect, const DebugLoc &DL,
11537 unsigned O, bool IsVariadic) {
11538 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
11539 "Expected inlined-at fields to agree");
11540 return new (DbgInfo->getAlloc())
11541 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
11542 DL, O, IsVariadic);
11543}
11544
11545void SelectionDAG::transferDbgValues(SDValue From, SDValue To,
11546 unsigned OffsetInBits, unsigned SizeInBits,
11547 bool InvalidateDbg) {
11548 SDNode *FromNode = From.getNode();
11549 SDNode *ToNode = To.getNode();
11550 assert(FromNode && ToNode && "Can't modify dbg values");
11551
11552 // PR35338
11553 // TODO: assert(From != To && "Redundant dbg value transfer");
11554 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
11555 if (From == To || FromNode == ToNode)
11556 return;
11557
11558 if (!FromNode->getHasDebugValue())
11559 return;
11560
11561 SDDbgOperand FromLocOp =
11562 SDDbgOperand::fromNode(Node: From.getNode(), ResNo: From.getResNo());
11563 SDDbgOperand ToLocOp = SDDbgOperand::fromNode(Node: To.getNode(), ResNo: To.getResNo());
11564
11565 SmallVector<SDDbgValue *, 2> ClonedDVs;
11566 for (SDDbgValue *Dbg : GetDbgValues(SD: FromNode)) {
11567 if (Dbg->isInvalidated())
11568 continue;
11569
11570 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
11571
11572 // Create a new location ops vector that is equal to the old vector, but
11573 // with each instance of FromLocOp replaced with ToLocOp.
11574 bool Changed = false;
11575 auto NewLocOps = Dbg->copyLocationOps();
11576 std::replace_if(
11577 first: NewLocOps.begin(), last: NewLocOps.end(),
11578 pred: [&Changed, FromLocOp](const SDDbgOperand &Op) {
11579 bool Match = Op == FromLocOp;
11580 Changed |= Match;
11581 return Match;
11582 },
11583 new_value: ToLocOp);
11584 // Ignore this SDDbgValue if we didn't find a matching location.
11585 if (!Changed)
11586 continue;
11587
11588 DIVariable *Var = Dbg->getVariable();
11589 auto *Expr = Dbg->getExpression();
11590 // If a fragment is requested, update the expression.
11591 if (SizeInBits) {
11592 // When splitting a larger (e.g., sign-extended) value whose
11593 // lower bits are described with an SDDbgValue, do not attempt
11594 // to transfer the SDDbgValue to the upper bits.
11595 if (auto FI = Expr->getFragmentInfo())
11596 if (OffsetInBits + SizeInBits > FI->SizeInBits)
11597 continue;
11598 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
11599 SizeInBits);
11600 if (!Fragment)
11601 continue;
11602 Expr = *Fragment;
11603 }
11604
11605 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
11606 // Clone the SDDbgValue and move it to To.
11607 SDDbgValue *Clone = getDbgValueList(
11608 Var, Expr, Locs: NewLocOps, Dependencies: AdditionalDependencies, IsIndirect: Dbg->isIndirect(),
11609 DL: Dbg->getDebugLoc(), O: std::max(a: ToNode->getIROrder(), b: Dbg->getOrder()),
11610 IsVariadic: Dbg->isVariadic());
11611 ClonedDVs.push_back(Elt: Clone);
11612
11613 if (InvalidateDbg) {
11614 // Invalidate value and indicate the SDDbgValue should not be emitted.
11615 Dbg->setIsInvalidated();
11616 Dbg->setIsEmitted();
11617 }
11618 }
11619
11620 for (SDDbgValue *Dbg : ClonedDVs) {
11621 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
11622 "Transferred DbgValues should depend on the new SDNode");
11623 AddDbgValue(DB: Dbg, isParameter: false);
11624 }
11625}
11626
11627void SelectionDAG::salvageDebugInfo(SDNode &N) {
11628 if (!N.getHasDebugValue())
11629 return;
11630
11631 auto GetLocationOperand = [](SDNode *Node, unsigned ResNo) {
11632 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(Val: Node))
11633 return SDDbgOperand::fromFrameIdx(FrameIdx: FISDN->getIndex());
11634 return SDDbgOperand::fromNode(Node, ResNo);
11635 };
11636
11637 SmallVector<SDDbgValue *, 2> ClonedDVs;
11638 for (auto *DV : GetDbgValues(SD: &N)) {
11639 if (DV->isInvalidated())
11640 continue;
11641 switch (N.getOpcode()) {
11642 default:
11643 break;
11644 case ISD::ADD: {
11645 SDValue N0 = N.getOperand(Num: 0);
11646 SDValue N1 = N.getOperand(Num: 1);
11647 if (!isa<ConstantSDNode>(Val: N0)) {
11648 bool RHSConstant = isa<ConstantSDNode>(Val: N1);
11649 uint64_t Offset;
11650 if (RHSConstant)
11651 Offset = N.getConstantOperandVal(Num: 1);
11652 // We are not allowed to turn indirect debug values variadic, so
11653 // don't salvage those.
11654 if (!RHSConstant && DV->isIndirect())
11655 continue;
11656
11657 // Rewrite an ADD constant node into a DIExpression. Since we are
11658 // performing arithmetic to compute the variable's *value* in the
11659 // DIExpression, we need to mark the expression with a
11660 // DW_OP_stack_value.
11661 auto *DIExpr = DV->getExpression();
11662 auto NewLocOps = DV->copyLocationOps();
11663 bool Changed = false;
11664 size_t OrigLocOpsSize = NewLocOps.size();
11665 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
11666 // We're not given a ResNo to compare against because the whole
11667 // node is going away. We know that any ISD::ADD only has one
11668 // result, so we can assume any node match is using the result.
11669 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
11670 NewLocOps[i].getSDNode() != &N)
11671 continue;
11672 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
11673 if (RHSConstant) {
11674 SmallVector<uint64_t, 3> ExprOps;
11675 DIExpression::appendOffset(Ops&: ExprOps, Offset);
11676 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
11677 } else {
11678 // Convert to a variadic expression (if not already).
11679 // convertToVariadicExpression() returns a const pointer, so we use
11680 // a temporary const variable here.
11681 const auto *TmpDIExpr =
11682 DIExpression::convertToVariadicExpression(Expr: DIExpr);
11683 SmallVector<uint64_t, 3> ExprOps;
11684 ExprOps.push_back(Elt: dwarf::DW_OP_LLVM_arg);
11685 ExprOps.push_back(Elt: NewLocOps.size());
11686 ExprOps.push_back(Elt: dwarf::DW_OP_plus);
11687 SDDbgOperand RHS =
11688 SDDbgOperand::fromNode(Node: N1.getNode(), ResNo: N1.getResNo());
11689 NewLocOps.push_back(Elt: RHS);
11690 DIExpr = DIExpression::appendOpsToArg(Expr: TmpDIExpr, Ops: ExprOps, ArgNo: i, StackValue: true);
11691 }
11692 Changed = true;
11693 }
11694 (void)Changed;
11695 assert(Changed && "Salvage target doesn't use N");
11696
11697 bool IsVariadic =
11698 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
11699
11700 auto AdditionalDependencies = DV->getAdditionalDependencies();
11701 SDDbgValue *Clone = getDbgValueList(
11702 Var: DV->getVariable(), Expr: DIExpr, Locs: NewLocOps, Dependencies: AdditionalDependencies,
11703 IsIndirect: DV->isIndirect(), DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic);
11704 ClonedDVs.push_back(Elt: Clone);
11705 DV->setIsInvalidated();
11706 DV->setIsEmitted();
11707 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
11708 N0.getNode()->dumprFull(this);
11709 dbgs() << " into " << *DIExpr << '\n');
11710 }
11711 break;
11712 }
11713 case ISD::TRUNCATE: {
11714 SDValue N0 = N.getOperand(Num: 0);
11715 TypeSize FromSize = N0.getValueSizeInBits();
11716 TypeSize ToSize = N.getValueSizeInBits(ResNo: 0);
11717
11718 DIExpression *DbgExpression = DV->getExpression();
11719 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, Signed: false);
11720 auto NewLocOps = DV->copyLocationOps();
11721 bool Changed = false;
11722 for (size_t i = 0; i < NewLocOps.size(); ++i) {
11723 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
11724 NewLocOps[i].getSDNode() != &N)
11725 continue;
11726
11727 NewLocOps[i] = GetLocationOperand(N0.getNode(), N0.getResNo());
11728 DbgExpression = DIExpression::appendOpsToArg(Expr: DbgExpression, Ops: ExtOps, ArgNo: i);
11729 Changed = true;
11730 }
11731 assert(Changed && "Salvage target doesn't use N");
11732 (void)Changed;
11733
11734 SDDbgValue *Clone =
11735 getDbgValueList(Var: DV->getVariable(), Expr: DbgExpression, Locs: NewLocOps,
11736 Dependencies: DV->getAdditionalDependencies(), IsIndirect: DV->isIndirect(),
11737 DL: DV->getDebugLoc(), O: DV->getOrder(), IsVariadic: DV->isVariadic());
11738
11739 ClonedDVs.push_back(Elt: Clone);
11740 DV->setIsInvalidated();
11741 DV->setIsEmitted();
11742 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
11743 dbgs() << " into " << *DbgExpression << '\n');
11744 break;
11745 }
11746 }
11747 }
11748
11749 for (SDDbgValue *Dbg : ClonedDVs) {
11750 assert((!Dbg->getSDNodes().empty() ||
11751 llvm::any_of(Dbg->getLocationOps(),
11752 [&](const SDDbgOperand &Op) {
11753 return Op.getKind() == SDDbgOperand::FRAMEIX;
11754 })) &&
11755 "Salvaged DbgValue should depend on a new SDNode");
11756 AddDbgValue(DB: Dbg, isParameter: false);
11757 }
11758}
11759
11760/// Creates a SDDbgLabel node.
11761SDDbgLabel *SelectionDAG::getDbgLabel(DILabel *Label,
11762 const DebugLoc &DL, unsigned O) {
11763 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
11764 "Expected inlined-at fields to agree");
11765 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
11766}
11767
11768namespace {
11769
11770/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
11771/// pointed to by a use iterator is deleted, increment the use iterator
11772/// so that it doesn't dangle.
11773///
11774class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
11775 SDNode::use_iterator &UI;
11776 SDNode::use_iterator &UE;
11777
11778 void NodeDeleted(SDNode *N, SDNode *E) override {
11779 // Increment the iterator as needed.
11780 while (UI != UE && N == UI->getUser())
11781 ++UI;
11782 }
11783
11784public:
11785 RAUWUpdateListener(SelectionDAG &d,
11786 SDNode::use_iterator &ui,
11787 SDNode::use_iterator &ue)
11788 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
11789};
11790
11791} // end anonymous namespace
11792
11793/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11794/// This can cause recursive merging of nodes in the DAG.
11795///
11796/// This version assumes From has a single result value.
11797///
11798void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) {
11799 SDNode *From = FromN.getNode();
11800 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
11801 "Cannot replace with this method!");
11802 assert(From != To.getNode() && "Cannot replace uses of with self");
11803
11804 // Preserve Debug Values
11805 transferDbgValues(From: FromN, To);
11806 // Preserve extra info.
11807 copyExtraInfo(From, To: To.getNode());
11808
11809 // Iterate over all the existing uses of From. New uses will be added
11810 // to the beginning of the use list, which we avoid visiting.
11811 // This specifically avoids visiting uses of From that arise while the
11812 // replacement is happening, because any such uses would be the result
11813 // of CSE: If an existing node looks like From after one of its operands
11814 // is replaced by To, we don't want to replace of all its users with To
11815 // too. See PR3018 for more info.
11816 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11817 RAUWUpdateListener Listener(*this, UI, UE);
11818 while (UI != UE) {
11819 SDNode *User = UI->getUser();
11820
11821 // This node is about to morph, remove its old self from the CSE maps.
11822 RemoveNodeFromCSEMaps(N: User);
11823
11824 // A user can appear in a use list multiple times, and when this
11825 // happens the uses are usually next to each other in the list.
11826 // To help reduce the number of CSE recomputations, process all
11827 // the uses of this user that we can find this way.
11828 do {
11829 SDUse &Use = *UI;
11830 ++UI;
11831 Use.set(To);
11832 if (To->isDivergent() != From->isDivergent())
11833 updateDivergence(N: User);
11834 } while (UI != UE && UI->getUser() == User);
11835 // Now that we have modified User, add it back to the CSE maps. If it
11836 // already exists there, recursively merge the results together.
11837 AddModifiedNodeToCSEMaps(N: User);
11838 }
11839
11840 // If we just RAUW'd the root, take note.
11841 if (FromN == getRoot())
11842 setRoot(To);
11843}
11844
11845/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11846/// This can cause recursive merging of nodes in the DAG.
11847///
11848/// This version assumes that for each value of From, there is a
11849/// corresponding value in To in the same position with the same type.
11850///
11851void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) {
11852#ifndef NDEBUG
11853 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11854 assert((!From->hasAnyUseOfValue(i) ||
11855 From->getValueType(i) == To->getValueType(i)) &&
11856 "Cannot use this version of ReplaceAllUsesWith!");
11857#endif
11858
11859 // Handle the trivial case.
11860 if (From == To)
11861 return;
11862
11863 // Preserve Debug Info. Only do this if there's a use.
11864 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11865 if (From->hasAnyUseOfValue(Value: i)) {
11866 assert((i < To->getNumValues()) && "Invalid To location");
11867 transferDbgValues(From: SDValue(From, i), To: SDValue(To, i));
11868 }
11869 // Preserve extra info.
11870 copyExtraInfo(From, To);
11871
11872 // Iterate over just the existing users of From. See the comments in
11873 // the ReplaceAllUsesWith above.
11874 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11875 RAUWUpdateListener Listener(*this, UI, UE);
11876 while (UI != UE) {
11877 SDNode *User = UI->getUser();
11878
11879 // This node is about to morph, remove its old self from the CSE maps.
11880 RemoveNodeFromCSEMaps(N: User);
11881
11882 // A user can appear in a use list multiple times, and when this
11883 // happens the uses are usually next to each other in the list.
11884 // To help reduce the number of CSE recomputations, process all
11885 // the uses of this user that we can find this way.
11886 do {
11887 SDUse &Use = *UI;
11888 ++UI;
11889 Use.setNode(To);
11890 if (To->isDivergent() != From->isDivergent())
11891 updateDivergence(N: User);
11892 } while (UI != UE && UI->getUser() == User);
11893
11894 // Now that we have modified User, add it back to the CSE maps. If it
11895 // already exists there, recursively merge the results together.
11896 AddModifiedNodeToCSEMaps(N: User);
11897 }
11898
11899 // If we just RAUW'd the root, take note.
11900 if (From == getRoot().getNode())
11901 setRoot(SDValue(To, getRoot().getResNo()));
11902}
11903
11904/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11905/// This can cause recursive merging of nodes in the DAG.
11906///
11907/// This version can replace From with any result values. To must match the
11908/// number and types of values returned by From.
11909void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
11910 if (From->getNumValues() == 1) // Handle the simple case efficiently.
11911 return ReplaceAllUsesWith(FromN: SDValue(From, 0), To: To[0]);
11912
11913 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11914 // Preserve Debug Info.
11915 transferDbgValues(From: SDValue(From, i), To: To[i]);
11916 // Preserve extra info.
11917 copyExtraInfo(From, To: To[i].getNode());
11918 }
11919
11920 // Iterate over just the existing users of From. See the comments in
11921 // the ReplaceAllUsesWith above.
11922 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11923 RAUWUpdateListener Listener(*this, UI, UE);
11924 while (UI != UE) {
11925 SDNode *User = UI->getUser();
11926
11927 // This node is about to morph, remove its old self from the CSE maps.
11928 RemoveNodeFromCSEMaps(N: User);
11929
11930 // A user can appear in a use list multiple times, and when this happens the
11931 // uses are usually next to each other in the list. To help reduce the
11932 // number of CSE and divergence recomputations, process all the uses of this
11933 // user that we can find this way.
11934 bool To_IsDivergent = false;
11935 do {
11936 SDUse &Use = *UI;
11937 const SDValue &ToOp = To[Use.getResNo()];
11938 ++UI;
11939 Use.set(ToOp);
11940 To_IsDivergent |= ToOp->isDivergent();
11941 } while (UI != UE && UI->getUser() == User);
11942
11943 if (To_IsDivergent != From->isDivergent())
11944 updateDivergence(N: User);
11945
11946 // Now that we have modified User, add it back to the CSE maps. If it
11947 // already exists there, recursively merge the results together.
11948 AddModifiedNodeToCSEMaps(N: User);
11949 }
11950
11951 // If we just RAUW'd the root, take note.
11952 if (From == getRoot().getNode())
11953 setRoot(SDValue(To[getRoot().getResNo()]));
11954}
11955
11956/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11957/// uses of other values produced by From.getNode() alone. The Deleted
11958/// vector is handled the same way as for ReplaceAllUsesWith.
11959void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){
11960 // Handle the really simple, really trivial case efficiently.
11961 if (From == To) return;
11962
11963 // Handle the simple, trivial, case efficiently.
11964 if (From.getNode()->getNumValues() == 1) {
11965 ReplaceAllUsesWith(FromN: From, To);
11966 return;
11967 }
11968
11969 // Preserve Debug Info.
11970 transferDbgValues(From, To);
11971 copyExtraInfo(From: From.getNode(), To: To.getNode());
11972
11973 // Iterate over just the existing users of From. See the comments in
11974 // the ReplaceAllUsesWith above.
11975 SDNode::use_iterator UI = From.getNode()->use_begin(),
11976 UE = From.getNode()->use_end();
11977 RAUWUpdateListener Listener(*this, UI, UE);
11978 while (UI != UE) {
11979 SDNode *User = UI->getUser();
11980 bool UserRemovedFromCSEMaps = false;
11981
11982 // A user can appear in a use list multiple times, and when this
11983 // happens the uses are usually next to each other in the list.
11984 // To help reduce the number of CSE recomputations, process all
11985 // the uses of this user that we can find this way.
11986 do {
11987 SDUse &Use = *UI;
11988
11989 // Skip uses of different values from the same node.
11990 if (Use.getResNo() != From.getResNo()) {
11991 ++UI;
11992 continue;
11993 }
11994
11995 // If this node hasn't been modified yet, it's still in the CSE maps,
11996 // so remove its old self from the CSE maps.
11997 if (!UserRemovedFromCSEMaps) {
11998 RemoveNodeFromCSEMaps(N: User);
11999 UserRemovedFromCSEMaps = true;
12000 }
12001
12002 ++UI;
12003 Use.set(To);
12004 if (To->isDivergent() != From->isDivergent())
12005 updateDivergence(N: User);
12006 } while (UI != UE && UI->getUser() == User);
12007 // We are iterating over all uses of the From node, so if a use
12008 // doesn't use the specific value, no changes are made.
12009 if (!UserRemovedFromCSEMaps)
12010 continue;
12011
12012 // Now that we have modified User, add it back to the CSE maps. If it
12013 // already exists there, recursively merge the results together.
12014 AddModifiedNodeToCSEMaps(N: User);
12015 }
12016
12017 // If we just RAUW'd the root, take note.
12018 if (From == getRoot())
12019 setRoot(To);
12020}
12021
12022namespace {
12023
12024/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
12025/// to record information about a use.
12026struct UseMemo {
12027 SDNode *User;
12028 unsigned Index;
12029 SDUse *Use;
12030};
12031
12032/// operator< - Sort Memos by User.
12033bool operator<(const UseMemo &L, const UseMemo &R) {
12034 return (intptr_t)L.User < (intptr_t)R.User;
12035}
12036
12037/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
12038/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
12039/// the node already has been taken care of recursively.
12040class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
12041 SmallVectorImpl<UseMemo> &Uses;
12042
12043 void NodeDeleted(SDNode *N, SDNode *E) override {
12044 for (UseMemo &Memo : Uses)
12045 if (Memo.User == N)
12046 Memo.User = nullptr;
12047 }
12048
12049public:
12050 RAUOVWUpdateListener(SelectionDAG &d, SmallVectorImpl<UseMemo> &uses)
12051 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
12052};
12053
12054} // end anonymous namespace
12055
12056/// Return true if a glue output should propagate divergence information.
12057static bool gluePropagatesDivergence(const SDNode *Node) {
12058 switch (Node->getOpcode()) {
12059 case ISD::CopyFromReg:
12060 case ISD::CopyToReg:
12061 return false;
12062 default:
12063 return true;
12064 }
12065
12066 llvm_unreachable("covered opcode switch");
12067}
12068
12069bool SelectionDAG::calculateDivergence(SDNode *N) {
12070 if (TLI->isSDNodeAlwaysUniform(N)) {
12071 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
12072 "Conflicting divergence information!");
12073 return false;
12074 }
12075 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
12076 return true;
12077 for (const auto &Op : N->ops()) {
12078 EVT VT = Op.getValueType();
12079
12080 // Skip Chain. It does not carry divergence.
12081 if (VT != MVT::Other && Op.getNode()->isDivergent() &&
12082 (VT != MVT::Glue || gluePropagatesDivergence(Node: Op.getNode())))
12083 return true;
12084 }
12085 return false;
12086}
12087
12088void SelectionDAG::updateDivergence(SDNode *N) {
12089 SmallVector<SDNode *, 16> Worklist(1, N);
12090 do {
12091 N = Worklist.pop_back_val();
12092 bool IsDivergent = calculateDivergence(N);
12093 if (N->SDNodeBits.IsDivergent != IsDivergent) {
12094 N->SDNodeBits.IsDivergent = IsDivergent;
12095 llvm::append_range(C&: Worklist, R: N->users());
12096 }
12097 } while (!Worklist.empty());
12098}
12099
12100void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
12101 DenseMap<SDNode *, unsigned> Degree;
12102 Order.reserve(n: AllNodes.size());
12103 for (auto &N : allnodes()) {
12104 unsigned NOps = N.getNumOperands();
12105 Degree[&N] = NOps;
12106 if (0 == NOps)
12107 Order.push_back(x: &N);
12108 }
12109 for (size_t I = 0; I != Order.size(); ++I) {
12110 SDNode *N = Order[I];
12111 for (auto *U : N->users()) {
12112 unsigned &UnsortedOps = Degree[U];
12113 if (0 == --UnsortedOps)
12114 Order.push_back(x: U);
12115 }
12116 }
12117}
12118
12119#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
12120void SelectionDAG::VerifyDAGDivergence() {
12121 std::vector<SDNode *> TopoOrder;
12122 CreateTopologicalOrder(TopoOrder);
12123 for (auto *N : TopoOrder) {
12124 assert(calculateDivergence(N) == N->isDivergent() &&
12125 "Divergence bit inconsistency detected");
12126 }
12127}
12128#endif
12129
12130/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
12131/// uses of other values produced by From.getNode() alone. The same value
12132/// may appear in both the From and To list. The Deleted vector is
12133/// handled the same way as for ReplaceAllUsesWith.
12134void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From,
12135 const SDValue *To,
12136 unsigned Num){
12137 // Handle the simple, trivial case efficiently.
12138 if (Num == 1)
12139 return ReplaceAllUsesOfValueWith(From: *From, To: *To);
12140
12141 transferDbgValues(From: *From, To: *To);
12142 copyExtraInfo(From: From->getNode(), To: To->getNode());
12143
12144 // Read up all the uses and make records of them. This helps
12145 // processing new uses that are introduced during the
12146 // replacement process.
12147 SmallVector<UseMemo, 4> Uses;
12148 for (unsigned i = 0; i != Num; ++i) {
12149 unsigned FromResNo = From[i].getResNo();
12150 SDNode *FromNode = From[i].getNode();
12151 for (SDUse &Use : FromNode->uses()) {
12152 if (Use.getResNo() == FromResNo) {
12153 UseMemo Memo = {.User: Use.getUser(), .Index: i, .Use: &Use};
12154 Uses.push_back(Elt: Memo);
12155 }
12156 }
12157 }
12158
12159 // Sort the uses, so that all the uses from a given User are together.
12160 llvm::sort(C&: Uses);
12161 RAUOVWUpdateListener Listener(*this, Uses);
12162
12163 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
12164 UseIndex != UseIndexEnd; ) {
12165 // We know that this user uses some value of From. If it is the right
12166 // value, update it.
12167 SDNode *User = Uses[UseIndex].User;
12168 // If the node has been deleted by recursive CSE updates when updating
12169 // another node, then just skip this entry.
12170 if (User == nullptr) {
12171 ++UseIndex;
12172 continue;
12173 }
12174
12175 // This node is about to morph, remove its old self from the CSE maps.
12176 RemoveNodeFromCSEMaps(N: User);
12177
12178 // The Uses array is sorted, so all the uses for a given User
12179 // are next to each other in the list.
12180 // To help reduce the number of CSE recomputations, process all
12181 // the uses of this user that we can find this way.
12182 do {
12183 unsigned i = Uses[UseIndex].Index;
12184 SDUse &Use = *Uses[UseIndex].Use;
12185 ++UseIndex;
12186
12187 Use.set(To[i]);
12188 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
12189
12190 // Now that we have modified User, add it back to the CSE maps. If it
12191 // already exists there, recursively merge the results together.
12192 AddModifiedNodeToCSEMaps(N: User);
12193 }
12194}
12195
12196/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
12197/// based on their topological order. It returns the maximum id and a vector
12198/// of the SDNodes* in assigned order by reference.
12199unsigned SelectionDAG::AssignTopologicalOrder() {
12200 unsigned DAGSize = 0;
12201
12202 // SortedPos tracks the progress of the algorithm. Nodes before it are
12203 // sorted, nodes after it are unsorted. When the algorithm completes
12204 // it is at the end of the list.
12205 allnodes_iterator SortedPos = allnodes_begin();
12206
12207 // Visit all the nodes. Move nodes with no operands to the front of
12208 // the list immediately. Annotate nodes that do have operands with their
12209 // operand count. Before we do this, the Node Id fields of the nodes
12210 // may contain arbitrary values. After, the Node Id fields for nodes
12211 // before SortedPos will contain the topological sort index, and the
12212 // Node Id fields for nodes At SortedPos and after will contain the
12213 // count of outstanding operands.
12214 for (SDNode &N : llvm::make_early_inc_range(Range: allnodes())) {
12215 checkForCycles(N: &N, DAG: this);
12216 unsigned Degree = N.getNumOperands();
12217 if (Degree == 0) {
12218 // A node with no uses, add it to the result array immediately.
12219 N.setNodeId(DAGSize++);
12220 allnodes_iterator Q(&N);
12221 if (Q != SortedPos)
12222 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT&: Q));
12223 assert(SortedPos != AllNodes.end() && "Overran node list");
12224 ++SortedPos;
12225 } else {
12226 // Temporarily use the Node Id as scratch space for the degree count.
12227 N.setNodeId(Degree);
12228 }
12229 }
12230
12231 // Visit all the nodes. As we iterate, move nodes into sorted order,
12232 // such that by the time the end is reached all nodes will be sorted.
12233 for (SDNode &Node : allnodes()) {
12234 SDNode *N = &Node;
12235 checkForCycles(N, DAG: this);
12236 // N is in sorted position, so all its uses have one less operand
12237 // that needs to be sorted.
12238 for (SDNode *P : N->users()) {
12239 unsigned Degree = P->getNodeId();
12240 assert(Degree != 0 && "Invalid node degree");
12241 --Degree;
12242 if (Degree == 0) {
12243 // All of P's operands are sorted, so P may sorted now.
12244 P->setNodeId(DAGSize++);
12245 if (P->getIterator() != SortedPos)
12246 SortedPos = AllNodes.insert(where: SortedPos, New: AllNodes.remove(IT: P));
12247 assert(SortedPos != AllNodes.end() && "Overran node list");
12248 ++SortedPos;
12249 } else {
12250 // Update P's outstanding operand count.
12251 P->setNodeId(Degree);
12252 }
12253 }
12254 if (Node.getIterator() == SortedPos) {
12255#ifndef NDEBUG
12256 allnodes_iterator I(N);
12257 SDNode *S = &*++I;
12258 dbgs() << "Overran sorted position:\n";
12259 S->dumprFull(this); dbgs() << "\n";
12260 dbgs() << "Checking if this is due to cycles\n";
12261 checkForCycles(this, true);
12262#endif
12263 llvm_unreachable(nullptr);
12264 }
12265 }
12266
12267 assert(SortedPos == AllNodes.end() &&
12268 "Topological sort incomplete!");
12269 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
12270 "First node in topological sort is not the entry token!");
12271 assert(AllNodes.front().getNodeId() == 0 &&
12272 "First node in topological sort has non-zero id!");
12273 assert(AllNodes.front().getNumOperands() == 0 &&
12274 "First node in topological sort has operands!");
12275 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
12276 "Last node in topologic sort has unexpected id!");
12277 assert(AllNodes.back().use_empty() &&
12278 "Last node in topologic sort has users!");
12279 assert(DAGSize == allnodes_size() && "Node count mismatch!");
12280 return DAGSize;
12281}
12282
12283/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
12284/// value is produced by SD.
12285void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
12286 for (SDNode *SD : DB->getSDNodes()) {
12287 if (!SD)
12288 continue;
12289 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
12290 SD->setHasDebugValue(true);
12291 }
12292 DbgInfo->add(V: DB, isParameter);
12293}
12294
12295void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(L: DB); }
12296
12297SDValue SelectionDAG::makeEquivalentMemoryOrdering(SDValue OldChain,
12298 SDValue NewMemOpChain) {
12299 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
12300 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
12301 // The new memory operation must have the same position as the old load in
12302 // terms of memory dependency. Create a TokenFactor for the old load and new
12303 // memory operation and update uses of the old load's output chain to use that
12304 // TokenFactor.
12305 if (OldChain == NewMemOpChain || OldChain.use_empty())
12306 return NewMemOpChain;
12307
12308 SDValue TokenFactor = getNode(Opcode: ISD::TokenFactor, DL: SDLoc(OldChain), VT: MVT::Other,
12309 N1: OldChain, N2: NewMemOpChain);
12310 ReplaceAllUsesOfValueWith(From: OldChain, To: TokenFactor);
12311 UpdateNodeOperands(N: TokenFactor.getNode(), Op1: OldChain, Op2: NewMemOpChain);
12312 return TokenFactor;
12313}
12314
12315SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
12316 SDValue NewMemOp) {
12317 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
12318 SDValue OldChain = SDValue(OldLoad, 1);
12319 SDValue NewMemOpChain = NewMemOp.getValue(R: 1);
12320 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
12321}
12322
12323SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
12324 Function **OutFunction) {
12325 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
12326
12327 auto *Symbol = cast<ExternalSymbolSDNode>(Val&: Op)->getSymbol();
12328 auto *Module = MF->getFunction().getParent();
12329 auto *Function = Module->getFunction(Name: Symbol);
12330
12331 if (OutFunction != nullptr)
12332 *OutFunction = Function;
12333
12334 if (Function != nullptr) {
12335 auto PtrTy = TLI->getPointerTy(DL: getDataLayout(), AS: Function->getAddressSpace());
12336 return getGlobalAddress(GV: Function, DL: SDLoc(Op), VT: PtrTy);
12337 }
12338
12339 std::string ErrorStr;
12340 raw_string_ostream ErrorFormatter(ErrorStr);
12341 ErrorFormatter << "Undefined external symbol ";
12342 ErrorFormatter << '"' << Symbol << '"';
12343 report_fatal_error(reason: Twine(ErrorStr));
12344}
12345
12346//===----------------------------------------------------------------------===//
12347// SDNode Class
12348//===----------------------------------------------------------------------===//
12349
12350bool llvm::isNullConstant(SDValue V) {
12351 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
12352 return Const != nullptr && Const->isZero();
12353}
12354
12355bool llvm::isNullConstantOrUndef(SDValue V) {
12356 return V.isUndef() || isNullConstant(V);
12357}
12358
12359bool llvm::isNullFPConstant(SDValue V) {
12360 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(Val&: V);
12361 return Const != nullptr && Const->isZero() && !Const->isNegative();
12362}
12363
12364bool llvm::isAllOnesConstant(SDValue V) {
12365 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
12366 return Const != nullptr && Const->isAllOnes();
12367}
12368
12369bool llvm::isOneConstant(SDValue V) {
12370 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
12371 return Const != nullptr && Const->isOne();
12372}
12373
12374bool llvm::isMinSignedConstant(SDValue V) {
12375 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: V);
12376 return Const != nullptr && Const->isMinSignedValue();
12377}
12378
12379bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
12380 unsigned OperandNo) {
12381 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
12382 // TODO: Target-specific opcodes could be added.
12383 if (auto *ConstV = isConstOrConstSplat(N: V, /*AllowUndefs*/ false,
12384 /*AllowTruncation*/ true)) {
12385 APInt Const = ConstV->getAPIntValue().trunc(width: V.getScalarValueSizeInBits());
12386 switch (Opcode) {
12387 case ISD::ADD:
12388 case ISD::OR:
12389 case ISD::XOR:
12390 case ISD::UMAX:
12391 return Const.isZero();
12392 case ISD::MUL:
12393 return Const.isOne();
12394 case ISD::AND:
12395 case ISD::UMIN:
12396 return Const.isAllOnes();
12397 case ISD::SMAX:
12398 return Const.isMinSignedValue();
12399 case ISD::SMIN:
12400 return Const.isMaxSignedValue();
12401 case ISD::SUB:
12402 case ISD::SHL:
12403 case ISD::SRA:
12404 case ISD::SRL:
12405 return OperandNo == 1 && Const.isZero();
12406 case ISD::UDIV:
12407 case ISD::SDIV:
12408 return OperandNo == 1 && Const.isOne();
12409 }
12410 } else if (auto *ConstFP = isConstOrConstSplatFP(N: V)) {
12411 switch (Opcode) {
12412 case ISD::FADD:
12413 return ConstFP->isZero() &&
12414 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
12415 case ISD::FSUB:
12416 return OperandNo == 1 && ConstFP->isZero() &&
12417 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
12418 case ISD::FMUL:
12419 return ConstFP->isExactlyValue(V: 1.0);
12420 case ISD::FDIV:
12421 return OperandNo == 1 && ConstFP->isExactlyValue(V: 1.0);
12422 case ISD::FMINNUM:
12423 case ISD::FMAXNUM: {
12424 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12425 EVT VT = V.getValueType();
12426 const fltSemantics &Semantics = VT.getFltSemantics();
12427 APFloat NeutralAF = !Flags.hasNoNaNs()
12428 ? APFloat::getQNaN(Sem: Semantics)
12429 : !Flags.hasNoInfs()
12430 ? APFloat::getInf(Sem: Semantics)
12431 : APFloat::getLargest(Sem: Semantics);
12432 if (Opcode == ISD::FMAXNUM)
12433 NeutralAF.changeSign();
12434
12435 return ConstFP->isExactlyValue(V: NeutralAF);
12436 }
12437 }
12438 }
12439 return false;
12440}
12441
12442SDValue llvm::peekThroughBitcasts(SDValue V) {
12443 while (V.getOpcode() == ISD::BITCAST)
12444 V = V.getOperand(i: 0);
12445 return V;
12446}
12447
12448SDValue llvm::peekThroughOneUseBitcasts(SDValue V) {
12449 while (V.getOpcode() == ISD::BITCAST && V.getOperand(i: 0).hasOneUse())
12450 V = V.getOperand(i: 0);
12451 return V;
12452}
12453
12454SDValue llvm::peekThroughExtractSubvectors(SDValue V) {
12455 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
12456 V = V.getOperand(i: 0);
12457 return V;
12458}
12459
12460SDValue llvm::peekThroughTruncates(SDValue V) {
12461 while (V.getOpcode() == ISD::TRUNCATE)
12462 V = V.getOperand(i: 0);
12463 return V;
12464}
12465
12466bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
12467 if (V.getOpcode() != ISD::XOR)
12468 return false;
12469 V = peekThroughBitcasts(V: V.getOperand(i: 1));
12470 unsigned NumBits = V.getScalarValueSizeInBits();
12471 ConstantSDNode *C =
12472 isConstOrConstSplat(N: V, AllowUndefs, /*AllowTruncation*/ true);
12473 return C && (C->getAPIntValue().countr_one() >= NumBits);
12474}
12475
12476ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs,
12477 bool AllowTruncation) {
12478 EVT VT = N.getValueType();
12479 APInt DemandedElts = VT.isFixedLengthVector()
12480 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
12481 : APInt(1, 1);
12482 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
12483}
12484
12485ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
12486 bool AllowUndefs,
12487 bool AllowTruncation) {
12488 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N))
12489 return CN;
12490
12491 // SplatVectors can truncate their operands. Ignore that case here unless
12492 // AllowTruncation is set.
12493 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
12494 EVT VecEltVT = N->getValueType(ResNo: 0).getVectorElementType();
12495 if (auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
12496 EVT CVT = CN->getValueType(ResNo: 0);
12497 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
12498 if (AllowTruncation || CVT == VecEltVT)
12499 return CN;
12500 }
12501 }
12502
12503 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
12504 BitVector UndefElements;
12505 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, UndefElements: &UndefElements);
12506
12507 // BuildVectors can truncate their operands. Ignore that case here unless
12508 // AllowTruncation is set.
12509 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
12510 if (CN && (UndefElements.none() || AllowUndefs)) {
12511 EVT CVT = CN->getValueType(ResNo: 0);
12512 EVT NSVT = N.getValueType().getScalarType();
12513 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
12514 if (AllowTruncation || (CVT == NSVT))
12515 return CN;
12516 }
12517 }
12518
12519 return nullptr;
12520}
12521
12522ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
12523 EVT VT = N.getValueType();
12524 APInt DemandedElts = VT.isFixedLengthVector()
12525 ? APInt::getAllOnes(numBits: VT.getVectorMinNumElements())
12526 : APInt(1, 1);
12527 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
12528}
12529
12530ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N,
12531 const APInt &DemandedElts,
12532 bool AllowUndefs) {
12533 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
12534 return CN;
12535
12536 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N)) {
12537 BitVector UndefElements;
12538 ConstantFPSDNode *CN =
12539 BV->getConstantFPSplatNode(DemandedElts, UndefElements: &UndefElements);
12540 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
12541 if (CN && (UndefElements.none() || AllowUndefs))
12542 return CN;
12543 }
12544
12545 if (N.getOpcode() == ISD::SPLAT_VECTOR)
12546 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
12547 return CN;
12548
12549 return nullptr;
12550}
12551
12552bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
12553 // TODO: may want to use peekThroughBitcast() here.
12554 ConstantSDNode *C =
12555 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
12556 return C && C->isZero();
12557}
12558
12559bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
12560 ConstantSDNode *C =
12561 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
12562 return C && C->isOne();
12563}
12564
12565bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
12566 N = peekThroughBitcasts(V: N);
12567 unsigned BitWidth = N.getScalarValueSizeInBits();
12568 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
12569 return C && C->isAllOnes() && C->getValueSizeInBits(ResNo: 0) == BitWidth;
12570}
12571
12572HandleSDNode::~HandleSDNode() {
12573 DropOperands();
12574}
12575
12576MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
12577 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
12578 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
12579 MemSDNodeBits.IsVolatile = MMO->isVolatile();
12580 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
12581 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
12582 MemSDNodeBits.IsInvariant = MMO->isInvariant();
12583
12584 // We check here that the size of the memory operand fits within the size of
12585 // the MMO. This is because the MMO might indicate only a possible address
12586 // range instead of specifying the affected memory addresses precisely.
12587 assert(
12588 (!MMO->getType().isValid() ||
12589 TypeSize::isKnownLE(memvt.getStoreSize(), MMO->getSize().getValue())) &&
12590 "Size mismatch!");
12591}
12592
12593/// Profile - Gather unique data for the node.
12594///
12595void SDNode::Profile(FoldingSetNodeID &ID) const {
12596 AddNodeIDNode(ID, N: this);
12597}
12598
12599namespace {
12600
12601 struct EVTArray {
12602 std::vector<EVT> VTs;
12603
12604 EVTArray() {
12605 VTs.reserve(n: MVT::VALUETYPE_SIZE);
12606 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
12607 VTs.push_back(x: MVT((MVT::SimpleValueType)i));
12608 }
12609 };
12610
12611} // end anonymous namespace
12612
12613/// getValueTypeList - Return a pointer to the specified value type.
12614///
12615const EVT *SDNode::getValueTypeList(MVT VT) {
12616 static EVTArray SimpleVTArray;
12617
12618 assert(VT < MVT::VALUETYPE_SIZE && "Value type out of range!");
12619 return &SimpleVTArray.VTs[VT.SimpleTy];
12620}
12621
12622/// hasAnyUseOfValue - Return true if there are any use of the indicated
12623/// value. This method ignores uses of other values defined by this operation.
12624bool SDNode::hasAnyUseOfValue(unsigned Value) const {
12625 assert(Value < getNumValues() && "Bad value!");
12626
12627 for (SDUse &U : uses())
12628 if (U.getResNo() == Value)
12629 return true;
12630
12631 return false;
12632}
12633
12634/// isOnlyUserOf - Return true if this node is the only use of N.
12635bool SDNode::isOnlyUserOf(const SDNode *N) const {
12636 bool Seen = false;
12637 for (const SDNode *User : N->users()) {
12638 if (User == this)
12639 Seen = true;
12640 else
12641 return false;
12642 }
12643
12644 return Seen;
12645}
12646
12647/// Return true if the only users of N are contained in Nodes.
12648bool SDNode::areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N) {
12649 bool Seen = false;
12650 for (const SDNode *User : N->users()) {
12651 if (llvm::is_contained(Range&: Nodes, Element: User))
12652 Seen = true;
12653 else
12654 return false;
12655 }
12656
12657 return Seen;
12658}
12659
12660/// isOperand - Return true if this node is an operand of N.
12661bool SDValue::isOperandOf(const SDNode *N) const {
12662 return is_contained(Range: N->op_values(), Element: *this);
12663}
12664
12665bool SDNode::isOperandOf(const SDNode *N) const {
12666 return any_of(Range: N->op_values(),
12667 P: [this](SDValue Op) { return this == Op.getNode(); });
12668}
12669
12670/// reachesChainWithoutSideEffects - Return true if this operand (which must
12671/// be a chain) reaches the specified operand without crossing any
12672/// side-effecting instructions on any chain path. In practice, this looks
12673/// through token factors and non-volatile loads. In order to remain efficient,
12674/// this only looks a couple of nodes in, it does not do an exhaustive search.
12675///
12676/// Note that we only need to examine chains when we're searching for
12677/// side-effects; SelectionDAG requires that all side-effects are represented
12678/// by chains, even if another operand would force a specific ordering. This
12679/// constraint is necessary to allow transformations like splitting loads.
12680bool SDValue::reachesChainWithoutSideEffects(SDValue Dest,
12681 unsigned Depth) const {
12682 if (*this == Dest) return true;
12683
12684 // Don't search too deeply, we just want to be able to see through
12685 // TokenFactor's etc.
12686 if (Depth == 0) return false;
12687
12688 // If this is a token factor, all inputs to the TF happen in parallel.
12689 if (getOpcode() == ISD::TokenFactor) {
12690 // First, try a shallow search.
12691 if (is_contained(Range: (*this)->ops(), Element: Dest)) {
12692 // We found the chain we want as an operand of this TokenFactor.
12693 // Essentially, we reach the chain without side-effects if we could
12694 // serialize the TokenFactor into a simple chain of operations with
12695 // Dest as the last operation. This is automatically true if the
12696 // chain has one use: there are no other ordering constraints.
12697 // If the chain has more than one use, we give up: some other
12698 // use of Dest might force a side-effect between Dest and the current
12699 // node.
12700 if (Dest.hasOneUse())
12701 return true;
12702 }
12703 // Next, try a deep search: check whether every operand of the TokenFactor
12704 // reaches Dest.
12705 return llvm::all_of(Range: (*this)->ops(), P: [=](SDValue Op) {
12706 return Op.reachesChainWithoutSideEffects(Dest, Depth: Depth - 1);
12707 });
12708 }
12709
12710 // Loads don't have side effects, look through them.
12711 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Val: *this)) {
12712 if (Ld->isUnordered())
12713 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth: Depth-1);
12714 }
12715 return false;
12716}
12717
12718bool SDNode::hasPredecessor(const SDNode *N) const {
12719 SmallPtrSet<const SDNode *, 32> Visited;
12720 SmallVector<const SDNode *, 16> Worklist;
12721 Worklist.push_back(Elt: this);
12722 return hasPredecessorHelper(N, Visited, Worklist);
12723}
12724
12725void SDNode::intersectFlagsWith(const SDNodeFlags Flags) {
12726 this->Flags &= Flags;
12727}
12728
12729SDValue
12730SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
12731 ArrayRef<ISD::NodeType> CandidateBinOps,
12732 bool AllowPartials) {
12733 // The pattern must end in an extract from index 0.
12734 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
12735 !isNullConstant(V: Extract->getOperand(Num: 1)))
12736 return SDValue();
12737
12738 // Match against one of the candidate binary ops.
12739 SDValue Op = Extract->getOperand(Num: 0);
12740 if (llvm::none_of(Range&: CandidateBinOps, P: [Op](ISD::NodeType BinOp) {
12741 return Op.getOpcode() == unsigned(BinOp);
12742 }))
12743 return SDValue();
12744
12745 // Floating-point reductions may require relaxed constraints on the final step
12746 // of the reduction because they may reorder intermediate operations.
12747 unsigned CandidateBinOp = Op.getOpcode();
12748 if (Op.getValueType().isFloatingPoint()) {
12749 SDNodeFlags Flags = Op->getFlags();
12750 switch (CandidateBinOp) {
12751 case ISD::FADD:
12752 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
12753 return SDValue();
12754 break;
12755 default:
12756 llvm_unreachable("Unhandled FP opcode for binop reduction");
12757 }
12758 }
12759
12760 // Matching failed - attempt to see if we did enough stages that a partial
12761 // reduction from a subvector is possible.
12762 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
12763 if (!AllowPartials || !Op)
12764 return SDValue();
12765 EVT OpVT = Op.getValueType();
12766 EVT OpSVT = OpVT.getScalarType();
12767 EVT SubVT = EVT::getVectorVT(Context&: *getContext(), VT: OpSVT, NumElements: NumSubElts);
12768 if (!TLI->isExtractSubvectorCheap(ResVT: SubVT, SrcVT: OpVT, Index: 0))
12769 return SDValue();
12770 BinOp = (ISD::NodeType)CandidateBinOp;
12771 return getExtractSubvector(DL: SDLoc(Op), VT: SubVT, Vec: Op, Idx: 0);
12772 };
12773
12774 // At each stage, we're looking for something that looks like:
12775 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
12776 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
12777 // i32 undef, i32 undef, i32 undef, i32 undef>
12778 // %a = binop <8 x i32> %op, %s
12779 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
12780 // we expect something like:
12781 // <4,5,6,7,u,u,u,u>
12782 // <2,3,u,u,u,u,u,u>
12783 // <1,u,u,u,u,u,u,u>
12784 // While a partial reduction match would be:
12785 // <2,3,u,u,u,u,u,u>
12786 // <1,u,u,u,u,u,u,u>
12787 unsigned Stages = Log2_32(Value: Op.getValueType().getVectorNumElements());
12788 SDValue PrevOp;
12789 for (unsigned i = 0; i < Stages; ++i) {
12790 unsigned MaskEnd = (1 << i);
12791
12792 if (Op.getOpcode() != CandidateBinOp)
12793 return PartialReduction(PrevOp, MaskEnd);
12794
12795 SDValue Op0 = Op.getOperand(i: 0);
12796 SDValue Op1 = Op.getOperand(i: 1);
12797
12798 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
12799 if (Shuffle) {
12800 Op = Op1;
12801 } else {
12802 Shuffle = dyn_cast<ShuffleVectorSDNode>(Val&: Op1);
12803 Op = Op0;
12804 }
12805
12806 // The first operand of the shuffle should be the same as the other operand
12807 // of the binop.
12808 if (!Shuffle || Shuffle->getOperand(Num: 0) != Op)
12809 return PartialReduction(PrevOp, MaskEnd);
12810
12811 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
12812 for (int Index = 0; Index < (int)MaskEnd; ++Index)
12813 if (Shuffle->getMaskElt(Idx: Index) != (int)(MaskEnd + Index))
12814 return PartialReduction(PrevOp, MaskEnd);
12815
12816 PrevOp = Op;
12817 }
12818
12819 // Handle subvector reductions, which tend to appear after the shuffle
12820 // reduction stages.
12821 while (Op.getOpcode() == CandidateBinOp) {
12822 unsigned NumElts = Op.getValueType().getVectorNumElements();
12823 SDValue Op0 = Op.getOperand(i: 0);
12824 SDValue Op1 = Op.getOperand(i: 1);
12825 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12826 Op1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12827 Op0.getOperand(i: 0) != Op1.getOperand(i: 0))
12828 break;
12829 SDValue Src = Op0.getOperand(i: 0);
12830 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12831 if (NumSrcElts != (2 * NumElts))
12832 break;
12833 if (!(Op0.getConstantOperandAPInt(i: 1) == 0 &&
12834 Op1.getConstantOperandAPInt(i: 1) == NumElts) &&
12835 !(Op1.getConstantOperandAPInt(i: 1) == 0 &&
12836 Op0.getConstantOperandAPInt(i: 1) == NumElts))
12837 break;
12838 Op = Src;
12839 }
12840
12841 BinOp = (ISD::NodeType)CandidateBinOp;
12842 return Op;
12843}
12844
12845SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) {
12846 EVT VT = N->getValueType(ResNo: 0);
12847 EVT EltVT = VT.getVectorElementType();
12848 unsigned NE = VT.getVectorNumElements();
12849
12850 SDLoc dl(N);
12851
12852 // If ResNE is 0, fully unroll the vector op.
12853 if (ResNE == 0)
12854 ResNE = NE;
12855 else if (NE > ResNE)
12856 NE = ResNE;
12857
12858 if (N->getNumValues() == 2) {
12859 SmallVector<SDValue, 8> Scalars0, Scalars1;
12860 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12861 EVT VT1 = N->getValueType(ResNo: 1);
12862 EVT EltVT1 = VT1.getVectorElementType();
12863
12864 unsigned i;
12865 for (i = 0; i != NE; ++i) {
12866 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12867 SDValue Operand = N->getOperand(Num: j);
12868 EVT OperandVT = Operand.getValueType();
12869
12870 // A vector operand; extract a single element.
12871 EVT OperandEltVT = OperandVT.getVectorElementType();
12872 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
12873 }
12874
12875 SDValue EltOp = getNode(Opcode: N->getOpcode(), DL: dl, ResultTys: {EltVT, EltVT1}, Ops: Operands);
12876 Scalars0.push_back(Elt: EltOp);
12877 Scalars1.push_back(Elt: EltOp.getValue(R: 1));
12878 }
12879
12880 for (; i < ResNE; ++i) {
12881 Scalars0.push_back(Elt: getUNDEF(VT: EltVT));
12882 Scalars1.push_back(Elt: getUNDEF(VT: EltVT1));
12883 }
12884
12885 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
12886 EVT VecVT1 = EVT::getVectorVT(Context&: *getContext(), VT: EltVT1, NumElements: ResNE);
12887 SDValue Vec0 = getBuildVector(VT: VecVT, DL: dl, Ops: Scalars0);
12888 SDValue Vec1 = getBuildVector(VT: VecVT1, DL: dl, Ops: Scalars1);
12889 return getMergeValues(Ops: {Vec0, Vec1}, dl);
12890 }
12891
12892 assert(N->getNumValues() == 1 &&
12893 "Can't unroll a vector with multiple results!");
12894
12895 SmallVector<SDValue, 8> Scalars;
12896 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12897
12898 unsigned i;
12899 for (i= 0; i != NE; ++i) {
12900 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12901 SDValue Operand = N->getOperand(Num: j);
12902 EVT OperandVT = Operand.getValueType();
12903 if (OperandVT.isVector()) {
12904 // A vector operand; extract a single element.
12905 EVT OperandEltVT = OperandVT.getVectorElementType();
12906 Operands[j] = getExtractVectorElt(DL: dl, VT: OperandEltVT, Vec: Operand, Idx: i);
12907 } else {
12908 // A scalar operand; just use it as is.
12909 Operands[j] = Operand;
12910 }
12911 }
12912
12913 switch (N->getOpcode()) {
12914 default: {
12915 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, Ops: Operands,
12916 Flags: N->getFlags()));
12917 break;
12918 }
12919 case ISD::VSELECT:
12920 Scalars.push_back(Elt: getNode(Opcode: ISD::SELECT, DL: dl, VT: EltVT, Ops: Operands));
12921 break;
12922 case ISD::SHL:
12923 case ISD::SRA:
12924 case ISD::SRL:
12925 case ISD::ROTL:
12926 case ISD::ROTR:
12927 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT, N1: Operands[0],
12928 N2: getShiftAmountOperand(LHSTy: Operands[0].getValueType(),
12929 Op: Operands[1])));
12930 break;
12931 case ISD::SIGN_EXTEND_INREG: {
12932 EVT ExtVT = cast<VTSDNode>(Val&: Operands[1])->getVT().getVectorElementType();
12933 Scalars.push_back(Elt: getNode(Opcode: N->getOpcode(), DL: dl, VT: EltVT,
12934 N1: Operands[0],
12935 N2: getValueType(VT: ExtVT)));
12936 break;
12937 }
12938 case ISD::ADDRSPACECAST: {
12939 const auto *ASC = cast<AddrSpaceCastSDNode>(Val: N);
12940 Scalars.push_back(Elt: getAddrSpaceCast(dl, VT: EltVT, Ptr: Operands[0],
12941 SrcAS: ASC->getSrcAddressSpace(),
12942 DestAS: ASC->getDestAddressSpace()));
12943 break;
12944 }
12945 }
12946 }
12947
12948 for (; i < ResNE; ++i)
12949 Scalars.push_back(Elt: getUNDEF(VT: EltVT));
12950
12951 EVT VecVT = EVT::getVectorVT(Context&: *getContext(), VT: EltVT, NumElements: ResNE);
12952 return getBuildVector(VT: VecVT, DL: dl, Ops: Scalars);
12953}
12954
12955std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12956 SDNode *N, unsigned ResNE) {
12957 unsigned Opcode = N->getOpcode();
12958 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12959 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12960 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12961 "Expected an overflow opcode");
12962
12963 EVT ResVT = N->getValueType(ResNo: 0);
12964 EVT OvVT = N->getValueType(ResNo: 1);
12965 EVT ResEltVT = ResVT.getVectorElementType();
12966 EVT OvEltVT = OvVT.getVectorElementType();
12967 SDLoc dl(N);
12968
12969 // If ResNE is 0, fully unroll the vector op.
12970 unsigned NE = ResVT.getVectorNumElements();
12971 if (ResNE == 0)
12972 ResNE = NE;
12973 else if (NE > ResNE)
12974 NE = ResNE;
12975
12976 SmallVector<SDValue, 8> LHSScalars;
12977 SmallVector<SDValue, 8> RHSScalars;
12978 ExtractVectorElements(Op: N->getOperand(Num: 0), Args&: LHSScalars, Start: 0, Count: NE);
12979 ExtractVectorElements(Op: N->getOperand(Num: 1), Args&: RHSScalars, Start: 0, Count: NE);
12980
12981 EVT SVT = TLI->getSetCCResultType(DL: getDataLayout(), Context&: *getContext(), VT: ResEltVT);
12982 SDVTList VTs = getVTList(VT1: ResEltVT, VT2: SVT);
12983 SmallVector<SDValue, 8> ResScalars;
12984 SmallVector<SDValue, 8> OvScalars;
12985 for (unsigned i = 0; i < NE; ++i) {
12986 SDValue Res = getNode(Opcode, DL: dl, VTList: VTs, N1: LHSScalars[i], N2: RHSScalars[i]);
12987 SDValue Ov =
12988 getSelect(DL: dl, VT: OvEltVT, Cond: Res.getValue(R: 1),
12989 LHS: getBoolConstant(V: true, DL: dl, VT: OvEltVT, OpVT: ResVT),
12990 RHS: getConstant(Val: 0, DL: dl, VT: OvEltVT));
12991
12992 ResScalars.push_back(Elt: Res);
12993 OvScalars.push_back(Elt: Ov);
12994 }
12995
12996 ResScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: ResEltVT));
12997 OvScalars.append(NumInputs: ResNE - NE, Elt: getUNDEF(VT: OvEltVT));
12998
12999 EVT NewResVT = EVT::getVectorVT(Context&: *getContext(), VT: ResEltVT, NumElements: ResNE);
13000 EVT NewOvVT = EVT::getVectorVT(Context&: *getContext(), VT: OvEltVT, NumElements: ResNE);
13001 return std::make_pair(x: getBuildVector(VT: NewResVT, DL: dl, Ops: ResScalars),
13002 y: getBuildVector(VT: NewOvVT, DL: dl, Ops: OvScalars));
13003}
13004
13005bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD,
13006 LoadSDNode *Base,
13007 unsigned Bytes,
13008 int Dist) const {
13009 if (LD->isVolatile() || Base->isVolatile())
13010 return false;
13011 // TODO: probably too restrictive for atomics, revisit
13012 if (!LD->isSimple())
13013 return false;
13014 if (LD->isIndexed() || Base->isIndexed())
13015 return false;
13016 if (LD->getChain() != Base->getChain())
13017 return false;
13018 EVT VT = LD->getMemoryVT();
13019 if (VT.getSizeInBits() / 8 != Bytes)
13020 return false;
13021
13022 auto BaseLocDecomp = BaseIndexOffset::match(N: Base, DAG: *this);
13023 auto LocDecomp = BaseIndexOffset::match(N: LD, DAG: *this);
13024
13025 int64_t Offset = 0;
13026 if (BaseLocDecomp.equalBaseIndex(Other: LocDecomp, DAG: *this, Off&: Offset))
13027 return (Dist * (int64_t)Bytes == Offset);
13028 return false;
13029}
13030
13031/// InferPtrAlignment - Infer alignment of a load / store address. Return
13032/// std::nullopt if it cannot be inferred.
13033MaybeAlign SelectionDAG::InferPtrAlign(SDValue Ptr) const {
13034 // If this is a GlobalAddress + cst, return the alignment.
13035 const GlobalValue *GV = nullptr;
13036 int64_t GVOffset = 0;
13037 if (TLI->isGAPlusOffset(N: Ptr.getNode(), GA&: GV, Offset&: GVOffset)) {
13038 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
13039 KnownBits Known(PtrWidth);
13040 llvm::computeKnownBits(V: GV, Known, DL: getDataLayout());
13041 unsigned AlignBits = Known.countMinTrailingZeros();
13042 if (AlignBits)
13043 return commonAlignment(A: Align(1ull << std::min(a: 31U, b: AlignBits)), Offset: GVOffset);
13044 }
13045
13046 // If this is a direct reference to a stack slot, use information about the
13047 // stack slot's alignment.
13048 int FrameIdx = INT_MIN;
13049 int64_t FrameOffset = 0;
13050 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Ptr)) {
13051 FrameIdx = FI->getIndex();
13052 } else if (isBaseWithConstantOffset(Op: Ptr) &&
13053 isa<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))) {
13054 // Handle FI+Cst
13055 FrameIdx = cast<FrameIndexSDNode>(Val: Ptr.getOperand(i: 0))->getIndex();
13056 FrameOffset = Ptr.getConstantOperandVal(i: 1);
13057 }
13058
13059 if (FrameIdx != INT_MIN) {
13060 const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo();
13061 return commonAlignment(A: MFI.getObjectAlign(ObjectIdx: FrameIdx), Offset: FrameOffset);
13062 }
13063
13064 return std::nullopt;
13065}
13066
13067/// Split the scalar node with EXTRACT_ELEMENT using the provided
13068/// VTs and return the low/high part.
13069std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
13070 const SDLoc &DL,
13071 const EVT &LoVT,
13072 const EVT &HiVT) {
13073 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
13074 "Split node must be a scalar type");
13075 SDValue Lo =
13076 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: LoVT, N1: N, N2: getIntPtrConstant(Val: 0, DL));
13077 SDValue Hi =
13078 getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: HiVT, N1: N, N2: getIntPtrConstant(Val: 1, DL));
13079 return std::make_pair(x&: Lo, y&: Hi);
13080}
13081
13082/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
13083/// which is split (or expanded) into two not necessarily identical pieces.
13084std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
13085 // Currently all types are split in half.
13086 EVT LoVT, HiVT;
13087 if (!VT.isVector())
13088 LoVT = HiVT = TLI->getTypeToTransformTo(Context&: *getContext(), VT);
13089 else
13090 LoVT = HiVT = VT.getHalfNumVectorElementsVT(Context&: *getContext());
13091
13092 return std::make_pair(x&: LoVT, y&: HiVT);
13093}
13094
13095/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
13096/// type, dependent on an enveloping VT that has been split into two identical
13097/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
13098std::pair<EVT, EVT>
13099SelectionDAG::GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT,
13100 bool *HiIsEmpty) const {
13101 EVT EltTp = VT.getVectorElementType();
13102 // Examples:
13103 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
13104 // custom VL=9 with enveloping VL=8/8 yields 8/1
13105 // custom VL=10 with enveloping VL=8/8 yields 8/2
13106 // etc.
13107 ElementCount VTNumElts = VT.getVectorElementCount();
13108 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
13109 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
13110 "Mixing fixed width and scalable vectors when enveloping a type");
13111 EVT LoVT, HiVT;
13112 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
13113 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
13114 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts - EnvNumElts);
13115 *HiIsEmpty = false;
13116 } else {
13117 // Flag that hi type has zero storage size, but return split envelop type
13118 // (this would be easier if vector types with zero elements were allowed).
13119 LoVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: VTNumElts);
13120 HiVT = EVT::getVectorVT(Context&: *getContext(), VT: EltTp, EC: EnvNumElts);
13121 *HiIsEmpty = true;
13122 }
13123 return std::make_pair(x&: LoVT, y&: HiVT);
13124}
13125
13126/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
13127/// low/high part.
13128std::pair<SDValue, SDValue>
13129SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
13130 const EVT &HiVT) {
13131 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
13132 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
13133 "Splitting vector with an invalid mixture of fixed and scalable "
13134 "vector types");
13135 assert(LoVT.getVectorMinNumElements() + HiVT.getVectorMinNumElements() <=
13136 N.getValueType().getVectorMinNumElements() &&
13137 "More vector elements requested than available!");
13138 SDValue Lo, Hi;
13139 Lo = getExtractSubvector(DL, VT: LoVT, Vec: N, Idx: 0);
13140 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
13141 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
13142 // IDX with the runtime scaling factor of the result vector type. For
13143 // fixed-width result vectors, that runtime scaling factor is 1.
13144 Hi = getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HiVT, N1: N,
13145 N2: getVectorIdxConstant(Val: LoVT.getVectorMinNumElements(), DL));
13146 return std::make_pair(x&: Lo, y&: Hi);
13147}
13148
13149std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
13150 const SDLoc &DL) {
13151 // Split the vector length parameter.
13152 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
13153 EVT VT = N.getValueType();
13154 assert(VecVT.getVectorElementCount().isKnownEven() &&
13155 "Expecting the mask to be an evenly-sized vector");
13156 unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
13157 SDValue HalfNumElts =
13158 VecVT.isFixedLengthVector()
13159 ? getConstant(Val: HalfMinNumElts, DL, VT)
13160 : getVScale(DL, VT, MulImm: APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
13161 SDValue Lo = getNode(Opcode: ISD::UMIN, DL, VT, N1: N, N2: HalfNumElts);
13162 SDValue Hi = getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N, N2: HalfNumElts);
13163 return std::make_pair(x&: Lo, y&: Hi);
13164}
13165
13166/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
13167SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) {
13168 EVT VT = N.getValueType();
13169 EVT WideVT = EVT::getVectorVT(Context&: *getContext(), VT: VT.getVectorElementType(),
13170 NumElements: NextPowerOf2(A: VT.getVectorNumElements()));
13171 return getInsertSubvector(DL, Vec: getUNDEF(VT: WideVT), SubVec: N, Idx: 0);
13172}
13173
13174void SelectionDAG::ExtractVectorElements(SDValue Op,
13175 SmallVectorImpl<SDValue> &Args,
13176 unsigned Start, unsigned Count,
13177 EVT EltVT) {
13178 EVT VT = Op.getValueType();
13179 if (Count == 0)
13180 Count = VT.getVectorNumElements();
13181 if (EltVT == EVT())
13182 EltVT = VT.getVectorElementType();
13183 SDLoc SL(Op);
13184 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
13185 Args.push_back(Elt: getExtractVectorElt(DL: SL, VT: EltVT, Vec: Op, Idx: i));
13186 }
13187}
13188
13189// getAddressSpace - Return the address space this GlobalAddress belongs to.
13190unsigned GlobalAddressSDNode::getAddressSpace() const {
13191 return getGlobal()->getType()->getAddressSpace();
13192}
13193
13194Type *ConstantPoolSDNode::getType() const {
13195 if (isMachineConstantPoolEntry())
13196 return Val.MachineCPVal->getType();
13197 return Val.ConstVal->getType();
13198}
13199
13200bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
13201 unsigned &SplatBitSize,
13202 bool &HasAnyUndefs,
13203 unsigned MinSplatBits,
13204 bool IsBigEndian) const {
13205 EVT VT = getValueType(ResNo: 0);
13206 assert(VT.isVector() && "Expected a vector type");
13207 unsigned VecWidth = VT.getSizeInBits();
13208 if (MinSplatBits > VecWidth)
13209 return false;
13210
13211 // FIXME: The widths are based on this node's type, but build vectors can
13212 // truncate their operands.
13213 SplatValue = APInt(VecWidth, 0);
13214 SplatUndef = APInt(VecWidth, 0);
13215
13216 // Get the bits. Bits with undefined values (when the corresponding element
13217 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
13218 // in SplatValue. If any of the values are not constant, give up and return
13219 // false.
13220 unsigned int NumOps = getNumOperands();
13221 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
13222 unsigned EltWidth = VT.getScalarSizeInBits();
13223
13224 for (unsigned j = 0; j < NumOps; ++j) {
13225 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
13226 SDValue OpVal = getOperand(Num: i);
13227 unsigned BitPos = j * EltWidth;
13228
13229 if (OpVal.isUndef())
13230 SplatUndef.setBits(loBit: BitPos, hiBit: BitPos + EltWidth);
13231 else if (auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal))
13232 SplatValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth), bitPosition: BitPos);
13233 else if (auto *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal))
13234 SplatValue.insertBits(SubBits: CN->getValueAPF().bitcastToAPInt(), bitPosition: BitPos);
13235 else
13236 return false;
13237 }
13238
13239 // The build_vector is all constants or undefs. Find the smallest element
13240 // size that splats the vector.
13241 HasAnyUndefs = (SplatUndef != 0);
13242
13243 // FIXME: This does not work for vectors with elements less than 8 bits.
13244 while (VecWidth > 8) {
13245 // If we can't split in half, stop here.
13246 if (VecWidth & 1)
13247 break;
13248
13249 unsigned HalfSize = VecWidth / 2;
13250 APInt HighValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: HalfSize);
13251 APInt LowValue = SplatValue.extractBits(numBits: HalfSize, bitPosition: 0);
13252 APInt HighUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: HalfSize);
13253 APInt LowUndef = SplatUndef.extractBits(numBits: HalfSize, bitPosition: 0);
13254
13255 // If the two halves do not match (ignoring undef bits), stop here.
13256 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
13257 MinSplatBits > HalfSize)
13258 break;
13259
13260 SplatValue = HighValue | LowValue;
13261 SplatUndef = HighUndef & LowUndef;
13262
13263 VecWidth = HalfSize;
13264 }
13265
13266 // FIXME: The loop above only tries to split in halves. But if the input
13267 // vector for example is <3 x i16> it wouldn't be able to detect a
13268 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
13269 // optimizations. I guess that back in the days when this helper was created
13270 // vectors normally was power-of-2 sized.
13271
13272 SplatBitSize = VecWidth;
13273 return true;
13274}
13275
13276SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts,
13277 BitVector *UndefElements) const {
13278 unsigned NumOps = getNumOperands();
13279 if (UndefElements) {
13280 UndefElements->clear();
13281 UndefElements->resize(N: NumOps);
13282 }
13283 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
13284 if (!DemandedElts)
13285 return SDValue();
13286 SDValue Splatted;
13287 for (unsigned i = 0; i != NumOps; ++i) {
13288 if (!DemandedElts[i])
13289 continue;
13290 SDValue Op = getOperand(Num: i);
13291 if (Op.isUndef()) {
13292 if (UndefElements)
13293 (*UndefElements)[i] = true;
13294 } else if (!Splatted) {
13295 Splatted = Op;
13296 } else if (Splatted != Op) {
13297 return SDValue();
13298 }
13299 }
13300
13301 if (!Splatted) {
13302 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
13303 assert(getOperand(FirstDemandedIdx).isUndef() &&
13304 "Can only have a splat without a constant for all undefs.");
13305 return getOperand(Num: FirstDemandedIdx);
13306 }
13307
13308 return Splatted;
13309}
13310
13311SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const {
13312 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
13313 return getSplatValue(DemandedElts, UndefElements);
13314}
13315
13316bool BuildVectorSDNode::getRepeatedSequence(const APInt &DemandedElts,
13317 SmallVectorImpl<SDValue> &Sequence,
13318 BitVector *UndefElements) const {
13319 unsigned NumOps = getNumOperands();
13320 Sequence.clear();
13321 if (UndefElements) {
13322 UndefElements->clear();
13323 UndefElements->resize(N: NumOps);
13324 }
13325 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
13326 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(Value: NumOps))
13327 return false;
13328
13329 // Set the undefs even if we don't find a sequence (like getSplatValue).
13330 if (UndefElements)
13331 for (unsigned I = 0; I != NumOps; ++I)
13332 if (DemandedElts[I] && getOperand(Num: I).isUndef())
13333 (*UndefElements)[I] = true;
13334
13335 // Iteratively widen the sequence length looking for repetitions.
13336 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
13337 Sequence.append(NumInputs: SeqLen, Elt: SDValue());
13338 for (unsigned I = 0; I != NumOps; ++I) {
13339 if (!DemandedElts[I])
13340 continue;
13341 SDValue &SeqOp = Sequence[I % SeqLen];
13342 SDValue Op = getOperand(Num: I);
13343 if (Op.isUndef()) {
13344 if (!SeqOp)
13345 SeqOp = Op;
13346 continue;
13347 }
13348 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
13349 Sequence.clear();
13350 break;
13351 }
13352 SeqOp = Op;
13353 }
13354 if (!Sequence.empty())
13355 return true;
13356 }
13357
13358 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
13359 return false;
13360}
13361
13362bool BuildVectorSDNode::getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
13363 BitVector *UndefElements) const {
13364 APInt DemandedElts = APInt::getAllOnes(numBits: getNumOperands());
13365 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
13366}
13367
13368ConstantSDNode *
13369BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts,
13370 BitVector *UndefElements) const {
13371 return dyn_cast_or_null<ConstantSDNode>(
13372 Val: getSplatValue(DemandedElts, UndefElements));
13373}
13374
13375ConstantSDNode *
13376BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const {
13377 return dyn_cast_or_null<ConstantSDNode>(Val: getSplatValue(UndefElements));
13378}
13379
13380ConstantFPSDNode *
13381BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts,
13382 BitVector *UndefElements) const {
13383 return dyn_cast_or_null<ConstantFPSDNode>(
13384 Val: getSplatValue(DemandedElts, UndefElements));
13385}
13386
13387ConstantFPSDNode *
13388BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const {
13389 return dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements));
13390}
13391
13392int32_t
13393BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
13394 uint32_t BitWidth) const {
13395 if (ConstantFPSDNode *CN =
13396 dyn_cast_or_null<ConstantFPSDNode>(Val: getSplatValue(UndefElements))) {
13397 bool IsExact;
13398 APSInt IntVal(BitWidth);
13399 const APFloat &APF = CN->getValueAPF();
13400 if (APF.convertToInteger(Result&: IntVal, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
13401 APFloat::opOK ||
13402 !IsExact)
13403 return -1;
13404
13405 return IntVal.exactLogBase2();
13406 }
13407 return -1;
13408}
13409
13410bool BuildVectorSDNode::getConstantRawBits(
13411 bool IsLittleEndian, unsigned DstEltSizeInBits,
13412 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
13413 // Early-out if this contains anything but Undef/Constant/ConstantFP.
13414 if (!isConstant())
13415 return false;
13416
13417 unsigned NumSrcOps = getNumOperands();
13418 unsigned SrcEltSizeInBits = getValueType(ResNo: 0).getScalarSizeInBits();
13419 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
13420 "Invalid bitcast scale");
13421
13422 // Extract raw src bits.
13423 SmallVector<APInt> SrcBitElements(NumSrcOps,
13424 APInt::getZero(numBits: SrcEltSizeInBits));
13425 BitVector SrcUndeElements(NumSrcOps, false);
13426
13427 for (unsigned I = 0; I != NumSrcOps; ++I) {
13428 SDValue Op = getOperand(Num: I);
13429 if (Op.isUndef()) {
13430 SrcUndeElements.set(I);
13431 continue;
13432 }
13433 auto *CInt = dyn_cast<ConstantSDNode>(Val&: Op);
13434 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op);
13435 assert((CInt || CFP) && "Unknown constant");
13436 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(width: SrcEltSizeInBits)
13437 : CFP->getValueAPF().bitcastToAPInt();
13438 }
13439
13440 // Recast to dst width.
13441 recastRawBits(IsLittleEndian, DstEltSizeInBits, DstBitElements&: RawBitElements,
13442 SrcBitElements, DstUndefElements&: UndefElements, SrcUndefElements: SrcUndeElements);
13443 return true;
13444}
13445
13446void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
13447 unsigned DstEltSizeInBits,
13448 SmallVectorImpl<APInt> &DstBitElements,
13449 ArrayRef<APInt> SrcBitElements,
13450 BitVector &DstUndefElements,
13451 const BitVector &SrcUndefElements) {
13452 unsigned NumSrcOps = SrcBitElements.size();
13453 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
13454 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
13455 "Invalid bitcast scale");
13456 assert(NumSrcOps == SrcUndefElements.size() &&
13457 "Vector size mismatch");
13458
13459 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
13460 DstUndefElements.clear();
13461 DstUndefElements.resize(N: NumDstOps, t: false);
13462 DstBitElements.assign(NumElts: NumDstOps, Elt: APInt::getZero(numBits: DstEltSizeInBits));
13463
13464 // Concatenate src elements constant bits together into dst element.
13465 if (SrcEltSizeInBits <= DstEltSizeInBits) {
13466 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
13467 for (unsigned I = 0; I != NumDstOps; ++I) {
13468 DstUndefElements.set(I);
13469 APInt &DstBits = DstBitElements[I];
13470 for (unsigned J = 0; J != Scale; ++J) {
13471 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13472 if (SrcUndefElements[Idx])
13473 continue;
13474 DstUndefElements.reset(Idx: I);
13475 const APInt &SrcBits = SrcBitElements[Idx];
13476 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
13477 "Illegal constant bitwidths");
13478 DstBits.insertBits(SubBits: SrcBits, bitPosition: J * SrcEltSizeInBits);
13479 }
13480 }
13481 return;
13482 }
13483
13484 // Split src element constant bits into dst elements.
13485 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
13486 for (unsigned I = 0; I != NumSrcOps; ++I) {
13487 if (SrcUndefElements[I]) {
13488 DstUndefElements.set(I: I * Scale, E: (I + 1) * Scale);
13489 continue;
13490 }
13491 const APInt &SrcBits = SrcBitElements[I];
13492 for (unsigned J = 0; J != Scale; ++J) {
13493 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
13494 APInt &DstBits = DstBitElements[Idx];
13495 DstBits = SrcBits.extractBits(numBits: DstEltSizeInBits, bitPosition: J * DstEltSizeInBits);
13496 }
13497 }
13498}
13499
13500bool BuildVectorSDNode::isConstant() const {
13501 for (const SDValue &Op : op_values()) {
13502 unsigned Opc = Op.getOpcode();
13503 if (!Op.isUndef() && Opc != ISD::Constant && Opc != ISD::ConstantFP)
13504 return false;
13505 }
13506 return true;
13507}
13508
13509std::optional<std::pair<APInt, APInt>>
13510BuildVectorSDNode::isConstantSequence() const {
13511 unsigned NumOps = getNumOperands();
13512 if (NumOps < 2)
13513 return std::nullopt;
13514
13515 if (!isa<ConstantSDNode>(Val: getOperand(Num: 0)) ||
13516 !isa<ConstantSDNode>(Val: getOperand(Num: 1)))
13517 return std::nullopt;
13518
13519 unsigned EltSize = getValueType(ResNo: 0).getScalarSizeInBits();
13520 APInt Start = getConstantOperandAPInt(Num: 0).trunc(width: EltSize);
13521 APInt Stride = getConstantOperandAPInt(Num: 1).trunc(width: EltSize) - Start;
13522
13523 if (Stride.isZero())
13524 return std::nullopt;
13525
13526 for (unsigned i = 2; i < NumOps; ++i) {
13527 if (!isa<ConstantSDNode>(Val: getOperand(Num: i)))
13528 return std::nullopt;
13529
13530 APInt Val = getConstantOperandAPInt(Num: i).trunc(width: EltSize);
13531 if (Val != (Start + (Stride * i)))
13532 return std::nullopt;
13533 }
13534
13535 return std::make_pair(x&: Start, y&: Stride);
13536}
13537
13538bool ShuffleVectorSDNode::isSplatMask(ArrayRef<int> Mask) {
13539 // Find the first non-undef value in the shuffle mask.
13540 unsigned i, e;
13541 for (i = 0, e = Mask.size(); i != e && Mask[i] < 0; ++i)
13542 /* search */;
13543
13544 // If all elements are undefined, this shuffle can be considered a splat
13545 // (although it should eventually get simplified away completely).
13546 if (i == e)
13547 return true;
13548
13549 // Make sure all remaining elements are either undef or the same as the first
13550 // non-undef value.
13551 for (int Idx = Mask[i]; i != e; ++i)
13552 if (Mask[i] >= 0 && Mask[i] != Idx)
13553 return false;
13554 return true;
13555}
13556
13557// Returns true if it is a constant integer BuildVector or constant integer,
13558// possibly hidden by a bitcast.
13559bool SelectionDAG::isConstantIntBuildVectorOrConstantInt(
13560 SDValue N, bool AllowOpaques) const {
13561 N = peekThroughBitcasts(V: N);
13562
13563 if (auto *C = dyn_cast<ConstantSDNode>(Val&: N))
13564 return AllowOpaques || !C->isOpaque();
13565
13566 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode()))
13567 return true;
13568
13569 // Treat a GlobalAddress supporting constant offset folding as a
13570 // constant integer.
13571 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: N))
13572 if (GA->getOpcode() == ISD::GlobalAddress &&
13573 TLI->isOffsetFoldingLegal(GA))
13574 return true;
13575
13576 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
13577 isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
13578 return true;
13579 return false;
13580}
13581
13582// Returns true if it is a constant float BuildVector or constant float.
13583bool SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) const {
13584 if (isa<ConstantFPSDNode>(Val: N))
13585 return true;
13586
13587 if (ISD::isBuildVectorOfConstantFPSDNodes(N: N.getNode()))
13588 return true;
13589
13590 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
13591 isa<ConstantFPSDNode>(Val: N.getOperand(i: 0)))
13592 return true;
13593
13594 return false;
13595}
13596
13597std::optional<bool> SelectionDAG::isBoolConstant(SDValue N,
13598 bool AllowTruncation) const {
13599 ConstantSDNode *Const = isConstOrConstSplat(N, AllowUndefs: false, AllowTruncation);
13600 if (!Const)
13601 return std::nullopt;
13602
13603 const APInt &CVal = Const->getAPIntValue();
13604 switch (TLI->getBooleanContents(Type: N.getValueType())) {
13605 case TargetLowering::ZeroOrOneBooleanContent:
13606 if (CVal.isOne())
13607 return true;
13608 if (CVal.isZero())
13609 return false;
13610 return std::nullopt;
13611 case TargetLowering::ZeroOrNegativeOneBooleanContent:
13612 if (CVal.isAllOnes())
13613 return true;
13614 if (CVal.isZero())
13615 return false;
13616 return std::nullopt;
13617 case TargetLowering::UndefinedBooleanContent:
13618 return CVal[0];
13619 }
13620 llvm_unreachable("Unknown BooleanContent enum");
13621}
13622
13623void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
13624 assert(!Node->OperandList && "Node already has operands");
13625 assert(SDNode::getMaxNumOperands() >= Vals.size() &&
13626 "too many operands to fit into SDNode");
13627 SDUse *Ops = OperandRecycler.allocate(
13628 Cap: ArrayRecycler<SDUse>::Capacity::get(N: Vals.size()), Allocator&: OperandAllocator);
13629
13630 bool IsDivergent = false;
13631 for (unsigned I = 0; I != Vals.size(); ++I) {
13632 Ops[I].setUser(Node);
13633 Ops[I].setInitial(Vals[I]);
13634 EVT VT = Ops[I].getValueType();
13635
13636 // Skip Chain. It does not carry divergence.
13637 if (VT != MVT::Other &&
13638 (VT != MVT::Glue || gluePropagatesDivergence(Node: Ops[I].getNode())) &&
13639 Ops[I].getNode()->isDivergent()) {
13640 IsDivergent = true;
13641 }
13642 }
13643 Node->NumOperands = Vals.size();
13644 Node->OperandList = Ops;
13645 if (!TLI->isSDNodeAlwaysUniform(N: Node)) {
13646 IsDivergent |= TLI->isSDNodeSourceOfDivergence(N: Node, FLI, UA);
13647 Node->SDNodeBits.IsDivergent = IsDivergent;
13648 }
13649 checkForCycles(N: Node);
13650}
13651
13652SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
13653 SmallVectorImpl<SDValue> &Vals) {
13654 size_t Limit = SDNode::getMaxNumOperands();
13655 while (Vals.size() > Limit) {
13656 unsigned SliceIdx = Vals.size() - Limit;
13657 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(N: SliceIdx, M: Limit);
13658 SDValue NewTF = getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: ExtractedTFs);
13659 Vals.erase(CS: Vals.begin() + SliceIdx, CE: Vals.end());
13660 Vals.emplace_back(Args&: NewTF);
13661 }
13662 return getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Vals);
13663}
13664
13665SDValue SelectionDAG::getNeutralElement(unsigned Opcode, const SDLoc &DL,
13666 EVT VT, SDNodeFlags Flags) {
13667 switch (Opcode) {
13668 default:
13669 return SDValue();
13670 case ISD::ADD:
13671 case ISD::OR:
13672 case ISD::XOR:
13673 case ISD::UMAX:
13674 return getConstant(Val: 0, DL, VT);
13675 case ISD::MUL:
13676 return getConstant(Val: 1, DL, VT);
13677 case ISD::AND:
13678 case ISD::UMIN:
13679 return getAllOnesConstant(DL, VT);
13680 case ISD::SMAX:
13681 return getConstant(Val: APInt::getSignedMinValue(numBits: VT.getSizeInBits()), DL, VT);
13682 case ISD::SMIN:
13683 return getConstant(Val: APInt::getSignedMaxValue(numBits: VT.getSizeInBits()), DL, VT);
13684 case ISD::FADD:
13685 // If flags allow, prefer positive zero since it's generally cheaper
13686 // to materialize on most targets.
13687 return getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT);
13688 case ISD::FMUL:
13689 return getConstantFP(Val: 1.0, DL, VT);
13690 case ISD::FMINNUM:
13691 case ISD::FMAXNUM: {
13692 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
13693 const fltSemantics &Semantics = VT.getFltSemantics();
13694 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Sem: Semantics) :
13695 !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics) :
13696 APFloat::getLargest(Sem: Semantics);
13697 if (Opcode == ISD::FMAXNUM)
13698 NeutralAF.changeSign();
13699
13700 return getConstantFP(V: NeutralAF, DL, VT);
13701 }
13702 case ISD::FMINIMUM:
13703 case ISD::FMAXIMUM: {
13704 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
13705 const fltSemantics &Semantics = VT.getFltSemantics();
13706 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Sem: Semantics)
13707 : APFloat::getLargest(Sem: Semantics);
13708 if (Opcode == ISD::FMAXIMUM)
13709 NeutralAF.changeSign();
13710
13711 return getConstantFP(V: NeutralAF, DL, VT);
13712 }
13713
13714 }
13715}
13716
13717/// Helper used to make a call to a library function that has one argument of
13718/// pointer type.
13719///
13720/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
13721/// used to get or set floating-point state. They have one argument of pointer
13722/// type, which points to the memory region containing bits of the
13723/// floating-point state. The value returned by such function is ignored in the
13724/// created call.
13725///
13726/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
13727/// \param Ptr Pointer used to save/load state.
13728/// \param InChain Ingoing token chain.
13729/// \returns Outgoing chain token.
13730SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
13731 SDValue InChain,
13732 const SDLoc &DLoc) {
13733 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
13734 TargetLowering::ArgListTy Args;
13735 TargetLowering::ArgListEntry Entry;
13736 Entry.Node = Ptr;
13737 Entry.Ty = Ptr.getValueType().getTypeForEVT(Context&: *getContext());
13738 Args.push_back(x: Entry);
13739 RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
13740 SDValue Callee = getExternalSymbol(Sym: TLI->getLibcallName(Call: LC),
13741 VT: TLI->getPointerTy(DL: getDataLayout()));
13742 TargetLowering::CallLoweringInfo CLI(*this);
13743 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
13744 CC: TLI->getLibcallCallingConv(Call: LC), ResultType: Type::getVoidTy(C&: *getContext()), Target: Callee,
13745 ArgsList: std::move(Args));
13746 return TLI->LowerCallTo(CLI).second;
13747}
13748
13749void SelectionDAG::copyExtraInfo(SDNode *From, SDNode *To) {
13750 assert(From && To && "Invalid SDNode; empty source SDValue?");
13751 auto I = SDEI.find(Val: From);
13752 if (I == SDEI.end())
13753 return;
13754
13755 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
13756 // the iterator, hence the need to make a copy to prevent a use-after-free.
13757 NodeExtraInfo NEI = I->second;
13758 if (LLVM_LIKELY(!NEI.PCSections)) {
13759 // No deep copy required for the types of extra info set.
13760 //
13761 // FIXME: Investigate if other types of extra info also need deep copy. This
13762 // depends on the types of nodes they can be attached to: if some extra info
13763 // is only ever attached to nodes where a replacement To node is always the
13764 // node where later use and propagation of the extra info has the intended
13765 // semantics, no deep copy is required.
13766 SDEI[To] = std::move(NEI);
13767 return;
13768 }
13769
13770 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
13771 // through the replacement of From with To. Otherwise, replacements of a node
13772 // (From) with more complex nodes (To and its operands) may result in lost
13773 // extra info where the root node (To) is insignificant in further propagating
13774 // and using extra info when further lowering to MIR.
13775 //
13776 // In the first step pre-populate the visited set with the nodes reachable
13777 // from the old From node. This avoids copying NodeExtraInfo to parts of the
13778 // DAG that is not new and should be left untouched.
13779 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
13780 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
13781 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
13782 if (MaxDepth == 0) {
13783 // Remember this node in case we need to increase MaxDepth and continue
13784 // populating FromReach from this node.
13785 Leafs.emplace_back(Args&: N);
13786 return;
13787 }
13788 if (!FromReach.insert(V: N).second)
13789 return;
13790 for (const SDValue &Op : N->op_values())
13791 Self(Self, Op.getNode(), MaxDepth - 1);
13792 };
13793
13794 // Copy extra info to To and all its transitive operands (that are new).
13795 SmallPtrSet<const SDNode *, 8> Visited;
13796 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
13797 if (FromReach.contains(V: N))
13798 return true;
13799 if (!Visited.insert(Ptr: N).second)
13800 return true;
13801 if (getEntryNode().getNode() == N)
13802 return false;
13803 for (const SDValue &Op : N->op_values()) {
13804 if (!Self(Self, Op.getNode()))
13805 return false;
13806 }
13807 // Copy only if entry node was not reached.
13808 SDEI[N] = NEI;
13809 return true;
13810 };
13811
13812 // We first try with a lower MaxDepth, assuming that the path to common
13813 // operands between From and To is relatively short. This significantly
13814 // improves performance in the common case. The initial MaxDepth is big
13815 // enough to avoid retry in the common case; the last MaxDepth is large
13816 // enough to avoid having to use the fallback below (and protects from
13817 // potential stack exhaustion from recursion).
13818 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
13819 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
13820 // StartFrom is the previous (or initial) set of leafs reachable at the
13821 // previous maximum depth.
13822 SmallVector<const SDNode *> StartFrom;
13823 std::swap(LHS&: StartFrom, RHS&: Leafs);
13824 for (const SDNode *N : StartFrom)
13825 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
13826 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
13827 return;
13828 // This should happen very rarely (reached the entry node).
13829 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
13830 assert(!Leafs.empty());
13831 }
13832
13833 // This should not happen - but if it did, that means the subgraph reachable
13834 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
13835 // could not visit all reachable common operands. Consequently, we were able
13836 // to reach the entry node.
13837 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
13838 assert(false && "From subgraph too complex - increase max. MaxDepth?");
13839 // Best-effort fallback if assertions disabled.
13840 SDEI[To] = std::move(NEI);
13841}
13842
13843#ifndef NDEBUG
13844static void checkForCyclesHelper(const SDNode *N,
13845 SmallPtrSetImpl<const SDNode*> &Visited,
13846 SmallPtrSetImpl<const SDNode*> &Checked,
13847 const llvm::SelectionDAG *DAG) {
13848 // If this node has already been checked, don't check it again.
13849 if (Checked.count(N))
13850 return;
13851
13852 // If a node has already been visited on this depth-first walk, reject it as
13853 // a cycle.
13854 if (!Visited.insert(N).second) {
13855 errs() << "Detected cycle in SelectionDAG\n";
13856 dbgs() << "Offending node:\n";
13857 N->dumprFull(DAG); dbgs() << "\n";
13858 abort();
13859 }
13860
13861 for (const SDValue &Op : N->op_values())
13862 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
13863
13864 Checked.insert(N);
13865 Visited.erase(N);
13866}
13867#endif
13868
13869void llvm::checkForCycles(const llvm::SDNode *N,
13870 const llvm::SelectionDAG *DAG,
13871 bool force) {
13872#ifndef NDEBUG
13873 bool check = force;
13874#ifdef EXPENSIVE_CHECKS
13875 check = true;
13876#endif // EXPENSIVE_CHECKS
13877 if (check) {
13878 assert(N && "Checking nonexistent SDNode");
13879 SmallPtrSet<const SDNode*, 32> visited;
13880 SmallPtrSet<const SDNode*, 32> checked;
13881 checkForCyclesHelper(N, visited, checked, DAG);
13882 }
13883#endif // !NDEBUG
13884}
13885
13886void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
13887 checkForCycles(N: DAG->getRoot().getNode(), DAG, force);
13888}
13889