1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/Analysis/VectorUtils.h"
17#include "llvm/CodeGen/Analysis.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/CodeGenCommonISel.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/SDPatternMatch.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/GlobalVariable.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCExpr.h"
33#include "llvm/Support/DivisionByConstantInfo.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Target/TargetMachine.h"
38#include <cctype>
39#include <deque>
40#include <list>
41using namespace llvm;
42using namespace llvm::SDPatternMatch;
43
44/// NOTE: The TargetMachine owns TLOF.
45TargetLowering::TargetLowering(const TargetMachine &tm,
46 const TargetSubtargetInfo &STI)
47 : TargetLoweringBase(tm, STI) {}
48
49// Define the virtual destructor out-of-line for build efficiency.
50TargetLowering::~TargetLowering() = default;
51
52const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
53 return nullptr;
54}
55
56bool TargetLowering::isPositionIndependent() const {
57 return getTargetMachine().isPositionIndependent();
58}
59
60/// Check whether a given call node is in tail position within its function. If
61/// so, it sets Chain to the input chain of the tail call.
62bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
63 SDValue &Chain) const {
64 const Function &F = DAG.getMachineFunction().getFunction();
65
66 // First, check if tail calls have been disabled in this function.
67 if (F.getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
68 return false;
69
70 // Conservatively require the attributes of the call to match those of
71 // the return. Ignore following attributes because they don't affect the
72 // call sequence.
73 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
74 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
75 Attribute::DereferenceableOrNull, Attribute::NoAlias,
76 Attribute::NonNull, Attribute::NoUndef,
77 Attribute::Range, Attribute::NoFPClass})
78 CallerAttrs.removeAttribute(Val: Attr);
79
80 if (CallerAttrs.hasAttributes())
81 return false;
82
83 // It's not safe to eliminate the sign / zero extension of the return value.
84 if (CallerAttrs.contains(A: Attribute::ZExt) ||
85 CallerAttrs.contains(A: Attribute::SExt))
86 return false;
87
88 // Check if the only use is a function return node.
89 return isUsedByReturnOnly(Node, Chain);
90}
91
92bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
93 const uint32_t *CallerPreservedMask,
94 const SmallVectorImpl<CCValAssign> &ArgLocs,
95 const SmallVectorImpl<SDValue> &OutVals) const {
96 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
97 const CCValAssign &ArgLoc = ArgLocs[I];
98 if (!ArgLoc.isRegLoc())
99 continue;
100 MCRegister Reg = ArgLoc.getLocReg();
101 // Only look at callee saved registers.
102 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg: Reg))
103 continue;
104 // Check that we pass the value used for the caller.
105 // (We look for a CopyFromReg reading a virtual register that is used
106 // for the function live-in value of register Reg)
107 SDValue Value = OutVals[I];
108 if (Value->getOpcode() == ISD::AssertZext)
109 Value = Value.getOperand(i: 0);
110 if (Value->getOpcode() != ISD::CopyFromReg)
111 return false;
112 Register ArgReg = cast<RegisterSDNode>(Val: Value->getOperand(Num: 1))->getReg();
113 if (MRI.getLiveInPhysReg(VReg: ArgReg) != Reg)
114 return false;
115 }
116 return true;
117}
118
119static bool paramHasAttr(const CallBase &Call, unsigned ArgIdx,
120 Attribute::AttrKind Kind) {
121 return Call.paramHasAttr(ArgNo: ArgIdx, Kind);
122}
123
124static bool paramHasAttr(const AttributeList &Attrs, unsigned ArgIdx,
125 Attribute::AttrKind Kind) {
126 return Attrs.hasParamAttr(ArgNo: ArgIdx, Kind);
127}
128
129static MaybeAlign getParamStackAlign(const CallBase &Call, unsigned ArgIdx) {
130 return Call.getParamStackAlign(ArgNo: ArgIdx);
131}
132
133static MaybeAlign getParamStackAlign(const AttributeList &Attrs,
134 unsigned ArgIdx) {
135 return Attrs.getParamStackAlignment(ArgNo: ArgIdx);
136}
137
138static MaybeAlign getParamAlign(const CallBase &Call, unsigned ArgIdx) {
139 return Call.getParamAlign(ArgNo: ArgIdx);
140}
141
142static MaybeAlign getParamAlign(const AttributeList &Attrs, unsigned ArgIdx) {
143 return Attrs.getParamAlignment(ArgNo: ArgIdx);
144}
145
146/// Set CallLoweringInfo attribute flags based on a call instruction
147/// and called function attributes.
148template <typename SourceT>
149static void setArgListEntryAttributes(TargetLoweringBase::ArgListEntry &Entry,
150 const SourceT &Src, unsigned ArgIdx) {
151 Entry.IsSExt = paramHasAttr(Src, ArgIdx, Attribute::SExt);
152 Entry.IsZExt = paramHasAttr(Src, ArgIdx, Attribute::ZExt);
153 Entry.IsNoExt = paramHasAttr(Src, ArgIdx, Attribute::NoExt);
154 Entry.IsInReg = paramHasAttr(Src, ArgIdx, Attribute::InReg);
155 Entry.IsSRet = paramHasAttr(Src, ArgIdx, Attribute::StructRet);
156 Entry.IsNest = paramHasAttr(Src, ArgIdx, Attribute::Nest);
157 Entry.IsByVal = paramHasAttr(Src, ArgIdx, Attribute::ByVal);
158 Entry.IsPreallocated = paramHasAttr(Src, ArgIdx, Attribute::Preallocated);
159 Entry.IsInAlloca = paramHasAttr(Src, ArgIdx, Attribute::InAlloca);
160 Entry.IsReturned = paramHasAttr(Src, ArgIdx, Attribute::Returned);
161 Entry.IsSwiftSelf = paramHasAttr(Src, ArgIdx, Attribute::SwiftSelf);
162 Entry.IsSwiftAsync = paramHasAttr(Src, ArgIdx, Attribute::SwiftAsync);
163 Entry.IsSwiftError = paramHasAttr(Src, ArgIdx, Attribute::SwiftError);
164 Entry.Alignment = getParamStackAlign(Src, ArgIdx);
165 Entry.IndirectType = nullptr;
166 assert(Entry.IsByVal + Entry.IsPreallocated + Entry.IsInAlloca +
167 Entry.IsSRet <=
168 1 &&
169 "multiple ABI attributes?");
170 if (Entry.IsByVal) {
171 Entry.IndirectType = Src.getParamByValType(ArgIdx);
172 if (!Entry.Alignment)
173 Entry.Alignment = getParamAlign(Src, ArgIdx);
174 }
175 if (Entry.IsPreallocated)
176 Entry.IndirectType = Src.getParamPreallocatedType(ArgIdx);
177 if (Entry.IsInAlloca)
178 Entry.IndirectType = Src.getParamInAllocaType(ArgIdx);
179 if (Entry.IsSRet)
180 Entry.IndirectType = Src.getParamStructRetType(ArgIdx);
181}
182
183void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
184 unsigned ArgIdx) {
185 setArgListEntryAttributes(Entry&: *this, Src: *Call, ArgIdx);
186}
187
188void TargetLoweringBase::ArgListEntry::setAttributes(const AttributeList &Attrs,
189 unsigned ArgIdx) {
190 setArgListEntryAttributes(Entry&: *this, Src: Attrs, ArgIdx);
191}
192
193void TargetLoweringBase::ArgListEntry::setExtAttribute(
194 Attribute::AttrKind Attr) {
195 if (Attr != Attribute::None) {
196 if (Attr == Attribute::SExt)
197 IsSExt = true;
198 else if (Attr == Attribute::ZExt)
199 IsZExt = true;
200 else if (Attr == Attribute::NoExt)
201 IsNoExt = true;
202 else
203 llvm_unreachable("Unexpected extension attribute!");
204 }
205}
206
207TargetLowering::ArgListTy
208TargetLowering::getArgListForFunctionType(FunctionType *FuncTy,
209 const AttributeList &FuncAttrs,
210 ArrayRef<SDValue> Ops) {
211 // TODO: This assumes each parameter maps to exactly one operand node, which
212 // does not hold when an argument requires type splitting.
213 assert(Ops.size() == FuncTy->getNumParams() &&
214 "argument count does not match the function type");
215 ArgListTy Args;
216 Args.reserve(n: Ops.size());
217 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
218 ArgListEntry Entry(Ops[I], FuncTy->getParamType(i: I));
219 Entry.setAttributes(Attrs: FuncAttrs, ArgIdx: I);
220 Args.push_back(x: Entry);
221 }
222 return Args;
223}
224
225/// Generate a libcall taking the given operands as arguments and returning a
226/// result of type RetVT.
227std::pair<SDValue, SDValue>
228TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl,
229 EVT RetVT, ArrayRef<SDValue> Ops,
230 MakeLibCallOptions CallOptions, const SDLoc &dl,
231 SDValue InChain) const {
232 if (LibcallImpl == RTLIB::Unsupported)
233 reportFatalInternalError(reason: "unsupported library call operation");
234
235 if (!InChain)
236 InChain = DAG.getEntryNode();
237
238 TargetLowering::ArgListTy Args;
239 Args.reserve(n: Ops.size());
240
241 ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
242 for (unsigned i = 0; i < Ops.size(); ++i) {
243 SDValue NewOp = Ops[i];
244 Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
245 ? OpsTypeOverrides[i]
246 : NewOp.getValueType().getTypeForEVT(Context&: *DAG.getContext());
247 TargetLowering::ArgListEntry Entry(NewOp, Ty);
248 if (CallOptions.IsSoften)
249 Entry.OrigTy =
250 CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(Context&: *DAG.getContext());
251
252 Entry.IsSExt =
253 shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned: CallOptions.IsSigned);
254 Entry.IsZExt = !Entry.IsSExt;
255
256 if (CallOptions.IsSoften &&
257 !shouldExtendTypeInLibCall(Type: CallOptions.OpsVTBeforeSoften[i])) {
258 Entry.IsSExt = Entry.IsZExt = false;
259 }
260 Args.push_back(x: Entry);
261 }
262
263 SDValue Callee =
264 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
265
266 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
267 Type *OrigRetTy = RetTy;
268 TargetLowering::CallLoweringInfo CLI(DAG);
269 bool signExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: CallOptions.IsSigned);
270 bool zeroExtend = !signExtend;
271
272 if (CallOptions.IsSoften) {
273 OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(Context&: *DAG.getContext());
274 if (!shouldExtendTypeInLibCall(Type: CallOptions.RetVTBeforeSoften))
275 signExtend = zeroExtend = false;
276 }
277
278 CLI.setDebugLoc(dl)
279 .setChain(InChain)
280 .setLibCallee(CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetTy, OrigResultType: OrigRetTy,
281 Target: Callee, ArgsList: std::move(Args))
282 .setNoReturn(CallOptions.DoesNotReturn)
283 .setDiscardResult(!CallOptions.IsReturnValueUsed)
284 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
285 .setSExtResult(signExtend)
286 .setZExtResult(zeroExtend);
287 return LowerCallTo(CLI);
288}
289
290bool TargetLowering::findOptimalMemOpLowering(
291 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
292 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
293 const AttributeList &FuncAttributes, EVT *LargestVT) const {
294 EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);
295
296 if (VT == MVT::Other) {
297 // Use the largest integer type whose alignment constraints are satisfied.
298 VT = MVT::LAST_INTEGER_VALUETYPE;
299 if (Op.isFixedDstAlign()) {
300 bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
301 while (VT != MVT::i8) {
302 unsigned VTSize = VT.getSizeInBits() / 8;
303 bool DstOk =
304 Op.getDstAlign() >= VTSize ||
305 allowsMisalignedMemoryAccesses(VT, AddrSpace: DstAS, Alignment: Op.getDstAlign());
306 bool SrcOk =
307 !LoadsFromSrc || Op.getSrcAlign() >= VTSize ||
308 allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcAS, Alignment: Op.getSrcAlign());
309 if (DstOk && SrcOk)
310 break;
311 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
312 }
313 }
314 assert(VT.isInteger());
315
316 // Find the largest legal integer type.
317 MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
318 while (!isTypeLegal(VT: LVT))
319 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
320 assert(LVT.isInteger());
321
322 // If the type we've chosen is larger than the largest legal integer type
323 // then use the largest legal type.
324 if (VT.bitsGT(VT: LVT))
325 VT = LVT;
326 }
327
328 unsigned NumMemOps = 0;
329 uint64_t Size = Op.size();
330 while (Size) {
331 unsigned VTSize = VT.getSizeInBits() / 8;
332 while (VTSize > Size) {
333 // For now, only use non-vector load / store's for the left-over pieces.
334 EVT NewVT = VT;
335 unsigned NewVTSize;
336
337 bool Found = false;
338 if (VT.isVector() || VT.isFloatingPoint()) {
339 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
340 if (isOperationLegalOrCustom(Op: ISD::STORE, VT: NewVT) &&
341 isSafeMemOpType(NewVT.getSimpleVT()))
342 Found = true;
343 else if (NewVT == MVT::i64 &&
344 isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::f64) &&
345 isSafeMemOpType(MVT::f64)) {
346 // i64 is usually not legal on 32-bit targets, but f64 may be.
347 NewVT = MVT::f64;
348 Found = true;
349 }
350 }
351
352 if (!Found) {
353 do {
354 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
355 if (NewVT == MVT::i8)
356 break;
357 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
358 }
359 NewVTSize = NewVT.getSizeInBits() / 8;
360
361 // If the new VT cannot cover all of the remaining bits, then consider
362 // issuing a (or a pair of) unaligned and overlapping load / store.
363 unsigned Fast;
364 if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
365 allowsMisalignedMemoryAccesses(
366 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
367 Flags: MachineMemOperand::MONone, &Fast) &&
368 Fast)
369 VTSize = Size;
370 else {
371 VT = NewVT;
372 VTSize = NewVTSize;
373 }
374 }
375
376 if (++NumMemOps > Limit)
377 return false;
378
379 MemOps.push_back(x: VT);
380 Size -= VTSize;
381 }
382
383 return true;
384}
385
386/// Soften the operands of a comparison. This code is shared among BR_CC,
387/// SELECT_CC, and SETCC handlers.
388void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
389 SDValue &NewLHS, SDValue &NewRHS,
390 ISD::CondCode &CCCode,
391 const SDLoc &dl, const SDValue OldLHS,
392 const SDValue OldRHS) const {
393 SDValue Chain;
394 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: dl, OldLHS,
395 OldRHS, Chain);
396}
397
398/// Select the libcall and the condition code to test its result against 0 for
399/// an ordered floating-point compare. \p BoolLC is the boolean helper (result
400/// is 0/1). \p TriStateLC is the per-predicate three-way helper and \p
401/// GenericLC the generic single-symbol three-way helper (both return -1/0/1,
402/// tested against 0 with \p TriStateCC). The boolean form is preferred, then
403/// the per-predicate three-way, then the generic three-way.
404static std::pair<RTLIB::Libcall, ISD::CondCode>
405selectFPCmpLibcall(const LibcallLoweringInfo &Libcalls, RTLIB::Libcall BoolLC,
406 RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC,
407 ISD::CondCode TriStateCC) {
408 if (Libcalls.getLibcallImpl(Call: BoolLC) != RTLIB::Unsupported)
409 return {BoolLC, ISD::SETNE};
410 if (Libcalls.getLibcallImpl(Call: TriStateLC) != RTLIB::Unsupported)
411 return {TriStateLC, TriStateCC};
412 return {GenericLC, TriStateCC};
413}
414
415void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
416 SDValue &NewLHS, SDValue &NewRHS,
417 ISD::CondCode &CCCode,
418 const SDLoc &dl, const SDValue OldLHS,
419 const SDValue OldRHS,
420 SDValue &Chain,
421 bool IsSignaling) const {
422 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
423 // not supporting it. We can update this code when libgcc provides such
424 // functions.
425
426 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
427 && "Unsupported setcc type!");
428
429 // Expand into one or more soft-fp libcall(s).
430 ISD::CondCode OrigCCCode = CCCode;
431 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
432 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
433 bool ShouldInvertCC = false;
434
435 // Expand a compare libcall family name (e.g. OEQ, FCMP3_PRED_OEQ) to the
436 // RTLIB::Libcall for VT.
437#define FP_CMP_LIBCALL(BASE) \
438 RTLIB::getFPLibCall(VT, RTLIB::BASE##_F32, RTLIB::BASE##_F64, \
439 RTLIB::UNKNOWN_LIBCALL, RTLIB::BASE##_F128, \
440 RTLIB::BASE##_PPCF128)
441
442 switch (CCCode) {
443 case ISD::SETEQ:
444 case ISD::SETOEQ:
445 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
446 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
447 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
448 break;
449 case ISD::SETNE:
450 case ISD::SETUNE:
451 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
452 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(UNE), FP_CMP_LIBCALL(FCMP3_PRED_UNE),
453 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETNE);
454 // Some ABIs (e.g. AEABI) provide neither a not-equal nor a three-way
455 // compare; obtain not-equal (UNE = !OEQ) by inverting ordered-equal.
456 if (DAG.getLibcalls().getLibcallImpl(Call: LC1) == RTLIB::Unsupported) {
457 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
458 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ),
459 FP_CMP_LIBCALL(FCMP3_PRED_OEQ), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
460 ShouldInvertCC = true;
461 }
462 break;
463 case ISD::SETGE:
464 case ISD::SETOGE:
465 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
466 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE), FP_CMP_LIBCALL(FCMP3_PRED_OGE),
467 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
468 break;
469 case ISD::SETLT:
470 case ISD::SETOLT:
471 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
472 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT), FP_CMP_LIBCALL(FCMP3_PRED_OLT),
473 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
474 break;
475 case ISD::SETLE:
476 case ISD::SETOLE:
477 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
478 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE), FP_CMP_LIBCALL(FCMP3_PRED_OLE),
479 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
480 break;
481 case ISD::SETGT:
482 case ISD::SETOGT:
483 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
484 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT), FP_CMP_LIBCALL(FCMP3_PRED_OGT),
485 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
486 break;
487 case ISD::SETO:
488 ShouldInvertCC = true;
489 [[fallthrough]];
490 case ISD::SETUO:
491 // Unordered is a boolean everywhere (__unordXf2 returns 0/1).
492 LC1 = FP_CMP_LIBCALL(UO);
493 CC1 = ISD::SETNE;
494 break;
495 case ISD::SETONE:
496 // SETONE = O && UNE
497 ShouldInvertCC = true;
498 [[fallthrough]];
499 case ISD::SETUEQ:
500 LC1 = FP_CMP_LIBCALL(UO);
501 CC1 = ISD::SETNE;
502 std::tie(args&: LC2, args&: CC2) = selectFPCmpLibcall(
503 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
504 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
505 break;
506 default:
507 // Invert CC for unordered comparisons, handled by the ordered inverse.
508 ShouldInvertCC = true;
509 switch (CCCode) {
510 case ISD::SETULT:
511 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
512 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE),
513 FP_CMP_LIBCALL(FCMP3_PRED_OGE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
514 break;
515 case ISD::SETULE:
516 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
517 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT),
518 FP_CMP_LIBCALL(FCMP3_PRED_OGT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
519 break;
520 case ISD::SETUGT:
521 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
522 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE),
523 FP_CMP_LIBCALL(FCMP3_PRED_OLE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
524 break;
525 case ISD::SETUGE:
526 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
527 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT),
528 FP_CMP_LIBCALL(FCMP3_PRED_OLT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
529 break;
530 default:
531 llvm_unreachable("Do not know how to soften this setcc!");
532 }
533 }
534
535#undef FP_CMP_LIBCALL
536
537 // Use the target specific return value for comparison lib calls.
538 EVT RetVT = EVT::getEVT(Ty: getCmpLibcallReturnType(Ctx&: *DAG.getContext()));
539 SDValue Ops[2] = {NewLHS, NewRHS};
540 TargetLowering::MakeLibCallOptions CallOptions;
541 EVT OpsVT[2] = { OldLHS.getValueType(),
542 OldRHS.getValueType() };
543 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
544
545 auto ReportNoLibcall = [&]() {
546 DAG.getContext()->emitError(
547 ErrorStr: Twine("no libcall available to soften floating-point ") +
548 ISD::getCondCodeName(Operation: OrigCCCode) + " compare with type " +
549 VT.getEVTString());
550 NewLHS = DAG.getPOISON(VT: RetVT);
551 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
552 CCCode = ISD::SETNE;
553 };
554
555 // Check availability before makeLibCall, which fatally errors otherwise.
556 RTLIB::LibcallImpl LC1Impl = DAG.getLibcalls().getLibcallImpl(Call: LC1);
557 if (LC1Impl == RTLIB::Unsupported) {
558 ReportNoLibcall();
559 return;
560 }
561
562 auto Call = makeLibCall(DAG, LibcallImpl: LC1Impl, RetVT, Ops, CallOptions, dl, InChain: Chain);
563 NewLHS = Call.first;
564 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
565
566 CCCode = CC1;
567 if (ShouldInvertCC) {
568 assert(RetVT.isInteger());
569 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
570 }
571
572 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
573 // Update Chain.
574 Chain = Call.second;
575 } else {
576 RTLIB::LibcallImpl LC2Impl = DAG.getLibcalls().getLibcallImpl(Call: LC2);
577 if (LC2Impl == RTLIB::Unsupported) {
578 ReportNoLibcall();
579 return;
580 }
581
582 assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
583 "unordered call should be simple boolean");
584
585 EVT SetCCVT =
586 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: RetVT);
587 if (getBooleanContents(Type: RetVT) == ZeroOrOneBooleanContent) {
588 NewLHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: RetVT, N1: Call.first,
589 N2: DAG.getValueType(MVT::i1));
590 }
591
592 SDValue Tmp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NewLHS, RHS: NewRHS, Cond: CCCode);
593 auto Call2 = makeLibCall(DAG, LibcallImpl: LC2Impl, RetVT, Ops, CallOptions, dl, InChain: Chain);
594 CCCode = CC2;
595 if (ShouldInvertCC)
596 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
597 NewLHS = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Call2.first, RHS: NewRHS, Cond: CCCode);
598 if (Chain)
599 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Call.second,
600 N2: Call2.second);
601 NewLHS = DAG.getNode(Opcode: ShouldInvertCC ? ISD::AND : ISD::OR, DL: dl,
602 VT: Tmp.getValueType(), N1: Tmp, N2: NewLHS);
603 NewRHS = SDValue();
604 }
605}
606
607/// Return the entry encoding for a jump table in the current function. The
608/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
609unsigned TargetLowering::getJumpTableEncoding() const {
610 // In non-pic modes, just use the address of a block.
611 if (!isPositionIndependent())
612 return MachineJumpTableInfo::EK_BlockAddress;
613
614 // Otherwise, use a label difference.
615 return MachineJumpTableInfo::EK_LabelDifference32;
616}
617
618SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
619 SelectionDAG &DAG) const {
620 return Table;
621}
622
623/// This returns the relocation base for the given PIC jumptable, the same as
624/// getPICJumpTableRelocBase, but as an MCExpr.
625const MCExpr *
626TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
627 unsigned JTI,MCContext &Ctx) const{
628 // The normal PIC reloc base is the label at the start of the jump table.
629 return MCSymbolRefExpr::create(Symbol: MF->getJTISymbol(JTI, Ctx), Ctx);
630}
631
632SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
633 SDValue Addr, int JTI,
634 SelectionDAG &DAG) const {
635 SDValue Chain = Value;
636 // Jump table debug info is only needed if CodeView is enabled.
637 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF()) {
638 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
639 }
640 return DAG.getNode(Opcode: ISD::BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
641}
642
643bool
644TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
645 const TargetMachine &TM = getTargetMachine();
646 const GlobalValue *GV = GA->getGlobal();
647
648 // If the address is not even local to this DSO we will have to load it from
649 // a got and then add the offset.
650 if (!TM.shouldAssumeDSOLocal(GV))
651 return false;
652
653 // If the code is position independent we will have to add a base register.
654 if (isPositionIndependent())
655 return false;
656
657 // Otherwise we can do it.
658 return true;
659}
660
661//===----------------------------------------------------------------------===//
662// Optimization Methods
663//===----------------------------------------------------------------------===//
664
665/// If the specified instruction has a constant integer operand and there are
666/// bits set in that constant that are not demanded, then clear those bits and
667/// return true.
668bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
669 const APInt &DemandedBits,
670 const APInt &DemandedElts,
671 TargetLoweringOpt &TLO) const {
672 SDLoc DL(Op);
673 unsigned Opcode = Op.getOpcode();
674
675 // Early-out if we've ended up calling an undemanded node, leave this to
676 // constant folding.
677 if (DemandedBits.isZero() || DemandedElts.isZero())
678 return false;
679
680 // Do target-specific constant optimization.
681 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
682 return TLO.New.getNode();
683
684 // FIXME: ISD::SELECT, ISD::SELECT_CC
685 switch (Opcode) {
686 default:
687 break;
688 case ISD::XOR:
689 case ISD::AND:
690 case ISD::OR: {
691 auto *Op1C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
692 if (!Op1C || Op1C->isOpaque())
693 return false;
694
695 // If this is a 'not' op, don't touch it because that's a canonical form.
696 const APInt &C = Op1C->getAPIntValue();
697 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(RHS: C))
698 return false;
699
700 if (!C.isSubsetOf(RHS: DemandedBits)) {
701 EVT VT = Op.getValueType();
702 SDValue NewC = TLO.DAG.getConstant(Val: DemandedBits & C, DL, VT);
703 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 0), N2: NewC,
704 Flags: Op->getFlags());
705 return TLO.CombineTo(O: Op, N: NewOp);
706 }
707
708 break;
709 }
710 }
711
712 return false;
713}
714
715bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
716 const APInt &DemandedBits,
717 TargetLoweringOpt &TLO) const {
718 EVT VT = Op.getValueType();
719 APInt DemandedElts = VT.isVector()
720 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
721 : APInt(1, 1);
722 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
723}
724
725/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
726/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
727/// but it could be generalized for targets with other types of implicit
728/// widening casts.
729bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
730 const APInt &DemandedBits,
731 TargetLoweringOpt &TLO) const {
732 assert(Op.getNumOperands() == 2 &&
733 "ShrinkDemandedOp only supports binary operators!");
734 assert(Op.getNode()->getNumValues() == 1 &&
735 "ShrinkDemandedOp only supports nodes with one result!");
736
737 EVT VT = Op.getValueType();
738 SelectionDAG &DAG = TLO.DAG;
739 SDLoc dl(Op);
740
741 // Early return, as this function cannot handle vector types.
742 if (VT.isVector())
743 return false;
744
745 assert(Op.getOperand(0).getValueType().getScalarSizeInBits() == BitWidth &&
746 Op.getOperand(1).getValueType().getScalarSizeInBits() == BitWidth &&
747 "ShrinkDemandedOp only supports operands that have the same size!");
748
749 // Don't do this if the node has another user, which may require the
750 // full value.
751 if (!Op.getNode()->hasOneUse())
752 return false;
753
754 // Search for the smallest integer type with free casts to and from
755 // Op's type. For expedience, just check power-of-2 integer types.
756 unsigned DemandedSize = DemandedBits.getActiveBits();
757 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
758 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
759 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SmallVTBits);
760 if (isTruncateFree(Val: Op, VT2: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT)) {
761 // We found a type with free casts.
762
763 // If the operation has the 'disjoint' flag, then the
764 // operands on the new node are also disjoint.
765 SDNodeFlags Flags(Op->getFlags().hasDisjoint() ? SDNodeFlags::Disjoint
766 : SDNodeFlags::None);
767 unsigned Opcode = Op.getOpcode();
768 if (Opcode == ISD::PTRADD) {
769 // It isn't a ptradd anymore if it doesn't operate on the entire
770 // pointer.
771 Opcode = ISD::ADD;
772 }
773 SDValue X = DAG.getNode(
774 Opcode, DL: dl, VT: SmallVT,
775 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
776 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 1)), Flags);
777 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
778 SDValue Z = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: X);
779 return TLO.CombineTo(O: Op, N: Z);
780 }
781 }
782 return false;
783}
784
785bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
786 DAGCombinerInfo &DCI) const {
787 SelectionDAG &DAG = DCI.DAG;
788 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
789 !DCI.isBeforeLegalizeOps());
790 KnownBits Known;
791
792 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
793 if (Simplified) {
794 DCI.AddToWorklist(N: Op.getNode());
795 DCI.CommitTargetLoweringOpt(TLO);
796 }
797 return Simplified;
798}
799
800bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
801 const APInt &DemandedElts,
802 DAGCombinerInfo &DCI) const {
803 SelectionDAG &DAG = DCI.DAG;
804 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
805 !DCI.isBeforeLegalizeOps());
806 KnownBits Known;
807
808 bool Simplified =
809 SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
810 if (Simplified) {
811 DCI.AddToWorklist(N: Op.getNode());
812 DCI.CommitTargetLoweringOpt(TLO);
813 }
814 return Simplified;
815}
816
817bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
818 KnownBits &Known,
819 TargetLoweringOpt &TLO,
820 unsigned Depth,
821 bool AssumeSingleUse) const {
822 EVT VT = Op.getValueType();
823
824 // Since the number of lanes in a scalable vector is unknown at compile time,
825 // we track one bit which is implicitly broadcast to all lanes. This means
826 // that all lanes in a scalable vector are considered demanded.
827 APInt DemandedElts = VT.isFixedLengthVector()
828 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
829 : APInt(1, 1);
830 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
831 AssumeSingleUse);
832}
833
834// TODO: Under what circumstances can we create nodes? Constant folding?
835SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
836 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
837 SelectionDAG &DAG, unsigned Depth) const {
838 EVT VT = Op.getValueType();
839
840 // Limit search depth.
841 if (Depth >= SelectionDAG::MaxRecursionDepth)
842 return SDValue();
843
844 // Ignore UNDEFs.
845 if (Op.isUndef())
846 return SDValue();
847
848 // Not demanding any bits/elts from Op.
849 if (DemandedBits == 0 || DemandedElts == 0)
850 return DAG.getUNDEF(VT);
851
852 bool IsLE = DAG.getDataLayout().isLittleEndian();
853 unsigned NumElts = DemandedElts.getBitWidth();
854 unsigned BitWidth = DemandedBits.getBitWidth();
855 KnownBits LHSKnown, RHSKnown;
856 switch (Op.getOpcode()) {
857 case ISD::Constant: {
858 const APInt &Value = Op->getAsAPIntVal();
859 if (!Value.isZero() && (Value & DemandedBits).isZero())
860 return DAG.getConstant(Val: 0, DL: SDLoc(Op), VT);
861 break;
862 }
863 case ISD::BITCAST: {
864 if (VT.isScalableVector())
865 return SDValue();
866
867 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
868 EVT SrcVT = Src.getValueType();
869 EVT DstVT = Op.getValueType();
870 if (SrcVT == DstVT)
871 return Src;
872
873 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
874 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
875 if (NumSrcEltBits == NumDstEltBits)
876 if (SDValue V = SimplifyMultipleUseDemandedBits(
877 Op: Src, DemandedBits, DemandedElts, DAG, Depth: Depth + 1))
878 return DAG.getBitcast(VT: DstVT, V);
879
880 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
881 unsigned Scale = NumDstEltBits / NumSrcEltBits;
882 unsigned NumSrcElts = SrcVT.getVectorNumElements();
883 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
884 for (unsigned i = 0; i != Scale; ++i) {
885 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
886 unsigned BitOffset = EltOffset * NumSrcEltBits;
887 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
888 }
889 // Recursive calls below may turn not demanded elements into poison, so we
890 // need to demand all smaller source elements that maps to a demanded
891 // destination element.
892 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
893
894 if (SDValue V = SimplifyMultipleUseDemandedBits(
895 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
896 return DAG.getBitcast(VT: DstVT, V);
897 }
898
899 // TODO - bigendian once we have test coverage.
900 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
901 unsigned Scale = NumSrcEltBits / NumDstEltBits;
902 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
903 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
904 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
905 for (unsigned i = 0; i != NumElts; ++i)
906 if (DemandedElts[i]) {
907 unsigned Offset = (i % Scale) * NumDstEltBits;
908 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
909 DemandedSrcElts.setBit(i / Scale);
910 }
911
912 if (SDValue V = SimplifyMultipleUseDemandedBits(
913 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
914 return DAG.getBitcast(VT: DstVT, V);
915 }
916
917 break;
918 }
919 case ISD::SCALAR_TO_VECTOR: {
920 if (!VT.isScalableVector() && !DemandedElts[0])
921 return DAG.getPOISON(VT);
922 break;
923 }
924 case ISD::AND: {
925 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
926 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
927
928 // If all of the demanded bits are known 1 on one side, return the other.
929 // These bits cannot contribute to the result of the 'and' in this
930 // context.
931 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero | RHSKnown.One))
932 return Op.getOperand(i: 0);
933 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.One))
934 return Op.getOperand(i: 1);
935 break;
936 }
937 case ISD::OR: {
938 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
939 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
940
941 // If all of the demanded bits are known zero on one side, return the
942 // other. These bits cannot contribute to the result of the 'or' in this
943 // context.
944 if (DemandedBits.isSubsetOf(RHS: LHSKnown.One | RHSKnown.Zero))
945 return Op.getOperand(i: 0);
946 if (DemandedBits.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
947 return Op.getOperand(i: 1);
948 break;
949 }
950 case ISD::XOR: {
951 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
952 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
953
954 // If all of the demanded bits are known zero on one side, return the
955 // other.
956 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero))
957 return Op.getOperand(i: 0);
958 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero))
959 return Op.getOperand(i: 1);
960 break;
961 }
962 case ISD::ADD:
963 case ISD::MUL:
964 case ISD::SMIN:
965 case ISD::SMAX:
966 case ISD::UMIN:
967 case ISD::UMAX: {
968 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 1),
969 DemandedElts, OperandNo: 1, Depth: Depth + 1))
970 return Op.getOperand(i: 0);
971
972 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 0),
973 DemandedElts, OperandNo: 0, Depth: Depth + 1))
974 return Op.getOperand(i: 1);
975 break;
976 }
977 case ISD::SHL: {
978 // If we are only demanding sign bits then we can use the shift source
979 // directly.
980 if (std::optional<unsigned> MaxSA =
981 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
982 SDValue Op0 = Op.getOperand(i: 0);
983 unsigned ShAmt = *MaxSA;
984 unsigned NumSignBits =
985 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
986 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
987 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
988 return Op0;
989 }
990 break;
991 }
992 case ISD::SRL: {
993 // If we are only demanding sign bits then we can use the shift source
994 // directly.
995 if (std::optional<unsigned> MaxSA =
996 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
997 SDValue Op0 = Op.getOperand(i: 0);
998 unsigned ShAmt = *MaxSA;
999 // Must already be signbits in DemandedBits bounds, and can't demand any
1000 // shifted in zeroes.
1001 if (DemandedBits.countl_zero() >= ShAmt) {
1002 unsigned NumSignBits =
1003 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
1004 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
1005 return Op0;
1006 }
1007 }
1008 break;
1009 }
1010 case ISD::SETCC: {
1011 SDValue Op0 = Op.getOperand(i: 0);
1012 SDValue Op1 = Op.getOperand(i: 1);
1013 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1014 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1015 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1016 // -1, we may be able to bypass the setcc.
1017 if (DemandedBits.isSignMask() &&
1018 Op0.getScalarValueSizeInBits() == BitWidth &&
1019 getBooleanContents(Type: Op0.getValueType()) ==
1020 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1021 // If we're testing X < 0, then this compare isn't needed - just use X!
1022 // FIXME: We're limiting to integer types here, but this should also work
1023 // if we don't care about FP signed-zero. The use of SETLT with FP means
1024 // that we don't care about NaNs.
1025 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1026 (isNullConstant(V: Op1) || ISD::isBuildVectorAllZeros(N: Op1.getNode())))
1027 return Op0;
1028 }
1029 break;
1030 }
1031 case ISD::SIGN_EXTEND_INREG: {
1032 // If none of the extended bits are demanded, eliminate the sextinreg.
1033 SDValue Op0 = Op.getOperand(i: 0);
1034 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1035 unsigned ExBits = ExVT.getScalarSizeInBits();
1036 if (DemandedBits.getActiveBits() <= ExBits &&
1037 shouldRemoveRedundantExtend(Op))
1038 return Op0;
1039 // If the input is already sign extended, just drop the extension.
1040 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
1041 if (NumSignBits >= (BitWidth - ExBits + 1))
1042 return Op0;
1043 break;
1044 }
1045 case ISD::ANY_EXTEND_VECTOR_INREG:
1046 case ISD::SIGN_EXTEND_VECTOR_INREG:
1047 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1048 if (VT.isScalableVector())
1049 return SDValue();
1050
1051 // If we only want the lowest element and none of extended bits, then we can
1052 // return the bitcasted source vector.
1053 SDValue Src = Op.getOperand(i: 0);
1054 EVT SrcVT = Src.getValueType();
1055 EVT DstVT = Op.getValueType();
1056 if (IsLE && DemandedElts == 1 &&
1057 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
1058 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
1059 return DAG.getBitcast(VT: DstVT, V: Src);
1060 }
1061 break;
1062 }
1063 case ISD::INSERT_VECTOR_ELT: {
1064 if (VT.isScalableVector())
1065 return SDValue();
1066
1067 // If we don't demand the inserted element, return the base vector.
1068 SDValue Vec = Op.getOperand(i: 0);
1069 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1070 EVT VecVT = Vec.getValueType();
1071 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements()) &&
1072 !DemandedElts[CIdx->getZExtValue()])
1073 return Vec;
1074 break;
1075 }
1076 case ISD::INSERT_SUBVECTOR: {
1077 if (VT.isScalableVector())
1078 return SDValue();
1079
1080 SDValue Vec = Op.getOperand(i: 0);
1081 SDValue Sub = Op.getOperand(i: 1);
1082 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1083 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1084 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1085 // If we don't demand the inserted subvector, return the base vector.
1086 if (DemandedSubElts == 0)
1087 return Vec;
1088 break;
1089 }
1090 case ISD::VECTOR_SHUFFLE: {
1091 assert(!VT.isScalableVector());
1092 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1093
1094 // If all the demanded elts are from one operand and are inline,
1095 // then we can use the operand directly.
1096 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
1097 for (unsigned i = 0; i != NumElts; ++i) {
1098 int M = ShuffleMask[i];
1099 if (M < 0 || !DemandedElts[i])
1100 continue;
1101 AllUndef = false;
1102 IdentityLHS &= (M == (int)i);
1103 IdentityRHS &= ((M - NumElts) == i);
1104 }
1105
1106 if (AllUndef)
1107 return DAG.getUNDEF(VT: Op.getValueType());
1108 if (IdentityLHS)
1109 return Op.getOperand(i: 0);
1110 if (IdentityRHS)
1111 return Op.getOperand(i: 1);
1112 break;
1113 }
1114 default:
1115 // TODO: Probably okay to remove after audit; here to reduce change size
1116 // in initial enablement patch for scalable vectors
1117 if (VT.isScalableVector())
1118 return SDValue();
1119
1120 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
1121 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
1122 Op, DemandedBits, DemandedElts, DAG, Depth))
1123 return V;
1124 break;
1125 }
1126 return SDValue();
1127}
1128
1129SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
1130 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
1131 unsigned Depth) const {
1132 EVT VT = Op.getValueType();
1133 // Since the number of lanes in a scalable vector is unknown at compile time,
1134 // we track one bit which is implicitly broadcast to all lanes. This means
1135 // that all lanes in a scalable vector are considered demanded.
1136 APInt DemandedElts = VT.isFixedLengthVector()
1137 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1138 : APInt(1, 1);
1139 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1140 Depth);
1141}
1142
1143SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
1144 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
1145 unsigned Depth) const {
1146 APInt DemandedBits = APInt::getAllOnes(numBits: Op.getScalarValueSizeInBits());
1147 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1148 Depth);
1149}
1150
1151// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
1152// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
1153static SDValue combineShiftToAVG(SDValue Op,
1154 TargetLowering::TargetLoweringOpt &TLO,
1155 const TargetLowering &TLI,
1156 const APInt &DemandedBits,
1157 const APInt &DemandedElts, unsigned Depth) {
1158 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1159 "SRL or SRA node is required here!");
1160 // Is the right shift using an immediate value of 1?
1161 ConstantSDNode *N1C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
1162 if (!N1C || !N1C->isOne())
1163 return SDValue();
1164
1165 // We are looking for an avgfloor
1166 // add(ext, ext)
1167 // or one of these as a avgceil
1168 // add(add(ext, ext), 1)
1169 // add(add(ext, 1), ext)
1170 // add(ext, add(ext, 1))
1171 SDValue Add = Op.getOperand(i: 0);
1172 if (Add.getOpcode() != ISD::ADD)
1173 return SDValue();
1174
1175 SDValue ExtOpA = Add.getOperand(i: 0);
1176 SDValue ExtOpB = Add.getOperand(i: 1);
1177 SDValue Add2;
1178 auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
1179 ConstantSDNode *ConstOp;
1180 if ((ConstOp = isConstOrConstSplat(N: Op2, DemandedElts)) &&
1181 ConstOp->isOne()) {
1182 ExtOpA = Op1;
1183 ExtOpB = Op3;
1184 Add2 = A;
1185 return true;
1186 }
1187 if ((ConstOp = isConstOrConstSplat(N: Op3, DemandedElts)) &&
1188 ConstOp->isOne()) {
1189 ExtOpA = Op1;
1190 ExtOpB = Op2;
1191 Add2 = A;
1192 return true;
1193 }
1194 return false;
1195 };
1196 bool IsCeil =
1197 (ExtOpA.getOpcode() == ISD::ADD &&
1198 MatchOperands(ExtOpA.getOperand(i: 0), ExtOpA.getOperand(i: 1), ExtOpB, ExtOpA)) ||
1199 (ExtOpB.getOpcode() == ISD::ADD &&
1200 MatchOperands(ExtOpB.getOperand(i: 0), ExtOpB.getOperand(i: 1), ExtOpA, ExtOpB));
1201
1202 // If the shift is signed (sra):
1203 // - Needs >= 2 sign bit for both operands.
1204 // - Needs >= 2 zero bits.
1205 // If the shift is unsigned (srl):
1206 // - Needs >= 1 zero bit for both operands.
1207 // - Needs 1 demanded bit zero and >= 2 sign bits.
1208 SelectionDAG &DAG = TLO.DAG;
1209 unsigned ShiftOpc = Op.getOpcode();
1210 bool IsSigned = false;
1211 unsigned KnownBits;
1212 unsigned NumSignedA = DAG.ComputeNumSignBits(Op: ExtOpA, DemandedElts, Depth);
1213 unsigned NumSignedB = DAG.ComputeNumSignBits(Op: ExtOpB, DemandedElts, Depth);
1214 unsigned NumSigned = std::min(a: NumSignedA, b: NumSignedB) - 1;
1215 unsigned NumZeroA =
1216 DAG.computeKnownBits(Op: ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
1217 unsigned NumZeroB =
1218 DAG.computeKnownBits(Op: ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
1219 unsigned NumZero = std::min(a: NumZeroA, b: NumZeroB);
1220
1221 switch (ShiftOpc) {
1222 default:
1223 llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
1224 case ISD::SRA: {
1225 if (NumZero >= 2 && NumSigned < NumZero) {
1226 IsSigned = false;
1227 KnownBits = NumZero;
1228 break;
1229 }
1230 if (NumSigned >= 1) {
1231 IsSigned = true;
1232 KnownBits = NumSigned;
1233 break;
1234 }
1235 return SDValue();
1236 }
1237 case ISD::SRL: {
1238 if (NumZero >= 1 && NumSigned < NumZero) {
1239 IsSigned = false;
1240 KnownBits = NumZero;
1241 break;
1242 }
1243 if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
1244 IsSigned = true;
1245 KnownBits = NumSigned;
1246 break;
1247 }
1248 return SDValue();
1249 }
1250 }
1251
1252 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
1253 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);
1254
1255 // Find the smallest power-2 type that is legal for this vector size and
1256 // operation, given the original type size and the number of known sign/zero
1257 // bits.
1258 EVT VT = Op.getValueType();
1259 unsigned MinWidth =
1260 std::max<unsigned>(a: VT.getScalarSizeInBits() - KnownBits, b: 8);
1261 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_ceil(Value: MinWidth));
1262 if (NVT.getScalarSizeInBits() > VT.getScalarSizeInBits())
1263 return SDValue();
1264 if (VT.isVector())
1265 NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT, EC: VT.getVectorElementCount());
1266 if (TLO.LegalTypes() && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT)) {
1267 // If we could not transform, and (both) adds are nuw/nsw, we can use the
1268 // larger type size to do the transform.
1269 if (TLO.LegalOperations() && !TLI.isOperationLegal(Op: AVGOpc, VT))
1270 return SDValue();
1271 if (DAG.willNotOverflowAdd(IsSigned, N0: Add.getOperand(i: 0),
1272 N1: Add.getOperand(i: 1)) &&
1273 (!Add2 || DAG.willNotOverflowAdd(IsSigned, N0: Add2.getOperand(i: 0),
1274 N1: Add2.getOperand(i: 1))))
1275 NVT = VT;
1276 else
1277 return SDValue();
1278 }
1279
1280 // Don't create a AVGFLOOR node with a scalar constant unless its legal as
1281 // this is likely to stop other folds (reassociation, value tracking etc.)
1282 if (!IsCeil && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT) &&
1283 (isa<ConstantSDNode>(Val: ExtOpA) || isa<ConstantSDNode>(Val: ExtOpB)))
1284 return SDValue();
1285
1286 SDLoc DL(Op);
1287 SDValue ResultAVG =
1288 DAG.getNode(Opcode: AVGOpc, DL, VT: NVT, N1: DAG.getExtOrTrunc(IsSigned, Op: ExtOpA, DL, VT: NVT),
1289 N2: DAG.getExtOrTrunc(IsSigned, Op: ExtOpB, DL, VT: NVT));
1290 return DAG.getExtOrTrunc(IsSigned, Op: ResultAVG, DL, VT);
1291}
1292
1293/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
1294/// result of Op are ever used downstream. If we can use this information to
1295/// simplify Op, create a new simplified DAG node and return true, returning the
1296/// original and new nodes in Old and New. Otherwise, analyze the expression and
1297/// return a mask of Known bits for the expression (used to simplify the
1298/// caller). The Known bits may only be accurate for those bits in the
1299/// OriginalDemandedBits and OriginalDemandedElts.
1300bool TargetLowering::SimplifyDemandedBits(
1301 SDValue Op, const APInt &OriginalDemandedBits,
1302 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
1303 unsigned Depth, bool AssumeSingleUse) const {
1304 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
1305 assert(Op.getScalarValueSizeInBits() == BitWidth &&
1306 "Mask size mismatches value type size!");
1307
1308 // Don't know anything.
1309 Known = KnownBits(BitWidth);
1310
1311 EVT VT = Op.getValueType();
1312 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
1313 unsigned NumElts = OriginalDemandedElts.getBitWidth();
1314 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
1315 "Unexpected vector size");
1316
1317 APInt DemandedBits = OriginalDemandedBits;
1318 APInt DemandedElts = OriginalDemandedElts;
1319 SDLoc dl(Op);
1320
1321 // Undef operand.
1322 if (Op.isUndef())
1323 return false;
1324
1325 // We can't simplify target constants.
1326 if (Op.getOpcode() == ISD::TargetConstant)
1327 return false;
1328
1329 if (Op.getOpcode() == ISD::Constant) {
1330 // We know all of the bits for a constant!
1331 Known = KnownBits::makeConstant(C: Op->getAsAPIntVal());
1332 return false;
1333 }
1334
1335 if (Op.getOpcode() == ISD::ConstantFP) {
1336 // We know all of the bits for a floating point constant!
1337 Known = KnownBits::makeConstant(
1338 C: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF().bitcastToAPInt());
1339 return false;
1340 }
1341
1342 // Other users may use these bits.
1343 bool HasMultiUse = false;
1344 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1345 if (Depth >= SelectionDAG::MaxRecursionDepth) {
1346 // Limit search depth.
1347 return false;
1348 }
1349 // Allow multiple uses, just set the DemandedBits/Elts to all bits.
1350 DemandedBits = APInt::getAllOnes(numBits: BitWidth);
1351 DemandedElts = APInt::getAllOnes(numBits: NumElts);
1352 HasMultiUse = true;
1353 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1354 // Not demanding any bits/elts from Op.
1355 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
1356 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
1357 // Limit search depth.
1358 return false;
1359 }
1360
1361 // Before folding to the LHS, recheck bits excluded from its earlier demand
1362 // using the full result demand. Mask identifies these excluded bits.
1363 // KnownMask selects the Zero/One member of the recomputed LHS known bits
1364 // that must cover Mask & DemandedBits.
1365 auto TryFoldToLHS = [&](const APInt &Mask,
1366 const APInt KnownBits::*KnownMask) {
1367 SDValue LHS = Op.getOperand(i: 0);
1368 APInt MaskedBits = Mask & DemandedBits;
1369 if (MaskedBits.isZero())
1370 return TLO.CombineTo(O: Op, N: LHS);
1371
1372 KnownBits LHSKnown;
1373 if (SimplifyDemandedBits(Op: LHS, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known&: LHSKnown, TLO,
1374 Depth: Depth + 1))
1375 return true;
1376
1377 return MaskedBits.isSubsetOf(RHS: LHSKnown.*KnownMask) && TLO.CombineTo(O: Op, N: LHS);
1378 };
1379
1380 // Try reducing the RHS demand using LHS known bits.
1381 // KnownMask selects the LHS Zero/One bits that make RHS bits irrelevant.
1382 auto TryReduceRHSDemand = [&](const KnownBits &LHSKnown,
1383 const APInt KnownBits::*KnownMask) {
1384 SDValue RHS = Op.getOperand(i: 1);
1385 if (!RHS.hasOneUse() || isConstOrConstSplat(N: RHS, DemandedElts))
1386 return false;
1387
1388 // Reuse LHS facts if the RHS did not reduce its earlier demand. Otherwise,
1389 // query independent facts to avoid using bits outside that demand.
1390 SDValue LHS = Op.getOperand(i: 0);
1391 // Keep only the selected mask, then compute DemandedBits & ~Mask in place.
1392 // This avoids copying the unused KnownBits mask and extra APInt
1393 // temporaries, which can allocate memory for masks wider than one word.
1394 APInt RHSDemandedBits =
1395 DemandedBits.intersects(RHS: Known.*KnownMask)
1396 ? TLO.DAG.computeKnownBits(Op: LHS, DemandedElts, Depth: Depth + 1).*KnownMask
1397 : LHSKnown.*KnownMask;
1398 RHSDemandedBits.flipAllBits();
1399 RHSDemandedBits &= DemandedBits;
1400 if (RHSDemandedBits == DemandedBits)
1401 return false;
1402
1403 // A failed retry must not replace Known with reduced-demand facts.
1404 KnownBits RetryKnown;
1405 return SimplifyDemandedBits(Op: RHS, OriginalDemandedBits: RHSDemandedBits, OriginalDemandedElts: DemandedElts, Known&: RetryKnown,
1406 TLO, Depth: Depth + 1);
1407 };
1408
1409 KnownBits Known2;
1410 switch (Op.getOpcode()) {
1411 case ISD::SCALAR_TO_VECTOR: {
1412 if (VT.isScalableVector())
1413 return false;
1414 if (!DemandedElts[0])
1415 return TLO.CombineTo(O: Op, N: TLO.DAG.getPOISON(VT));
1416
1417 KnownBits SrcKnown;
1418 SDValue Src = Op.getOperand(i: 0);
1419 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1420 APInt SrcDemandedBits = DemandedBits.zext(width: SrcBitWidth);
1421 if (SimplifyDemandedBits(Op: Src, DemandedBits: SrcDemandedBits, Known&: SrcKnown, TLO, Depth: Depth + 1))
1422 return true;
1423
1424 // Upper elements are poison, so only get the knownbits if we just demand
1425 // the bottom element.
1426 if (DemandedElts == 1)
1427 Known = SrcKnown.anyextOrTrunc(BitWidth);
1428 break;
1429 }
1430 case ISD::BUILD_VECTOR:
1431 // Collect the known bits that are shared by every demanded element.
1432 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1433 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1434 return false; // Don't fall through, will infinitely loop.
1435 case ISD::SPLAT_VECTOR: {
1436 SDValue Scl = Op.getOperand(i: 0);
1437 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: Scl.getValueSizeInBits());
1438 KnownBits KnownScl;
1439 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1440 return true;
1441
1442 // Implicitly truncate the bits to match the official semantics of
1443 // SPLAT_VECTOR.
1444 Known = KnownScl.trunc(BitWidth);
1445 break;
1446 }
1447 case ISD::FREEZE: {
1448 SDValue N0 = Op.getOperand(i: 0);
1449 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
1450 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
1451 return TLO.CombineTo(O: Op, N: N0);
1452 break;
1453 }
1454 case ISD::LOAD: {
1455 auto *LD = cast<LoadSDNode>(Val&: Op);
1456 if (getTargetConstantFromLoad(LD)) {
1457 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1458 return false; // Don't fall through, will infinitely loop.
1459 }
1460 if (ISD::isZEXTLoad(N: Op.getNode()) && Op.getResNo() == 0) {
1461 // If this is a ZEXTLoad and we are looking at the loaded value.
1462 EVT MemVT = LD->getMemoryVT();
1463 unsigned MemBits = MemVT.getScalarSizeInBits();
1464 Known.Zero.setBitsFrom(MemBits);
1465 return false; // Don't fall through, will infinitely loop.
1466 }
1467 break;
1468 }
1469 case ISD::INSERT_VECTOR_ELT: {
1470 if (VT.isScalableVector())
1471 return false;
1472 SDValue Vec = Op.getOperand(i: 0);
1473 SDValue Scl = Op.getOperand(i: 1);
1474 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1475 EVT VecVT = Vec.getValueType();
1476
1477 // If index isn't constant, assume we need all vector elements AND the
1478 // inserted element.
1479 APInt DemandedVecElts(DemandedElts);
1480 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements())) {
1481 unsigned Idx = CIdx->getZExtValue();
1482 DemandedVecElts.clearBit(BitPosition: Idx);
1483
1484 // Inserted element is not required.
1485 if (!DemandedElts[Idx])
1486 return TLO.CombineTo(O: Op, N: Vec);
1487 }
1488
1489 KnownBits KnownScl;
1490 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1491 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: NumSclBits);
1492 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1493 return true;
1494
1495 Known = KnownScl.anyextOrTrunc(BitWidth);
1496
1497 KnownBits KnownVec;
1498 if (SimplifyDemandedBits(Op: Vec, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedVecElts, Known&: KnownVec, TLO,
1499 Depth: Depth + 1))
1500 return true;
1501
1502 if (!!DemandedVecElts)
1503 Known = Known.intersectWith(RHS: KnownVec);
1504
1505 return false;
1506 }
1507 case ISD::INSERT_SUBVECTOR: {
1508 if (VT.isScalableVector())
1509 return false;
1510 // Demand any elements from the subvector and the remainder from the src its
1511 // inserted into.
1512 SDValue Src = Op.getOperand(i: 0);
1513 SDValue Sub = Op.getOperand(i: 1);
1514 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1515 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1516 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1517 APInt DemandedSrcElts = DemandedElts;
1518 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
1519
1520 KnownBits KnownSub, KnownSrc;
1521 if (SimplifyDemandedBits(Op: Sub, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts, Known&: KnownSub, TLO,
1522 Depth: Depth + 1))
1523 return true;
1524 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known&: KnownSrc, TLO,
1525 Depth: Depth + 1))
1526 return true;
1527
1528 Known.setAllConflict();
1529 if (!!DemandedSubElts)
1530 Known = Known.intersectWith(RHS: KnownSub);
1531 if (!!DemandedSrcElts)
1532 Known = Known.intersectWith(RHS: KnownSrc);
1533
1534 // Attempt to avoid multi-use src if we don't need anything from it.
1535 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1536 !DemandedSrcElts.isAllOnes()) {
1537 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1538 Op: Sub, DemandedBits, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
1539 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1540 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1541 if (NewSub || NewSrc) {
1542 NewSub = NewSub ? NewSub : Sub;
1543 NewSrc = NewSrc ? NewSrc : Src;
1544 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: NewSrc, N2: NewSub,
1545 N3: Op.getOperand(i: 2));
1546 return TLO.CombineTo(O: Op, N: NewOp);
1547 }
1548 }
1549 break;
1550 }
1551 case ISD::EXTRACT_SUBVECTOR: {
1552 if (VT.isScalableVector())
1553 return false;
1554 // Offset the demanded elts by the subvector index.
1555 SDValue Src = Op.getOperand(i: 0);
1556 if (Src.getValueType().isScalableVector())
1557 break;
1558 uint64_t Idx = Op.getConstantOperandVal(i: 1);
1559 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1560 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
1561
1562 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known, TLO,
1563 Depth: Depth + 1))
1564 return true;
1565
1566 // Attempt to avoid multi-use src if we don't need anything from it.
1567 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1568 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1569 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1570 if (DemandedSrc) {
1571 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc,
1572 N2: Op.getOperand(i: 1));
1573 return TLO.CombineTo(O: Op, N: NewOp);
1574 }
1575 }
1576 break;
1577 }
1578 case ISD::CONCAT_VECTORS: {
1579 if (VT.isScalableVector())
1580 return false;
1581 Known.setAllConflict();
1582 EVT SubVT = Op.getOperand(i: 0).getValueType();
1583 unsigned NumSubVecs = Op.getNumOperands();
1584 unsigned NumSubElts = SubVT.getVectorNumElements();
1585 for (unsigned i = 0; i != NumSubVecs; ++i) {
1586 APInt DemandedSubElts =
1587 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
1588 if (SimplifyDemandedBits(Op: Op.getOperand(i), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts,
1589 Known&: Known2, TLO, Depth: Depth + 1))
1590 return true;
1591 // Known bits are shared by every demanded subvector element.
1592 if (!!DemandedSubElts)
1593 Known = Known.intersectWith(RHS: Known2);
1594 }
1595 break;
1596 }
1597 case ISD::VECTOR_SHUFFLE: {
1598 assert(!VT.isScalableVector());
1599 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1600
1601 // Collect demanded elements from shuffle operands..
1602 APInt DemandedLHS, DemandedRHS;
1603 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: ShuffleMask, DemandedElts, DemandedLHS,
1604 DemandedRHS))
1605 break;
1606
1607 if (!!DemandedLHS || !!DemandedRHS) {
1608 SDValue Op0 = Op.getOperand(i: 0);
1609 SDValue Op1 = Op.getOperand(i: 1);
1610
1611 Known.setAllConflict();
1612 if (!!DemandedLHS) {
1613 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedLHS, Known&: Known2, TLO,
1614 Depth: Depth + 1))
1615 return true;
1616 Known = Known.intersectWith(RHS: Known2);
1617 }
1618 if (!!DemandedRHS) {
1619 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedRHS, Known&: Known2, TLO,
1620 Depth: Depth + 1))
1621 return true;
1622 Known = Known.intersectWith(RHS: Known2);
1623 }
1624
1625 // Attempt to avoid multi-use ops if we don't need anything from them.
1626 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1627 Op: Op0, DemandedBits, DemandedElts: DemandedLHS, DAG&: TLO.DAG, Depth: Depth + 1);
1628 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1629 Op: Op1, DemandedBits, DemandedElts: DemandedRHS, DAG&: TLO.DAG, Depth: Depth + 1);
1630 if (DemandedOp0 || DemandedOp1) {
1631 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1632 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1633 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, N1: Op0, N2: Op1, Mask: ShuffleMask);
1634 return TLO.CombineTo(O: Op, N: NewOp);
1635 }
1636 }
1637 break;
1638 }
1639 case ISD::AND: {
1640 SDValue Op0 = Op.getOperand(i: 0);
1641 SDValue Op1 = Op.getOperand(i: 1);
1642
1643 // If the RHS is a constant, check to see if the LHS would be zero without
1644 // using the bits from the RHS. Below, we use knowledge about the RHS to
1645 // simplify the LHS, here we're using information from the LHS to simplify
1646 // the RHS.
1647 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: Op1, DemandedElts)) {
1648 // Do not increment Depth here; that can cause an infinite loop.
1649 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth);
1650 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1651 if ((LHSKnown.Zero & DemandedBits) ==
1652 (~RHSC->getAPIntValue() & DemandedBits))
1653 return TLO.CombineTo(O: Op, N: Op0);
1654
1655 // If any of the set bits in the RHS are known zero on the LHS, shrink
1656 // the constant.
1657 if (ShrinkDemandedConstant(Op, DemandedBits: ~LHSKnown.Zero & DemandedBits,
1658 DemandedElts, TLO))
1659 return true;
1660
1661 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1662 // constant, but if this 'and' is only clearing bits that were just set by
1663 // the xor, then this 'and' can be eliminated by shrinking the mask of
1664 // the xor. For example, for a 32-bit X:
1665 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1666 if (isBitwiseNot(V: Op0) && Op0.hasOneUse() &&
1667 LHSKnown.One == ~RHSC->getAPIntValue()) {
1668 SDValue Xor = TLO.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1669 return TLO.CombineTo(O: Op, N: Xor);
1670 }
1671 }
1672
1673 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2 (or zero).
1674 SDValue X, Y;
1675 if (sd_match(N: Op,
1676 P: m_And(L: m_Value(N&: Y),
1677 R: m_OneUse(P: m_AnyOf(preds: m_Add(L: m_Value(N&: X), R: m_Deferred(V&: Y)),
1678 preds: m_Sub(L: m_Value(N&: X), R: m_Deferred(V&: Y)))))) &&
1679 TLO.DAG.isKnownToBeAPowerOfTwo(Val: Y, DemandedElts, /*OrZero=*/true)) {
1680 return TLO.CombineTo(
1681 O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TLO.DAG.getNOT(DL: dl, Val: X, VT), N2: Y));
1682 }
1683
1684 // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
1685 // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
1686 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
1687 (Op0.getOperand(i: 0).isUndef() ||
1688 ISD::isBuildVectorOfConstantSDNodes(N: Op0.getOperand(i: 0).getNode())) &&
1689 Op0->hasOneUse()) {
1690 unsigned NumSubElts =
1691 Op0.getOperand(i: 1).getValueType().getVectorNumElements();
1692 unsigned SubIdx = Op0.getConstantOperandVal(i: 2);
1693 APInt DemandedSub =
1694 APInt::getBitsSet(numBits: NumElts, loBit: SubIdx, hiBit: SubIdx + NumSubElts);
1695 KnownBits KnownSubMask =
1696 TLO.DAG.computeKnownBits(Op: Op1, DemandedElts: DemandedSub & DemandedElts, Depth: Depth + 1);
1697 if (DemandedBits.isSubsetOf(RHS: KnownSubMask.One)) {
1698 SDValue NewAnd =
1699 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1700 SDValue NewInsert =
1701 TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: NewAnd,
1702 N2: Op0.getOperand(i: 1), N3: Op0.getOperand(i: 2));
1703 return TLO.CombineTo(O: Op, N: NewInsert);
1704 }
1705 }
1706
1707 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1708 Depth: Depth + 1))
1709 return true;
1710 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.Zero & DemandedBits, OriginalDemandedElts: DemandedElts,
1711 Known&: Known2, TLO, Depth: Depth + 1))
1712 return true;
1713
1714 // If all of the demanded bits are known one on one side, return the other.
1715 // These bits cannot contribute to the result of the 'and'.
1716 if (DemandedBits.isSubsetOf(RHS: Known2.Zero | Known.One)) {
1717 if (TryFoldToLHS(Known.Zero, &KnownBits::Zero))
1718 return true;
1719 }
1720 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.One))
1721 return TLO.CombineTo(O: Op, N: Op1);
1722 // If all of the demanded bits in the inputs are known zeros, return zero.
1723 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1724 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: dl, VT));
1725 // If the RHS is a constant, see if we can simplify it.
1726 if (ShrinkDemandedConstant(Op, DemandedBits: ~Known2.Zero & DemandedBits, DemandedElts,
1727 TLO))
1728 return true;
1729 // If the operation can be done in a smaller type, do so.
1730 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1731 return true;
1732
1733 // Attempt to avoid multi-use ops if we don't need anything from them.
1734 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1735 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1736 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1737 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1738 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1739 if (DemandedOp0 || DemandedOp1) {
1740 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1741 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1742 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1743 return TLO.CombineTo(O: Op, N: NewOp);
1744 }
1745 }
1746
1747 if (TryReduceRHSDemand(Known2, &KnownBits::Zero))
1748 return true;
1749
1750 Known &= Known2;
1751 break;
1752 }
1753 case ISD::OR: {
1754 SDValue Op0 = Op.getOperand(i: 0);
1755 SDValue Op1 = Op.getOperand(i: 1);
1756 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1757 Depth: Depth + 1)) {
1758 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1759 return true;
1760 }
1761
1762 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.One & DemandedBits, OriginalDemandedElts: DemandedElts,
1763 Known&: Known2, TLO, Depth: Depth + 1)) {
1764 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1765 return true;
1766 }
1767
1768 // If all of the demanded bits are known zero on one side, return the other.
1769 // These bits cannot contribute to the result of the 'or'.
1770 if (DemandedBits.isSubsetOf(RHS: Known2.One | Known.Zero)) {
1771 if (TryFoldToLHS(Known.One, &KnownBits::One)) {
1772 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1773 return true;
1774 }
1775 }
1776 if (DemandedBits.isSubsetOf(RHS: Known.One | Known2.Zero))
1777 return TLO.CombineTo(O: Op, N: Op1);
1778 // If the RHS is a constant, see if we can simplify it.
1779 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1780 return true;
1781 // If the operation can be done in a smaller type, do so.
1782 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1783 return true;
1784
1785 // Attempt to avoid multi-use ops if we don't need anything from them.
1786 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1787 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1788 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1789 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1790 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1791 if (DemandedOp0 || DemandedOp1) {
1792 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1793 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1794 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1795 return TLO.CombineTo(O: Op, N: NewOp);
1796 }
1797 }
1798
1799 // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
1800 // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
1801 SDValue X, Y, C1, C2;
1802 if (sd_match(N: Op, P: m_Or(L: m_OneUse(P: m_And(L: m_Value(N&: X), R: m_Value(N&: C1))),
1803 R: m_OneUse(P: m_And(L: m_Or(L: m_Deferred(V&: X), R: m_Value(N&: Y)),
1804 R: m_Value(N&: C2)))))) {
1805 if (SDValue C12 =
1806 TLO.DAG.FoldConstantArithmetic(Opcode: ISD::OR, DL: dl, VT, Ops: {C1, C2})) {
1807 SDValue MaskX = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: C12);
1808 SDValue MaskY = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Y, N2: C2);
1809 return TLO.CombineTo(O: Op,
1810 N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: MaskX, N2: MaskY));
1811 }
1812 }
1813
1814 if (TryReduceRHSDemand(Known2, &KnownBits::One)) {
1815 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1816 return true;
1817 }
1818
1819 Known |= Known2;
1820 break;
1821 }
1822 case ISD::XOR: {
1823 SDValue Op0 = Op.getOperand(i: 0);
1824 SDValue Op1 = Op.getOperand(i: 1);
1825
1826 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1827 Depth: Depth + 1))
1828 return true;
1829 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
1830 Depth: Depth + 1))
1831 return true;
1832
1833 // If all of the demanded bits are known zero on one side, return the other.
1834 // These bits cannot contribute to the result of the 'xor'.
1835 if (DemandedBits.isSubsetOf(RHS: Known.Zero))
1836 return TLO.CombineTo(O: Op, N: Op0);
1837 if (DemandedBits.isSubsetOf(RHS: Known2.Zero))
1838 return TLO.CombineTo(O: Op, N: Op1);
1839 // If the operation can be done in a smaller type, do so.
1840 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1841 return true;
1842
1843 // If all of the unknown bits are known to be zero on one side or the other
1844 // turn this into an *inclusive* or.
1845 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1846 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1847 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op0, N2: Op1));
1848
1849 ConstantSDNode *C = isConstOrConstSplat(N: Op1, DemandedElts);
1850 if (C) {
1851 // If one side is a constant, and all of the set bits in the constant are
1852 // also known set on the other side, turn this into an AND, as we know
1853 // the bits will be cleared.
1854 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1855 // NB: it is okay if more bits are known than are requested
1856 if (C->getAPIntValue() == Known2.One) {
1857 SDValue ANDC =
1858 TLO.DAG.getConstant(Val: ~C->getAPIntValue() & DemandedBits, DL: dl, VT);
1859 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0, N2: ANDC));
1860 }
1861
1862 // If the RHS is a constant, see if we can change it. Don't alter a -1
1863 // constant because that's a 'not' op, and that is better for combining
1864 // and codegen.
1865 if (!C->isAllOnes() && DemandedBits.isSubsetOf(RHS: C->getAPIntValue())) {
1866 // We're flipping all demanded bits. Flip the undemanded bits too.
1867 SDValue New = TLO.DAG.getNOT(DL: dl, Val: Op0, VT);
1868 return TLO.CombineTo(O: Op, N: New);
1869 }
1870
1871 unsigned Op0Opcode = Op0.getOpcode();
1872 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
1873 if (ConstantSDNode *ShiftC =
1874 isConstOrConstSplat(N: Op0.getOperand(i: 1), DemandedElts)) {
1875 // Don't crash on an oversized shift. We can not guarantee that a
1876 // bogus shift has been simplified to undef.
1877 if (ShiftC->getAPIntValue().ult(RHS: BitWidth)) {
1878 uint64_t ShiftAmt = ShiftC->getZExtValue();
1879 APInt Ones = APInt::getAllOnes(numBits: BitWidth);
1880 Ones = Op0Opcode == ISD::SHL ? Ones.shl(shiftAmt: ShiftAmt)
1881 : Ones.lshr(shiftAmt: ShiftAmt);
1882 if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
1883 isDesirableToCommuteXorWithShift(N: Op.getNode())) {
1884 // If the xor constant is a demanded mask, do a 'not' before the
1885 // shift:
1886 // xor (X << ShiftC), XorC --> (not X) << ShiftC
1887 // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
1888 SDValue Not = TLO.DAG.getNOT(DL: dl, Val: Op0.getOperand(i: 0), VT);
1889 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op0Opcode, DL: dl, VT, N1: Not,
1890 N2: Op0.getOperand(i: 1)));
1891 }
1892 }
1893 }
1894 }
1895 }
1896
1897 // If we can't turn this into a 'not', try to shrink the constant.
1898 if (!C || !C->isAllOnes())
1899 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1900 return true;
1901
1902 // Attempt to avoid multi-use ops if we don't need anything from them.
1903 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1904 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1905 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1906 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1907 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1908 if (DemandedOp0 || DemandedOp1) {
1909 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1910 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1911 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1912 return TLO.CombineTo(O: Op, N: NewOp);
1913 }
1914 }
1915
1916 Known ^= Known2;
1917 break;
1918 }
1919 case ISD::SELECT:
1920 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1921 Known, TLO, Depth: Depth + 1))
1922 return true;
1923 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1924 Known&: Known2, TLO, Depth: Depth + 1))
1925 return true;
1926
1927 // If the operands are constants, see if we can simplify them.
1928 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1929 return true;
1930
1931 // Only known if known in both the LHS and RHS.
1932 Known = Known.intersectWith(RHS: Known2);
1933 break;
1934 case ISD::VSELECT:
1935 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1936 Known, TLO, Depth: Depth + 1))
1937 return true;
1938 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1939 Known&: Known2, TLO, Depth: Depth + 1))
1940 return true;
1941
1942 // Only known if known in both the LHS and RHS.
1943 Known = Known.intersectWith(RHS: Known2);
1944 break;
1945 case ISD::SELECT_CC:
1946 if (SimplifyDemandedBits(Op: Op.getOperand(i: 3), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1947 Known, TLO, Depth: Depth + 1))
1948 return true;
1949 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1950 Known&: Known2, TLO, Depth: Depth + 1))
1951 return true;
1952
1953 // If the operands are constants, see if we can simplify them.
1954 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1955 return true;
1956
1957 // Only known if known in both the LHS and RHS.
1958 Known = Known.intersectWith(RHS: Known2);
1959 break;
1960 case ISD::SETCC: {
1961 SDValue Op0 = Op.getOperand(i: 0);
1962 SDValue Op1 = Op.getOperand(i: 1);
1963 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1964 // If we're testing X < 0, X >= 0, X <= -1 or X > -1
1965 // (X is of integer type) then we only need the sign mask of the previous
1966 // result
1967 if (Op1.getValueType().isInteger() &&
1968 (((CC == ISD::SETLT || CC == ISD::SETGE) && isNullOrNullSplat(V: Op1)) ||
1969 ((CC == ISD::SETLE || CC == ISD::SETGT) &&
1970 isAllOnesOrAllOnesSplat(V: Op1)))) {
1971 KnownBits KnownOp0;
1972 if (SimplifyDemandedBits(
1973 Op: Op0, OriginalDemandedBits: APInt::getSignMask(BitWidth: Op0.getScalarValueSizeInBits()),
1974 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1))
1975 return true;
1976 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1977 // width as the setcc result, and (3) the result of a setcc conforms to 0
1978 // or -1, we may be able to bypass the setcc.
1979 if (DemandedBits.isSignMask() &&
1980 Op0.getScalarValueSizeInBits() == BitWidth &&
1981 getBooleanContents(Type: Op0.getValueType()) ==
1982 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1983 // If we remove a >= 0 or > -1 (for integers), we need to introduce a
1984 // NOT Operation
1985 if (CC == ISD::SETGE || CC == ISD::SETGT) {
1986 SDLoc DL(Op);
1987 EVT VT = Op0.getValueType();
1988 SDValue NotOp0 = TLO.DAG.getNOT(DL, Val: Op0, VT);
1989 return TLO.CombineTo(O: Op, N: NotOp0);
1990 }
1991 return TLO.CombineTo(O: Op, N: Op0);
1992 }
1993 }
1994 if (getBooleanContents(Type: Op0.getValueType()) ==
1995 TargetLowering::ZeroOrOneBooleanContent &&
1996 BitWidth > 1)
1997 Known.Zero.setBitsFrom(1);
1998 break;
1999 }
2000 case ISD::SHL: {
2001 SDValue Op0 = Op.getOperand(i: 0);
2002 SDValue Op1 = Op.getOperand(i: 1);
2003 EVT ShiftVT = Op1.getValueType();
2004
2005 if (std::optional<unsigned> KnownSA =
2006 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2007 unsigned ShAmt = *KnownSA;
2008 if (ShAmt == 0)
2009 return TLO.CombineTo(O: Op, N: Op0);
2010
2011 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
2012 // single shift. We can do this if the bottom bits (which are shifted
2013 // out) are never demanded.
2014 // TODO - support non-uniform vector amounts.
2015 if (Op0.getOpcode() == ISD::SRL) {
2016 if (!DemandedBits.intersects(RHS: APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ShAmt))) {
2017 if (std::optional<unsigned> InnerSA =
2018 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2019 unsigned C1 = *InnerSA;
2020 unsigned Opc = ISD::SHL;
2021 int Diff = ShAmt - C1;
2022 if (Diff < 0) {
2023 Diff = -Diff;
2024 Opc = ISD::SRL;
2025 }
2026 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
2027 return TLO.CombineTo(
2028 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
2029 }
2030 }
2031 }
2032
2033 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
2034 // are not demanded. This will likely allow the anyext to be folded away.
2035 // TODO - support non-uniform vector amounts.
2036 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
2037 SDValue InnerOp = Op0.getOperand(i: 0);
2038 EVT InnerVT = InnerOp.getValueType();
2039 unsigned InnerBits = InnerVT.getScalarSizeInBits();
2040 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
2041 isTypeDesirableForOp(ISD::SHL, VT: InnerVT)) {
2042 SDValue NarrowShl = TLO.DAG.getNode(
2043 Opcode: ISD::SHL, DL: dl, VT: InnerVT, N1: InnerOp,
2044 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: InnerVT, DL: dl));
2045 return TLO.CombineTo(
2046 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
2047 }
2048
2049 // Repeat the SHL optimization above in cases where an extension
2050 // intervenes: (shl (anyext (shr x, c1)), c2) to
2051 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
2052 // aren't demanded (as above) and that the shifted upper c1 bits of
2053 // x aren't demanded.
2054 // TODO - support non-uniform vector amounts.
2055 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
2056 InnerOp.hasOneUse()) {
2057 if (std::optional<unsigned> SA2 = TLO.DAG.getValidShiftAmount(
2058 V: InnerOp, DemandedElts, Depth: Depth + 2)) {
2059 unsigned InnerShAmt = *SA2;
2060 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
2061 DemandedBits.getActiveBits() <=
2062 (InnerBits - InnerShAmt + ShAmt) &&
2063 DemandedBits.countr_zero() >= ShAmt) {
2064 SDValue NewSA =
2065 TLO.DAG.getConstant(Val: ShAmt - InnerShAmt, DL: dl, VT: ShiftVT);
2066 SDValue NewExt = TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT,
2067 Operand: InnerOp.getOperand(i: 0));
2068 return TLO.CombineTo(
2069 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: NewExt, N2: NewSA));
2070 }
2071 }
2072 }
2073 }
2074
2075 APInt InDemandedMask = DemandedBits.lshr(shiftAmt: ShAmt);
2076 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2077 Depth: Depth + 1)) {
2078 // Disable the nsw and nuw flags. We can no longer guarantee that we
2079 // won't wrap after simplification.
2080 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2081 return true;
2082 }
2083 Known <<= ShAmt;
2084 // low bits known zero.
2085 Known.Zero.setLowBits(ShAmt);
2086
2087 // Attempt to avoid multi-use ops if we don't need anything from them.
2088 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2089 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2090 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2091 if (DemandedOp0) {
2092 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2093 return TLO.CombineTo(O: Op, N: NewOp);
2094 }
2095 }
2096
2097 // TODO: Can we merge this fold with the one below?
2098 // Try shrinking the operation as long as the shift amount will still be
2099 // in range.
2100 if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() &&
2101 Op.getNode()->hasOneUse()) {
2102 // Search for the smallest integer type with free casts to and from
2103 // Op's type. For expedience, just check power-of-2 integer types.
2104 unsigned DemandedSize = DemandedBits.getActiveBits();
2105 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
2106 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
2107 EVT SmallVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: SmallVTBits);
2108 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: SmallVT) &&
2109 isTypeDesirableForOp(ISD::SHL, VT: SmallVT) &&
2110 isTruncateFree(FromVT: VT, ToVT: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT) &&
2111 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: SmallVT))) {
2112 assert(DemandedSize <= SmallVTBits &&
2113 "Narrowed below demanded bits?");
2114 // We found a type with free casts.
2115 SDValue NarrowShl = TLO.DAG.getNode(
2116 Opcode: ISD::SHL, DL: dl, VT: SmallVT,
2117 N1: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
2118 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: SmallVT, DL: dl));
2119 return TLO.CombineTo(
2120 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
2121 }
2122 }
2123 }
2124
2125 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2126 // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
2127 // Only do this if we demand the upper half so the knownbits are correct.
2128 unsigned HalfWidth = BitWidth / 2;
2129 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
2130 DemandedBits.countLeadingOnes() >= HalfWidth) {
2131 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: HalfWidth);
2132 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2133 isTypeDesirableForOp(ISD::SHL, VT: HalfVT) &&
2134 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2135 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: HalfVT))) {
2136 // If we're demanding the upper bits at all, we must ensure
2137 // that the upper bits of the shift result are known to be zero,
2138 // which is equivalent to the narrow shift being NUW.
2139 if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
2140 bool IsNSW = Known.countMinSignBits() > HalfWidth;
2141 SDNodeFlags Flags;
2142 Flags.setNoSignedWrap(IsNSW);
2143 Flags.setNoUnsignedWrap(IsNUW);
2144 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2145 SDValue NewShiftAmt =
2146 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2147 SDValue NewShift = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HalfVT, N1: NewOp,
2148 N2: NewShiftAmt, Flags);
2149 SDValue NewExt =
2150 TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift);
2151 return TLO.CombineTo(O: Op, N: NewExt);
2152 }
2153 }
2154 }
2155 } else {
2156 // This is a variable shift, so we can't shift the demand mask by a known
2157 // amount. But if we are not demanding high bits, then we are not
2158 // demanding those bits from the pre-shifted operand either.
2159 if (unsigned CTLZ = DemandedBits.countl_zero()) {
2160 APInt DemandedFromOp(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - CTLZ));
2161 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedFromOp, OriginalDemandedElts: DemandedElts, Known, TLO,
2162 Depth: Depth + 1)) {
2163 // Disable the nsw and nuw flags. We can no longer guarantee that we
2164 // won't wrap after simplification.
2165 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2166 return true;
2167 }
2168 Known.resetAll();
2169 }
2170 }
2171
2172 // If we are only demanding sign bits then we can use the shift source
2173 // directly.
2174 if (std::optional<unsigned> MaxSA =
2175 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2176 unsigned ShAmt = *MaxSA;
2177 unsigned NumSignBits =
2178 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2179 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
2180 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
2181 return TLO.CombineTo(O: Op, N: Op0);
2182 }
2183 break;
2184 }
2185 case ISD::SRL: {
2186 SDValue Op0 = Op.getOperand(i: 0);
2187 SDValue Op1 = Op.getOperand(i: 1);
2188 EVT ShiftVT = Op1.getValueType();
2189
2190 if (std::optional<unsigned> KnownSA =
2191 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2192 unsigned ShAmt = *KnownSA;
2193 if (ShAmt == 0)
2194 return TLO.CombineTo(O: Op, N: Op0);
2195
2196 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
2197 // single shift. We can do this if the top bits (which are shifted out)
2198 // are never demanded.
2199 // TODO - support non-uniform vector amounts.
2200 if (Op0.getOpcode() == ISD::SHL) {
2201 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2202 if (std::optional<unsigned> InnerSA =
2203 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2204 unsigned C1 = *InnerSA;
2205 unsigned Opc = ISD::SRL;
2206 int Diff = ShAmt - C1;
2207 if (Diff < 0) {
2208 Diff = -Diff;
2209 Opc = ISD::SHL;
2210 }
2211 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
2212 return TLO.CombineTo(
2213 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
2214 }
2215 }
2216 }
2217
2218 // If this is (srl (sra X, C1), ShAmt), see if we can combine this into a
2219 // single sra. We can do this if the top bits are never demanded.
2220 if (Op0.getOpcode() == ISD::SRA && Op0.hasOneUse()) {
2221 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2222 if (std::optional<unsigned> InnerSA =
2223 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2224 unsigned C1 = *InnerSA;
2225 // Clamp the combined shift amount if it exceeds the bit width.
2226 unsigned Combined = std::min(a: C1 + ShAmt, b: BitWidth - 1);
2227 SDValue NewSA = TLO.DAG.getConstant(Val: Combined, DL: dl, VT: ShiftVT);
2228 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT,
2229 N1: Op0.getOperand(i: 0), N2: NewSA));
2230 }
2231 }
2232 }
2233
2234 APInt InDemandedMask = (DemandedBits << ShAmt);
2235
2236 // If the shift is exact, then it does demand the low bits (and knows that
2237 // they are zero).
2238 if (Op->getFlags().hasExact())
2239 InDemandedMask.setLowBits(ShAmt);
2240
2241 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2242 // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
2243 if ((BitWidth % 2) == 0 && !VT.isVector()) {
2244 APInt HiBits = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth / 2);
2245 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: BitWidth / 2);
2246 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2247 isTypeDesirableForOp(ISD::SRL, VT: HalfVT) &&
2248 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2249 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT: HalfVT)) &&
2250 ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
2251 TLO.DAG.MaskedValueIsZero(Op: Op0, Mask: HiBits))) {
2252 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2253 SDValue NewShiftAmt =
2254 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2255 SDValue NewShift =
2256 TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HalfVT, N1: NewOp, N2: NewShiftAmt);
2257 return TLO.CombineTo(
2258 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift));
2259 }
2260 }
2261
2262 // Compute the new bits that are at the top now.
2263 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2264 Depth: Depth + 1))
2265 return true;
2266 Known >>= ShAmt;
2267 // High bits known zero.
2268 Known.Zero.setHighBits(ShAmt);
2269
2270 // Attempt to avoid multi-use ops if we don't need anything from them.
2271 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2272 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2273 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2274 if (DemandedOp0) {
2275 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2276 return TLO.CombineTo(O: Op, N: NewOp);
2277 }
2278 }
2279 } else {
2280 // Use generic knownbits computation as it has support for non-uniform
2281 // shift amounts.
2282 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2283 }
2284
2285 // If we are only demanding sign bits then we can use the shift source
2286 // directly.
2287 if (std::optional<unsigned> MaxSA =
2288 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2289 unsigned ShAmt = *MaxSA;
2290 // Must already be signbits in DemandedBits bounds, and can't demand any
2291 // shifted in zeroes.
2292 if (DemandedBits.countl_zero() >= ShAmt) {
2293 unsigned NumSignBits =
2294 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2295 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
2296 return TLO.CombineTo(O: Op, N: Op0);
2297 }
2298 }
2299
2300 // Try to match AVG patterns (after shift simplification).
2301 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2302 DemandedElts, Depth: Depth + 1))
2303 return TLO.CombineTo(O: Op, N: AVG);
2304
2305 break;
2306 }
2307 case ISD::SRA: {
2308 SDValue Op0 = Op.getOperand(i: 0);
2309 SDValue Op1 = Op.getOperand(i: 1);
2310 EVT ShiftVT = Op1.getValueType();
2311
2312 // If we only want bits that already match the signbit then we don't need
2313 // to shift.
2314 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
2315 if (TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1) >=
2316 NumHiDemandedBits)
2317 return TLO.CombineTo(O: Op, N: Op0);
2318
2319 // If this is an arithmetic shift right and only the low-bit is set, we can
2320 // always convert this into a logical shr, even if the shift amount is
2321 // variable. The low bit of the shift cannot be an input sign bit unless
2322 // the shift amount is >= the size of the datatype, which is undefined.
2323 if (DemandedBits.isOne())
2324 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2325
2326 if (std::optional<unsigned> KnownSA =
2327 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2328 unsigned ShAmt = *KnownSA;
2329 if (ShAmt == 0)
2330 return TLO.CombineTo(O: Op, N: Op0);
2331
2332 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
2333 // supports sext_inreg.
2334 if (Op0.getOpcode() == ISD::SHL) {
2335 if (std::optional<unsigned> InnerSA =
2336 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2337 unsigned LowBits = BitWidth - ShAmt;
2338 EVT ExtVT = VT.changeElementType(
2339 Context&: *TLO.DAG.getContext(),
2340 EltVT: EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: LowBits));
2341
2342 if (*InnerSA == ShAmt) {
2343 if (!TLO.LegalOperations() ||
2344 getOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: ExtVT) == Legal)
2345 return TLO.CombineTo(
2346 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT,
2347 N1: Op0.getOperand(i: 0),
2348 N2: TLO.DAG.getValueType(ExtVT)));
2349
2350 // Even if we can't convert to sext_inreg, we might be able to
2351 // remove this shift pair if the input is already sign extended.
2352 unsigned NumSignBits =
2353 TLO.DAG.ComputeNumSignBits(Op: Op0.getOperand(i: 0), DemandedElts);
2354 if (NumSignBits > ShAmt)
2355 return TLO.CombineTo(O: Op, N: Op0.getOperand(i: 0));
2356 }
2357 }
2358 }
2359
2360 APInt InDemandedMask = (DemandedBits << ShAmt);
2361
2362 // If the shift is exact, then it does demand the low bits (and knows that
2363 // they are zero).
2364 if (Op->getFlags().hasExact())
2365 InDemandedMask.setLowBits(ShAmt);
2366
2367 // If any of the demanded bits are produced by the sign extension, we also
2368 // demand the input sign bit.
2369 if (DemandedBits.countl_zero() < ShAmt)
2370 InDemandedMask.setSignBit();
2371
2372 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2373 Depth: Depth + 1))
2374 return true;
2375 Known >>= ShAmt;
2376
2377 // If the input sign bit is known to be zero, or if none of the top bits
2378 // are demanded, turn this into an unsigned shift right.
2379 if (Known.Zero[BitWidth - ShAmt - 1] ||
2380 DemandedBits.countl_zero() >= ShAmt) {
2381 SDNodeFlags Flags;
2382 Flags.setExact(Op->getFlags().hasExact());
2383 return TLO.CombineTo(
2384 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1, Flags));
2385 }
2386
2387 int Log2 = DemandedBits.exactLogBase2();
2388 if (Log2 >= 0) {
2389 // The bit must come from the sign.
2390 SDValue NewSA = TLO.DAG.getConstant(Val: BitWidth - 1 - Log2, DL: dl, VT: ShiftVT);
2391 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: NewSA));
2392 }
2393
2394 if (Known.One[BitWidth - ShAmt - 1])
2395 // New bits are known one.
2396 Known.One.setHighBits(ShAmt);
2397
2398 // Attempt to avoid multi-use ops if we don't need anything from them.
2399 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2400 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2401 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2402 if (DemandedOp0) {
2403 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2404 return TLO.CombineTo(O: Op, N: NewOp);
2405 }
2406 }
2407 }
2408
2409 // Try to match AVG patterns (after shift simplification).
2410 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2411 DemandedElts, Depth: Depth + 1))
2412 return TLO.CombineTo(O: Op, N: AVG);
2413
2414 break;
2415 }
2416 case ISD::FSHL:
2417 case ISD::FSHR: {
2418 SDValue Op0 = Op.getOperand(i: 0);
2419 SDValue Op1 = Op.getOperand(i: 1);
2420 SDValue Op2 = Op.getOperand(i: 2);
2421 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
2422
2423 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op2, DemandedElts)) {
2424 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2425
2426 // For fshl, 0-shift returns the 1st arg.
2427 // For fshr, 0-shift returns the 2nd arg.
2428 if (Amt == 0) {
2429 if (SimplifyDemandedBits(Op: IsFSHL ? Op0 : Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2430 Known, TLO, Depth: Depth + 1))
2431 return true;
2432 break;
2433 }
2434
2435 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
2436 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
2437 APInt Demanded0 = DemandedBits.lshr(shiftAmt: IsFSHL ? Amt : (BitWidth - Amt));
2438 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
2439 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2440 Depth: Depth + 1))
2441 return true;
2442 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: Demanded1, OriginalDemandedElts: DemandedElts, Known, TLO,
2443 Depth: Depth + 1))
2444 return true;
2445
2446 Known2 <<= (IsFSHL ? Amt : (BitWidth - Amt));
2447 Known >>= (IsFSHL ? (BitWidth - Amt) : Amt);
2448 Known = Known.unionWith(RHS: Known2);
2449
2450 // Attempt to avoid multi-use ops if we don't need anything from them.
2451 if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
2452 !DemandedElts.isAllOnes()) {
2453 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2454 Op: Op0, DemandedBits: Demanded0, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2455 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2456 Op: Op1, DemandedBits: Demanded1, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2457 if (DemandedOp0 || DemandedOp1) {
2458 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2459 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2460 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedOp0,
2461 N2: DemandedOp1, N3: Op2);
2462 return TLO.CombineTo(O: Op, N: NewOp);
2463 }
2464 }
2465 }
2466
2467 if (isPowerOf2_32(Value: BitWidth)) {
2468 // Fold FSHR(Op0,Op1,Op2) -> SRL(Op1,Op2)
2469 // iff we're guaranteed not to use Op0.
2470 // TODO: Add FSHL equivalent?
2471 if (!IsFSHL && !DemandedBits.isAllOnes() &&
2472 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT))) {
2473 KnownBits KnownAmt =
2474 TLO.DAG.computeKnownBits(Op: Op2, DemandedElts, Depth: Depth + 1);
2475 unsigned MaxShiftAmt =
2476 KnownAmt.getMaxValue().getLimitedValue(Limit: BitWidth - 1);
2477 // Check we don't demand any shifted bits outside Op1.
2478 if (DemandedBits.countl_zero() >= MaxShiftAmt) {
2479 EVT AmtVT = Op2.getValueType();
2480 SDValue NewAmt =
2481 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Op2,
2482 N2: TLO.DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
2483 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op1, N2: NewAmt);
2484 return TLO.CombineTo(O: Op, N: NewOp);
2485 }
2486 }
2487
2488 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2489 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
2490 if (SimplifyDemandedBits(Op: Op2, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2491 Depth: Depth + 1))
2492 return true;
2493 }
2494 break;
2495 }
2496 case ISD::ROTL:
2497 case ISD::ROTR: {
2498 SDValue Op0 = Op.getOperand(i: 0);
2499 SDValue Op1 = Op.getOperand(i: 1);
2500 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
2501
2502 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
2503 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1))
2504 return TLO.CombineTo(O: Op, N: Op0);
2505
2506 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op1, DemandedElts)) {
2507 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2508 unsigned RevAmt = BitWidth - Amt;
2509
2510 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
2511 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
2512 APInt Demanded0 = DemandedBits.rotr(rotateAmt: IsROTL ? Amt : RevAmt);
2513 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2514 Depth: Depth + 1))
2515 return true;
2516
2517 // rot*(x, 0) --> x
2518 if (Amt == 0)
2519 return TLO.CombineTo(O: Op, N: Op0);
2520
2521 // See if we don't demand either half of the rotated bits.
2522 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT)) &&
2523 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2524 Op1 = TLO.DAG.getConstant(Val: IsROTL ? Amt : RevAmt, DL: dl, VT: Op1.getValueType());
2525 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: Op1));
2526 }
2527 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT)) &&
2528 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2529 Op1 = TLO.DAG.getConstant(Val: IsROTL ? RevAmt : Amt, DL: dl, VT: Op1.getValueType());
2530 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2531 }
2532 }
2533
2534 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2535 if (isPowerOf2_32(Value: BitWidth)) {
2536 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
2537 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2538 Depth: Depth + 1))
2539 return true;
2540 }
2541 break;
2542 }
2543 case ISD::SMIN:
2544 case ISD::SMAX:
2545 case ISD::UMIN:
2546 case ISD::UMAX: {
2547 unsigned Opc = Op.getOpcode();
2548 SDValue Op0 = Op.getOperand(i: 0);
2549 SDValue Op1 = Op.getOperand(i: 1);
2550
2551 // If we're only demanding signbits, then we can simplify to OR/AND node.
2552 unsigned BitOp =
2553 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
2554 unsigned NumSignBits =
2555 std::min(a: TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1),
2556 b: TLO.DAG.ComputeNumSignBits(Op: Op1, DemandedElts, Depth: Depth + 1));
2557 unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
2558 if (NumSignBits >= NumDemandedUpperBits)
2559 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: BitOp, DL: SDLoc(Op), VT, N1: Op0, N2: Op1));
2560
2561 // Check if one arg is always less/greater than (or equal) to the other arg.
2562 KnownBits Known0 = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2563 KnownBits Known1 = TLO.DAG.computeKnownBits(Op: Op1, DemandedElts, Depth: Depth + 1);
2564 switch (Opc) {
2565 case ISD::SMIN:
2566 if (std::optional<bool> IsSLE = KnownBits::sle(LHS: Known0, RHS: Known1))
2567 return TLO.CombineTo(O: Op, N: *IsSLE ? Op0 : Op1);
2568 if (std::optional<bool> IsSLT = KnownBits::slt(LHS: Known0, RHS: Known1))
2569 return TLO.CombineTo(O: Op, N: *IsSLT ? Op0 : Op1);
2570 Known = KnownBits::smin(LHS: Known0, RHS: Known1);
2571 break;
2572 case ISD::SMAX:
2573 if (std::optional<bool> IsSGE = KnownBits::sge(LHS: Known0, RHS: Known1))
2574 return TLO.CombineTo(O: Op, N: *IsSGE ? Op0 : Op1);
2575 if (std::optional<bool> IsSGT = KnownBits::sgt(LHS: Known0, RHS: Known1))
2576 return TLO.CombineTo(O: Op, N: *IsSGT ? Op0 : Op1);
2577 Known = KnownBits::smax(LHS: Known0, RHS: Known1);
2578 break;
2579 case ISD::UMIN:
2580 if (std::optional<bool> IsULE = KnownBits::ule(LHS: Known0, RHS: Known1))
2581 return TLO.CombineTo(O: Op, N: *IsULE ? Op0 : Op1);
2582 if (std::optional<bool> IsULT = KnownBits::ult(LHS: Known0, RHS: Known1))
2583 return TLO.CombineTo(O: Op, N: *IsULT ? Op0 : Op1);
2584 Known = KnownBits::umin(LHS: Known0, RHS: Known1);
2585 break;
2586 case ISD::UMAX:
2587 if (std::optional<bool> IsUGE = KnownBits::uge(LHS: Known0, RHS: Known1))
2588 return TLO.CombineTo(O: Op, N: *IsUGE ? Op0 : Op1);
2589 if (std::optional<bool> IsUGT = KnownBits::ugt(LHS: Known0, RHS: Known1))
2590 return TLO.CombineTo(O: Op, N: *IsUGT ? Op0 : Op1);
2591 Known = KnownBits::umax(LHS: Known0, RHS: Known1);
2592 break;
2593 }
2594 break;
2595 }
2596 case ISD::BITREVERSE: {
2597 SDValue Src = Op.getOperand(i: 0);
2598 APInt DemandedSrcBits = DemandedBits.reverseBits();
2599 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2600 Depth: Depth + 1))
2601 return true;
2602 Known = Known2.reverseBits();
2603 break;
2604 }
2605 case ISD::BSWAP: {
2606 SDValue Src = Op.getOperand(i: 0);
2607
2608 // If the only bits demanded come from one byte of the bswap result,
2609 // just shift the input byte into position to eliminate the bswap.
2610 unsigned NLZ = DemandedBits.countl_zero();
2611 unsigned NTZ = DemandedBits.countr_zero();
2612
2613 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
2614 // we need all the bits down to bit 8. Likewise, round NLZ. If we
2615 // have 14 leading zeros, round to 8.
2616 NLZ = alignDown(Value: NLZ, Align: 8);
2617 NTZ = alignDown(Value: NTZ, Align: 8);
2618 // If we need exactly one byte, we can do this transformation.
2619 if (BitWidth - NLZ - NTZ == 8) {
2620 // Replace this with either a left or right shift to get the byte into
2621 // the right place.
2622 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
2623 if (!TLO.LegalOperations() || isOperationLegal(Op: ShiftOpcode, VT)) {
2624 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2625 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
2626 SDValue NewOp = TLO.DAG.getNode(Opcode: ShiftOpcode, DL: dl, VT, N1: Src, N2: ShAmt);
2627 return TLO.CombineTo(O: Op, N: NewOp);
2628 }
2629 }
2630
2631 APInt DemandedSrcBits = DemandedBits.byteSwap();
2632 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2633 Depth: Depth + 1))
2634 return true;
2635 Known = Known2.byteSwap();
2636 break;
2637 }
2638 case ISD::CTPOP: {
2639 // If only 1 bit is demanded, replace with PARITY as long as we're before
2640 // op legalization.
2641 // FIXME: Limit to scalars for now.
2642 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
2643 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT,
2644 Operand: Op.getOperand(i: 0)));
2645
2646 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2647 break;
2648 }
2649 case ISD::PDEP: {
2650 SDValue Op0 = Op.getOperand(i: 0);
2651 SDValue Op1 = Op.getOperand(i: 1);
2652
2653 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2654 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
2655
2656 // If the demanded bits has leading zeroes, we don't demand those from the
2657 // mask.
2658 if (SimplifyDemandedBits(Op: Op1, DemandedBits: LoMask, Known, TLO, Depth: Depth + 1))
2659 return true;
2660
2661 // The number of possible 1s in the mask determines the number of LSBs of
2662 // operand 0 used. Undemanded bits from the mask don't matter so filter
2663 // them before counting.
2664 KnownBits Known2;
2665 uint64_t Count = (~Known.Zero & LoMask).popcount();
2666 APInt DemandedMask(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: Count));
2667 if (SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, Known&: Known2, TLO, Depth: Depth + 1))
2668 return true;
2669
2670 // Zeroes are retained from the mask, but not ones.
2671 Known.One.clearAllBits();
2672 // The result will have at least as many trailing zeros as the non-mask
2673 // operand since bits can only map to the same or higher bit position.
2674 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
2675 break;
2676 }
2677 case ISD::SIGN_EXTEND_INREG: {
2678 SDValue Op0 = Op.getOperand(i: 0);
2679 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2680 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2681
2682 // If we only care about the highest bit, don't bother shifting right.
2683 if (DemandedBits.isSignMask()) {
2684 unsigned MinSignedBits =
2685 TLO.DAG.ComputeMaxSignificantBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2686 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2687 // However if the input is already sign extended we expect the sign
2688 // extension to be dropped altogether later and do not simplify.
2689 if (!AlreadySignExtended) {
2690 // Compute the correct shift amount type, which must be getShiftAmountTy
2691 // for scalar types after legalization.
2692 SDValue ShiftAmt =
2693 TLO.DAG.getShiftAmountConstant(Val: BitWidth - ExVTBits, VT, DL: dl);
2694 return TLO.CombineTo(O: Op,
2695 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: ShiftAmt));
2696 }
2697 }
2698
2699 // If none of the extended bits are demanded, eliminate the sextinreg.
2700 if (DemandedBits.getActiveBits() <= ExVTBits)
2701 return TLO.CombineTo(O: Op, N: Op0);
2702
2703 APInt InputDemandedBits = DemandedBits.getLoBits(numBits: ExVTBits);
2704
2705 // Since the sign extended bits are demanded, we know that the sign
2706 // bit is demanded.
2707 InputDemandedBits.setBit(ExVTBits - 1);
2708
2709 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InputDemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
2710 Depth: Depth + 1))
2711 return true;
2712
2713 // If the sign bit of the input is known set or clear, then we know the
2714 // top bits of the result.
2715
2716 // If the input sign bit is known zero, convert this into a zero extension.
2717 if (Known.Zero[ExVTBits - 1])
2718 return TLO.CombineTo(O: Op, N: TLO.DAG.getZeroExtendInReg(Op: Op0, DL: dl, VT: ExVT));
2719
2720 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ExVTBits);
2721 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
2722 Known.One.setBitsFrom(ExVTBits);
2723 Known.Zero &= Mask;
2724 } else { // Input sign bit unknown
2725 Known.Zero &= Mask;
2726 Known.One &= Mask;
2727 }
2728 break;
2729 }
2730 case ISD::BUILD_PAIR: {
2731 EVT HalfVT = Op.getOperand(i: 0).getValueType();
2732 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
2733
2734 APInt MaskLo = DemandedBits.getLoBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2735 APInt MaskHi = DemandedBits.getHiBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2736
2737 KnownBits KnownLo, KnownHi;
2738
2739 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: MaskLo, Known&: KnownLo, TLO, Depth: Depth + 1))
2740 return true;
2741
2742 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: MaskHi, Known&: KnownHi, TLO, Depth: Depth + 1))
2743 return true;
2744
2745 Known = KnownHi.concat(Lo: KnownLo);
2746 break;
2747 }
2748 case ISD::ZERO_EXTEND_VECTOR_INREG:
2749 if (VT.isScalableVector())
2750 return false;
2751 [[fallthrough]];
2752 case ISD::ZERO_EXTEND: {
2753 SDValue Src = Op.getOperand(i: 0);
2754 EVT SrcVT = Src.getValueType();
2755 unsigned InBits = SrcVT.getScalarSizeInBits();
2756 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2757 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
2758
2759 // If none of the top bits are demanded, convert this into an any_extend.
2760 if (DemandedBits.getActiveBits() <= InBits) {
2761 // If we only need the non-extended bits of the bottom element
2762 // then we can just bitcast to the result.
2763 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2764 VT.getSizeInBits() == SrcVT.getSizeInBits())
2765 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2766
2767 unsigned Opc =
2768 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2769 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2770 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2771 }
2772
2773 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2774 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2775 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2776 Depth: Depth + 1)) {
2777 Op->dropFlags(Mask: SDNodeFlags::NonNeg);
2778 return true;
2779 }
2780 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2781 Known = Known.zext(BitWidth);
2782
2783 // Attempt to avoid multi-use ops if we don't need anything from them.
2784 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2785 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2786 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2787 break;
2788 }
2789 case ISD::SIGN_EXTEND_VECTOR_INREG:
2790 if (VT.isScalableVector())
2791 return false;
2792 [[fallthrough]];
2793 case ISD::SIGN_EXTEND: {
2794 SDValue Src = Op.getOperand(i: 0);
2795 EVT SrcVT = Src.getValueType();
2796 unsigned InBits = SrcVT.getScalarSizeInBits();
2797 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2798 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
2799
2800 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2801 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2802
2803 // Since some of the sign extended bits are demanded, we know that the sign
2804 // bit is demanded.
2805 InDemandedBits.setBit(InBits - 1);
2806
2807 // If none of the top bits are demanded, convert this into an any_extend.
2808 if (DemandedBits.getActiveBits() <= InBits) {
2809 // If we only need the non-extended bits of the bottom element
2810 // then we can just bitcast to the result.
2811 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2812 VT.getSizeInBits() == SrcVT.getSizeInBits())
2813 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2814
2815 // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
2816 if (getBooleanContents(Type: VT) != ZeroOrNegativeOneBooleanContent ||
2817 TLO.DAG.ComputeNumSignBits(Op: Src, DemandedElts: InDemandedElts, Depth: Depth + 1) !=
2818 InBits) {
2819 unsigned Opc =
2820 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2821 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2822 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2823 }
2824 }
2825
2826 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2827 Depth: Depth + 1))
2828 return true;
2829 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2830
2831 // If the sign bit is known one, the top bits match.
2832 Known = Known.sext(BitWidth);
2833
2834 // If the sign bit is known zero, convert this to a zero extend.
2835 if (Known.isNonNegative()) {
2836 unsigned Opc =
2837 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
2838 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT)) {
2839 SDNodeFlags Flags;
2840 if (!IsVecInReg)
2841 Flags |= SDNodeFlags::NonNeg;
2842 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src, Flags));
2843 }
2844 }
2845
2846 // Attempt to avoid multi-use ops if we don't need anything from them.
2847 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2848 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2849 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2850 break;
2851 }
2852 case ISD::ANY_EXTEND_VECTOR_INREG:
2853 if (VT.isScalableVector())
2854 return false;
2855 [[fallthrough]];
2856 case ISD::ANY_EXTEND: {
2857 SDValue Src = Op.getOperand(i: 0);
2858 EVT SrcVT = Src.getValueType();
2859 unsigned InBits = SrcVT.getScalarSizeInBits();
2860 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2861 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2862
2863 // If we only need the bottom element then we can just bitcast.
2864 // TODO: Handle ANY_EXTEND?
2865 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2866 VT.getSizeInBits() == SrcVT.getSizeInBits())
2867 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2868
2869 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2870 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2871 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2872 Depth: Depth + 1))
2873 return true;
2874 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2875 Known = Known.anyext(BitWidth);
2876
2877 // Attempt to avoid multi-use ops if we don't need anything from them.
2878 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2879 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2880 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2881 break;
2882 }
2883 case ISD::TRUNCATE: {
2884 SDValue Src = Op.getOperand(i: 0);
2885
2886 // Simplify the input, using demanded bit information, and compute the known
2887 // zero/one bits live out.
2888 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2889 APInt TruncMask = DemandedBits.zext(width: OperandBitWidth);
2890 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: TruncMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2891 Depth: Depth + 1)) {
2892 // Disable the nsw and nuw flags. We can no longer guarantee that we
2893 // won't wrap after simplification.
2894 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2895 return true;
2896 }
2897 Known = Known.trunc(BitWidth);
2898
2899 // Attempt to avoid multi-use ops if we don't need anything from them.
2900 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2901 Op: Src, DemandedBits: TruncMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2902 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: NewSrc));
2903
2904 // If the input is only used by this truncate, see if we can shrink it based
2905 // on the known demanded bits.
2906 switch (Src.getOpcode()) {
2907 default:
2908 break;
2909 case ISD::SRL:
2910 // Shrink SRL by a constant if none of the high bits shifted in are
2911 // demanded.
2912 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2913 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2914 // undesirable.
2915 break;
2916
2917 if (Src.getNode()->hasOneUse()) {
2918 if (isTruncateFree(Val: Src, VT2: VT) &&
2919 !isTruncateFree(FromVT: Src.getValueType(), ToVT: VT)) {
2920 // If truncate is only free at trunc(srl), do not turn it into
2921 // srl(trunc). The check is done by first check the truncate is free
2922 // at Src's opcode(srl), then check the truncate is not done by
2923 // referencing sub-register. In test, if both trunc(srl) and
2924 // srl(trunc)'s trunc are free, srl(trunc) performs better. If only
2925 // trunc(srl)'s trunc is free, trunc(srl) is better.
2926 break;
2927 }
2928
2929 std::optional<unsigned> ShAmtC =
2930 TLO.DAG.getValidShiftAmount(V: Src, DemandedElts, Depth: Depth + 2);
2931 if (!ShAmtC || *ShAmtC >= BitWidth)
2932 break;
2933 unsigned ShVal = *ShAmtC;
2934
2935 APInt HighBits =
2936 APInt::getHighBitsSet(numBits: OperandBitWidth, hiBitsSet: OperandBitWidth - BitWidth);
2937 HighBits.lshrInPlace(ShiftAmt: ShVal);
2938 HighBits = HighBits.trunc(width: BitWidth);
2939 if (!(HighBits & DemandedBits)) {
2940 // None of the shifted in bits are needed. Add a truncate of the
2941 // shift input, then shift it.
2942 SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
2943 SDValue NewTrunc =
2944 TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Src.getOperand(i: 0));
2945 return TLO.CombineTo(
2946 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NewTrunc, N2: NewShAmt));
2947 }
2948 }
2949 break;
2950 }
2951
2952 break;
2953 }
2954 case ISD::AssertZext: {
2955 // AssertZext demands all of the high bits, plus any of the low bits
2956 // demanded by its users.
2957 EVT ZVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2958 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ZVT.getSizeInBits());
2959 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: ~InMask | DemandedBits, Known,
2960 TLO, Depth: Depth + 1))
2961 return true;
2962
2963 Known.Zero |= ~InMask;
2964 Known.One &= (~Known.Zero);
2965 break;
2966 }
2967 case ISD::EXTRACT_VECTOR_ELT: {
2968 SDValue Src = Op.getOperand(i: 0);
2969 SDValue Idx = Op.getOperand(i: 1);
2970 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2971 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2972
2973 if (SrcEltCnt.isScalable())
2974 return false;
2975
2976 // Demand the bits from every vector element without a constant index.
2977 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2978 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
2979 if (auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx))
2980 if (CIdx->getAPIntValue().ult(RHS: NumSrcElts))
2981 DemandedSrcElts = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: CIdx->getZExtValue());
2982
2983 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2984 // anything about the extended bits.
2985 APInt DemandedSrcBits = DemandedBits;
2986 if (BitWidth > EltBitWidth)
2987 DemandedSrcBits = DemandedSrcBits.trunc(width: EltBitWidth);
2988
2989 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts, Known&: Known2, TLO,
2990 Depth: Depth + 1))
2991 return true;
2992
2993 // Attempt to avoid multi-use ops if we don't need anything from them.
2994 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2995 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2996 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
2997 SDValue NewOp =
2998 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc, N2: Idx);
2999 return TLO.CombineTo(O: Op, N: NewOp);
3000 }
3001 }
3002
3003 Known = Known2;
3004 if (BitWidth > EltBitWidth)
3005 Known = Known.anyext(BitWidth);
3006 break;
3007 }
3008 case ISD::BITCAST: {
3009 if (VT.isScalableVector())
3010 return false;
3011 SDValue Src = Op.getOperand(i: 0);
3012 EVT SrcVT = Src.getValueType();
3013 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
3014
3015 // If this is an FP->Int bitcast and if the sign bit is the only
3016 // thing demanded, turn this into a FGETSIGN.
3017 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
3018 DemandedBits == APInt::getSignMask(BitWidth: Op.getValueSizeInBits()) &&
3019 SrcVT.isFloatingPoint()) {
3020 if (isOperationLegalOrCustom(Op: ISD::FGETSIGN, VT)) {
3021 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
3022 // place. We expect the SHL to be eliminated by other optimizations.
3023 SDValue Sign = TLO.DAG.getNode(Opcode: ISD::FGETSIGN, DL: dl, VT, Operand: Src);
3024 unsigned ShVal = Op.getValueSizeInBits() - 1;
3025 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
3026 return TLO.CombineTo(O: Op,
3027 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Sign, N2: ShAmt));
3028 }
3029 }
3030
3031 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
3032 // Demand the elt/bit if any of the original elts/bits are demanded.
3033 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
3034 unsigned Scale = BitWidth / NumSrcEltBits;
3035 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3036 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
3037 for (unsigned i = 0; i != Scale; ++i) {
3038 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
3039 unsigned BitOffset = EltOffset * NumSrcEltBits;
3040 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
3041 }
3042 // Recursive calls below may turn not demanded elements into poison, so we
3043 // need to demand all smaller source elements that maps to a demanded
3044 // destination element.
3045 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3046
3047 APInt KnownSrcUndef, KnownSrcZero;
3048 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
3049 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
3050 return true;
3051
3052 KnownBits KnownSrcBits;
3053 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
3054 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
3055 return true;
3056 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
3057 // TODO - bigendian once we have test coverage.
3058 unsigned Scale = NumSrcEltBits / BitWidth;
3059 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
3060 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
3061 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
3062 for (unsigned i = 0; i != NumElts; ++i)
3063 if (DemandedElts[i]) {
3064 unsigned Offset = (i % Scale) * BitWidth;
3065 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
3066 DemandedSrcElts.setBit(i / Scale);
3067 }
3068
3069 if (SrcVT.isVector()) {
3070 APInt KnownSrcUndef, KnownSrcZero;
3071 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
3072 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
3073 return true;
3074 }
3075
3076 KnownBits KnownSrcBits;
3077 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
3078 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
3079 return true;
3080
3081 // Attempt to avoid multi-use ops if we don't need anything from them.
3082 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
3083 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
3084 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
3085 SDValue NewOp = TLO.DAG.getBitcast(VT, V: DemandedSrc);
3086 return TLO.CombineTo(O: Op, N: NewOp);
3087 }
3088 }
3089 }
3090
3091 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
3092 // recursive call where Known may be useful to the caller.
3093 if (Depth > 0) {
3094 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3095 return false;
3096 }
3097 break;
3098 }
3099 case ISD::MUL:
3100 if (DemandedBits.isPowerOf2()) {
3101 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
3102 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
3103 // odd (has LSB set), then the left-shifted low bit of X is the answer.
3104 unsigned CTZ = DemandedBits.countr_zero();
3105 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
3106 if (C && C->getAPIntValue().countr_zero() == CTZ) {
3107 SDValue AmtC = TLO.DAG.getShiftAmountConstant(Val: CTZ, VT, DL: dl);
3108 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op.getOperand(i: 0), N2: AmtC);
3109 return TLO.CombineTo(O: Op, N: Shl);
3110 }
3111 }
3112 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
3113 // X * X is odd iff X is odd.
3114 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
3115 if (Op.getOperand(i: 0) == Op.getOperand(i: 1) && DemandedBits.ult(RHS: 4)) {
3116 SDValue One = TLO.DAG.getConstant(Val: 1, DL: dl, VT);
3117 SDValue And1 = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op.getOperand(i: 0), N2: One);
3118 return TLO.CombineTo(O: Op, N: And1);
3119 }
3120 [[fallthrough]];
3121 case ISD::PTRADD:
3122 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
3123 break;
3124 // PTRADD behaves like ADD if pointers are represented as integers.
3125 [[fallthrough]];
3126 case ISD::ADD:
3127 case ISD::SUB: {
3128 // Add, Sub, and Mul don't demand any bits in positions beyond that
3129 // of the highest bit demanded of them.
3130 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
3131 SDNodeFlags Flags = Op.getNode()->getFlags();
3132 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
3133 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
3134 KnownBits KnownOp0, KnownOp1;
3135 auto GetDemandedBitsLHSMask = [&](APInt Demanded,
3136 const KnownBits &KnownRHS) {
3137 if (Op.getOpcode() == ISD::MUL)
3138 Demanded.clearHighBits(hiBits: KnownRHS.countMinTrailingZeros());
3139 return Demanded;
3140 };
3141 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: LoMask, OriginalDemandedElts: DemandedElts, Known&: KnownOp1, TLO,
3142 Depth: Depth + 1) ||
3143 SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: GetDemandedBitsLHSMask(LoMask, KnownOp1),
3144 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1) ||
3145 // See if the operation should be performed at a smaller bit width.
3146 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
3147 // Disable the nsw and nuw flags. We can no longer guarantee that we
3148 // won't wrap after simplification.
3149 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
3150 return true;
3151 }
3152
3153 // neg x with only low bit demanded is simply x.
3154 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
3155 isNullConstant(V: Op0))
3156 return TLO.CombineTo(O: Op, N: Op1);
3157
3158 // Attempt to avoid multi-use ops if we don't need anything from them.
3159 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
3160 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
3161 Op: Op0, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3162 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
3163 Op: Op1, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3164 if (DemandedOp0 || DemandedOp1) {
3165 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
3166 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
3167 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1,
3168 Flags: Flags & ~SDNodeFlags::NoWrap);
3169 return TLO.CombineTo(O: Op, N: NewOp);
3170 }
3171 }
3172
3173 // If we have a constant operand, we may be able to turn it into -1 if we
3174 // do not demand the high bits. This can make the constant smaller to
3175 // encode, allow more general folding, or match specialized instruction
3176 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
3177 // is probably not useful (and could be detrimental).
3178 ConstantSDNode *C = isConstOrConstSplat(N: Op1);
3179 APInt HighMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: DemandedBitsLZ);
3180 if (C && !C->isAllOnes() && !C->isOne() &&
3181 (C->getAPIntValue() | HighMask).isAllOnes()) {
3182 SDValue Neg1 = TLO.DAG.getAllOnesConstant(DL: dl, VT);
3183 // Disable the nsw and nuw flags. We can no longer guarantee that we
3184 // won't wrap after simplification.
3185 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Neg1,
3186 Flags: Flags & ~SDNodeFlags::NoWrap);
3187 return TLO.CombineTo(O: Op, N: NewOp);
3188 }
3189
3190 // Match a multiply with a disguised negated-power-of-2 and convert to a
3191 // an equivalent shift-left amount.
3192 // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3193 auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
3194 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
3195 return 0;
3196
3197 // Don't touch opaque constants. Also, ignore zero and power-of-2
3198 // multiplies. Those will get folded later.
3199 ConstantSDNode *MulC = isConstOrConstSplat(N: Mul.getOperand(i: 1));
3200 if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
3201 !MulC->getAPIntValue().isPowerOf2()) {
3202 APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
3203 if (UnmaskedC.isNegatedPowerOf2())
3204 return (-UnmaskedC).logBase2();
3205 }
3206 return 0;
3207 };
3208
3209 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
3210 unsigned ShlAmt) {
3211 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(Val: ShlAmt, VT, DL: dl);
3212 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: X, N2: ShlAmtC);
3213 SDValue Res = TLO.DAG.getNode(Opcode: NT, DL: dl, VT, N1: Y, N2: Shl);
3214 return TLO.CombineTo(O: Op, N: Res);
3215 };
3216
3217 if (isOperationLegalOrCustom(Op: ISD::SHL, VT)) {
3218 if (Op.getOpcode() == ISD::ADD) {
3219 // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3220 if (unsigned ShAmt = getShiftLeftAmt(Op0))
3221 return foldMul(ISD::SUB, Op0.getOperand(i: 0), Op1, ShAmt);
3222 // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
3223 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3224 return foldMul(ISD::SUB, Op1.getOperand(i: 0), Op0, ShAmt);
3225 }
3226 if (Op.getOpcode() == ISD::SUB) {
3227 // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
3228 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3229 return foldMul(ISD::ADD, Op1.getOperand(i: 0), Op0, ShAmt);
3230 }
3231 }
3232
3233 if (Op.getOpcode() == ISD::MUL) {
3234 Known = KnownBits::mul(LHS: KnownOp0, RHS: KnownOp1);
3235 } else { // Op.getOpcode() is either ISD::ADD, ISD::PTRADD, or ISD::SUB.
3236 Known = KnownBits::computeForAddSub(
3237 Add: Op.getOpcode() != ISD::SUB, NSW: Flags.hasNoSignedWrap(),
3238 NUW: Flags.hasNoUnsignedWrap(), LHS: KnownOp0, RHS: KnownOp1);
3239 }
3240 break;
3241 }
3242 case ISD::FABS: {
3243 SDValue Op0 = Op.getOperand(i: 0);
3244 APInt SignMask = APInt::getSignMask(BitWidth);
3245
3246 if (!DemandedBits.intersects(RHS: SignMask))
3247 return TLO.CombineTo(O: Op, N: Op0);
3248
3249 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3250 Depth: Depth + 1))
3251 return true;
3252
3253 if (Known.isNonNegative())
3254 return TLO.CombineTo(O: Op, N: Op0);
3255 if (Known.isNegative())
3256 return TLO.CombineTo(
3257 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3258
3259 Known.Zero |= SignMask;
3260 Known.One &= ~SignMask;
3261
3262 break;
3263 }
3264 case ISD::FCOPYSIGN: {
3265 SDValue Op0 = Op.getOperand(i: 0);
3266 SDValue Op1 = Op.getOperand(i: 1);
3267
3268 unsigned BitWidth0 = Op0.getScalarValueSizeInBits();
3269 unsigned BitWidth1 = Op1.getScalarValueSizeInBits();
3270 APInt SignMask0 = APInt::getSignMask(BitWidth: BitWidth0);
3271 APInt SignMask1 = APInt::getSignMask(BitWidth: BitWidth1);
3272
3273 if (!DemandedBits.intersects(RHS: SignMask0))
3274 return TLO.CombineTo(O: Op, N: Op0);
3275
3276 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~SignMask0 & DemandedBits, OriginalDemandedElts: DemandedElts,
3277 Known, TLO, Depth: Depth + 1) ||
3278 SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: SignMask1, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
3279 Depth: Depth + 1))
3280 return true;
3281
3282 if (Known2.isNonNegative())
3283 return TLO.CombineTo(
3284 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3285
3286 if (Known2.isNegative())
3287 return TLO.CombineTo(
3288 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT,
3289 Operand: TLO.DAG.getNode(Opcode: ISD::FABS, DL: SDLoc(Op0), VT, Operand: Op0)));
3290
3291 Known.Zero &= ~SignMask0;
3292 Known.One &= ~SignMask0;
3293 break;
3294 }
3295 case ISD::FNEG: {
3296 SDValue Op0 = Op.getOperand(i: 0);
3297 APInt SignMask = APInt::getSignMask(BitWidth);
3298
3299 if (!DemandedBits.intersects(RHS: SignMask))
3300 return TLO.CombineTo(O: Op, N: Op0);
3301
3302 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3303 Depth: Depth + 1))
3304 return true;
3305
3306 if (!Known.isSignUnknown()) {
3307 Known.Zero ^= SignMask;
3308 Known.One ^= SignMask;
3309 }
3310
3311 break;
3312 }
3313 default:
3314 // We also ask the target about intrinsics (which could be specific to it).
3315 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3316 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
3317 // TODO: Probably okay to remove after audit; here to reduce change size
3318 // in initial enablement patch for scalable vectors
3319 if (Op.getValueType().isScalableVector())
3320 break;
3321 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
3322 Known, TLO, Depth))
3323 return true;
3324 break;
3325 }
3326
3327 // Just use computeKnownBits to compute output bits.
3328 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3329 break;
3330 }
3331
3332 // If we know the value of all of the demanded bits, return this as a
3333 // constant.
3334 if (!isTargetCanonicalConstantNode(Op) &&
3335 DemandedBits.isSubsetOf(RHS: Known.Zero | Known.One)) {
3336 // Avoid folding to a constant if any OpaqueConstant is involved.
3337 if (llvm::any_of(Range: Op->ops(), P: [](SDValue V) {
3338 auto *C = dyn_cast<ConstantSDNode>(Val&: V);
3339 return C && C->isOpaque();
3340 }))
3341 return false;
3342 if (VT.isInteger())
3343 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: Known.One, DL: dl, VT));
3344 if (VT.isFloatingPoint())
3345 return TLO.CombineTo(
3346 O: Op, N: TLO.DAG.getConstantFP(Val: APFloat(VT.getFltSemantics(), Known.One),
3347 DL: dl, VT));
3348 }
3349
3350 // A multi use 'all demanded elts' simplify failed to find any knownbits.
3351 // Try again just for the original demanded elts.
3352 // Ensure we do this AFTER constant folding above.
3353 if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
3354 Known = TLO.DAG.computeKnownBits(Op, DemandedElts: OriginalDemandedElts, Depth);
3355
3356 return false;
3357}
3358
3359bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
3360 const APInt &DemandedElts,
3361 DAGCombinerInfo &DCI) const {
3362 SelectionDAG &DAG = DCI.DAG;
3363 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
3364 !DCI.isBeforeLegalizeOps());
3365
3366 APInt KnownUndef, KnownZero;
3367 bool Simplified =
3368 SimplifyDemandedVectorElts(Op, DemandedEltMask: DemandedElts, KnownUndef, KnownZero, TLO);
3369 if (Simplified) {
3370 DCI.AddToWorklist(N: Op.getNode());
3371 DCI.CommitTargetLoweringOpt(TLO);
3372 }
3373
3374 return Simplified;
3375}
3376
3377/// Given a vector binary operation and known undefined elements for each input
3378/// operand, compute whether each element of the output is undefined.
3379static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
3380 const APInt &UndefOp0,
3381 const APInt &UndefOp1) {
3382 EVT VT = BO.getValueType();
3383 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
3384 "Vector binop only");
3385
3386 EVT EltVT = VT.getVectorElementType();
3387 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
3388 assert(UndefOp0.getBitWidth() == NumElts &&
3389 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
3390
3391 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
3392 const APInt &UndefVals) {
3393 if (UndefVals[Index])
3394 return DAG.getUNDEF(VT: EltVT);
3395
3396 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
3397 // Try hard to make sure that the getNode() call is not creating temporary
3398 // nodes. Ignore opaque integers because they do not constant fold.
3399 SDValue Elt = BV->getOperand(Num: Index);
3400 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
3401 if (isa<ConstantFPSDNode>(Val: Elt) || Elt.isUndef() || (C && !C->isOpaque()))
3402 return Elt;
3403 }
3404
3405 return SDValue();
3406 };
3407
3408 APInt KnownUndef = APInt::getZero(numBits: NumElts);
3409 for (unsigned i = 0; i != NumElts; ++i) {
3410 // If both inputs for this element are either constant or undef and match
3411 // the element type, compute the constant/undef result for this element of
3412 // the vector.
3413 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
3414 // not handle FP constants. The code within getNode() should be refactored
3415 // to avoid the danger of creating a bogus temporary node here.
3416 SDValue C0 = getUndefOrConstantElt(BO.getOperand(i: 0), i, UndefOp0);
3417 SDValue C1 = getUndefOrConstantElt(BO.getOperand(i: 1), i, UndefOp1);
3418 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
3419 if (DAG.getNode(Opcode: BO.getOpcode(), DL: SDLoc(BO), VT: EltVT, N1: C0, N2: C1).isUndef())
3420 KnownUndef.setBit(i);
3421 }
3422 return KnownUndef;
3423}
3424
3425bool TargetLowering::SimplifyDemandedVectorElts(
3426 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
3427 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
3428 bool AssumeSingleUse) const {
3429 EVT VT = Op.getValueType();
3430 unsigned Opcode = Op.getOpcode();
3431 APInt DemandedElts = OriginalDemandedElts;
3432 unsigned NumElts = DemandedElts.getBitWidth();
3433 assert(VT.isVector() && "Expected vector op");
3434
3435 KnownUndef = KnownZero = APInt::getZero(numBits: NumElts);
3436
3437 if (!shouldSimplifyDemandedVectorElts(Op, TLO))
3438 return false;
3439
3440 // TODO: For now we assume we know nothing about scalable vectors.
3441 if (VT.isScalableVector())
3442 return false;
3443
3444 assert(VT.getVectorNumElements() == NumElts &&
3445 "Mask size mismatches value type element count!");
3446
3447 // Undef operand.
3448 if (Op.isUndef()) {
3449 KnownUndef.setAllBits();
3450 return false;
3451 }
3452
3453 // If Op has other users, assume that all elements are needed.
3454 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
3455 DemandedElts.setAllBits();
3456
3457 // Not demanding any elements from Op.
3458 if (DemandedElts == 0) {
3459 KnownUndef.setAllBits();
3460 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3461 }
3462
3463 // Limit search depth.
3464 if (Depth >= SelectionDAG::MaxRecursionDepth)
3465 return false;
3466
3467 SDLoc DL(Op);
3468 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3469 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
3470
3471 auto TryShrinkBinOp = [&](SDValue Op0, SDValue Op1) {
3472 unsigned ShrunkSize = getPreferredShrunkVectorSizeInBits(Op, DemandedElts);
3473 if (!ShrunkSize)
3474 return false;
3475
3476 assert(ShrunkSize % EltSizeInBits == 0 &&
3477 "Shrunk size not a multiple of element size");
3478 assert(ShrunkSize < VT.getSizeInBits() &&
3479 "Shrunk size must be < original vector size");
3480 assert(ShrunkSize >= EltSizeInBits * DemandedElts.getActiveBits() &&
3481 "Shrunk size must be >= demanded size");
3482
3483 EVT ShrunkVT = VT.changeVectorElementCount(
3484 Context&: *TLO.DAG.getContext(),
3485 EC: ElementCount::getFixed(MinVal: ShrunkSize / EltSizeInBits));
3486 Op0 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op0, Idx: 0);
3487 Op1 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op1, Idx: 0);
3488 SDValue NewOp =
3489 TLO.DAG.getNode(Opcode, DL, VT: ShrunkVT, N1: Op0, N2: Op1, Flags: Op->getFlags());
3490 return TLO.CombineTo(
3491 O: Op, N: TLO.DAG.getInsertSubvector(DL, Vec: TLO.DAG.getUNDEF(VT), SubVec: NewOp, Idx: 0));
3492 };
3493
3494 // Helper for demanding the specified elements and all the bits of both binary
3495 // operands.
3496 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
3497 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op: Op0, DemandedElts,
3498 DAG&: TLO.DAG, Depth: Depth + 1);
3499 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op: Op1, DemandedElts,
3500 DAG&: TLO.DAG, Depth: Depth + 1);
3501 if (NewOp0 || NewOp1) {
3502 SDValue NewOp =
3503 TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, N1: NewOp0 ? NewOp0 : Op0,
3504 N2: NewOp1 ? NewOp1 : Op1, Flags: Op->getFlags());
3505 return TLO.CombineTo(O: Op, N: NewOp);
3506 }
3507
3508 if (TryShrinkBinOp(Op0, Op1))
3509 return true;
3510
3511 return false;
3512 };
3513
3514 switch (Opcode) {
3515 case ISD::SCALAR_TO_VECTOR: {
3516 if (!DemandedElts[0])
3517 return TLO.CombineTo(O: Op, N: TLO.DAG.getPOISON(VT));
3518 // Upper elements are poison, not undef - don't mark them as KnownUndef.
3519 break;
3520 }
3521 case ISD::BITCAST: {
3522 SDValue Src = Op.getOperand(i: 0);
3523 EVT SrcVT = Src.getValueType();
3524
3525 if (!SrcVT.isVector()) {
3526 // TODO - bigendian once we have test coverage.
3527 if (IsLE) {
3528 APInt DemandedSrcBits = APInt::getZero(numBits: SrcVT.getSizeInBits());
3529 unsigned EltSize = VT.getScalarSizeInBits();
3530 for (unsigned I = 0; I != NumElts; ++I) {
3531 if (DemandedElts[I]) {
3532 unsigned Offset = I * EltSize;
3533 DemandedSrcBits.setBits(loBit: Offset, hiBit: Offset + EltSize);
3534 }
3535 }
3536 KnownBits Known;
3537 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, Known, TLO, Depth: Depth + 1))
3538 return true;
3539 }
3540 break;
3541 }
3542
3543 // Fast handling of 'identity' bitcasts.
3544 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3545 if (NumSrcElts == NumElts)
3546 return SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedElts, KnownUndef,
3547 KnownZero, TLO, Depth: Depth + 1);
3548
3549 APInt SrcDemandedElts, SrcZero, SrcUndef;
3550
3551 // Bitcast from 'large element' src vector to 'small element' vector, we
3552 // must demand a source element if any DemandedElt maps to it.
3553 if ((NumElts % NumSrcElts) == 0) {
3554 unsigned Scale = NumElts / NumSrcElts;
3555 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3556 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3557 TLO, Depth: Depth + 1))
3558 return true;
3559
3560 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
3561 // of the large element.
3562 // TODO - bigendian once we have test coverage.
3563 if (IsLE) {
3564 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
3565 APInt SrcDemandedBits = APInt::getZero(numBits: SrcEltSizeInBits);
3566 for (unsigned i = 0; i != NumElts; ++i)
3567 if (DemandedElts[i]) {
3568 unsigned Ofs = (i % Scale) * EltSizeInBits;
3569 SrcDemandedBits.setBits(loBit: Ofs, hiBit: Ofs + EltSizeInBits);
3570 }
3571
3572 KnownBits Known;
3573 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: SrcDemandedBits, OriginalDemandedElts: SrcDemandedElts, Known,
3574 TLO, Depth: Depth + 1))
3575 return true;
3576
3577 // The bitcast has split each wide element into a number of
3578 // narrow subelements. We have just computed the Known bits
3579 // for wide elements. See if element splitting results in
3580 // some subelements being zero. Only for demanded elements!
3581 for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3582 if (!Known.Zero.extractBits(numBits: EltSizeInBits, bitPosition: SubElt * EltSizeInBits)
3583 .isAllOnes())
3584 continue;
3585 for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3586 unsigned Elt = Scale * SrcElt + SubElt;
3587 // A wholly-undef source lane is reported as undef below; don't also
3588 // flag it as zero, keeping the undef and zero sets disjoint.
3589 if (DemandedElts[Elt] && !SrcUndef[SrcElt])
3590 KnownZero.setBit(Elt);
3591 }
3592 }
3593 }
3594
3595 // If the src element is zero/undef then all the output elements will be -
3596 // only demanded elements are guaranteed to be correct.
3597 for (unsigned i = 0; i != NumSrcElts; ++i) {
3598 if (SrcDemandedElts[i]) {
3599 if (SrcZero[i])
3600 KnownZero.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3601 if (SrcUndef[i])
3602 KnownUndef.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3603 }
3604 }
3605 }
3606
3607 // Bitcast from 'small element' src vector to 'large element' vector, we
3608 // demand all smaller source elements covered by the larger demanded element
3609 // of this vector.
3610 if ((NumSrcElts % NumElts) == 0) {
3611 unsigned Scale = NumSrcElts / NumElts;
3612 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3613 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3614 TLO, Depth: Depth + 1))
3615 return true;
3616
3617 // If all the src elements covering an output element are zero/undef, then
3618 // the output element will be as well, assuming it was demanded.
3619 for (unsigned i = 0; i != NumElts; ++i) {
3620 if (DemandedElts[i]) {
3621 if (SrcZero.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3622 KnownZero.setBit(i);
3623 if (SrcUndef.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3624 KnownUndef.setBit(i);
3625 }
3626 }
3627 }
3628 break;
3629 }
3630 case ISD::FREEZE: {
3631 SDValue N0 = Op.getOperand(i: 0);
3632 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
3633 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
3634 return TLO.CombineTo(O: Op, N: N0);
3635
3636 // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE
3637 // freeze(op(x, ...)) -> op(freeze(x), ...).
3638 // Don't sink the freeze below SCALAR_TO_VECTOR when the scalar is a load
3639 // of a promoted (wider than the element) type: freeze(load) can never be
3640 // folded away (the loaded value may be poison in memory), and the extra
3641 // freeze node then blocks ISel patterns matching scalar_to_vector of a
3642 // load, e.g. the AArch64 scalar_to_vector(extload) -> ldr b/h forms.
3643 // freeze(scalar_to_vector(load)) is equivalent for the demanded element
3644 // zero, and ISel selects the freeze as a plain copy.
3645 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1) {
3646 SDValue Scalar = N0.getOperand(i: 0);
3647 bool IsPromotedLoad = Scalar.getOpcode() == ISD::LOAD &&
3648 Scalar.getValueType() != VT.getVectorElementType();
3649 if (!IsPromotedLoad)
3650 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT,
3651 Operand: TLO.DAG.getFreeze(V: Scalar)));
3652 }
3653 break;
3654 }
3655 case ISD::BUILD_VECTOR: {
3656 // Check all elements and simplify any unused elements with UNDEF.
3657 if (!DemandedElts.isAllOnes()) {
3658 // Don't simplify BROADCASTS.
3659 if (llvm::any_of(Range: Op->op_values(),
3660 P: [&](SDValue Elt) { return Op.getOperand(i: 0) != Elt; })) {
3661 SmallVector<SDValue, 32> Ops(Op->ops());
3662 bool Updated = false;
3663 for (unsigned i = 0; i != NumElts; ++i) {
3664 if (!DemandedElts[i] && !Ops[i].isUndef()) {
3665 Ops[i] = TLO.DAG.getUNDEF(VT: Ops[0].getValueType());
3666 KnownUndef.setBit(i);
3667 Updated = true;
3668 }
3669 }
3670 if (Updated)
3671 return TLO.CombineTo(O: Op, N: TLO.DAG.getBuildVector(VT, DL, Ops));
3672 }
3673 }
3674 for (unsigned i = 0; i != NumElts; ++i) {
3675 SDValue SrcOp = Op.getOperand(i);
3676 if (SrcOp.isUndef()) {
3677 KnownUndef.setBit(i);
3678 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
3679 (isNullConstant(V: SrcOp) || isNullFPConstant(V: SrcOp))) {
3680 KnownZero.setBit(i);
3681 }
3682 }
3683 break;
3684 }
3685 case ISD::CONCAT_VECTORS: {
3686 EVT SubVT = Op.getOperand(i: 0).getValueType();
3687 unsigned NumSubVecs = Op.getNumOperands();
3688 unsigned NumSubElts = SubVT.getVectorNumElements();
3689 for (unsigned i = 0; i != NumSubVecs; ++i) {
3690 SDValue SubOp = Op.getOperand(i);
3691 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3692 APInt SubUndef, SubZero;
3693 if (SimplifyDemandedVectorElts(Op: SubOp, OriginalDemandedElts: SubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3694 Depth: Depth + 1))
3695 return true;
3696 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: i * NumSubElts);
3697 KnownZero.insertBits(SubBits: SubZero, bitPosition: i * NumSubElts);
3698 }
3699
3700 // Attempt to avoid multi-use ops if we don't need anything from them.
3701 if (!DemandedElts.isAllOnes()) {
3702 bool FoundNewSub = false;
3703 SmallVector<SDValue, 2> DemandedSubOps;
3704 for (unsigned i = 0; i != NumSubVecs; ++i) {
3705 SDValue SubOp = Op.getOperand(i);
3706 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3707 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3708 Op: SubOp, DemandedElts: SubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3709 DemandedSubOps.push_back(Elt: NewSubOp ? NewSubOp : SubOp);
3710 FoundNewSub = NewSubOp ? true : FoundNewSub;
3711 }
3712 if (FoundNewSub) {
3713 SDValue NewOp =
3714 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, Ops: DemandedSubOps);
3715 return TLO.CombineTo(O: Op, N: NewOp);
3716 }
3717 }
3718 break;
3719 }
3720 case ISD::INSERT_SUBVECTOR: {
3721 // Demand any elements from the subvector and the remainder from the src it
3722 // is inserted into.
3723 SDValue Src = Op.getOperand(i: 0);
3724 SDValue Sub = Op.getOperand(i: 1);
3725 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3726 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3727 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3728 APInt DemandedSrcElts = DemandedElts;
3729 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3730
3731 // If none of the sub operand elements are demanded, bypass the insert.
3732 if (!DemandedSubElts)
3733 return TLO.CombineTo(O: Op, N: Src);
3734
3735 APInt SubUndef, SubZero;
3736 if (SimplifyDemandedVectorElts(Op: Sub, OriginalDemandedElts: DemandedSubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3737 Depth: Depth + 1))
3738 return true;
3739
3740 // If none of the src operand elements are demanded, replace it with undef.
3741 if (!DemandedSrcElts && !Src.isUndef())
3742 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT,
3743 N1: TLO.DAG.getUNDEF(VT), N2: Sub,
3744 N3: Op.getOperand(i: 2)));
3745
3746 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef, KnownZero,
3747 TLO, Depth: Depth + 1))
3748 return true;
3749 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: Idx);
3750 KnownZero.insertBits(SubBits: SubZero, bitPosition: Idx);
3751
3752 // Attempt to avoid multi-use ops if we don't need anything from them.
3753 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
3754 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3755 Op: Src, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
3756 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3757 Op: Sub, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3758 if (NewSrc || NewSub) {
3759 NewSrc = NewSrc ? NewSrc : Src;
3760 NewSub = NewSub ? NewSub : Sub;
3761 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3762 N2: NewSub, N3: Op.getOperand(i: 2));
3763 return TLO.CombineTo(O: Op, N: NewOp);
3764 }
3765 }
3766 break;
3767 }
3768 case ISD::EXTRACT_SUBVECTOR: {
3769 // Offset the demanded elts by the subvector index.
3770 SDValue Src = Op.getOperand(i: 0);
3771 if (Src.getValueType().isScalableVector())
3772 break;
3773 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3774 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3775 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3776
3777 APInt SrcUndef, SrcZero;
3778 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3779 Depth: Depth + 1))
3780 return true;
3781 KnownUndef = SrcUndef.extractBits(numBits: NumElts, bitPosition: Idx);
3782 KnownZero = SrcZero.extractBits(numBits: NumElts, bitPosition: Idx);
3783
3784 // Attempt to avoid multi-use ops if we don't need anything from them.
3785 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(Op: Src, DemandedElts: DemandedSrcElts,
3786 DAG&: TLO.DAG, Depth: Depth + 1);
3787 if (NewSrc) {
3788 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3789 N2: Op.getOperand(i: 1));
3790 return TLO.CombineTo(O: Op, N: NewOp);
3791 }
3792 break;
3793 }
3794 case ISD::INSERT_VECTOR_ELT: {
3795 SDValue Vec = Op.getOperand(i: 0);
3796 SDValue Scl = Op.getOperand(i: 1);
3797 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
3798
3799 // For a legal, constant insertion index, if we don't need this insertion
3800 // then strip it, else remove it from the demanded elts.
3801 if (CIdx && CIdx->getAPIntValue().ult(RHS: NumElts)) {
3802 unsigned Idx = CIdx->getZExtValue();
3803 if (!DemandedElts[Idx])
3804 return TLO.CombineTo(O: Op, N: Vec);
3805
3806 APInt DemandedVecElts(DemandedElts);
3807 DemandedVecElts.clearBit(BitPosition: Idx);
3808 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedVecElts, KnownUndef,
3809 KnownZero, TLO, Depth: Depth + 1))
3810 return true;
3811
3812 KnownUndef.setBitVal(BitPosition: Idx, BitValue: Scl.isUndef());
3813
3814 KnownZero.setBitVal(BitPosition: Idx, BitValue: isNullConstant(V: Scl) || isNullFPConstant(V: Scl));
3815 break;
3816 }
3817
3818 APInt VecUndef, VecZero;
3819 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedElts, KnownUndef&: VecUndef, KnownZero&: VecZero, TLO,
3820 Depth: Depth + 1))
3821 return true;
3822 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
3823 break;
3824 }
3825 case ISD::VSELECT: {
3826 SDValue Sel = Op.getOperand(i: 0);
3827 SDValue LHS = Op.getOperand(i: 1);
3828 SDValue RHS = Op.getOperand(i: 2);
3829
3830 // Try to transform the select condition based on the current demanded
3831 // elements.
3832 APInt UndefSel, ZeroSel;
3833 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3834 Depth: Depth + 1))
3835 return true;
3836
3837 // See if we can simplify either vselect operand.
3838 APInt DemandedLHS(DemandedElts);
3839 APInt DemandedRHS(DemandedElts);
3840 APInt UndefLHS, ZeroLHS;
3841 APInt UndefRHS, ZeroRHS;
3842 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3843 Depth: Depth + 1))
3844 return true;
3845 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3846 Depth: Depth + 1))
3847 return true;
3848
3849 KnownUndef = UndefLHS & UndefRHS;
3850 KnownZero = ZeroLHS & ZeroRHS;
3851
3852 // If we know that the selected element is always zero, we don't need the
3853 // select value element.
3854 APInt DemandedSel = DemandedElts & ~KnownZero;
3855 if (DemandedSel != DemandedElts)
3856 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedSel, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3857 Depth: Depth + 1))
3858 return true;
3859
3860 break;
3861 }
3862 case ISD::VECTOR_SHUFFLE: {
3863 SDValue LHS = Op.getOperand(i: 0);
3864 SDValue RHS = Op.getOperand(i: 1);
3865 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
3866
3867 // Collect demanded elements from shuffle operands..
3868 APInt DemandedLHS(NumElts, 0);
3869 APInt DemandedRHS(NumElts, 0);
3870 for (unsigned i = 0; i != NumElts; ++i) {
3871 int M = ShuffleMask[i];
3872 if (M < 0 || !DemandedElts[i])
3873 continue;
3874 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
3875 if (M < (int)NumElts)
3876 DemandedLHS.setBit(M);
3877 else
3878 DemandedRHS.setBit(M - NumElts);
3879 }
3880
3881 // If either side isn't demanded, replace it by UNDEF. We handle this
3882 // explicitly here to also simplify in case of multiple uses (on the
3883 // contrary to the SimplifyDemandedVectorElts calls below).
3884 bool FoldLHS = !DemandedLHS && !LHS.isUndef();
3885 bool FoldRHS = !DemandedRHS && !RHS.isUndef();
3886 if (FoldLHS || FoldRHS) {
3887 LHS = FoldLHS ? TLO.DAG.getUNDEF(VT: LHS.getValueType()) : LHS;
3888 RHS = FoldRHS ? TLO.DAG.getUNDEF(VT: RHS.getValueType()) : RHS;
3889 SDValue NewOp =
3890 TLO.DAG.getVectorShuffle(VT, dl: SDLoc(Op), N1: LHS, N2: RHS, Mask: ShuffleMask);
3891 return TLO.CombineTo(O: Op, N: NewOp);
3892 }
3893
3894 // See if we can simplify either shuffle operand.
3895 APInt UndefLHS, ZeroLHS;
3896 APInt UndefRHS, ZeroRHS;
3897 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3898 Depth: Depth + 1))
3899 return true;
3900 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3901 Depth: Depth + 1))
3902 return true;
3903
3904 // Simplify mask using undef elements from LHS/RHS.
3905 bool Updated = false;
3906 bool IdentityLHS = true, IdentityRHS = true;
3907 SmallVector<int, 32> NewMask(ShuffleMask);
3908 for (unsigned i = 0; i != NumElts; ++i) {
3909 int &M = NewMask[i];
3910 if (M < 0)
3911 continue;
3912 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
3913 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
3914 Updated = true;
3915 M = -1;
3916 }
3917 IdentityLHS &= (M < 0) || (M == (int)i);
3918 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3919 }
3920
3921 // Update legal shuffle masks based on demanded elements if it won't reduce
3922 // to Identity which can cause premature removal of the shuffle mask.
3923 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
3924 SDValue LegalShuffle =
3925 buildLegalVectorShuffle(VT, DL, N0: LHS, N1: RHS, Mask: NewMask, DAG&: TLO.DAG);
3926 if (LegalShuffle)
3927 return TLO.CombineTo(O: Op, N: LegalShuffle);
3928 }
3929
3930 // Propagate undef/zero elements from LHS/RHS.
3931 for (unsigned i = 0; i != NumElts; ++i) {
3932 int M = ShuffleMask[i];
3933 if (M < 0) {
3934 KnownUndef.setBit(i);
3935 } else if (M < (int)NumElts) {
3936 if (UndefLHS[M])
3937 KnownUndef.setBit(i);
3938 if (ZeroLHS[M])
3939 KnownZero.setBit(i);
3940 } else {
3941 if (UndefRHS[M - NumElts])
3942 KnownUndef.setBit(i);
3943 if (ZeroRHS[M - NumElts])
3944 KnownZero.setBit(i);
3945 }
3946 }
3947 break;
3948 }
3949 case ISD::ANY_EXTEND_VECTOR_INREG:
3950 case ISD::SIGN_EXTEND_VECTOR_INREG:
3951 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3952 APInt SrcUndef, SrcZero;
3953 SDValue Src = Op.getOperand(i: 0);
3954 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3955 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3956 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3957 Depth: Depth + 1))
3958 return true;
3959 KnownZero = SrcZero.zextOrTrunc(width: NumElts);
3960 KnownUndef = SrcUndef.zextOrTrunc(width: NumElts);
3961
3962 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
3963 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3964 DemandedSrcElts == 1) {
3965 // aext - if we just need the bottom element then we can bitcast.
3966 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
3967 }
3968
3969 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
3970 // zext(undef) upper bits are guaranteed to be zero.
3971 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
3972 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3973 KnownUndef.clearAllBits();
3974
3975 // zext - if we just need the bottom element then we can mask:
3976 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
3977 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
3978 Op->isOnlyUserOf(N: Src.getNode()) &&
3979 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3980 SDLoc DL(Op);
3981 EVT SrcVT = Src.getValueType();
3982 EVT SrcSVT = SrcVT.getScalarType();
3983
3984 // If we're after type legalization and SrcSVT is not legal, use the
3985 // promoted type for creating constants to avoid creating nodes with
3986 // illegal types.
3987 if (TLO.LegalTypes())
3988 SrcSVT = getLegalTypeToTransformTo(Context&: *TLO.DAG.getContext(), VT: SrcSVT);
3989
3990 SmallVector<SDValue> MaskElts;
3991 MaskElts.push_back(Elt: TLO.DAG.getAllOnesConstant(DL, VT: SrcSVT));
3992 MaskElts.append(NumInputs: NumSrcElts - 1, Elt: TLO.DAG.getConstant(Val: 0, DL, VT: SrcSVT));
3993 SDValue Mask = TLO.DAG.getBuildVector(VT: SrcVT, DL, Ops: MaskElts);
3994 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
3995 Opcode: ISD::AND, DL, VT: SrcVT, Ops: {Src.getOperand(i: 1), Mask})) {
3996 Fold = TLO.DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src.getOperand(i: 0), N2: Fold);
3997 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Fold));
3998 }
3999 }
4000 }
4001 break;
4002 }
4003
4004 // TODO: There are more binop opcodes that could be handled here - MIN,
4005 // MAX, saturated math, etc.
4006 case ISD::ADD: {
4007 SDValue Op0 = Op.getOperand(i: 0);
4008 SDValue Op1 = Op.getOperand(i: 1);
4009 if (Op0 == Op1 && Op->isOnlyUserOf(N: Op0.getNode())) {
4010 APInt UndefLHS, ZeroLHS;
4011 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4012 Depth: Depth + 1, /*AssumeSingleUse*/ true))
4013 return true;
4014 }
4015 [[fallthrough]];
4016 }
4017 case ISD::AVGCEILS:
4018 case ISD::AVGCEILU:
4019 case ISD::AVGFLOORS:
4020 case ISD::AVGFLOORU:
4021 case ISD::AVGSUBFLOORS:
4022 case ISD::AVGSUBFLOORU:
4023 case ISD::OR:
4024 case ISD::XOR:
4025 case ISD::SUB:
4026 case ISD::FADD:
4027 case ISD::FSUB:
4028 case ISD::FMUL:
4029 case ISD::FDIV:
4030 case ISD::FREM:
4031 case ISD::PSEUDO_FMIN:
4032 case ISD::PSEUDO_FMAX: {
4033 SDValue Op0 = Op.getOperand(i: 0);
4034 SDValue Op1 = Op.getOperand(i: 1);
4035
4036 APInt UndefRHS, ZeroRHS;
4037 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
4038 Depth: Depth + 1))
4039 return true;
4040 APInt UndefLHS, ZeroLHS;
4041 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4042 Depth: Depth + 1))
4043 return true;
4044
4045 KnownZero = ZeroLHS & ZeroRHS;
4046 KnownUndef = getKnownUndefForVectorBinop(BO: Op, DAG&: TLO.DAG, UndefOp0: UndefLHS, UndefOp1: UndefRHS);
4047
4048 // Attempt to avoid multi-use ops if we don't need anything from them.
4049 // TODO - use KnownUndef to relax the demandedelts?
4050 if (!DemandedElts.isAllOnes())
4051 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4052 return true;
4053 break;
4054 }
4055 case ISD::SHL:
4056 case ISD::SRL:
4057 case ISD::SRA:
4058 case ISD::ROTL:
4059 case ISD::ROTR: {
4060 SDValue Op0 = Op.getOperand(i: 0);
4061 SDValue Op1 = Op.getOperand(i: 1);
4062
4063 APInt UndefRHS, ZeroRHS;
4064 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
4065 Depth: Depth + 1))
4066 return true;
4067 APInt UndefLHS, ZeroLHS;
4068 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4069 Depth: Depth + 1))
4070 return true;
4071
4072 KnownZero = ZeroLHS & ~UndefRHS;
4073 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
4074
4075 // Attempt to avoid multi-use ops if we don't need anything from them.
4076 // TODO - use KnownUndef to relax the demandedelts?
4077 if (!DemandedElts.isAllOnes())
4078 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4079 return true;
4080 break;
4081 }
4082 case ISD::MUL:
4083 case ISD::MULHU:
4084 case ISD::MULHS:
4085 case ISD::AND: {
4086 SDValue Op0 = Op.getOperand(i: 0);
4087 SDValue Op1 = Op.getOperand(i: 1);
4088
4089 APInt SrcUndef, SrcZero;
4090 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
4091 Depth: Depth + 1))
4092 return true;
4093 // FIXME: If we know that a demanded element was zero in Op1 we don't need
4094 // to demand it in Op0 - its guaranteed to be zero. There is however a
4095 // restriction, as we must not make any of the originally demanded elements
4096 // more poisonous. We could reduce amount of elements demanded, but then we
4097 // also need a to inform SimplifyDemandedVectorElts that some elements must
4098 // not be made more poisonous.
4099 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef, KnownZero,
4100 TLO, Depth: Depth + 1))
4101 return true;
4102
4103 KnownUndef &= DemandedElts;
4104 KnownZero &= DemandedElts;
4105
4106 // If every element pair has a zero/undef/poison then just fold to zero.
4107 // fold (and x, undef/poison) -> 0 / (and x, 0) -> 0
4108 // fold (mul x, undef/poison) -> 0 / (mul x, 0) -> 0
4109 if (DemandedElts.isSubsetOf(RHS: SrcZero | KnownZero | SrcUndef | KnownUndef))
4110 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
4111
4112 // If either side has a zero element, then the result element is zero, even
4113 // if the other is an UNDEF.
4114 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
4115 // and then handle 'and' nodes with the rest of the binop opcodes.
4116 KnownZero |= SrcZero;
4117 KnownUndef &= SrcUndef;
4118 KnownUndef &= ~KnownZero;
4119
4120 // Attempt to avoid multi-use ops if we don't need anything from them.
4121 if (!DemandedElts.isAllOnes())
4122 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4123 return true;
4124 break;
4125 }
4126 case ISD::TRUNCATE:
4127 case ISD::SIGN_EXTEND:
4128 case ISD::ZERO_EXTEND:
4129 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
4130 KnownZero, TLO, Depth: Depth + 1))
4131 return true;
4132
4133 if (!DemandedElts.isAllOnes())
4134 if (SDValue NewOp = SimplifyMultipleUseDemandedVectorElts(
4135 Op: Op.getOperand(i: 0), DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
4136 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, Operand: NewOp));
4137
4138 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
4139 // zext(undef) upper bits are guaranteed to be zero.
4140 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4141 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
4142 KnownUndef.clearAllBits();
4143 }
4144 break;
4145 case ISD::SINT_TO_FP:
4146 case ISD::UINT_TO_FP:
4147 case ISD::FP_TO_SINT:
4148 case ISD::FP_TO_UINT:
4149 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
4150 KnownZero, TLO, Depth: Depth + 1))
4151 return true;
4152 // Don't fall through to generic undef -> undef handling.
4153 return false;
4154 default: {
4155 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
4156 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
4157 KnownZero, TLO, Depth))
4158 return true;
4159 } else {
4160 KnownBits Known;
4161 APInt DemandedBits = APInt::getAllOnes(numBits: EltSizeInBits);
4162 if (SimplifyDemandedBits(Op, OriginalDemandedBits: DemandedBits, OriginalDemandedElts, Known,
4163 TLO, Depth, AssumeSingleUse))
4164 return true;
4165 }
4166 break;
4167 }
4168 }
4169
4170 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
4171
4172 // Constant fold all undef cases.
4173 // TODO: Handle zero cases as well.
4174 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4175 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
4176
4177 return false;
4178}
4179
4180/// Determine which of the bits specified in Mask are known to be either zero or
4181/// one and return them in the Known.
4182void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
4183 KnownBits &Known,
4184 const APInt &DemandedElts,
4185 const SelectionDAG &DAG,
4186 unsigned Depth) const {
4187 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4188 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4189 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4190 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4191 "Should use MaskedValueIsZero if you don't know whether Op"
4192 " is a target node!");
4193 Known.resetAll();
4194}
4195
4196void TargetLowering::computeKnownBitsForTargetInstr(
4197 GISelValueTracking &Analysis, Register R, KnownBits &Known,
4198 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4199 unsigned Depth) const {
4200 Known.resetAll();
4201}
4202
4203void TargetLowering::computeKnownFPClassForTargetInstr(
4204 GISelValueTracking &Analysis, Register R, KnownFPClass &Known,
4205 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4206 unsigned Depth) const {
4207 Known.resetAll();
4208}
4209
4210void TargetLowering::computeKnownBitsForStackObjectPointer(
4211 KnownBits &Known, const MachineFunction &, Align Alignment) const {
4212 // The low bits are known zero if the pointer is aligned.
4213 Known.Zero.setLowBits(Log2(A: Alignment));
4214}
4215
4216SDValue TargetLowering::annotateStackObjectPointer(SDValue Ptr,
4217 SelectionDAG &DAG,
4218 const SDLoc &DL,
4219 Align Alignment) const {
4220 // Materialize leading-zero stack object pointer facts as AssertZext.
4221 // Alignment-derived low zero bits are not represented on the returned DAG
4222 // value here.
4223 EVT PtrVT = Ptr.getValueType();
4224
4225 unsigned RegSize = PtrVT.getScalarSizeInBits();
4226 KnownBits Known(RegSize);
4227 computeKnownBitsForStackObjectPointer(Known, DAG.getMachineFunction(),
4228 Alignment);
4229
4230 unsigned NumZeroBits = Known.countMinLeadingZeros();
4231 if (!NumZeroBits)
4232 return Ptr;
4233
4234 EVT FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
4235 return DAG.getNode(Opcode: ISD::AssertZext, DL, VT: PtrVT, N1: Ptr, N2: DAG.getValueType(FromVT));
4236}
4237
4238Align TargetLowering::computeKnownAlignForTargetInstr(
4239 GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI,
4240 unsigned Depth) const {
4241 return Align(1);
4242}
4243
4244/// This method can be implemented by targets that want to expose additional
4245/// information about sign bits to the DAG Combiner.
4246unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
4247 const APInt &,
4248 const SelectionDAG &,
4249 unsigned Depth) const {
4250 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4251 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4252 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4253 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4254 "Should use ComputeNumSignBits if you don't know whether Op"
4255 " is a target node!");
4256 return 1;
4257}
4258
4259unsigned TargetLowering::computeNumSignBitsForTargetInstr(
4260 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4261 const MachineRegisterInfo &MRI, unsigned Depth) const {
4262 return 1;
4263}
4264
4265bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
4266 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
4267 TargetLoweringOpt &TLO, unsigned Depth) const {
4268 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4269 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4270 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4271 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4272 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
4273 " is a target node!");
4274 return false;
4275}
4276
4277bool TargetLowering::SimplifyDemandedBitsForTargetNode(
4278 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4279 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
4280 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4281 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4282 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4283 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4284 "Should use SimplifyDemandedBits if you don't know whether Op"
4285 " is a target node!");
4286 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
4287 return false;
4288}
4289
4290SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
4291 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4292 SelectionDAG &DAG, unsigned Depth) const {
4293 assert(
4294 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4295 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4296 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4297 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4298 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
4299 " is a target node!");
4300 return SDValue();
4301}
4302
4303SDValue
4304TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
4305 SDValue N1, MutableArrayRef<int> Mask,
4306 SelectionDAG &DAG) const {
4307 bool LegalMask = isShuffleMaskLegal(Mask, VT);
4308 if (!LegalMask) {
4309 std::swap(a&: N0, b&: N1);
4310 ShuffleVectorSDNode::commuteMask(Mask);
4311 LegalMask = isShuffleMaskLegal(Mask, VT);
4312 }
4313
4314 if (!LegalMask)
4315 return SDValue();
4316
4317 return DAG.getVectorShuffle(VT, dl: DL, N1: N0, N2: N1, Mask);
4318}
4319
4320const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
4321 return nullptr;
4322}
4323
4324bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4325 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4326 UndefPoisonKind Kind, unsigned Depth) const {
4327 assert(
4328 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4329 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4330 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4331 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4332 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
4333 " is a target node!");
4334
4335 // If Op can't create undef/poison and none of its operands are undef/poison
4336 // then Op is never undef/poison.
4337 return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, Kind,
4338 /*ConsiderFlags*/ true, Depth) &&
4339 all_of(Range: Op->ops(), P: [&](SDValue V) {
4340 return DAG.isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
4341 });
4342}
4343
4344bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
4345 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4346 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
4347 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4348 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4349 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4350 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4351 "Should use canCreateUndefOrPoison if you don't know whether Op"
4352 " is a target node!");
4353 // Be conservative and return true.
4354 return true;
4355}
4356
4357void TargetLowering::computeKnownFPClassForTargetNode(const SDValue Op,
4358 KnownFPClass &Known,
4359 const APInt &DemandedElts,
4360 const SelectionDAG &DAG,
4361 unsigned Depth) const {
4362 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4363 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4364 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4365 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4366 "Should use computeKnownFPClass if you don't know whether Op"
4367 " is a target node!");
4368}
4369
4370bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
4371 const APInt &DemandedElts,
4372 const SelectionDAG &DAG,
4373 bool SNaN,
4374 unsigned Depth) const {
4375 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4376 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4377 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4378 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4379 "Should use isKnownNeverNaN if you don't know whether Op"
4380 " is a target node!");
4381 return false;
4382}
4383
4384bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
4385 const APInt &DemandedElts,
4386 APInt &UndefElts,
4387 const SelectionDAG &DAG,
4388 unsigned Depth) const {
4389 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4390 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4391 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4392 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4393 "Should use isSplatValue if you don't know whether Op"
4394 " is a target node!");
4395 return false;
4396}
4397
4398// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
4399// work with truncating build vectors and vectors with elements of less than
4400// 8 bits.
4401bool TargetLowering::isConstTrueVal(SDValue N) const {
4402 if (!N)
4403 return false;
4404
4405 unsigned EltWidth;
4406 APInt CVal;
4407 if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
4408 /*AllowTruncation=*/true)) {
4409 CVal = CN->getAPIntValue();
4410 EltWidth = N.getValueType().getScalarSizeInBits();
4411 } else
4412 return false;
4413
4414 // If this is a truncating splat, truncate the splat value.
4415 // Otherwise, we may fail to match the expected values below.
4416 if (EltWidth < CVal.getBitWidth())
4417 CVal = CVal.trunc(width: EltWidth);
4418
4419 switch (getBooleanContents(Type: N.getValueType())) {
4420 case UndefinedBooleanContent:
4421 return CVal[0];
4422 case ZeroOrOneBooleanContent:
4423 return CVal.isOne();
4424 case ZeroOrNegativeOneBooleanContent:
4425 return CVal.isAllOnes();
4426 }
4427
4428 llvm_unreachable("Invalid boolean contents");
4429}
4430
4431bool TargetLowering::isConstFalseVal(SDValue N) const {
4432 if (!N)
4433 return false;
4434
4435 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N);
4436 if (!CN) {
4437 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
4438 if (!BV)
4439 return false;
4440
4441 // Only interested in constant splats, we don't care about undef
4442 // elements in identifying boolean constants and getConstantSplatNode
4443 // returns NULL if all ops are undef;
4444 CN = BV->getConstantSplatNode();
4445 if (!CN)
4446 return false;
4447 }
4448
4449 if (getBooleanContents(Type: N->getValueType(ResNo: 0)) == UndefinedBooleanContent)
4450 return !CN->getAPIntValue()[0];
4451
4452 return CN->isZero();
4453}
4454
4455bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
4456 bool SExt) const {
4457 if (VT == MVT::i1)
4458 return N->isOne();
4459
4460 TargetLowering::BooleanContent Cnt = getBooleanContents(Type: VT);
4461 switch (Cnt) {
4462 case TargetLowering::ZeroOrOneBooleanContent:
4463 // An extended value of 1 is always true, unless its original type is i1,
4464 // in which case it will be sign extended to -1.
4465 return (N->isOne() && !SExt) || (SExt && (N->getValueType(ResNo: 0) != MVT::i1));
4466 case TargetLowering::UndefinedBooleanContent:
4467 case TargetLowering::ZeroOrNegativeOneBooleanContent:
4468 return N->isAllOnes() && SExt;
4469 }
4470 llvm_unreachable("Unexpected enumeration.");
4471}
4472
4473/// This helper function of SimplifySetCC tries to optimize the comparison when
4474/// either operand of the SetCC node is a bitwise-and instruction.
4475SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
4476 ISD::CondCode Cond, const SDLoc &DL,
4477 DAGCombinerInfo &DCI) const {
4478 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
4479 std::swap(a&: N0, b&: N1);
4480
4481 SelectionDAG &DAG = DCI.DAG;
4482 EVT OpVT = N0.getValueType();
4483 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
4484 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4485 return SDValue();
4486
4487 // (X & Y) != 0 --> zextOrTrunc(X & Y)
4488 // iff everything but LSB is known zero:
4489 if (Cond == ISD::SETNE && isNullConstant(V: N1) &&
4490 (getBooleanContents(Type: OpVT) == TargetLowering::UndefinedBooleanContent ||
4491 getBooleanContents(Type: OpVT) == TargetLowering::ZeroOrOneBooleanContent)) {
4492 unsigned NumEltBits = OpVT.getScalarSizeInBits();
4493 APInt UpperBits = APInt::getHighBitsSet(numBits: NumEltBits, hiBitsSet: NumEltBits - 1);
4494 if (DAG.MaskedValueIsZero(Op: N0, Mask: UpperBits))
4495 return DAG.getBoolExtOrTrunc(Op: N0, SL: DL, VT, OpVT);
4496 }
4497
4498 // Try to eliminate a power-of-2 mask constant by converting to a signbit
4499 // test in a narrow type that we can truncate to with no cost. Examples:
4500 // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
4501 // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
4502 // TODO: This conservatively checks for type legality on the source and
4503 // destination types. That may inhibit optimizations, but it also
4504 // allows setcc->shift transforms that may be more beneficial.
4505 auto *AndC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
4506 if (AndC && isNullConstant(V: N1) && AndC->getAPIntValue().isPowerOf2() &&
4507 isTypeLegal(VT: OpVT) && N0.hasOneUse()) {
4508 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
4509 BitWidth: AndC->getAPIntValue().getActiveBits());
4510 if (isTruncateFree(FromVT: OpVT, ToVT: NarrowVT) && isTypeLegal(VT: NarrowVT)) {
4511 SDValue Trunc = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 0), DL, VT: NarrowVT);
4512 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: NarrowVT);
4513 return DAG.getSetCC(DL, VT, LHS: Trunc, RHS: Zero,
4514 Cond: Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT);
4515 }
4516 }
4517
4518 // Match these patterns in any of their permutations:
4519 // (X & Y) == Y
4520 // (X & Y) != Y
4521 SDValue X, Y;
4522 if (N0.getOperand(i: 0) == N1) {
4523 X = N0.getOperand(i: 1);
4524 Y = N0.getOperand(i: 0);
4525 } else if (N0.getOperand(i: 1) == N1) {
4526 X = N0.getOperand(i: 0);
4527 Y = N0.getOperand(i: 1);
4528 } else {
4529 return SDValue();
4530 }
4531
4532 // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
4533 // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
4534 // its liable to create and infinite loop.
4535 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: OpVT);
4536 if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
4537 DAG.isKnownToBeAPowerOfTwo(Val: Y)) {
4538 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
4539 // Note that where Y is variable and is known to have at most one bit set
4540 // (for example, if it is Z & 1) we cannot do this; the expressions are not
4541 // equivalent when Y == 0.
4542 assert(OpVT.isInteger());
4543 Cond = ISD::getSetCCInverse(Operation: Cond, Type: OpVT);
4544 if (DCI.isBeforeLegalizeOps() ||
4545 isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType()))
4546 return DAG.getSetCC(DL, VT, LHS: N0, RHS: Zero, Cond);
4547 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
4548 // If the target supports an 'and-not' or 'and-complement' logic operation,
4549 // try to use that to make a comparison operation more efficient.
4550 // But don't do this transform if the mask is a single bit because there are
4551 // more efficient ways to deal with that case (for example, 'bt' on x86 or
4552 // 'rlwinm' on PPC).
4553
4554 // Bail out if the compare operand that we want to turn into a zero is
4555 // already a zero (otherwise, infinite loop).
4556 if (isNullConstant(V: Y))
4557 return SDValue();
4558
4559 // Transform this into: ~X & Y == 0.
4560 SDValue NotX = DAG.getNOT(DL: SDLoc(X), Val: X, VT: OpVT);
4561 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: NotX, N2: Y);
4562 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: Zero, Cond);
4563 }
4564
4565 return SDValue();
4566}
4567
4568/// This helper function of SimplifySetCC tries to optimize the comparison when
4569/// either operand of the SetCC node is a bitwise-or instruction.
4570/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
4571SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
4572 ISD::CondCode Cond, const SDLoc &DL,
4573 DAGCombinerInfo &DCI) const {
4574 if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
4575 std::swap(a&: N0, b&: N1);
4576
4577 SelectionDAG &DAG = DCI.DAG;
4578 EVT OpVT = N0.getValueType();
4579 if (!N0.hasOneUse() || !OpVT.isInteger() ||
4580 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4581 return SDValue();
4582
4583 // (X | Y) == Y
4584 // (X | Y) != Y
4585 SDValue X;
4586 if (sd_match(N: N0, P: m_Or(L: m_Value(N&: X), R: m_Specific(N: N1))) && hasAndNotCompare(Y: X)) {
4587 // If the target supports an 'and-not' or 'and-complement' logic operation,
4588 // try to use that to make a comparison operation more efficient.
4589
4590 // Bail out if the compare operand that we want to turn into a zero is
4591 // already a zero (otherwise, infinite loop).
4592 if (isNullConstant(V: N1))
4593 return SDValue();
4594
4595 // Transform this into: X & ~Y ==/!= 0.
4596 SDValue NotY = DAG.getNOT(DL: SDLoc(N1), Val: N1, VT: OpVT);
4597 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: X, N2: NotY);
4598 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4599 }
4600
4601 return SDValue();
4602}
4603
4604/// There are multiple IR patterns that could be checking whether certain
4605/// truncation of a signed number would be lossy or not. The pattern which is
4606/// best at IR level, may not lower optimally. Thus, we want to unfold it.
4607/// We are looking for the following pattern: (KeptBits is a constant)
4608/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
4609/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
4610/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
4611/// We will unfold it into the natural trunc+sext pattern:
4612/// ((%x << C) a>> C) dstcond %x
4613/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
4614SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4615 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
4616 const SDLoc &DL) const {
4617 // We must be comparing with a constant.
4618 ConstantSDNode *C1;
4619 if (!(C1 = dyn_cast<ConstantSDNode>(Val&: N1)))
4620 return SDValue();
4621
4622 // N0 should be: add %x, (1 << (KeptBits-1))
4623 if (N0->getOpcode() != ISD::ADD)
4624 return SDValue();
4625
4626 // And we must be 'add'ing a constant.
4627 ConstantSDNode *C01;
4628 if (!(C01 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1))))
4629 return SDValue();
4630
4631 SDValue X = N0->getOperand(Num: 0);
4632 EVT XVT = X.getValueType();
4633
4634 // Validate constants ...
4635
4636 APInt I1 = C1->getAPIntValue();
4637
4638 ISD::CondCode NewCond;
4639 if (Cond == ISD::CondCode::SETULT) {
4640 NewCond = ISD::CondCode::SETEQ;
4641 } else if (Cond == ISD::CondCode::SETULE) {
4642 NewCond = ISD::CondCode::SETEQ;
4643 // But need to 'canonicalize' the constant.
4644 I1 += 1;
4645 } else if (Cond == ISD::CondCode::SETUGT) {
4646 NewCond = ISD::CondCode::SETNE;
4647 // But need to 'canonicalize' the constant.
4648 I1 += 1;
4649 } else if (Cond == ISD::CondCode::SETUGE) {
4650 NewCond = ISD::CondCode::SETNE;
4651 } else
4652 return SDValue();
4653
4654 APInt I01 = C01->getAPIntValue();
4655
4656 auto checkConstants = [&I1, &I01]() -> bool {
4657 // Both of them must be power-of-two, and the constant from setcc is bigger.
4658 return I1.ugt(RHS: I01) && I1.isPowerOf2() && I01.isPowerOf2();
4659 };
4660
4661 if (checkConstants()) {
4662 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
4663 } else {
4664 // What if we invert constants? (and the target predicate)
4665 I1.negate();
4666 I01.negate();
4667 assert(XVT.isInteger());
4668 NewCond = getSetCCInverse(Operation: NewCond, Type: XVT);
4669 if (!checkConstants())
4670 return SDValue();
4671 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
4672 }
4673
4674 // They are power-of-two, so which bit is set?
4675 const unsigned KeptBits = I1.logBase2();
4676 const unsigned KeptBitsMinusOne = I01.logBase2();
4677
4678 // Magic!
4679 if (KeptBits != (KeptBitsMinusOne + 1))
4680 return SDValue();
4681 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
4682
4683 // We don't want to do this in every single case.
4684 SelectionDAG &DAG = DCI.DAG;
4685 if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
4686 return SDValue();
4687
4688 // Unfold into: sext_inreg(%x) cond %x
4689 // Where 'cond' will be either 'eq' or 'ne'.
4690 SDValue SExtInReg = DAG.getNode(
4691 Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: XVT, N1: X,
4692 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: KeptBits)));
4693 return DAG.getSetCC(DL, VT: SCCVT, LHS: SExtInReg, RHS: X, Cond: NewCond);
4694}
4695
4696// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4697SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4698 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4699 DAGCombinerInfo &DCI, const SDLoc &DL) const {
4700 assert(isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->isZero() &&
4701 "Should be a comparison with 0.");
4702 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4703 "Valid only for [in]equality comparisons.");
4704
4705 unsigned NewShiftOpcode;
4706 SDValue X, C, Y;
4707
4708 SelectionDAG &DAG = DCI.DAG;
4709
4710 // Look for '(C l>>/<< Y)'.
4711 auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
4712 // The shift should be one-use.
4713 if (!V.hasOneUse())
4714 return false;
4715 unsigned OldShiftOpcode = V.getOpcode();
4716 switch (OldShiftOpcode) {
4717 case ISD::SHL:
4718 NewShiftOpcode = ISD::SRL;
4719 break;
4720 case ISD::SRL:
4721 NewShiftOpcode = ISD::SHL;
4722 break;
4723 default:
4724 return false; // must be a logical shift.
4725 }
4726 // We should be shifting a constant.
4727 // FIXME: best to use isConstantOrConstantVector().
4728 C = V.getOperand(i: 0);
4729 ConstantSDNode *CC =
4730 isConstOrConstSplat(N: C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4731 if (!CC)
4732 return false;
4733 Y = V.getOperand(i: 1);
4734
4735 ConstantSDNode *XC =
4736 isConstOrConstSplat(N: X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4737 return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
4738 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
4739 };
4740
4741 // LHS of comparison should be an one-use 'and'.
4742 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
4743 return SDValue();
4744
4745 X = N0.getOperand(i: 0);
4746 SDValue Mask = N0.getOperand(i: 1);
4747
4748 // 'and' is commutative!
4749 if (!Match(Mask)) {
4750 std::swap(a&: X, b&: Mask);
4751 if (!Match(Mask))
4752 return SDValue();
4753 }
4754
4755 EVT VT = X.getValueType();
4756
4757 // Produce:
4758 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
4759 SDValue T0 = DAG.getNode(Opcode: NewShiftOpcode, DL, VT, N1: X, N2: Y);
4760 SDValue T1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T0, N2: C);
4761 SDValue T2 = DAG.getSetCC(DL, VT: SCCVT, LHS: T1, RHS: N1C, Cond);
4762 return T2;
4763}
4764
4765/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
4766/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
4767/// handle the commuted versions of these patterns.
4768SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
4769 ISD::CondCode Cond, const SDLoc &DL,
4770 DAGCombinerInfo &DCI) const {
4771 unsigned BOpcode = N0.getOpcode();
4772 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
4773 "Unexpected binop");
4774 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
4775
4776 // (X + Y) == X --> Y == 0
4777 // (X - Y) == X --> Y == 0
4778 // (X ^ Y) == X --> Y == 0
4779 SelectionDAG &DAG = DCI.DAG;
4780 EVT OpVT = N0.getValueType();
4781 SDValue X = N0.getOperand(i: 0);
4782 SDValue Y = N0.getOperand(i: 1);
4783 if (X == N1)
4784 return DAG.getSetCC(DL, VT, LHS: Y, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4785
4786 if (Y != N1)
4787 return SDValue();
4788
4789 // (X + Y) == Y --> X == 0
4790 // (X ^ Y) == Y --> X == 0
4791 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
4792 return DAG.getSetCC(DL, VT, LHS: X, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4793
4794 // The shift would not be valid if the operands are boolean (i1).
4795 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
4796 return SDValue();
4797
4798 // (X - Y) == Y --> X == Y << 1
4799 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: OpVT, DL);
4800 SDValue YShl1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: N1.getValueType(), N1: Y, N2: One);
4801 if (!DCI.isCalledByLegalizer())
4802 DCI.AddToWorklist(N: YShl1.getNode());
4803 return DAG.getSetCC(DL, VT, LHS: X, RHS: YShl1, Cond);
4804}
4805
4806static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
4807 SDValue N0, const APInt &C1,
4808 ISD::CondCode Cond, const SDLoc &dl,
4809 SelectionDAG &DAG) {
4810 // Look through truncs that don't change the value of a ctpop.
4811 // FIXME: Add vector support? Need to be careful with setcc result type below.
4812 SDValue CTPOP = N0;
4813 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
4814 N0.getScalarValueSizeInBits() > Log2_32(Value: N0.getOperand(i: 0).getScalarValueSizeInBits()))
4815 CTPOP = N0.getOperand(i: 0);
4816
4817 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
4818 return SDValue();
4819
4820 EVT CTVT = CTPOP.getValueType();
4821 SDValue CTOp = CTPOP.getOperand(i: 0);
4822
4823 // Expand a power-of-2-or-zero comparison based on ctpop:
4824 // (ctpop x) u< 2 -> (x & x-1) == 0
4825 // (ctpop x) u> 1 -> (x & x-1) != 0
4826 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
4827 // Keep the CTPOP if it is a cheap vector op.
4828 if (CTVT.isVector() && TLI.isCtpopFast(VT: CTVT))
4829 return SDValue();
4830
4831 unsigned CostLimit = TLI.getCustomCtpopCost(VT: CTVT, Cond);
4832 if (C1.ugt(RHS: CostLimit + (Cond == ISD::SETULT)))
4833 return SDValue();
4834 if (C1 == 0 && (Cond == ISD::SETULT))
4835 return SDValue(); // This is handled elsewhere.
4836
4837 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
4838
4839 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4840 SDValue Result = CTOp;
4841 for (unsigned i = 0; i < Passes; i++) {
4842 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: Result, N2: NegOne);
4843 Result = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: Result, N2: Add);
4844 }
4845 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
4846 return DAG.getSetCC(DL: dl, VT, LHS: Result, RHS: DAG.getConstant(Val: 0, DL: dl, VT: CTVT), Cond: CC);
4847 }
4848
4849 // Expand a power-of-2 comparison based on ctpop
4850 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
4851 // Keep the CTPOP if it is cheap.
4852 if (TLI.isCtpopFast(VT: CTVT))
4853 return SDValue();
4854
4855 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: CTVT);
4856 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4857 assert(CTVT.isInteger());
4858 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: CTOp, N2: NegOne);
4859
4860 // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
4861 // check before emitting a potentially unnecessary op.
4862 if (DAG.isKnownNeverZero(Op: CTOp)) {
4863 // (ctpop x) == 1 --> (x & x-1) == 0
4864 // (ctpop x) != 1 --> (x & x-1) != 0
4865 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4866 SDValue RHS = DAG.getSetCC(DL: dl, VT, LHS: And, RHS: Zero, Cond);
4867 return RHS;
4868 }
4869
4870 // (ctpop x) == 1 --> (x ^ x-1) > x-1
4871 // (ctpop x) != 1 --> (x ^ x-1) <= x-1
4872 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4873 ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE;
4874 return DAG.getSetCC(DL: dl, VT, LHS: Xor, RHS: Add, Cond: CmpCond);
4875 }
4876
4877 return SDValue();
4878}
4879
4880static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1,
4881 ISD::CondCode Cond, const SDLoc &dl,
4882 SelectionDAG &DAG) {
4883 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4884 return SDValue();
4885
4886 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4887 if (!C1 || !(C1->isZero() || C1->isAllOnes()))
4888 return SDValue();
4889
4890 auto getRotateSource = [](SDValue X) {
4891 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
4892 return X.getOperand(i: 0);
4893 return SDValue();
4894 };
4895
4896 // Peek through a rotated value compared against 0 or -1:
4897 // (rot X, Y) == 0/-1 --> X == 0/-1
4898 // (rot X, Y) != 0/-1 --> X != 0/-1
4899 if (SDValue R = getRotateSource(N0))
4900 return DAG.getSetCC(DL: dl, VT, LHS: R, RHS: N1, Cond);
4901
4902 // Peek through an 'or' of a rotated value compared against 0:
4903 // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
4904 // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
4905 //
4906 // TODO: Add the 'and' with -1 sibling.
4907 // TODO: Recurse through a series of 'or' ops to find the rotate.
4908 EVT OpVT = N0.getValueType();
4909 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
4910 if (SDValue R = getRotateSource(N0.getOperand(i: 0))) {
4911 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 1));
4912 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4913 }
4914 if (SDValue R = getRotateSource(N0.getOperand(i: 1))) {
4915 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 0));
4916 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4917 }
4918 }
4919
4920 return SDValue();
4921}
4922
4923static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1,
4924 ISD::CondCode Cond, const SDLoc &dl,
4925 SelectionDAG &DAG) {
4926 // If we are testing for all-bits-clear, we might be able to do that with
4927 // less shifting since bit-order does not matter.
4928 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4929 return SDValue();
4930
4931 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4932 if (!C1 || !C1->isZero())
4933 return SDValue();
4934
4935 if (!N0.hasOneUse() ||
4936 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
4937 return SDValue();
4938
4939 unsigned BitWidth = N0.getScalarValueSizeInBits();
4940 auto *ShAmtC = isConstOrConstSplat(N: N0.getOperand(i: 2));
4941 if (!ShAmtC)
4942 return SDValue();
4943
4944 uint64_t ShAmt = ShAmtC->getAPIntValue().urem(RHS: BitWidth);
4945 if (ShAmt == 0)
4946 return SDValue();
4947
4948 // Canonicalize fshr as fshl to reduce pattern-matching.
4949 if (N0.getOpcode() == ISD::FSHR)
4950 ShAmt = BitWidth - ShAmt;
4951
4952 // Match an 'or' with a specific operand 'Other' in either commuted variant.
4953 SDValue X, Y;
4954 auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
4955 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
4956 return false;
4957 if (Or.getOperand(i: 0) == Other) {
4958 X = Or.getOperand(i: 0);
4959 Y = Or.getOperand(i: 1);
4960 return true;
4961 }
4962 if (Or.getOperand(i: 1) == Other) {
4963 X = Or.getOperand(i: 1);
4964 Y = Or.getOperand(i: 0);
4965 return true;
4966 }
4967 return false;
4968 };
4969
4970 EVT OpVT = N0.getValueType();
4971 EVT ShAmtVT = N0.getOperand(i: 2).getValueType();
4972 SDValue F0 = N0.getOperand(i: 0);
4973 SDValue F1 = N0.getOperand(i: 1);
4974 if (matchOr(F0, F1)) {
4975 // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
4976 SDValue NewShAmt = DAG.getConstant(Val: ShAmt, DL: dl, VT: ShAmtVT);
4977 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4978 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4979 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4980 }
4981 if (matchOr(F1, F0)) {
4982 // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
4983 SDValue NewShAmt = DAG.getConstant(Val: BitWidth - ShAmt, DL: dl, VT: ShAmtVT);
4984 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4985 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4986 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4987 }
4988
4989 return SDValue();
4990}
4991
4992/// Try to simplify a setcc built with the specified operands and cc. If it is
4993/// unable to simplify it, return a null SDValue.
4994SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
4995 ISD::CondCode Cond, bool foldBooleans,
4996 DAGCombinerInfo &DCI,
4997 const SDLoc &dl) const {
4998 SelectionDAG &DAG = DCI.DAG;
4999 const DataLayout &Layout = DAG.getDataLayout();
5000 EVT OpVT = N0.getValueType();
5001 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5002
5003 // Constant fold or commute setcc.
5004 if (SDValue Fold = DAG.FoldSetCC(VT, N1: N0, N2: N1, Cond, dl))
5005 return Fold;
5006
5007 bool N0ConstOrSplat =
5008 isConstOrConstSplat(N: N0, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
5009 bool N1ConstOrSplat =
5010 isConstOrConstSplat(N: N1, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
5011
5012 // Canonicalize toward having the constant on the RHS.
5013 // TODO: Handle non-splat vector constants. All undef causes trouble.
5014 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
5015 // infinite loop here when we encounter one.
5016 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Operation: Cond);
5017 if (N0ConstOrSplat && !N1ConstOrSplat &&
5018 (DCI.isBeforeLegalizeOps() ||
5019 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())))
5020 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
5021
5022 // If we have a subtract with the same 2 non-constant operands as this setcc
5023 // -- but in reverse order -- then try to commute the operands of this setcc
5024 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
5025 // instruction on some targets.
5026 if (!N0ConstOrSplat && !N1ConstOrSplat &&
5027 (DCI.isBeforeLegalizeOps() ||
5028 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())) &&
5029 DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N1, N0}) &&
5030 !DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N0, N1}))
5031 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
5032
5033 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
5034 return V;
5035
5036 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
5037 return V;
5038
5039 if (auto *N1C = isConstOrConstSplat(N: N1)) {
5040 const APInt &C1 = N1C->getAPIntValue();
5041
5042 // Optimize some CTPOP cases.
5043 if (SDValue V = simplifySetCCWithCTPOP(TLI: *this, VT, N0, C1, Cond, dl, DAG))
5044 return V;
5045
5046 // For equality to 0 of a no-wrap multiply, decompose and test each op:
5047 // X * Y == 0 --> (X == 0) || (Y == 0)
5048 // X * Y != 0 --> (X != 0) && (Y != 0)
5049 // TODO: This bails out if minsize is set, but if the target doesn't have a
5050 // single instruction multiply for this type, it would likely be
5051 // smaller to decompose.
5052 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5053 N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
5054 (N0->getFlags().hasNoUnsignedWrap() ||
5055 N0->getFlags().hasNoSignedWrap()) &&
5056 !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
5057 SDValue IsXZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
5058 SDValue IsYZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1, Cond);
5059 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
5060 return DAG.getNode(Opcode: LogicOp, DL: dl, VT, N1: IsXZero, N2: IsYZero);
5061 }
5062
5063 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
5064 // equality comparison, then we're just comparing whether X itself is
5065 // zero.
5066 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
5067 N0.getOperand(i: 0).getOpcode() == ISD::CTLZ &&
5068 llvm::has_single_bit<uint32_t>(Value: N0.getScalarValueSizeInBits())) {
5069 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
5070 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5071 ShAmt->getAPIntValue() == Log2_32(Value: N0.getScalarValueSizeInBits())) {
5072 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
5073 // (srl (ctlz x), 5) == 0 -> X != 0
5074 // (srl (ctlz x), 5) != 1 -> X != 0
5075 Cond = ISD::SETNE;
5076 } else {
5077 // (srl (ctlz x), 5) != 0 -> X == 0
5078 // (srl (ctlz x), 5) == 1 -> X == 0
5079 Cond = ISD::SETEQ;
5080 }
5081 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: N0.getValueType());
5082 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0).getOperand(i: 0), RHS: Zero,
5083 Cond);
5084 }
5085 }
5086 }
5087 }
5088
5089 // setcc X, 0, setlt --> X (when X is all sign bits)
5090 // setcc X, 0, setne --> X (when X is all sign bits)
5091 //
5092 // When we know that X has 0 or -1 in each element (or scalar), this
5093 // comparison will produce X. This is only true when boolean contents are
5094 // represented via 0s and -1s.
5095 if (VT == OpVT &&
5096 // Check that the result of setcc is 0 and -1.
5097 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent &&
5098 // Match only for checks X < 0 and X != 0
5099 (Cond == ISD::SETLT || Cond == ISD::SETNE) && isNullOrNullSplat(V: N1) &&
5100 // The identity holds iff we know all sign bits for all lanes.
5101 DAG.ComputeNumSignBits(Op: N0) == N0.getScalarValueSizeInBits())
5102 return N0;
5103
5104 // FIXME: Support vectors.
5105 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5106 const APInt &C1 = N1C->getAPIntValue();
5107
5108 // (zext x) == C --> x == (trunc C)
5109 // (sext x) == C --> x == (trunc C)
5110 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5111 DCI.isBeforeLegalize() && N0->hasOneUse()) {
5112 unsigned MinBits = N0.getValueSizeInBits();
5113 SDValue PreExt;
5114 bool Signed = false;
5115 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
5116 // ZExt
5117 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
5118 PreExt = N0->getOperand(Num: 0);
5119 } else if (N0->getOpcode() == ISD::AND) {
5120 // DAGCombine turns costly ZExts into ANDs
5121 if (auto *C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
5122 if ((C->getAPIntValue()+1).isPowerOf2()) {
5123 MinBits = C->getAPIntValue().countr_one();
5124 PreExt = N0->getOperand(Num: 0);
5125 }
5126 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
5127 // SExt
5128 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
5129 PreExt = N0->getOperand(Num: 0);
5130 Signed = true;
5131 } else if (auto *LN0 = dyn_cast<LoadSDNode>(Val&: N0)) {
5132 // ZEXTLOAD / SEXTLOAD
5133 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
5134 MinBits = LN0->getMemoryVT().getSizeInBits();
5135 PreExt = N0;
5136 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
5137 Signed = true;
5138 MinBits = LN0->getMemoryVT().getSizeInBits();
5139 PreExt = N0;
5140 }
5141 }
5142
5143 // Figure out how many bits we need to preserve this constant.
5144 unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();
5145
5146 // Make sure we're not losing bits from the constant.
5147 if (MinBits > 0 &&
5148 MinBits < C1.getBitWidth() &&
5149 MinBits >= ReqdBits) {
5150 EVT MinVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MinBits);
5151 if (isTypeDesirableForOp(ISD::SETCC, VT: MinVT)) {
5152 // Will get folded away.
5153 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MinVT, Operand: PreExt);
5154 if (MinBits == 1 && C1 == 1)
5155 // Invert the condition.
5156 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i1),
5157 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5158 SDValue C = DAG.getConstant(Val: C1.trunc(width: MinBits), DL: dl, VT: MinVT);
5159 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: C, Cond);
5160 }
5161
5162 // If truncating the setcc operands is not desirable, we can still
5163 // simplify the expression in some cases:
5164 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
5165 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
5166 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
5167 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
5168 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
5169 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
5170 SDValue TopSetCC = N0->getOperand(Num: 0);
5171 unsigned N0Opc = N0->getOpcode();
5172 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
5173 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
5174 TopSetCC.getOpcode() == ISD::SETCC &&
5175 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
5176 (isConstFalseVal(N: N1) ||
5177 isExtendedTrueVal(N: N1C, VT: N0->getValueType(ResNo: 0), SExt))) {
5178
5179 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
5180 (!N1C->isZero() && Cond == ISD::SETNE);
5181
5182 if (!Inverse)
5183 return TopSetCC;
5184
5185 ISD::CondCode InvCond = ISD::getSetCCInverse(
5186 Operation: cast<CondCodeSDNode>(Val: TopSetCC.getOperand(i: 2))->get(),
5187 Type: TopSetCC.getOperand(i: 0).getValueType());
5188 return DAG.getSetCC(DL: dl, VT, LHS: TopSetCC.getOperand(i: 0),
5189 RHS: TopSetCC.getOperand(i: 1),
5190 Cond: InvCond);
5191 }
5192 }
5193 }
5194
5195 // If the LHS is '(and load, const)', the RHS is 0, the test is for
5196 // equality or unsigned, and all 1 bits of the const are in the same
5197 // partial word, see if we can shorten the load.
5198 if (DCI.isBeforeLegalize() &&
5199 !ISD::isSignedIntSetCC(Code: Cond) &&
5200 N0.getOpcode() == ISD::AND && C1 == 0 &&
5201 N0.getNode()->hasOneUse() &&
5202 isa<LoadSDNode>(Val: N0.getOperand(i: 0)) &&
5203 N0.getOperand(i: 0).getNode()->hasOneUse() &&
5204 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5205 auto *Lod = cast<LoadSDNode>(Val: N0.getOperand(i: 0));
5206 APInt bestMask;
5207 unsigned bestWidth = 0, bestOffset = 0;
5208 if (Lod->isSimple() && Lod->isUnindexed() &&
5209 (Lod->getMemoryVT().isByteSized() ||
5210 isPaddedAtMostSignificantBitsWhenStored(VT: Lod->getMemoryVT()))) {
5211 unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
5212 unsigned origWidth = N0.getValueSizeInBits();
5213 unsigned maskWidth = origWidth;
5214 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
5215 // 8 bits, but have to be careful...
5216 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
5217 origWidth = Lod->getMemoryVT().getSizeInBits();
5218 const APInt &Mask = N0.getConstantOperandAPInt(i: 1);
5219 // Only consider power-of-2 widths (and at least one byte) as candiates
5220 // for the narrowed load.
5221 for (unsigned width = 8; width < origWidth; width *= 2) {
5222 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: width);
5223 APInt newMask = APInt::getLowBitsSet(numBits: maskWidth, loBitsSet: width);
5224 // Avoid accessing any padding here for now (we could use memWidth
5225 // instead of origWidth here otherwise).
5226 unsigned maxOffset = origWidth - width;
5227 for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
5228 if (Mask.isSubsetOf(RHS: newMask)) {
5229 unsigned ptrOffset =
5230 Layout.isLittleEndian() ? offset : memWidth - width - offset;
5231 unsigned IsFast = 0;
5232 assert((ptrOffset % 8) == 0 && "Non-Bytealigned pointer offset");
5233 Align NewAlign = commonAlignment(A: Lod->getAlign(), Offset: ptrOffset / 8);
5234 if (shouldReduceLoadWidth(Load: Lod, ExtTy: ISD::NON_EXTLOAD, NewVT: newVT,
5235 ByteOffset: ptrOffset / 8) &&
5236 allowsMemoryAccess(
5237 Context&: *DAG.getContext(), DL: Layout, VT: newVT, AddrSpace: Lod->getAddressSpace(),
5238 Alignment: NewAlign, Flags: Lod->getMemOperand()->getFlags(), Fast: &IsFast) &&
5239 IsFast) {
5240 bestOffset = ptrOffset / 8;
5241 bestMask = Mask.lshr(shiftAmt: offset);
5242 bestWidth = width;
5243 break;
5244 }
5245 }
5246 newMask <<= 8;
5247 }
5248 if (bestWidth)
5249 break;
5250 }
5251 }
5252 if (bestWidth) {
5253 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: bestWidth);
5254 SDValue Ptr = Lod->getBasePtr();
5255 if (bestOffset != 0)
5256 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: bestOffset));
5257 SDValue NewLoad =
5258 DAG.getLoad(VT: newVT, dl, Chain: Lod->getChain(), Ptr,
5259 PtrInfo: Lod->getPointerInfo().getWithOffset(O: bestOffset),
5260 Alignment: Lod->getBaseAlign(), MMOFlags: MachineMemOperand::MONone,
5261 Metadata: Lod->getMMOMetadataForSubAccess());
5262 SDValue And =
5263 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: newVT, N1: NewLoad,
5264 N2: DAG.getConstant(Val: bestMask.trunc(width: bestWidth), DL: dl, VT: newVT));
5265 return DAG.getSetCC(DL: dl, VT, LHS: And, RHS: DAG.getConstant(Val: 0LL, DL: dl, VT: newVT), Cond);
5266 }
5267 }
5268
5269 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
5270 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
5271 unsigned InSize = N0.getOperand(i: 0).getValueSizeInBits();
5272
5273 // If the comparison constant has bits in the upper part, the
5274 // zero-extended value could never match.
5275 if (C1.intersects(RHS: APInt::getHighBitsSet(numBits: C1.getBitWidth(),
5276 hiBitsSet: C1.getBitWidth() - InSize))) {
5277 switch (Cond) {
5278 case ISD::SETUGT:
5279 case ISD::SETUGE:
5280 case ISD::SETEQ:
5281 return DAG.getConstant(Val: 0, DL: dl, VT);
5282 case ISD::SETULT:
5283 case ISD::SETULE:
5284 case ISD::SETNE:
5285 return DAG.getConstant(Val: 1, DL: dl, VT);
5286 case ISD::SETGT:
5287 case ISD::SETGE:
5288 // True if the sign bit of C1 is set.
5289 return DAG.getConstant(Val: C1.isNegative(), DL: dl, VT);
5290 case ISD::SETLT:
5291 case ISD::SETLE:
5292 // True if the sign bit of C1 isn't set.
5293 return DAG.getConstant(Val: C1.isNonNegative(), DL: dl, VT);
5294 default:
5295 break;
5296 }
5297 }
5298
5299 // Otherwise, we can perform the comparison with the low bits.
5300 switch (Cond) {
5301 case ISD::SETEQ:
5302 case ISD::SETNE:
5303 case ISD::SETUGT:
5304 case ISD::SETUGE:
5305 case ISD::SETULT:
5306 case ISD::SETULE: {
5307 EVT newVT = N0.getOperand(i: 0).getValueType();
5308 // FIXME: Should use isNarrowingProfitable.
5309 if (DCI.isBeforeLegalizeOps() ||
5310 (isOperationLegal(Op: ISD::SETCC, VT: newVT) &&
5311 isCondCodeLegal(CC: Cond, VT: newVT.getSimpleVT()) &&
5312 isTypeDesirableForOp(ISD::SETCC, VT: newVT))) {
5313 EVT NewSetCCVT = getSetCCResultType(DL: Layout, Context&: *DAG.getContext(), VT: newVT);
5314 SDValue NewConst = DAG.getConstant(Val: C1.trunc(width: InSize), DL: dl, VT: newVT);
5315
5316 SDValue NewSetCC = DAG.getSetCC(DL: dl, VT: NewSetCCVT, LHS: N0.getOperand(i: 0),
5317 RHS: NewConst, Cond);
5318 return DAG.getBoolExtOrTrunc(Op: NewSetCC, SL: dl, VT, OpVT: N0.getValueType());
5319 }
5320 break;
5321 }
5322 default:
5323 break; // todo, be more careful with signed comparisons
5324 }
5325 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5326 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5327 !isSExtCheaperThanZExt(FromTy: cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT(),
5328 ToTy: OpVT)) {
5329 EVT ExtSrcTy = cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT();
5330 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
5331 EVT ExtDstTy = N0.getValueType();
5332 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
5333
5334 // If the constant doesn't fit into the number of bits for the source of
5335 // the sign extension, it is impossible for both sides to be equal.
5336 if (C1.getSignificantBits() > ExtSrcTyBits)
5337 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
5338
5339 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
5340 ExtDstTy != ExtSrcTy && "Unexpected types!");
5341 APInt Imm = APInt::getLowBitsSet(numBits: ExtDstTyBits, loBitsSet: ExtSrcTyBits);
5342 SDValue ZextOp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ExtDstTy, N1: N0.getOperand(i: 0),
5343 N2: DAG.getConstant(Val: Imm, DL: dl, VT: ExtDstTy));
5344 if (!DCI.isCalledByLegalizer())
5345 DCI.AddToWorklist(N: ZextOp.getNode());
5346 // Otherwise, make this a use of a zext.
5347 return DAG.getSetCC(DL: dl, VT, LHS: ZextOp,
5348 RHS: DAG.getConstant(Val: C1 & Imm, DL: dl, VT: ExtDstTy), Cond);
5349 } else if ((N1C->isZero() || N1C->isOne()) &&
5350 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5351 // SETCC (X), [0|1], [EQ|NE] -> X if X is known 0/1. i1 types are
5352 // excluded as they are handled below whilst checking for foldBooleans.
5353 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
5354 isTypeLegal(VT) && VT.bitsLE(VT: N0.getValueType()) &&
5355 (N0.getValueType() == MVT::i1 ||
5356 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5357 DAG.MaskedValueIsZero(
5358 Op: N0, Mask: APInt::getBitsSetFrom(numBits: N0.getValueSizeInBits(), loBit: 1))) {
5359 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
5360 if (TrueWhenTrue)
5361 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: N0);
5362 // Invert the condition.
5363 if (N0.getOpcode() == ISD::SETCC) {
5364 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
5365 CC = ISD::getSetCCInverse(Operation: CC, Type: N0.getOperand(i: 0).getValueType());
5366 if (DCI.isBeforeLegalizeOps() ||
5367 isCondCodeLegal(CC, VT: N0.getOperand(i: 0).getSimpleValueType()))
5368 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond: CC);
5369 }
5370 }
5371
5372 if ((N0.getOpcode() == ISD::XOR ||
5373 (N0.getOpcode() == ISD::AND &&
5374 N0.getOperand(i: 0).getOpcode() == ISD::XOR &&
5375 N0.getOperand(i: 1) == N0.getOperand(i: 0).getOperand(i: 1))) &&
5376 isOneConstant(V: N0.getOperand(i: 1))) {
5377 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
5378 // can only do this if the top bits are known zero.
5379 unsigned BitWidth = N0.getValueSizeInBits();
5380 if (DAG.MaskedValueIsZero(Op: N0,
5381 Mask: APInt::getHighBitsSet(numBits: BitWidth,
5382 hiBitsSet: BitWidth-1))) {
5383 // Okay, get the un-inverted input value.
5384 SDValue Val;
5385 if (N0.getOpcode() == ISD::XOR) {
5386 Val = N0.getOperand(i: 0);
5387 } else {
5388 assert(N0.getOpcode() == ISD::AND &&
5389 N0.getOperand(0).getOpcode() == ISD::XOR);
5390 // ((X^1)&1)^1 -> X & 1
5391 Val = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N0.getValueType(),
5392 N1: N0.getOperand(i: 0).getOperand(i: 0),
5393 N2: N0.getOperand(i: 1));
5394 }
5395
5396 return DAG.getSetCC(DL: dl, VT, LHS: Val, RHS: N1,
5397 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5398 }
5399 } else if (N1C->isOne()) {
5400 SDValue Op0 = N0;
5401 if (Op0.getOpcode() == ISD::TRUNCATE)
5402 Op0 = Op0.getOperand(i: 0);
5403
5404 if ((Op0.getOpcode() == ISD::XOR) &&
5405 Op0.getOperand(i: 0).getOpcode() == ISD::SETCC &&
5406 Op0.getOperand(i: 1).getOpcode() == ISD::SETCC) {
5407 SDValue XorLHS = Op0.getOperand(i: 0);
5408 SDValue XorRHS = Op0.getOperand(i: 1);
5409 // Ensure that the input setccs return an i1 type or 0/1 value.
5410 if (Op0.getValueType() == MVT::i1 ||
5411 (getBooleanContents(Type: XorLHS.getOperand(i: 0).getValueType()) ==
5412 ZeroOrOneBooleanContent &&
5413 getBooleanContents(Type: XorRHS.getOperand(i: 0).getValueType()) ==
5414 ZeroOrOneBooleanContent)) {
5415 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
5416 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
5417 return DAG.getSetCC(DL: dl, VT, LHS: XorLHS, RHS: XorRHS, Cond);
5418 }
5419 }
5420 if (Op0.getOpcode() == ISD::AND && isOneConstant(V: Op0.getOperand(i: 1))) {
5421 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
5422 if (Op0.getValueType().bitsGT(VT))
5423 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5424 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5425 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5426 else if (Op0.getValueType().bitsLT(VT))
5427 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5428 N1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5429 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5430
5431 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5432 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5433 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5434 }
5435 if (Op0.getOpcode() == ISD::AssertZext &&
5436 cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT() == MVT::i1)
5437 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5438 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5439 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5440 }
5441 }
5442
5443 // Given:
5444 // icmp eq/ne (urem %x, %y), 0
5445 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
5446 // icmp eq/ne %x, 0
5447 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
5448 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5449 KnownBits XKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 0));
5450 KnownBits YKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 1));
5451 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
5452 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
5453 }
5454
5455 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
5456 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
5457 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5458 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
5459 N0.getConstantOperandAPInt(i: 1) == OpVT.getScalarSizeInBits() - 1 &&
5460 N1C->isAllOnes()) {
5461 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
5462 RHS: DAG.getConstant(Val: 0, DL: dl, VT: OpVT),
5463 Cond: Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
5464 }
5465
5466 // fold (setcc (trunc x) c) -> (setcc x c)
5467 if (N0.getOpcode() == ISD::TRUNCATE &&
5468 ((N0->getFlags().hasNoUnsignedWrap() && !ISD::isSignedIntSetCC(Code: Cond)) ||
5469 (N0->getFlags().hasNoSignedWrap() &&
5470 !ISD::isUnsignedIntSetCC(Code: Cond))) &&
5471 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5472 EVT NewVT = N0.getOperand(i: 0).getValueType();
5473 SDValue NewConst = DAG.getConstant(
5474 Val: (N0->getFlags().hasNoSignedWrap() && !ISD::isUnsignedIntSetCC(Code: Cond))
5475 ? C1.sext(width: NewVT.getSizeInBits())
5476 : C1.zext(width: NewVT.getSizeInBits()),
5477 DL: dl, VT: NewVT);
5478 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NewConst, Cond);
5479 }
5480
5481 if (SDValue V =
5482 optimizeSetCCOfSignedTruncationCheck(SCCVT: VT, N0, N1, Cond, DCI, DL: dl))
5483 return V;
5484 }
5485
5486 // These simplifications apply to splat vectors as well.
5487 // TODO: Handle more splat vector cases.
5488 if (auto *N1C = isConstOrConstSplat(N: N1)) {
5489 const APInt &C1 = N1C->getAPIntValue();
5490
5491 APInt MinVal, MaxVal;
5492 unsigned OperandBitSize = N1C->getValueType(ResNo: 0).getScalarSizeInBits();
5493 if (ISD::isSignedIntSetCC(Code: Cond)) {
5494 MinVal = APInt::getSignedMinValue(numBits: OperandBitSize);
5495 MaxVal = APInt::getSignedMaxValue(numBits: OperandBitSize);
5496 } else {
5497 MinVal = APInt::getMinValue(numBits: OperandBitSize);
5498 MaxVal = APInt::getMaxValue(numBits: OperandBitSize);
5499 }
5500
5501 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
5502 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
5503 // X >= MIN --> true
5504 if (C1 == MinVal)
5505 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5506
5507 if (!VT.isVector()) { // TODO: Support this for vectors.
5508 // X >= C0 --> X > (C0 - 1)
5509 APInt C = C1 - 1;
5510 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
5511 if ((DCI.isBeforeLegalizeOps() ||
5512 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5513 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5514 isLegalICmpImmediate(C.getSExtValue())))) {
5515 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5516 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5517 Cond: NewCC);
5518 }
5519 }
5520 }
5521
5522 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
5523 // X <= MAX --> true
5524 if (C1 == MaxVal)
5525 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5526
5527 // X <= C0 --> X < (C0 + 1)
5528 if (!VT.isVector()) { // TODO: Support this for vectors.
5529 APInt C = C1 + 1;
5530 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
5531 if ((DCI.isBeforeLegalizeOps() ||
5532 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5533 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5534 isLegalICmpImmediate(C.getSExtValue())))) {
5535 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5536 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5537 Cond: NewCC);
5538 }
5539 }
5540 }
5541
5542 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
5543 if (C1 == MinVal)
5544 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X < MIN --> false
5545
5546 // TODO: Support this for vectors after legalize ops.
5547 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5548 // Canonicalize setlt X, Max --> setne X, Max
5549 if (C1 == MaxVal)
5550 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5551
5552 // If we have setult X, 1, turn it into seteq X, 0
5553 if (C1 == MinVal+1)
5554 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5555 RHS: DAG.getConstant(Val: MinVal, DL: dl, VT: N0.getValueType()),
5556 Cond: ISD::SETEQ);
5557 }
5558 }
5559
5560 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
5561 if (C1 == MaxVal)
5562 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X > MAX --> false
5563
5564 // TODO: Support this for vectors after legalize ops.
5565 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5566 // Canonicalize setgt X, Min --> setne X, Min
5567 if (C1 == MinVal)
5568 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5569
5570 // If we have setugt X, Max-1, turn it into seteq X, Max
5571 if (C1 == MaxVal-1)
5572 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5573 RHS: DAG.getConstant(Val: MaxVal, DL: dl, VT: N0.getValueType()),
5574 Cond: ISD::SETEQ);
5575 }
5576 }
5577
5578 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
5579 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
5580 if (C1.isZero())
5581 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
5582 SCCVT: VT, N0, N1C: N1, Cond, DCI, DL: dl))
5583 return CC;
5584
5585 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
5586 // For example, when high 32-bits of i64 X are known clear:
5587 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
5588 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
5589 bool CmpZero = N1C->isZero();
5590 bool CmpNegOne = N1C->isAllOnes();
5591 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
5592 // Match or(lo,shl(hi,bw/2)) pattern.
5593 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
5594 unsigned EltBits = V.getScalarValueSizeInBits();
5595 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
5596 return false;
5597 SDValue LHS = V.getOperand(i: 0);
5598 SDValue RHS = V.getOperand(i: 1);
5599 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: EltBits / 2);
5600 // Unshifted element must have zero upperbits.
5601 if (RHS.getOpcode() == ISD::SHL &&
5602 isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)) &&
5603 RHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5604 DAG.MaskedValueIsZero(Op: LHS, Mask: HiBits)) {
5605 Lo = LHS;
5606 Hi = RHS.getOperand(i: 0);
5607 return true;
5608 }
5609 if (LHS.getOpcode() == ISD::SHL &&
5610 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
5611 LHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5612 DAG.MaskedValueIsZero(Op: RHS, Mask: HiBits)) {
5613 Lo = RHS;
5614 Hi = LHS.getOperand(i: 0);
5615 return true;
5616 }
5617 return false;
5618 };
5619
5620 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
5621 unsigned EltBits = N0.getScalarValueSizeInBits();
5622 unsigned HalfBits = EltBits / 2;
5623 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: HalfBits);
5624 SDValue LoBits = DAG.getConstant(Val: ~HiBits, DL: dl, VT: OpVT);
5625 SDValue HiMask = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Hi, N2: LoBits);
5626 SDValue NewN0 =
5627 DAG.getNode(Opcode: CmpZero ? ISD::OR : ISD::AND, DL: dl, VT: OpVT, N1: Lo, N2: HiMask);
5628 SDValue NewN1 = CmpZero ? DAG.getConstant(Val: 0, DL: dl, VT: OpVT) : LoBits;
5629 return DAG.getSetCC(DL: dl, VT, LHS: NewN0, RHS: NewN1, Cond);
5630 };
5631
5632 SDValue Lo, Hi;
5633 if (IsConcat(N0, Lo, Hi))
5634 return MergeConcat(Lo, Hi);
5635
5636 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
5637 SDValue Lo0, Lo1, Hi0, Hi1;
5638 if (IsConcat(N0.getOperand(i: 0), Lo0, Hi0) &&
5639 IsConcat(N0.getOperand(i: 1), Lo1, Hi1)) {
5640 return MergeConcat(DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Lo0, N2: Lo1),
5641 DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Hi0, N2: Hi1));
5642 }
5643 }
5644 }
5645 }
5646
5647 // If we have "setcc X, C0", check to see if we can shrink the immediate
5648 // by changing cc.
5649 // TODO: Support this for vectors after legalize ops.
5650 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5651 // SETUGT X, SINTMAX -> SETLT X, 0
5652 // SETUGE X, SINTMIN -> SETLT X, 0
5653 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
5654 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
5655 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5656 RHS: DAG.getConstant(Val: 0, DL: dl, VT: N1.getValueType()),
5657 Cond: ISD::SETLT);
5658
5659 // SETULT X, SINTMIN -> SETGT X, -1
5660 // SETULE X, SINTMAX -> SETGT X, -1
5661 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
5662 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
5663 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5664 RHS: DAG.getAllOnesConstant(DL: dl, VT: N1.getValueType()),
5665 Cond: ISD::SETGT);
5666 }
5667 }
5668
5669 // Back to non-vector simplifications.
5670 // TODO: Can we do these for vector splats?
5671 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5672 const APInt &C1 = N1C->getAPIntValue();
5673 EVT ShValTy = N0.getValueType();
5674
5675 // Fold bit comparisons when we can. This will result in an
5676 // incorrect value when boolean false is negative one, unless
5677 // the bitsize is 1 in which case the false value is the same
5678 // in practice regardless of the representation.
5679 if ((VT.getSizeInBits() == 1 ||
5680 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5681 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5682 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(VT: ShValTy))) &&
5683 N0.getOpcode() == ISD::AND) {
5684 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5685 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
5686 // Perform the xform if the AND RHS is a single bit.
5687 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5688 if (AndRHS->getAPIntValue().isPowerOf2() &&
5689 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5690 return DAG.getNode(
5691 Opcode: ISD::TRUNCATE, DL: dl, VT,
5692 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5693 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5694 }
5695 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5696 // (X & 8) == 8 --> (X & 8) >> 3
5697 // Perform the xform if C1 is a single bit.
5698 unsigned ShCt = C1.logBase2();
5699 if (C1.isPowerOf2() && !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5700 return DAG.getNode(
5701 Opcode: ISD::TRUNCATE, DL: dl, VT,
5702 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5703 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5704 }
5705 }
5706 }
5707 }
5708
5709 if (C1.getSignificantBits() <= 64 &&
5710 !isLegalICmpImmediate(C1.getSExtValue())) {
5711 // (X & -256) == 256 -> (X >> 8) == 1
5712 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5713 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
5714 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5715 const APInt &AndRHSC = AndRHS->getAPIntValue();
5716 if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(RHS: AndRHSC)) {
5717 unsigned ShiftBits = AndRHSC.countr_zero();
5718 if (!shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5719 // If using an unsigned shift doesn't yield a legal compare
5720 // immediate, try using sra instead.
5721 APInt NewC = C1.lshr(shiftAmt: ShiftBits);
5722 if (NewC.getSignificantBits() <= 64 &&
5723 !isLegalICmpImmediate(NewC.getSExtValue())) {
5724 APInt SignedC = C1.ashr(ShiftAmt: ShiftBits);
5725 if (SignedC.getSignificantBits() <= 64 &&
5726 isLegalICmpImmediate(SignedC.getSExtValue())) {
5727 SDValue Shift = DAG.getNode(
5728 Opcode: ISD::SRA, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5729 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5730 SDValue CmpRHS = DAG.getConstant(Val: SignedC, DL: dl, VT: ShValTy);
5731 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5732 }
5733 }
5734 SDValue Shift = DAG.getNode(
5735 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5736 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5737 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5738 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5739 }
5740 }
5741 }
5742 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
5743 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
5744 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
5745 // X < 0x100000000 -> (X >> 32) < 1
5746 // X >= 0x100000000 -> (X >> 32) >= 1
5747 // X <= 0x0ffffffff -> (X >> 32) < 1
5748 // X > 0x0ffffffff -> (X >> 32) >= 1
5749 unsigned ShiftBits;
5750 APInt NewC = C1;
5751 ISD::CondCode NewCond = Cond;
5752 if (AdjOne) {
5753 ShiftBits = C1.countr_one();
5754 NewC = NewC + 1;
5755 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
5756 } else {
5757 ShiftBits = C1.countr_zero();
5758 }
5759 APInt RangeWidth = NewC;
5760 NewC.lshrInPlace(ShiftAmt: ShiftBits);
5761 if (ShiftBits && NewC.getSignificantBits() <= 64 &&
5762 isLegalICmpImmediate(NewC.getSExtValue()) &&
5763 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5764 // If this is an offset range check, try to move the offset after the
5765 // shift to avoid preserving the pre-shift add with a mask.
5766 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse()) {
5767 if (auto *AddC = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
5768 const APInt &AddVal = AddC->getAPIntValue();
5769 if (AddVal.countr_zero() >= ShiftBits) {
5770 APInt RangeLower = -AddVal;
5771 bool Overflow;
5772 (void)RangeLower.uadd_ov(RHS: RangeWidth, Overflow);
5773 if (!RangeWidth.isZero() && !Overflow) {
5774 SDValue Shift = DAG.getNode(
5775 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5776 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5777 APInt Offset = -RangeLower.lshr(shiftAmt: ShiftBits);
5778 SDValue ShiftedAdd =
5779 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ShValTy, N1: Shift,
5780 N2: DAG.getConstant(Val: Offset, DL: dl, VT: ShValTy));
5781 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5782 return DAG.getSetCC(DL: dl, VT, LHS: ShiftedAdd, RHS: CmpRHS, Cond: NewCond);
5783 }
5784 }
5785 }
5786 }
5787 SDValue Shift =
5788 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5789 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5790 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5791 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond: NewCond);
5792 }
5793 }
5794 }
5795 }
5796
5797 if (!isa<ConstantFPSDNode>(Val: N0) && isa<ConstantFPSDNode>(Val: N1)) {
5798 auto *CFP = cast<ConstantFPSDNode>(Val&: N1);
5799 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
5800
5801 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
5802 // constant if knowing that the operand is non-nan is enough. We prefer to
5803 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
5804 // materialize 0.0.
5805 if (Cond == ISD::SETO || Cond == ISD::SETUO)
5806 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N0, Cond);
5807
5808 // setcc (fneg x), C -> setcc swap(pred) x, -C
5809 if (N0.getOpcode() == ISD::FNEG) {
5810 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Operation: Cond);
5811 if (DCI.isBeforeLegalizeOps() ||
5812 isCondCodeLegal(CC: SwapCond, VT: N0.getSimpleValueType())) {
5813 SDValue NegN1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: N0.getValueType(), Operand: N1);
5814 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NegN1, Cond: SwapCond);
5815 }
5816 }
5817
5818 // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
5819 if (isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT: N0.getValueType()) &&
5820 !isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(ResNo: 0))) {
5821 bool IsFabs = N0.getOpcode() == ISD::FABS;
5822 SDValue Op = IsFabs ? N0.getOperand(i: 0) : N0;
5823 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
5824 FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
5825 : (IsFabs ? fcInf : fcPosInf);
5826 if (Cond == ISD::SETUEQ)
5827 Flag |= fcNan;
5828 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: dl, VT, N1: Op,
5829 N2: DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32));
5830 }
5831 }
5832
5833 // If the condition is not legal, see if we can find an equivalent one
5834 // which is legal.
5835 if (!isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType())) {
5836 // If the comparison was an awkward floating-point == or != and one of
5837 // the comparison operands is infinity or negative infinity, convert the
5838 // condition to a less-awkward <= or >=.
5839 if (CFP->getValueAPF().isInfinity()) {
5840 bool IsNegInf = CFP->getValueAPF().isNegative();
5841 ISD::CondCode NewCond = ISD::SETCC_INVALID;
5842 switch (Cond) {
5843 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
5844 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
5845 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
5846 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
5847 default: break;
5848 }
5849 if (NewCond != ISD::SETCC_INVALID &&
5850 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType()))
5851 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5852 }
5853 }
5854 }
5855
5856 if (N0 == N1) {
5857 // The sext(setcc()) => setcc() optimization relies on the appropriate
5858 // constant being emitted.
5859 assert(!N0.getValueType().isInteger() &&
5860 "Integer types should be handled by FoldSetCC");
5861
5862 bool EqTrue = ISD::isTrueWhenEqual(Cond);
5863 unsigned UOF = ISD::getUnorderedFlavor(Cond);
5864 if (UOF == 2) // FP operators that are undefined on NaNs.
5865 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5866 if (UOF == unsigned(EqTrue))
5867 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5868 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
5869 // if it is not already.
5870 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
5871 if (NewCond != Cond &&
5872 (DCI.isBeforeLegalizeOps() ||
5873 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType())))
5874 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5875 }
5876
5877 // ~X > ~Y --> Y > X
5878 // ~X < ~Y --> Y < X
5879 // ~X < C --> X > ~C
5880 // ~X > C --> X < ~C
5881 if ((isSignedIntSetCC(Code: Cond) || isUnsignedIntSetCC(Code: Cond)) &&
5882 N0.getValueType().isInteger()) {
5883 if (isBitwiseNot(V: N0)) {
5884 if (isBitwiseNot(V: N1))
5885 return DAG.getSetCC(DL: dl, VT, LHS: N1.getOperand(i: 0), RHS: N0.getOperand(i: 0), Cond);
5886
5887 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N1) &&
5888 !DAG.isConstantIntBuildVectorOrConstantInt(N: N0.getOperand(i: 0))) {
5889 SDValue Not = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5890 return DAG.getSetCC(DL: dl, VT, LHS: Not, RHS: N0.getOperand(i: 0), Cond);
5891 }
5892 }
5893 }
5894
5895 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5896 N0.getValueType().isInteger()) {
5897 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
5898 N0.getOpcode() == ISD::XOR) {
5899 // Simplify (X+Y) == (X+Z) --> Y == Z
5900 if (N0.getOpcode() == N1.getOpcode()) {
5901 if (N0.getOperand(i: 0) == N1.getOperand(i: 0))
5902 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 1), Cond);
5903 if (N0.getOperand(i: 1) == N1.getOperand(i: 1))
5904 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
5905 if (isCommutativeBinOp(Opcode: N0.getOpcode())) {
5906 // If X op Y == Y op X, try other combinations.
5907 if (N0.getOperand(i: 0) == N1.getOperand(i: 1))
5908 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 0),
5909 Cond);
5910 if (N0.getOperand(i: 1) == N1.getOperand(i: 0))
5911 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 1),
5912 Cond);
5913 }
5914 }
5915
5916 // If RHS is a legal immediate value for a compare instruction, we need
5917 // to be careful about increasing register pressure needlessly.
5918 bool LegalRHSImm = false;
5919
5920 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: N1)) {
5921 if (auto *LHSR = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5922 // Turn (X+C1) == C2 --> X == C2-C1
5923 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
5924 return DAG.getSetCC(
5925 DL: dl, VT, LHS: N0.getOperand(i: 0),
5926 RHS: DAG.getConstant(Val: RHSC->getAPIntValue() - LHSR->getAPIntValue(),
5927 DL: dl, VT: N0.getValueType()),
5928 Cond);
5929
5930 // Turn (X^C1) == C2 --> X == C1^C2
5931 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
5932 return DAG.getSetCC(
5933 DL: dl, VT, LHS: N0.getOperand(i: 0),
5934 RHS: DAG.getConstant(Val: LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
5935 DL: dl, VT: N0.getValueType()),
5936 Cond);
5937 }
5938
5939 // Turn (C1-X) == C2 --> X == C1-C2
5940 if (auto *SUBC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0)))
5941 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
5942 return DAG.getSetCC(
5943 DL: dl, VT, LHS: N0.getOperand(i: 1),
5944 RHS: DAG.getConstant(Val: SUBC->getAPIntValue() - RHSC->getAPIntValue(),
5945 DL: dl, VT: N0.getValueType()),
5946 Cond);
5947
5948 // Could RHSC fold directly into a compare?
5949 if (RHSC->getValueType(ResNo: 0).getSizeInBits() <= 64)
5950 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
5951 }
5952
5953 // (X+Y) == X --> Y == 0 and similar folds.
5954 // Don't do this if X is an immediate that can fold into a cmp
5955 // instruction and X+Y has other uses. It could be an induction variable
5956 // chain, and the transform would increase register pressure.
5957 if (!LegalRHSImm || N0.hasOneUse())
5958 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, DL: dl, DCI))
5959 return V;
5960 }
5961
5962 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
5963 N1.getOpcode() == ISD::XOR)
5964 if (SDValue V = foldSetCCWithBinOp(VT, N0: N1, N1: N0, Cond, DL: dl, DCI))
5965 return V;
5966
5967 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, DL: dl, DCI))
5968 return V;
5969
5970 if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, DL: dl, DCI))
5971 return V;
5972 }
5973
5974 // Fold remainder of division by a constant.
5975 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
5976 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5977 // When division is cheap or optimizing for minimum size,
5978 // fall through to DIVREM creation by skipping this fold.
5979 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
5980 if (N0.getOpcode() == ISD::UREM) {
5981 if (SDValue Folded = buildUREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5982 return Folded;
5983 } else if (N0.getOpcode() == ISD::SREM) {
5984 if (SDValue Folded = buildSREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5985 return Folded;
5986 }
5987 }
5988 }
5989
5990 // Fold away ALL boolean setcc's.
5991 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
5992 SDValue Temp;
5993 switch (Cond) {
5994 default: llvm_unreachable("Unknown integer setcc!");
5995 case ISD::SETEQ: // X == Y -> ~(X^Y)
5996 Temp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
5997 N0 = DAG.getNOT(DL: dl, Val: Temp, VT: OpVT);
5998 if (!DCI.isCalledByLegalizer())
5999 DCI.AddToWorklist(N: Temp.getNode());
6000 break;
6001 case ISD::SETNE: // X != Y --> (X^Y)
6002 N0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
6003 break;
6004 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
6005 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
6006 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
6007 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1, N2: Temp);
6008 if (!DCI.isCalledByLegalizer())
6009 DCI.AddToWorklist(N: Temp.getNode());
6010 break;
6011 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
6012 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
6013 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
6014 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: N0, N2: Temp);
6015 if (!DCI.isCalledByLegalizer())
6016 DCI.AddToWorklist(N: Temp.getNode());
6017 break;
6018 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
6019 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
6020 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
6021 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1, N2: Temp);
6022 if (!DCI.isCalledByLegalizer())
6023 DCI.AddToWorklist(N: Temp.getNode());
6024 break;
6025 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
6026 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
6027 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
6028 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: N0, N2: Temp);
6029 break;
6030 }
6031 if (VT.getScalarType() != MVT::i1) {
6032 if (!DCI.isCalledByLegalizer())
6033 DCI.AddToWorklist(N: N0.getNode());
6034 // FIXME: If running after legalize, we probably can't do this.
6035 ISD::NodeType ExtendCode = getExtendForContent(Content: getBooleanContents(Type: OpVT));
6036 N0 = DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: N0);
6037 }
6038 return N0;
6039 }
6040
6041 // Fold (setcc (trunc x) (trunc y)) -> (setcc x y)
6042 if (N0.getOpcode() == ISD::TRUNCATE && N1.getOpcode() == ISD::TRUNCATE &&
6043 N0.getOperand(i: 0).getValueType() == N1.getOperand(i: 0).getValueType() &&
6044 ((!ISD::isSignedIntSetCC(Code: Cond) && N0->getFlags().hasNoUnsignedWrap() &&
6045 N1->getFlags().hasNoUnsignedWrap()) ||
6046 (!ISD::isUnsignedIntSetCC(Code: Cond) && N0->getFlags().hasNoSignedWrap() &&
6047 N1->getFlags().hasNoSignedWrap())) &&
6048 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
6049 if (VT.getScalarType() == MVT::i1)
6050 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
6051 // For (legal) non vXi1 cases - ensure we adjust the cmp and result types.
6052 EVT OldCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
6053 VT: N0.getValueType());
6054 if (VT == OldCCVT) {
6055 EVT NewCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
6056 VT: N0.getOperand(i: 0).getValueType());
6057 return DAG.getBoolExtOrTrunc(
6058 Op: DAG.getSetCC(DL: dl, VT: NewCCVT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond),
6059 SL: dl, VT, OpVT: N0.getOperand(i: 0).getValueType());
6060 }
6061 }
6062
6063 // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
6064 // TODO: Remove that .isVector() check
6065 if (VT.isVector() && isZeroOrZeroSplat(N: N1) && N0.getOpcode() == ISD::SUB &&
6066 N0->getFlags().hasNoSignedWrap() && ISD::isSignedIntSetCC(Code: Cond)) {
6067 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond);
6068 }
6069
6070 // Could not fold it.
6071 return SDValue();
6072}
6073
6074/// Returns true (and the GlobalValue and the offset) if the node is a
6075/// GlobalAddress + offset.
6076bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
6077 int64_t &Offset) const {
6078
6079 SDNode *N = unwrapAddress(N: SDValue(WN, 0)).getNode();
6080
6081 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(Val: N)) {
6082 GA = GASD->getGlobal();
6083 Offset += GASD->getOffset();
6084 return true;
6085 }
6086
6087 if (N->isAnyAdd()) {
6088 SDValue N1 = N->getOperand(Num: 0);
6089 SDValue N2 = N->getOperand(Num: 1);
6090 if (isGAPlusOffset(WN: N1.getNode(), GA, Offset)) {
6091 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N2)) {
6092 Offset += V->getSExtValue();
6093 return true;
6094 }
6095 } else if (isGAPlusOffset(WN: N2.getNode(), GA, Offset)) {
6096 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N1)) {
6097 Offset += V->getSExtValue();
6098 return true;
6099 }
6100 }
6101 }
6102
6103 return false;
6104}
6105
6106SDValue TargetLowering::PerformDAGCombine(SDNode *N,
6107 DAGCombinerInfo &DCI) const {
6108 // Default implementation: no optimization.
6109 return SDValue();
6110}
6111
6112//===----------------------------------------------------------------------===//
6113// Inline Assembler Implementation Methods
6114//===----------------------------------------------------------------------===//
6115
6116TargetLowering::ConstraintType
6117TargetLowering::getConstraintType(StringRef Constraint) const {
6118 unsigned S = Constraint.size();
6119
6120 if (S == 1) {
6121 switch (Constraint[0]) {
6122 default: break;
6123 case 'r':
6124 return C_RegisterClass;
6125 case 'm': // memory
6126 case 'o': // offsetable
6127 case 'V': // not offsetable
6128 return C_Memory;
6129 case 'p': // Address.
6130 return C_Address;
6131 case 'n': // Simple Integer
6132 case 'E': // Floating Point Constant
6133 case 'F': // Floating Point Constant
6134 return C_Immediate;
6135 case 'i': // Simple Integer or Relocatable Constant
6136 case 's': // Relocatable Constant
6137 case 'X': // Allow ANY value.
6138 case 'I': // Target registers.
6139 case 'J':
6140 case 'K':
6141 case 'L':
6142 case 'M':
6143 case 'N':
6144 case 'O':
6145 case 'P':
6146 case '<':
6147 case '>':
6148 return C_Other;
6149 }
6150 }
6151
6152 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
6153 if (S == 8 && Constraint.substr(Start: 1, N: 6) == "memory") // "{memory}"
6154 return C_Memory;
6155 return C_Register;
6156 }
6157 return C_Unknown;
6158}
6159
6160/// Try to replace an X constraint, which matches anything, with another that
6161/// has more specific requirements based on the type of the corresponding
6162/// operand.
6163const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
6164 if (ConstraintVT.isInteger())
6165 return "r";
6166 if (ConstraintVT.isFloatingPoint())
6167 return "f"; // works for many targets
6168 return nullptr;
6169}
6170
6171SDValue TargetLowering::LowerAsmOutputForConstraint(
6172 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
6173 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
6174 return SDValue();
6175}
6176
6177/// Lower the specified operand into the Ops vector.
6178/// If it is invalid, don't add anything to Ops.
6179void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
6180 StringRef Constraint,
6181 std::vector<SDValue> &Ops,
6182 SelectionDAG &DAG) const {
6183
6184 if (Constraint.size() > 1)
6185 return;
6186
6187 char ConstraintLetter = Constraint[0];
6188 switch (ConstraintLetter) {
6189 default: break;
6190 case 'X': // Allows any operand
6191 case 'i': // Simple Integer or Relocatable Constant
6192 case 'n': // Simple Integer
6193 case 's': { // Relocatable Constant
6194
6195 ConstantSDNode *C;
6196 uint64_t Offset = 0;
6197
6198 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
6199 // etc., since getelementpointer is variadic. We can't use
6200 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
6201 // while in this case the GA may be furthest from the root node which is
6202 // likely an ISD::ADD.
6203 while (true) {
6204 if ((C = dyn_cast<ConstantSDNode>(Val&: Op)) && ConstraintLetter != 's') {
6205 // gcc prints these as sign extended. Sign extend value to 64 bits
6206 // now; without this it would get ZExt'd later in
6207 // ScheduleDAGSDNodes::EmitNode, which is very generic.
6208 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
6209 BooleanContent BCont = getBooleanContents(Type: MVT::i64);
6210 ISD::NodeType ExtOpc =
6211 IsBool ? getExtendForContent(Content: BCont) : ISD::SIGN_EXTEND;
6212 int64_t ExtVal =
6213 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
6214 Ops.push_back(
6215 x: DAG.getTargetConstant(Val: Offset + ExtVal, DL: SDLoc(C), VT: MVT::i64));
6216 return;
6217 }
6218 if (ConstraintLetter != 'n') {
6219 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
6220 Ops.push_back(x: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(Op),
6221 VT: GA->getValueType(ResNo: 0),
6222 offset: Offset + GA->getOffset()));
6223 return;
6224 }
6225 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
6226 Ops.push_back(x: DAG.getTargetBlockAddress(
6227 BA: BA->getBlockAddress(), VT: BA->getValueType(ResNo: 0),
6228 Offset: Offset + BA->getOffset(), TargetFlags: BA->getTargetFlags()));
6229 return;
6230 }
6231 if (isa<BasicBlockSDNode>(Val: Op)) {
6232 Ops.push_back(x: Op);
6233 return;
6234 }
6235 }
6236 const unsigned OpCode = Op.getOpcode();
6237 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
6238 if ((C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 0))))
6239 Op = Op.getOperand(i: 1);
6240 // Subtraction is not commutative.
6241 else if (OpCode == ISD::ADD &&
6242 (C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))))
6243 Op = Op.getOperand(i: 0);
6244 else
6245 return;
6246 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
6247 continue;
6248 }
6249 return;
6250 }
6251 break;
6252 }
6253 }
6254}
6255
6256void TargetLowering::CollectTargetIntrinsicOperands(
6257 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
6258}
6259
6260std::pair<unsigned, const TargetRegisterClass *>
6261TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
6262 StringRef Constraint,
6263 MVT VT) const {
6264 if (!Constraint.starts_with(Prefix: "{"))
6265 return std::make_pair(x: 0u, y: static_cast<TargetRegisterClass *>(nullptr));
6266 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
6267
6268 // Remove the braces from around the name.
6269 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
6270
6271 std::pair<unsigned, const TargetRegisterClass *> R =
6272 std::make_pair(x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
6273
6274 // Figure out which register class contains this reg.
6275 for (const TargetRegisterClass &RC : RI->regclasses()) {
6276 // If none of the value types for this register class are valid, we
6277 // can't use it. For example, 64-bit reg classes on 32-bit targets.
6278 if (!isLegalRC(TRI: *RI, RC))
6279 continue;
6280
6281 for (const MCPhysReg &PR : RC) {
6282 if (RegName.equals_insensitive(RHS: RI->getRegAsmName(Reg: PR))) {
6283 std::pair<unsigned, const TargetRegisterClass *> S =
6284 std::make_pair(x: PR, y: &RC);
6285
6286 // If this register class has the requested value type, return it,
6287 // otherwise keep searching and return the first class found
6288 // if no other is found which explicitly has the requested type.
6289 if (RI->isTypeLegalForClass(RC, T: VT))
6290 return S;
6291 if (!R.second)
6292 R = S;
6293 }
6294 }
6295 }
6296
6297 return R;
6298}
6299
6300//===----------------------------------------------------------------------===//
6301// Constraint Selection.
6302
6303/// Return true of this is an input operand that is a matching constraint like
6304/// "4".
6305bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
6306 assert(!ConstraintCode.empty() && "No known constraint!");
6307 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
6308}
6309
6310/// If this is an input matching constraint, this method returns the output
6311/// operand it matches.
6312unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
6313 assert(!ConstraintCode.empty() && "No known constraint!");
6314 return atoi(nptr: ConstraintCode.c_str());
6315}
6316
6317/// Split up the constraint string from the inline assembly value into the
6318/// specific constraints and their prefixes, and also tie in the associated
6319/// operand values.
6320/// If this returns an empty vector, and if the constraint string itself
6321/// isn't empty, there was an error parsing.
6322TargetLowering::AsmOperandInfoVector
6323TargetLowering::ParseConstraints(const DataLayout &DL,
6324 const TargetRegisterInfo *TRI,
6325 const CallBase &Call) const {
6326 /// Information about all of the constraints.
6327 AsmOperandInfoVector ConstraintOperands;
6328 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
6329 unsigned maCount = 0; // Largest number of multiple alternative constraints.
6330
6331 // Do a prepass over the constraints, canonicalizing them, and building up the
6332 // ConstraintOperands list.
6333 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6334 unsigned ResNo = 0; // ResNo - The result number of the next output.
6335 unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.
6336
6337 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
6338 ConstraintOperands.emplace_back(args: std::move(CI));
6339 AsmOperandInfo &OpInfo = ConstraintOperands.back();
6340
6341 // Update multiple alternative constraint count.
6342 if (OpInfo.multipleAlternatives.size() > maCount)
6343 maCount = OpInfo.multipleAlternatives.size();
6344
6345 OpInfo.ConstraintVT = MVT::Other;
6346
6347 // Compute the value type for each operand.
6348 switch (OpInfo.Type) {
6349 case InlineAsm::isOutput: {
6350 // Indirect outputs just consume an argument.
6351 if (OpInfo.isIndirect) {
6352 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6353 break;
6354 }
6355
6356 // The return value of the call is this value. As such, there is no
6357 // corresponding argument.
6358 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
6359 EVT VT;
6360 if (auto *STy = dyn_cast<StructType>(Val: Call.getType())) {
6361 VT = getAsmOperandValueType(DL, Ty: STy->getElementType(N: ResNo));
6362 } else {
6363 assert(ResNo == 0 && "Asm only has one result!");
6364 VT = getAsmOperandValueType(DL, Ty: Call.getType());
6365 }
6366 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6367 ++ResNo;
6368 break;
6369 }
6370 case InlineAsm::isInput:
6371 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6372 break;
6373 case InlineAsm::isLabel:
6374 OpInfo.CallOperandVal = cast<CallBrInst>(Val: &Call)->getIndirectDest(i: LabelNo);
6375 ++LabelNo;
6376 continue;
6377 case InlineAsm::isClobber:
6378 // Nothing to do.
6379 break;
6380 }
6381
6382 if (OpInfo.CallOperandVal) {
6383 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
6384 if (OpInfo.isIndirect) {
6385 OpTy = Call.getParamElementType(ArgNo);
6386 assert(OpTy && "Indirect operand must have elementtype attribute");
6387 }
6388
6389 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6390 if (StructType *STy = dyn_cast<StructType>(Val: OpTy))
6391 if (STy->getNumElements() == 1)
6392 OpTy = STy->getElementType(N: 0);
6393
6394 // If OpTy is not a single value, it may be a struct/union that we
6395 // can tile with integers.
6396 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6397 unsigned BitSize = DL.getTypeSizeInBits(Ty: OpTy);
6398 switch (BitSize) {
6399 default: break;
6400 case 1:
6401 case 8:
6402 case 16:
6403 case 32:
6404 case 64:
6405 case 128:
6406 OpTy = IntegerType::get(C&: OpTy->getContext(), NumBits: BitSize);
6407 break;
6408 }
6409 }
6410
6411 EVT VT = getAsmOperandValueType(DL, Ty: OpTy, AllowUnknown: true);
6412 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6413 ArgNo++;
6414 }
6415 }
6416
6417 // If we have multiple alternative constraints, select the best alternative.
6418 if (!ConstraintOperands.empty()) {
6419 if (maCount) {
6420 unsigned bestMAIndex = 0;
6421 int bestWeight = -1;
6422 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
6423 int weight = -1;
6424 unsigned maIndex;
6425 // Compute the sums of the weights for each alternative, keeping track
6426 // of the best (highest weight) one so far.
6427 for (maIndex = 0; maIndex < maCount; ++maIndex) {
6428 int weightSum = 0;
6429 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6430 cIndex != eIndex; ++cIndex) {
6431 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6432 if (OpInfo.Type == InlineAsm::isClobber)
6433 continue;
6434
6435 // If this is an output operand with a matching input operand,
6436 // look up the matching input. If their types mismatch, e.g. one
6437 // is an integer, the other is floating point, or their sizes are
6438 // different, flag it as an maCantMatch.
6439 if (OpInfo.hasMatchingInput()) {
6440 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6441 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6442 if ((OpInfo.ConstraintVT.isInteger() !=
6443 Input.ConstraintVT.isInteger()) ||
6444 (OpInfo.ConstraintVT.getSizeInBits() !=
6445 Input.ConstraintVT.getSizeInBits())) {
6446 weightSum = -1; // Can't match.
6447 break;
6448 }
6449 }
6450 }
6451 weight = getMultipleConstraintMatchWeight(info&: OpInfo, maIndex);
6452 if (weight == -1) {
6453 weightSum = -1;
6454 break;
6455 }
6456 weightSum += weight;
6457 }
6458 // Update best.
6459 if (weightSum > bestWeight) {
6460 bestWeight = weightSum;
6461 bestMAIndex = maIndex;
6462 }
6463 }
6464
6465 // Now select chosen alternative in each constraint.
6466 for (AsmOperandInfo &cInfo : ConstraintOperands)
6467 if (cInfo.Type != InlineAsm::isClobber)
6468 cInfo.selectAlternative(index: bestMAIndex);
6469 }
6470 }
6471
6472 // Check and hook up tied operands, choose constraint code to use.
6473 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6474 cIndex != eIndex; ++cIndex) {
6475 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6476
6477 // If this is an output operand with a matching input operand, look up the
6478 // matching input. If their types mismatch, e.g. one is an integer, the
6479 // other is floating point, or their sizes are different, flag it as an
6480 // error.
6481 if (OpInfo.hasMatchingInput()) {
6482 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6483
6484 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6485 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6486 getRegForInlineAsmConstraint(RI: TRI, Constraint: OpInfo.ConstraintCode,
6487 VT: OpInfo.ConstraintVT);
6488 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6489 getRegForInlineAsmConstraint(RI: TRI, Constraint: Input.ConstraintCode,
6490 VT: Input.ConstraintVT);
6491 const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
6492 OpInfo.ConstraintVT.isFloatingPoint();
6493 const bool InOpIsIntOrFP = Input.ConstraintVT.isInteger() ||
6494 Input.ConstraintVT.isFloatingPoint();
6495 if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
6496 (MatchRC.second != InputRC.second)) {
6497 report_fatal_error(reason: "Unsupported asm: input constraint"
6498 " with a matching output constraint of"
6499 " incompatible type!");
6500 }
6501 }
6502 }
6503 }
6504
6505 return ConstraintOperands;
6506}
6507
6508/// Return a number indicating our preference for chosing a type of constraint
6509/// over another, for the purpose of sorting them. Immediates are almost always
6510/// preferrable (when they can be emitted). A higher return value means a
6511/// stronger preference for one constraint type relative to another.
6512/// FIXME: We should prefer registers over memory but doing so may lead to
6513/// unrecoverable register exhaustion later.
6514/// https://github.com/llvm/llvm-project/issues/20571
6515static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) {
6516 switch (CT) {
6517 case TargetLowering::C_Immediate:
6518 case TargetLowering::C_Other:
6519 return 4;
6520 case TargetLowering::C_Memory:
6521 case TargetLowering::C_Address:
6522 return 3;
6523 case TargetLowering::C_RegisterClass:
6524 return 2;
6525 case TargetLowering::C_Register:
6526 return 1;
6527 case TargetLowering::C_Unknown:
6528 return 0;
6529 }
6530 llvm_unreachable("Invalid constraint type");
6531}
6532
6533/// Examine constraint type and operand type and determine a weight value.
6534/// This object must already have been set up with the operand type
6535/// and the current alternative constraint selected.
6536TargetLowering::ConstraintWeight
6537 TargetLowering::getMultipleConstraintMatchWeight(
6538 AsmOperandInfo &info, int maIndex) const {
6539 InlineAsm::ConstraintCodeVector *rCodes;
6540 if (maIndex >= (int)info.multipleAlternatives.size())
6541 rCodes = &info.Codes;
6542 else
6543 rCodes = &info.multipleAlternatives[maIndex].Codes;
6544 ConstraintWeight BestWeight = CW_Invalid;
6545
6546 // Loop over the options, keeping track of the most general one.
6547 for (const std::string &rCode : *rCodes) {
6548 ConstraintWeight weight =
6549 getSingleConstraintMatchWeight(info, constraint: rCode.c_str());
6550 if (weight > BestWeight)
6551 BestWeight = weight;
6552 }
6553
6554 return BestWeight;
6555}
6556
6557/// Examine constraint type and operand type and determine a weight value.
6558/// This object must already have been set up with the operand type
6559/// and the current alternative constraint selected.
6560TargetLowering::ConstraintWeight
6561 TargetLowering::getSingleConstraintMatchWeight(
6562 AsmOperandInfo &info, const char *constraint) const {
6563 ConstraintWeight weight = CW_Invalid;
6564 Value *CallOperandVal = info.CallOperandVal;
6565 // If we don't have a value, we can't do a match,
6566 // but allow it at the lowest weight.
6567 if (!CallOperandVal)
6568 return CW_Default;
6569 // Look at the constraint type.
6570 switch (*constraint) {
6571 case 'i': // immediate integer.
6572 case 'n': // immediate integer with a known value.
6573 if (isa<ConstantInt>(Val: CallOperandVal))
6574 weight = CW_Constant;
6575 break;
6576 case 's': // non-explicit intregal immediate.
6577 if (isa<GlobalValue>(Val: CallOperandVal))
6578 weight = CW_Constant;
6579 break;
6580 case 'E': // immediate float if host format.
6581 case 'F': // immediate float.
6582 if (isa<ConstantFP>(Val: CallOperandVal))
6583 weight = CW_Constant;
6584 break;
6585 case '<': // memory operand with autodecrement.
6586 case '>': // memory operand with autoincrement.
6587 case 'm': // memory operand.
6588 case 'o': // offsettable memory operand
6589 case 'V': // non-offsettable memory operand
6590 weight = CW_Memory;
6591 break;
6592 case 'r': // general register.
6593 case 'g': // general register, memory operand or immediate integer.
6594 // note: Clang converts "g" to "imr".
6595 if (CallOperandVal->getType()->isIntegerTy())
6596 weight = CW_Register;
6597 break;
6598 case 'X': // any operand.
6599 default:
6600 weight = CW_Default;
6601 break;
6602 }
6603 return weight;
6604}
6605
6606/// If there are multiple different constraints that we could pick for this
6607/// operand (e.g. "imr") try to pick the 'best' one.
6608/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
6609/// into seven classes:
6610/// Register -> one specific register
6611/// RegisterClass -> a group of regs
6612/// Memory -> memory
6613/// Address -> a symbolic memory reference
6614/// Immediate -> immediate values
6615/// Other -> magic values (such as "Flag Output Operands")
6616/// Unknown -> something we don't recognize yet and can't handle
6617/// Ideally, we would pick the most specific constraint possible: if we have
6618/// something that fits into a register, we would pick it. The problem here
6619/// is that if we have something that could either be in a register or in
6620/// memory that use of the register could cause selection of *other*
6621/// operands to fail: they might only succeed if we pick memory. Because of
6622/// this the heuristic we use is:
6623///
6624/// 1) If there is an 'other' constraint, and if the operand is valid for
6625/// that constraint, use it. This makes us take advantage of 'i'
6626/// constraints when available.
6627/// 2) Otherwise, pick the most general constraint present. This prefers
6628/// 'm' over 'r', for example.
6629///
6630TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
6631 TargetLowering::AsmOperandInfo &OpInfo) const {
6632 ConstraintGroup Ret;
6633
6634 Ret.reserve(N: OpInfo.Codes.size());
6635 for (StringRef Code : OpInfo.Codes) {
6636 TargetLowering::ConstraintType CType = getConstraintType(Constraint: Code);
6637
6638 // Indirect 'other' or 'immediate' constraints are not allowed.
6639 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6640 CType == TargetLowering::C_Register ||
6641 CType == TargetLowering::C_RegisterClass))
6642 continue;
6643
6644 // Things with matching constraints can only be registers, per gcc
6645 // documentation. This mainly affects "g" constraints.
6646 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
6647 continue;
6648
6649 Ret.emplace_back(Args&: Code, Args&: CType);
6650 }
6651
6652 llvm::stable_sort(Range&: Ret, C: [](ConstraintPair a, ConstraintPair b) {
6653 return getConstraintPiority(CT: a.second) > getConstraintPiority(CT: b.second);
6654 });
6655
6656 return Ret;
6657}
6658
6659/// If we have an immediate, see if we can lower it. Return true if we can,
6660/// false otherwise.
6661static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P,
6662 SDValue Op, SelectionDAG *DAG,
6663 const TargetLowering &TLI) {
6664
6665 assert((P.second == TargetLowering::C_Other ||
6666 P.second == TargetLowering::C_Immediate) &&
6667 "need immediate or other");
6668
6669 if (!Op.getNode())
6670 return false;
6671
6672 std::vector<SDValue> ResultOps;
6673 TLI.LowerAsmOperandForConstraint(Op, Constraint: P.first, Ops&: ResultOps, DAG&: *DAG);
6674 return !ResultOps.empty();
6675}
6676
6677/// Determines the constraint code and constraint type to use for the specific
6678/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
6679void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
6680 SDValue Op,
6681 SelectionDAG *DAG) const {
6682 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
6683
6684 // Single-letter constraints ('r') are very common.
6685 if (OpInfo.Codes.size() == 1) {
6686 OpInfo.ConstraintCode = OpInfo.Codes[0];
6687 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6688 } else {
6689 ConstraintGroup G = getConstraintPreferences(OpInfo);
6690 if (G.empty())
6691 return;
6692
6693 unsigned BestIdx = 0;
6694 for (const unsigned E = G.size();
6695 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
6696 G[BestIdx].second == TargetLowering::C_Immediate);
6697 ++BestIdx) {
6698 if (lowerImmediateIfPossible(P&: G[BestIdx], Op, DAG, TLI: *this))
6699 break;
6700 // If we're out of constraints, just pick the first one.
6701 if (BestIdx + 1 == E) {
6702 BestIdx = 0;
6703 break;
6704 }
6705 }
6706
6707 OpInfo.ConstraintCode = G[BestIdx].first;
6708 OpInfo.ConstraintType = G[BestIdx].second;
6709 }
6710
6711 // 'X' matches anything.
6712 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
6713 // Constants are handled elsewhere. For Functions, the type here is the
6714 // type of the result, which is not what we want to look at; leave them
6715 // alone.
6716 Value *v = OpInfo.CallOperandVal;
6717 if (isa<ConstantInt>(Val: v) || isa<Function>(Val: v)) {
6718 return;
6719 }
6720
6721 if (isa<BasicBlock>(Val: v) || isa<BlockAddress>(Val: v)) {
6722 OpInfo.ConstraintCode = "i";
6723 return;
6724 }
6725
6726 // Otherwise, try to resolve it to something we know about by looking at
6727 // the actual operand type.
6728 if (const char *Repl = LowerXConstraint(ConstraintVT: OpInfo.ConstraintVT)) {
6729 OpInfo.ConstraintCode = Repl;
6730 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6731 }
6732 }
6733}
6734
6735/// Given an exact SDIV by a constant, create a multiplication
6736/// with the multiplicative inverse of the constant.
6737/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6738static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
6739 const SDLoc &dl, SelectionDAG &DAG,
6740 SmallVectorImpl<SDNode *> &Created) {
6741 SDValue Op0 = N->getOperand(Num: 0);
6742 SDValue Op1 = N->getOperand(Num: 1);
6743 EVT VT = N->getValueType(ResNo: 0);
6744 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6745 EVT ShSVT = ShVT.getScalarType();
6746
6747 bool UseSRA = false;
6748 SmallVector<SDValue, 16> Shifts, Factors;
6749
6750 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6751 if (C->isZero())
6752 return false;
6753
6754 EVT CT = C->getValueType(ResNo: 0);
6755 APInt Divisor = C->getAPIntValue();
6756 unsigned Shift = Divisor.countr_zero();
6757 if (Shift) {
6758 Divisor.ashrInPlace(ShiftAmt: Shift);
6759 UseSRA = true;
6760 }
6761 APInt Factor = Divisor.multiplicativeInverse();
6762 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6763 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6764 return true;
6765 };
6766
6767 // Collect all magic values from the build vector.
6768 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildSDIVPattern))
6769 return SDValue();
6770
6771 SDValue Shift, Factor;
6772 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6773 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6774 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6775 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6776 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6777 "Expected matchUnaryPredicate to return one element for scalable "
6778 "vectors");
6779 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6780 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6781 } else {
6782 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6783 Shift = Shifts[0];
6784 Factor = Factors[0];
6785 }
6786
6787 SDValue Res = Op0;
6788 if (UseSRA) {
6789 Res = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6790 Created.push_back(Elt: Res.getNode());
6791 }
6792
6793 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6794}
6795
6796/// Given an exact UDIV by a constant, create a multiplication
6797/// with the multiplicative inverse of the constant.
6798/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6799static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
6800 const SDLoc &dl, SelectionDAG &DAG,
6801 SmallVectorImpl<SDNode *> &Created) {
6802 EVT VT = N->getValueType(ResNo: 0);
6803 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6804 EVT ShSVT = ShVT.getScalarType();
6805
6806 bool UseSRL = false;
6807 SmallVector<SDValue, 16> Shifts, Factors;
6808
6809 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6810 if (C->isZero())
6811 return false;
6812
6813 EVT CT = C->getValueType(ResNo: 0);
6814 APInt Divisor = C->getAPIntValue();
6815 unsigned Shift = Divisor.countr_zero();
6816 if (Shift) {
6817 Divisor.lshrInPlace(ShiftAmt: Shift);
6818 UseSRL = true;
6819 }
6820 // Calculate the multiplicative inverse modulo BW.
6821 APInt Factor = Divisor.multiplicativeInverse();
6822 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6823 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6824 return true;
6825 };
6826
6827 SDValue Op1 = N->getOperand(Num: 1);
6828
6829 // Collect all magic values from the build vector.
6830 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildUDIVPattern))
6831 return SDValue();
6832
6833 SDValue Shift, Factor;
6834 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6835 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6836 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6837 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6838 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6839 "Expected matchUnaryPredicate to return one element for scalable "
6840 "vectors");
6841 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6842 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6843 } else {
6844 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6845 Shift = Shifts[0];
6846 Factor = Factors[0];
6847 }
6848
6849 SDValue Res = N->getOperand(Num: 0);
6850 if (UseSRL) {
6851 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6852 Created.push_back(Elt: Res.getNode());
6853 }
6854
6855 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6856}
6857
6858SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
6859 SelectionDAG &DAG,
6860 SmallVectorImpl<SDNode *> &Created) const {
6861 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6862 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6863 return SDValue(N, 0); // Lower SDIV as SDIV
6864 return SDValue();
6865}
6866
6867SDValue
6868TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
6869 SelectionDAG &DAG,
6870 SmallVectorImpl<SDNode *> &Created) const {
6871 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6872 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6873 return SDValue(N, 0); // Lower SREM as SREM
6874 return SDValue();
6875}
6876
6877/// Build sdiv by power-of-2 with conditional move instructions
6878/// Ref: "Hacker's Delight" by Henry Warren 10-1
6879/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
6880/// bgez x, label
6881/// add x, x, 2**k-1
6882/// label:
6883/// sra res, x, k
6884/// neg res, res (when the divisor is negative)
6885SDValue TargetLowering::buildSDIVPow2WithCMov(
6886 SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
6887 SmallVectorImpl<SDNode *> &Created) const {
6888 unsigned Lg2 = Divisor.countr_zero();
6889 EVT VT = N->getValueType(ResNo: 0);
6890
6891 SDLoc DL(N);
6892 SDValue N0 = N->getOperand(Num: 0);
6893 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
6894 APInt Lg2Mask = APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: Lg2);
6895 SDValue Pow2MinusOne = DAG.getConstant(Val: Lg2Mask, DL, VT);
6896
6897 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
6898 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
6899 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: N0, RHS: Zero, Cond: ISD::SETLT);
6900 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: Pow2MinusOne);
6901 SDValue CMov = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cmp, N2: Add, N3: N0);
6902
6903 Created.push_back(Elt: Cmp.getNode());
6904 Created.push_back(Elt: Add.getNode());
6905 Created.push_back(Elt: CMov.getNode());
6906
6907 // Divide by pow2.
6908 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CMov,
6909 N2: DAG.getShiftAmountConstant(Val: Lg2, VT, DL));
6910
6911 // If we're dividing by a positive value, we're done. Otherwise, we must
6912 // negate the result.
6913 if (Divisor.isNonNegative())
6914 return SRA;
6915
6916 Created.push_back(Elt: SRA.getNode());
6917 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: SRA);
6918}
6919
6920/// Given an ISD::SDIV node expressing a divide by constant,
6921/// return a DAG expression to select that will generate the same value by
6922/// multiplying by a magic number.
6923/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6924SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
6925 bool IsAfterLegalization,
6926 bool IsAfterLegalTypes,
6927 SmallVectorImpl<SDNode *> &Created) const {
6928 SDLoc dl(N);
6929
6930 // If the sdiv has an 'exact' bit we can use a simpler lowering.
6931 if (N->getFlags().hasExact())
6932 return BuildExactSDIV(TLI: *this, N, dl, DAG, Created);
6933
6934 EVT VT = N->getValueType(ResNo: 0);
6935 EVT SVT = VT.getScalarType();
6936 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6937 EVT ShSVT = ShVT.getScalarType();
6938 unsigned EltBits = VT.getScalarSizeInBits();
6939 EVT MulVT;
6940
6941 // Check to see if we can do this.
6942 // FIXME: We should be more aggressive here.
6943 EVT QueryVT = VT;
6944 if (VT.isVector()) {
6945 // If the vector type will be legalized to a vector type with the same
6946 // element type, allow the transform before type legalization if MULHS or
6947 // SMUL_LOHI are supported.
6948 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
6949 if (!QueryVT.isVector() ||
6950 QueryVT.getVectorElementType() != VT.getVectorElementType())
6951 return SDValue();
6952 } else if (!isTypeLegal(VT)) {
6953 // Limit this to simple scalars for now.
6954 if (!VT.isSimple())
6955 return SDValue();
6956
6957 // If this type will be promoted to a large enough type with a legal
6958 // multiply operation, we can go ahead and do this transform.
6959 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
6960 return SDValue();
6961
6962 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6963 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6964 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
6965 return SDValue();
6966 }
6967
6968 bool HasMULHS =
6969 isOperationLegalOrCustom(Op: ISD::MULHS, VT: QueryVT, LegalOnly: IsAfterLegalization);
6970 bool HasSMUL_LOHI =
6971 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
6972
6973 if (isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
6974 // If type twice as wide legal, widen and use a mul plus a shift.
6975 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
6976 // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
6977 // custom lowered. This is very expensive so avoid it at all costs for
6978 // constant divisors.
6979 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::SDIV, VT) &&
6980 isOperationCustom(Op: ISD::SDIVREM, VT: VT.getScalarType())) ||
6981 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
6982 MulVT = WideVT;
6983 }
6984
6985 if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
6986 return SDValue();
6987
6988 // If we're after type legalization and SVT is not legal, use the
6989 // promoted type for creating constants to avoid creating nodes with
6990 // illegal types.
6991 if (IsAfterLegalTypes && VT.isVector()) {
6992 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
6993 if (SVT.bitsLT(VT: VT.getScalarType()))
6994 return SDValue();
6995 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
6996 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
6997 return SDValue();
6998 }
6999 const unsigned SVTBits = SVT.getSizeInBits();
7000
7001 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
7002
7003 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
7004 if (C->isZero())
7005 return false;
7006 // Truncate the divisor to the target scalar type in case it was promoted
7007 // during type legalization.
7008 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
7009 SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(D: Divisor);
7010 int NumeratorFactor = 0;
7011 int ShiftMask = -1;
7012
7013 if (Divisor.isOne() || Divisor.isAllOnes()) {
7014 // If d is +1/-1, we just multiply the numerator by +1/-1.
7015 NumeratorFactor = Divisor.getSExtValue();
7016 magics.Magic = 0;
7017 magics.ShiftAmount = 0;
7018 ShiftMask = 0;
7019 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
7020 // If d > 0 and m < 0, add the numerator.
7021 NumeratorFactor = 1;
7022 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
7023 // If d < 0 and m > 0, subtract the numerator.
7024 NumeratorFactor = -1;
7025 }
7026
7027 MagicFactors.push_back(
7028 Elt: DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT));
7029 Factors.push_back(Elt: DAG.getSignedConstant(Val: NumeratorFactor, DL: dl, VT: SVT));
7030 Shifts.push_back(Elt: DAG.getConstant(Val: magics.ShiftAmount, DL: dl, VT: ShSVT));
7031 ShiftMasks.push_back(Elt: DAG.getSignedConstant(Val: ShiftMask, DL: dl, VT: SVT));
7032 return true;
7033 };
7034
7035 SDValue N0 = N->getOperand(Num: 0);
7036 SDValue N1 = N->getOperand(Num: 1);
7037
7038 // Collect the shifts / magic values from each element.
7039 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildSDIVPattern, /*AllowUndefs=*/false,
7040 /*AllowTruncation=*/true))
7041 return SDValue();
7042
7043 SDValue MagicFactor, Factor, Shift, ShiftMask;
7044 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7045 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
7046 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
7047 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
7048 ShiftMask = DAG.getBuildVector(VT, DL: dl, Ops: ShiftMasks);
7049 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7050 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
7051 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
7052 "Expected matchUnaryPredicate to return one element for scalable "
7053 "vectors");
7054 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
7055 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
7056 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
7057 ShiftMask = DAG.getSplatVector(VT, DL: dl, Op: ShiftMasks[0]);
7058 } else {
7059 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7060 MagicFactor = MagicFactors[0];
7061 Factor = Factors[0];
7062 Shift = Shifts[0];
7063 ShiftMask = ShiftMasks[0];
7064 }
7065
7066 // Multiply the numerator (operand 0) by the magic value.
7067 auto GetMULHS = [&](SDValue X, SDValue Y) {
7068 if (HasMULHS)
7069 return DAG.getNode(Opcode: ISD::MULHS, DL: dl, VT, N1: X, N2: Y);
7070 if (HasSMUL_LOHI) {
7071 SDValue LoHi =
7072 DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7073 return LoHi.getValue(R: 1);
7074 }
7075
7076 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: X);
7077 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7078 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7079 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7080 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7081 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7082 };
7083
7084 SDValue Q = GetMULHS(N0, MagicFactor);
7085 if (!Q)
7086 return SDValue();
7087
7088 Created.push_back(Elt: Q.getNode());
7089
7090 // (Optionally) Add/subtract the numerator using Factor.
7091 Factor = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: N0, N2: Factor);
7092 Created.push_back(Elt: Factor.getNode());
7093 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: Factor);
7094 Created.push_back(Elt: Q.getNode());
7095
7096 // Shift right algebraic by shift value.
7097 Q = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Q, N2: Shift);
7098 Created.push_back(Elt: Q.getNode());
7099
7100 // Extract the sign bit, mask it and add it to the quotient.
7101 SDValue SignShift = DAG.getConstant(Val: EltBits - 1, DL: dl, VT: ShVT);
7102 SDValue T = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: SignShift);
7103 Created.push_back(Elt: T.getNode());
7104 T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: ShiftMask);
7105 Created.push_back(Elt: T.getNode());
7106 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: T);
7107}
7108
7109/// Given an ISD::UDIV node expressing a divide by constant,
7110/// return a DAG expression to select that will generate the same value by
7111/// multiplying by a magic number.
7112/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
7113SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
7114 bool IsAfterLegalization,
7115 bool IsAfterLegalTypes,
7116 SmallVectorImpl<SDNode *> &Created) const {
7117 SDLoc dl(N);
7118
7119 // If the udiv has an 'exact' bit we can use a simpler lowering.
7120 if (N->getFlags().hasExact())
7121 return BuildExactUDIV(TLI: *this, N, dl, DAG, Created);
7122
7123 EVT VT = N->getValueType(ResNo: 0);
7124 EVT SVT = VT.getScalarType();
7125 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7126 EVT ShSVT = ShVT.getScalarType();
7127 unsigned EltBits = VT.getScalarSizeInBits();
7128 EVT MulVT;
7129
7130 // Check to see if we can do this.
7131 // FIXME: We should be more aggressive here.
7132 EVT QueryVT = VT;
7133 if (VT.isVector()) {
7134 // If the vector type will be legalized to a vector type with the same
7135 // element type, allow the transform before type legalization if MULHU or
7136 // UMUL_LOHI are supported.
7137 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
7138 if (!QueryVT.isVector() ||
7139 QueryVT.getVectorElementType() != VT.getVectorElementType())
7140 return SDValue();
7141 } else if (!isTypeLegal(VT)) {
7142 // Limit this to simple scalars for now.
7143 if (!VT.isSimple())
7144 return SDValue();
7145
7146 // If this type will be promoted to a large enough type with a legal
7147 // multiply operation, we can go ahead and do this transform.
7148 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
7149 return SDValue();
7150
7151 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7152 if (MulVT.getSizeInBits() < (2 * EltBits) ||
7153 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
7154 return SDValue();
7155 }
7156
7157 bool HasMULHU =
7158 isOperationLegalOrCustom(Op: ISD::MULHU, VT: QueryVT, LegalOnly: IsAfterLegalization);
7159 bool HasUMUL_LOHI =
7160 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
7161
7162 if (isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
7163 // If type twice as wide legal, widen and use a mul plus a shift.
7164 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
7165 // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
7166 // custom lowered. This is very expensive so avoid it at all costs for
7167 // constant divisors.
7168 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::UDIV, VT) &&
7169 isOperationCustom(Op: ISD::UDIVREM, VT: VT.getScalarType())) ||
7170 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
7171 MulVT = WideVT;
7172 }
7173
7174 if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
7175 return SDValue();
7176
7177 SDValue N0 = N->getOperand(Num: 0);
7178 SDValue N1 = N->getOperand(Num: 1);
7179
7180 // Try to use leading zeros of the dividend to reduce the multiplier and
7181 // avoid expensive fixups.
7182 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
7183
7184 // If we're after type legalization and SVT is not legal, use the
7185 // promoted type for creating constants to avoid creating nodes with
7186 // illegal types.
7187 if (IsAfterLegalTypes && VT.isVector()) {
7188 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
7189 if (SVT.bitsLT(VT: VT.getScalarType()))
7190 return SDValue();
7191 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
7192 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
7193 return SDValue();
7194 }
7195 const unsigned SVTBits = SVT.getSizeInBits();
7196
7197 // Allow i32 to be widened to i64 for uncooperative divisors if i64 MULHU or
7198 // UMUL_LOHI is supported.
7199 const EVT WideSVT = MVT::i64;
7200 const bool HasWideMULHU =
7201 VT == MVT::i32 &&
7202 isOperationLegalOrCustom(Op: ISD::MULHU, VT: WideSVT, LegalOnly: IsAfterLegalization);
7203 const bool HasWideUMUL_LOHI =
7204 VT == MVT::i32 &&
7205 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: WideSVT, LegalOnly: IsAfterLegalization);
7206 const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
7207
7208 // For even divisors with a 33-bit magic number, the widened high-multiply
7209 // path is only worthwhile over the even-divisor rewrite on targets that
7210 // zero-extend i32 to i64 for free (e.g. x86-64 and AArch64). Elsewhere (e.g.
7211 // RISC-V) keep the even-divisor rewrite, which avoids the explicit extension.
7212 const bool AllowEvenToWiden = AllowWiden && isZExtFree(FromTy: VT, ToTy: WideSVT);
7213
7214 bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
7215 bool UseWiden = false;
7216 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
7217
7218 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
7219 if (C->isZero())
7220 return false;
7221 // Truncate the divisor to the target scalar type in case it was promoted
7222 // during type legalization.
7223 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
7224
7225 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
7226
7227 // Magic algorithm doesn't work for division by 1. We need to emit a select
7228 // at the end.
7229 if (Divisor.isOne()) {
7230 PreShift = PostShift = DAG.getUNDEF(VT: ShSVT);
7231 MagicFactor = NPQFactor = DAG.getUNDEF(VT: SVT);
7232 } else {
7233 UnsignedDivisionByConstantInfo magics =
7234 UnsignedDivisionByConstantInfo::get(
7235 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()),
7236 /*AllowEvenDivisorOptimization=*/!AllowEvenToWiden,
7237 /*AllowWidenOptimization=*/AllowWiden);
7238
7239 if (magics.Widen) {
7240 UseWiden = true;
7241 MagicFactor = DAG.getConstant(Val: magics.Magic, DL: dl, VT: WideSVT);
7242 } else {
7243 MagicFactor = DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT);
7244 }
7245
7246 assert(magics.PreShift < Divisor.getBitWidth() &&
7247 "We shouldn't generate an undefined shift!");
7248 assert(magics.PostShift < Divisor.getBitWidth() &&
7249 "We shouldn't generate an undefined shift!");
7250 assert((!magics.IsAdd || magics.PreShift == 0) &&
7251 "Unexpected pre-shift");
7252 PreShift = DAG.getConstant(Val: magics.PreShift, DL: dl, VT: ShSVT);
7253 PostShift = DAG.getConstant(Val: magics.PostShift, DL: dl, VT: ShSVT);
7254 NPQFactor = DAG.getConstant(
7255 Val: magics.IsAdd ? APInt::getOneBitSet(numBits: SVTBits, BitNo: EltBits - 1)
7256 : APInt::getZero(numBits: SVTBits),
7257 DL: dl, VT: SVT);
7258 UseNPQ |= magics.IsAdd;
7259 UsePreShift |= magics.PreShift != 0;
7260 UsePostShift |= magics.PostShift != 0;
7261 }
7262
7263 PreShifts.push_back(Elt: PreShift);
7264 MagicFactors.push_back(Elt: MagicFactor);
7265 NPQFactors.push_back(Elt: NPQFactor);
7266 PostShifts.push_back(Elt: PostShift);
7267 return true;
7268 };
7269
7270 // Collect the shifts/magic values from each element.
7271 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildUDIVPattern, /*AllowUndefs=*/false,
7272 /*AllowTruncation=*/true))
7273 return SDValue();
7274
7275 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
7276 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7277 PreShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PreShifts);
7278 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
7279 NPQFactor = DAG.getBuildVector(VT, DL: dl, Ops: NPQFactors);
7280 PostShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PostShifts);
7281 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7282 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
7283 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
7284 "Expected matchUnaryPredicate to return one for scalable vectors");
7285 PreShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PreShifts[0]);
7286 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
7287 NPQFactor = DAG.getSplatVector(VT, DL: dl, Op: NPQFactors[0]);
7288 PostShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PostShifts[0]);
7289 } else {
7290 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7291 PreShift = PreShifts[0];
7292 MagicFactor = MagicFactors[0];
7293 PostShift = PostShifts[0];
7294 }
7295
7296 if (UseWiden) {
7297 // Compute: (WideSVT(x) * MagicFactor) >> WideSVTBits.
7298 SDValue WideN0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: WideSVT, Operand: N0);
7299
7300 // Perform WideSVTxWideSVT -> 2*WideSVT multiplication and extract high
7301 // WideSVT bits
7302 SDValue High;
7303 if (HasWideMULHU) {
7304 High = DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT: WideSVT, N1: WideN0, N2: MagicFactor);
7305 } else {
7306 assert(HasWideUMUL_LOHI);
7307 SDValue LoHi =
7308 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: WideSVT, VT2: WideSVT),
7309 N1: WideN0, N2: MagicFactor);
7310 High = LoHi.getValue(R: 1);
7311 }
7312
7313 Created.push_back(Elt: High.getNode());
7314 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: High);
7315 }
7316
7317 SDValue Q = N0;
7318 if (UsePreShift) {
7319 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PreShift);
7320 Created.push_back(Elt: Q.getNode());
7321 }
7322
7323 auto GetMULHU = [&](SDValue X, SDValue Y) {
7324 if (HasMULHU)
7325 return DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT, N1: X, N2: Y);
7326 if (HasUMUL_LOHI) {
7327 SDValue LoHi =
7328 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7329 return LoHi.getValue(R: 1);
7330 }
7331
7332 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: X);
7333 Y = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7334 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7335 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7336 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7337 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7338 };
7339
7340 // Multiply the numerator (operand 0) by the magic value.
7341 Q = GetMULHU(Q, MagicFactor);
7342 if (!Q)
7343 return SDValue();
7344
7345 Created.push_back(Elt: Q.getNode());
7346
7347 if (UseNPQ) {
7348 SDValue NPQ = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: N0, N2: Q);
7349 Created.push_back(Elt: NPQ.getNode());
7350
7351 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
7352 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
7353 if (VT.isVector())
7354 NPQ = GetMULHU(NPQ, NPQFactor);
7355 else
7356 NPQ = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NPQ, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT));
7357
7358 Created.push_back(Elt: NPQ.getNode());
7359
7360 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: NPQ, N2: Q);
7361 Created.push_back(Elt: Q.getNode());
7362 }
7363
7364 if (UsePostShift) {
7365 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PostShift);
7366 Created.push_back(Elt: Q.getNode());
7367 }
7368
7369 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7370
7371 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
7372 SDValue IsOne = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N1, RHS: One, Cond: ISD::SETEQ);
7373 return DAG.getSelect(DL: dl, VT, Cond: IsOne, LHS: N0, RHS: Q);
7374}
7375
7376/// If all values in Values that *don't* match the predicate are same 'splat'
7377/// value, then replace all values with that splat value.
7378/// Else, if AlternativeReplacement was provided, then replace all values that
7379/// do match predicate with AlternativeReplacement value.
7380static void
7381turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
7382 std::function<bool(SDValue)> Predicate,
7383 SDValue AlternativeReplacement = SDValue()) {
7384 SDValue Replacement;
7385 // Is there a value for which the Predicate does *NOT* match? What is it?
7386 auto SplatValue = llvm::find_if_not(Range&: Values, P: Predicate);
7387 if (SplatValue != Values.end()) {
7388 // Does Values consist only of SplatValue's and values matching Predicate?
7389 if (llvm::all_of(Range&: Values, P: [Predicate, SplatValue](SDValue Value) {
7390 return Value == *SplatValue || Predicate(Value);
7391 })) // Then we shall replace values matching predicate with SplatValue.
7392 Replacement = *SplatValue;
7393 }
7394 if (!Replacement) {
7395 // Oops, we did not find the "baseline" splat value.
7396 if (!AlternativeReplacement)
7397 return; // Nothing to do.
7398 // Let's replace with provided value then.
7399 Replacement = AlternativeReplacement;
7400 }
7401 std::replace_if(first: Values.begin(), last: Values.end(), pred: Predicate, new_value: Replacement);
7402}
7403
7404/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
7405/// where the divisor and comparison target are constants,
7406/// return a DAG expression that will generate the same comparison result
7407/// using only multiplications, additions and shifts/rotations.
7408/// Ref: "Hacker's Delight" 10-17.
7409SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
7410 SDValue CompTargetNode,
7411 ISD::CondCode Cond,
7412 DAGCombinerInfo &DCI,
7413 const SDLoc &DL) const {
7414 SmallVector<SDNode *, 5> Built;
7415 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7416 DCI, DL, Created&: Built)) {
7417 for (SDNode *N : Built)
7418 DCI.AddToWorklist(N);
7419 return Folded;
7420 }
7421
7422 return SDValue();
7423}
7424
7425SDValue
7426TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
7427 SDValue CompTargetNode, ISD::CondCode Cond,
7428 DAGCombinerInfo &DCI, const SDLoc &DL,
7429 SmallVectorImpl<SDNode *> &Created) const {
7430 // fold (seteq/ne (urem N, D), C) ->
7431 // (setule/ugt (rotr (mul (sub N, C), P), K), Q)
7432 // - D must be constant, with D = D0 * 2^K where D0 is odd
7433 // - P is the multiplicative inverse of D0 modulo 2^W
7434 // - Q = floor(((2^W) - 1) / D)
7435 // where W is the width of the common type of N and D.
7436 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7437 "Only applicable for (in)equality comparisons.");
7438
7439 SelectionDAG &DAG = DCI.DAG;
7440
7441 EVT VT = REMNode.getValueType();
7442 EVT SVT = VT.getScalarType();
7443 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7444 EVT ShSVT = ShVT.getScalarType();
7445
7446 // If MUL is unavailable, we cannot proceed in any case.
7447 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7448 return SDValue();
7449
7450 bool ComparingWithAllZeros = true;
7451 bool AllComparisonsWithNonZerosAreTautological = true;
7452 bool HadTautologicalLanes = false;
7453 bool AllLanesAreTautological = true;
7454 bool HadEvenDivisor = false;
7455 bool AllDivisorsArePowerOfTwo = true;
7456 bool HadTautologicalInvertedLanes = false;
7457 SmallVector<SDValue, 16> PAmts, KAmts, QAmts;
7458
7459 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
7460 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7461 if (CDiv->isZero())
7462 return false;
7463
7464 const APInt &D = CDiv->getAPIntValue();
7465 const APInt &Cmp = CCmp->getAPIntValue();
7466
7467 ComparingWithAllZeros &= Cmp.isZero();
7468
7469 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7470 // if C2 is not less than C1, the comparison is always false.
7471 // But we will only be able to produce the comparison that will give the
7472 // opposive tautological answer. So this lane would need to be fixed up.
7473 bool TautologicalInvertedLane = D.ule(RHS: Cmp);
7474 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
7475
7476 // If all lanes are tautological (either all divisors are ones, or divisor
7477 // is not greater than the constant we are comparing with),
7478 // we will prefer to avoid the fold.
7479 bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
7480 HadTautologicalLanes |= TautologicalLane;
7481 AllLanesAreTautological &= TautologicalLane;
7482
7483 // If we are comparing with non-zero, we need'll need to subtract said
7484 // comparison value from the LHS. But there is no point in doing that if
7485 // every lane where we are comparing with non-zero is tautological..
7486 if (!Cmp.isZero())
7487 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
7488
7489 // Decompose D into D0 * 2^K
7490 unsigned K = D.countr_zero();
7491 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7492 APInt D0 = D.lshr(shiftAmt: K);
7493
7494 // D is even if it has trailing zeros.
7495 HadEvenDivisor |= (K != 0);
7496 // D is a power-of-two if D0 is one.
7497 // If all divisors are power-of-two, we will prefer to avoid the fold.
7498 AllDivisorsArePowerOfTwo &= D0.isOne();
7499
7500 // P = inv(D0, 2^W)
7501 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7502 unsigned W = D.getBitWidth();
7503 APInt P = D0.multiplicativeInverse();
7504 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7505
7506 // Q = floor((2^W - 1) u/ D)
7507 // R = ((2^W - 1) u% D)
7508 APInt Q, R;
7509 APInt::udivrem(LHS: APInt::getAllOnes(numBits: W), RHS: D, Quotient&: Q, Remainder&: R);
7510
7511 // If we are comparing with zero, then that comparison constant is okay,
7512 // else it may need to be one less than that.
7513 if (Cmp.ugt(RHS: R))
7514 Q -= 1;
7515
7516 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7517 "We are expecting that K is always less than all-ones for ShSVT");
7518
7519 // If the lane is tautological the result can be constant-folded.
7520 if (TautologicalLane) {
7521 // Set P and K amount to a bogus values so we can try to splat them.
7522 P = 0;
7523 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7524 // And ensure that comparison constant is tautological,
7525 // it will always compare true/false.
7526 Q.setAllBits();
7527 } else {
7528 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7529 }
7530
7531 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7532 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7533 return true;
7534 };
7535
7536 SDValue N = REMNode.getOperand(i: 0);
7537 SDValue D = REMNode.getOperand(i: 1);
7538
7539 // Collect the values from each element.
7540 if (!ISD::matchBinaryPredicate(LHS: D, RHS: CompTargetNode, Match: BuildUREMPattern))
7541 return SDValue();
7542
7543 // If all lanes are tautological, the result can be constant-folded.
7544 if (AllLanesAreTautological)
7545 return SDValue();
7546
7547 // If this is a urem by a powers-of-two, avoid the fold since it can be
7548 // best implemented as a bit test.
7549 if (AllDivisorsArePowerOfTwo)
7550 return SDValue();
7551
7552 SDValue PVal, KVal, QVal;
7553 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7554 if (HadTautologicalLanes) {
7555 // Try to turn PAmts into a splat, since we don't care about the values
7556 // that are currently '0'. If we can't, just keep '0'`s.
7557 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7558 // Try to turn KAmts into a splat, since we don't care about the values
7559 // that are currently '-1'. If we can't, change them to '0'`s.
7560 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7561 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7562 }
7563
7564 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7565 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7566 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7567 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7568 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
7569 "Expected matchBinaryPredicate to return one element for "
7570 "SPLAT_VECTORs");
7571 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7572 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7573 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7574 } else {
7575 PVal = PAmts[0];
7576 KVal = KAmts[0];
7577 QVal = QAmts[0];
7578 }
7579
7580 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
7581 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::SUB, VT))
7582 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
7583 assert(CompTargetNode.getValueType() == N.getValueType() &&
7584 "Expecting that the types on LHS and RHS of comparisons match.");
7585 N = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N, N2: CompTargetNode);
7586 }
7587
7588 // (mul N, P)
7589 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7590 Created.push_back(Elt: Op0.getNode());
7591
7592 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7593 // divisors as a performance improvement, since rotating by 0 is a no-op.
7594 if (HadEvenDivisor) {
7595 // We need ROTR to do this.
7596 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7597 return SDValue();
7598 // UREM: (rotr (mul N, P), K)
7599 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7600 Created.push_back(Elt: Op0.getNode());
7601 }
7602
7603 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
7604 SDValue NewCC =
7605 DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7606 Cond: ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
7607 if (!HadTautologicalInvertedLanes)
7608 return NewCC;
7609
7610 // If any lanes previously compared always-false, the NewCC will give
7611 // always-true result for them, so we need to fixup those lanes.
7612 // Or the other way around for inequality predicate.
7613 assert(VT.isVector() && "Can/should only get here for vectors.");
7614 Created.push_back(Elt: NewCC.getNode());
7615
7616 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7617 // if C2 is not less than C1, the comparison is always false.
7618 // But we have produced the comparison that will give the
7619 // opposive tautological answer. So these lanes would need to be fixed up.
7620 SDValue TautologicalInvertedChannels =
7621 DAG.getSetCC(DL, VT: SETCCVT, LHS: D, RHS: CompTargetNode, Cond: ISD::SETULE);
7622 Created.push_back(Elt: TautologicalInvertedChannels.getNode());
7623
7624 // NOTE: we avoid letting illegal types through even if we're before legalize
7625 // ops – legalization has a hard time producing good code for this.
7626 if (isOperationLegalOrCustom(Op: ISD::VSELECT, VT: SETCCVT)) {
7627 // If we have a vector select, let's replace the comparison results in the
7628 // affected lanes with the correct tautological result.
7629 SDValue Replacement = DAG.getBoolConstant(V: Cond == ISD::SETEQ ? false : true,
7630 DL, VT: SETCCVT, OpVT: SETCCVT);
7631 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: SETCCVT, N1: TautologicalInvertedChannels,
7632 N2: Replacement, N3: NewCC);
7633 }
7634
7635 // Else, we can just invert the comparison result in the appropriate lanes.
7636 //
7637 // NOTE: see the note above VSELECT above.
7638 if (isOperationLegalOrCustom(Op: ISD::XOR, VT: SETCCVT))
7639 return DAG.getNode(Opcode: ISD::XOR, DL, VT: SETCCVT, N1: NewCC,
7640 N2: TautologicalInvertedChannels);
7641
7642 return SDValue(); // Don't know how to lower.
7643}
7644
7645/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
7646/// where the divisor is constant and the comparison target is zero,
7647/// return a DAG expression that will generate the same comparison result
7648/// using only multiplications, additions and shifts/rotations.
7649/// Ref: "Hacker's Delight" 10-17.
7650SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
7651 SDValue CompTargetNode,
7652 ISD::CondCode Cond,
7653 DAGCombinerInfo &DCI,
7654 const SDLoc &DL) const {
7655 SmallVector<SDNode *, 7> Built;
7656 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7657 DCI, DL, Created&: Built)) {
7658 assert(Built.size() <= 7 && "Max size prediction failed.");
7659 for (SDNode *N : Built)
7660 DCI.AddToWorklist(N);
7661 return Folded;
7662 }
7663
7664 return SDValue();
7665}
7666
7667SDValue
7668TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
7669 SDValue CompTargetNode, ISD::CondCode Cond,
7670 DAGCombinerInfo &DCI, const SDLoc &DL,
7671 SmallVectorImpl<SDNode *> &Created) const {
7672 // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
7673 // Fold:
7674 // (seteq/ne (srem N, D), 0)
7675 // To:
7676 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
7677 //
7678 // - D must be constant, with D = D0 * 2^K where D0 is odd
7679 // - P is the multiplicative inverse of D0 modulo 2^W
7680 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
7681 // - Q = floor((2 * A) / (2^K))
7682 // where W is the width of the common type of N and D.
7683 //
7684 // When D is a power of two (and thus D0 is 1), the normal
7685 // formula for A and Q don't apply, because the derivation
7686 // depends on D not dividing 2^(W-1), and thus theorem ZRS
7687 // does not apply. This specifically fails when N = INT_MIN.
7688 //
7689 // Instead, for power-of-two D, we use:
7690 // - A = 0
7691 // | -> No offset needed. We're effectively treating it the same as urem.
7692 // - Q = 2^(W-K) - 1
7693 // |-> Test that the top K bits are zero after rotation
7694 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7695 "Only applicable for (in)equality comparisons.");
7696
7697 SelectionDAG &DAG = DCI.DAG;
7698
7699 EVT VT = REMNode.getValueType();
7700 EVT SVT = VT.getScalarType();
7701 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7702 EVT ShSVT = ShVT.getScalarType();
7703
7704 // If we are after ops legalization, and MUL is unavailable, we can not
7705 // proceed.
7706 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7707 return SDValue();
7708
7709 // TODO: Could support comparing with non-zero too.
7710 ConstantSDNode *CompTarget = isConstOrConstSplat(N: CompTargetNode);
7711 if (!CompTarget || !CompTarget->isZero())
7712 return SDValue();
7713
7714 bool HadOneDivisor = false;
7715 bool AllDivisorsAreOnes = true;
7716 bool HadEvenDivisor = false;
7717 bool AllDivisorsArePowerOfTwo = true;
7718 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
7719
7720 auto BuildSREMPattern = [&](ConstantSDNode *C) {
7721 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7722 if (C->isZero())
7723 return false;
7724
7725 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
7726
7727 // WARNING: this fold is only valid for positive divisors!
7728 // `rem %X, -C` is equivalent to `rem %X, C`
7729 APInt D = C->getAPIntValue().abs();
7730
7731 // If all divisors are ones, we will prefer to avoid the fold.
7732 HadOneDivisor |= D.isOne();
7733 AllDivisorsAreOnes &= D.isOne();
7734
7735 // Decompose D into D0 * 2^K
7736 unsigned K = D.countr_zero();
7737 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7738 APInt D0 = D.lshr(shiftAmt: K);
7739
7740 // D is even if it has trailing zeros.
7741 HadEvenDivisor |= (K != 0);
7742
7743 // D is a power-of-two if D0 is one. This includes INT_MIN.
7744 // If all divisors are power-of-two, we will prefer to avoid the fold.
7745 AllDivisorsArePowerOfTwo &= D0.isOne();
7746
7747 // P = inv(D0, 2^W)
7748 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7749 unsigned W = D.getBitWidth();
7750 APInt P = D0.multiplicativeInverse();
7751 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7752
7753 // A = floor((2^(W - 1) - 1) / D0) & -2^K
7754 APInt A = APInt::getSignedMaxValue(numBits: W).udiv(RHS: D0);
7755 A.clearLowBits(loBits: K);
7756
7757 // Q = floor((2 * A) / (2^K))
7758 APInt Q = (2 * A).udiv(RHS: APInt::getOneBitSet(numBits: W, BitNo: K));
7759
7760 assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
7761 "We are expecting that A is always less than all-ones for SVT");
7762 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7763 "We are expecting that K is always less than all-ones for ShSVT");
7764
7765 // If D was a power of two, apply the alternate constant derivation.
7766 if (D0.isOne()) {
7767 // A = 0
7768 A = APInt(W, 0);
7769 // - Q = 2^(W-K) - 1
7770 Q = APInt::getLowBitsSet(numBits: W, loBitsSet: W - K);
7771 }
7772
7773 // If the divisor is 1 the result can be constant-folded.
7774 if (D.isOne()) {
7775 // Set P, A and K to a bogus values so we can try to splat them.
7776 P = 0;
7777 A.setAllBits();
7778 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7779
7780 // x ?% 1 == 0 <--> true <--> x u<= -1
7781 Q.setAllBits();
7782 } else {
7783 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7784 }
7785
7786 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7787 AAmts.push_back(Elt: DAG.getConstant(Val: A, DL, VT: SVT));
7788 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7789 return true;
7790 };
7791
7792 SDValue N = REMNode.getOperand(i: 0);
7793 SDValue D = REMNode.getOperand(i: 1);
7794
7795 // Collect the values from each element.
7796 if (!ISD::matchUnaryPredicate(Op: D, Match: BuildSREMPattern))
7797 return SDValue();
7798
7799 // If this is a srem by a one, avoid the fold since it can be constant-folded.
7800 if (AllDivisorsAreOnes)
7801 return SDValue();
7802
7803 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
7804 // since it can be best implemented as a bit test.
7805 if (AllDivisorsArePowerOfTwo)
7806 return SDValue();
7807
7808 SDValue PVal, AVal, KVal, QVal;
7809 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7810 if (HadOneDivisor) {
7811 // Try to turn PAmts into a splat, since we don't care about the values
7812 // that are currently '0'. If we can't, just keep '0'`s.
7813 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7814 // Try to turn AAmts into a splat, since we don't care about the
7815 // values that are currently '-1'. If we can't, change them to '0'`s.
7816 turnVectorIntoSplatVector(Values: AAmts, Predicate: isAllOnesConstant,
7817 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: SVT));
7818 // Try to turn KAmts into a splat, since we don't care about the values
7819 // that are currently '-1'. If we can't, change them to '0'`s.
7820 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7821 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7822 }
7823
7824 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7825 AVal = DAG.getBuildVector(VT, DL, Ops: AAmts);
7826 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7827 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7828 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7829 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
7830 QAmts.size() == 1 &&
7831 "Expected matchUnaryPredicate to return one element for scalable "
7832 "vectors");
7833 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7834 AVal = DAG.getSplatVector(VT, DL, Op: AAmts[0]);
7835 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7836 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7837 } else {
7838 assert(isa<ConstantSDNode>(D) && "Expected a constant");
7839 PVal = PAmts[0];
7840 AVal = AAmts[0];
7841 KVal = KAmts[0];
7842 QVal = QAmts[0];
7843 }
7844
7845 // (mul N, P)
7846 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7847 Created.push_back(Elt: Op0.getNode());
7848
7849 // We need ADD to do this.
7850 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ADD, VT))
7851 return SDValue();
7852
7853 // (add (mul N, P), A)
7854 Op0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: AVal);
7855 Created.push_back(Elt: Op0.getNode());
7856
7857 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7858 // divisors as a performance improvement, since rotating by 0 is a no-op.
7859 if (HadEvenDivisor) {
7860 // We need ROTR to do this.
7861 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7862 return SDValue();
7863 // SREM: (rotr (add (mul N, P), A), K)
7864 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7865 Created.push_back(Elt: Op0.getNode());
7866 }
7867
7868 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
7869 return DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7870 Cond: (Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT);
7871}
7872
7873SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
7874 const DenormalMode &Mode,
7875 SDNodeFlags Flags) const {
7876 SDLoc DL(Op);
7877 EVT VT = Op.getValueType();
7878 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7879 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL, VT);
7880
7881 // This is specifically a check for the handling of denormal inputs, not the
7882 // result.
7883 if (Mode.Input == DenormalMode::PreserveSign ||
7884 Mode.Input == DenormalMode::PositiveZero) {
7885 // Test = X == 0.0
7886 return DAG.getSetCC(DL, VT: CCVT, LHS: Op, RHS: FPZero, Cond: ISD::SETEQ, /*Chain=*/{},
7887 /*Signaling=*/IsSignaling: false, Flags);
7888 }
7889
7890 // Testing it with denormal inputs to avoid wrong estimate.
7891 //
7892 // Test = fabs(X) < SmallestNormal
7893 const fltSemantics &FltSem = VT.getFltSemantics();
7894 APFloat SmallestNorm = APFloat::getSmallestNormalized(Sem: FltSem);
7895 SDValue NormC = DAG.getConstantFP(Val: SmallestNorm, DL, VT);
7896 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: Op, Flags);
7897 return DAG.getSetCC(DL, VT: CCVT, LHS: Fabs, RHS: NormC, Cond: ISD::SETLT, /*Chain=*/{},
7898 /*Signaling=*/IsSignaling: false, Flags);
7899}
7900
7901SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
7902 bool LegalOps, bool OptForSize,
7903 NegatibleCost &Cost,
7904 unsigned Depth) const {
7905 // fneg is removable even if it has multiple uses.
7906 if (Op.getOpcode() == ISD::FNEG) {
7907 Cost = NegatibleCost::Cheaper;
7908 return Op.getOperand(i: 0);
7909 }
7910
7911 // Don't recurse exponentially.
7912 if (Depth > SelectionDAG::MaxRecursionDepth)
7913 return SDValue();
7914
7915 // Pre-increment recursion depth for use in recursive calls.
7916 ++Depth;
7917 const SDNodeFlags Flags = Op->getFlags();
7918 EVT VT = Op.getValueType();
7919 unsigned Opcode = Op.getOpcode();
7920
7921 // Don't allow anything with multiple uses unless we know it is free.
7922 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
7923 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
7924 isFPExtFree(DestVT: VT, SrcVT: Op.getOperand(i: 0).getValueType());
7925 if (!IsFreeExtend)
7926 return SDValue();
7927 }
7928
7929 auto RemoveDeadNode = [&](SDValue N) {
7930 if (N && N.getNode()->use_empty())
7931 DAG.RemoveDeadNode(N: N.getNode());
7932 };
7933
7934 SDLoc DL(Op);
7935
7936 // Because getNegatedExpression can delete nodes we need a handle to keep
7937 // temporary nodes alive in case the recursion manages to create an identical
7938 // node.
7939 std::list<HandleSDNode> Handles;
7940
7941 switch (Opcode) {
7942 case ISD::ConstantFP: {
7943 // Don't invert constant FP values after legalization unless the target says
7944 // the negated constant is legal.
7945 bool IsOpLegal =
7946 isOperationLegal(Op: ISD::ConstantFP, VT) ||
7947 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF()), VT,
7948 ForCodeSize: OptForSize);
7949
7950 if (LegalOps && !IsOpLegal)
7951 break;
7952
7953 APFloat V = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
7954 V.changeSign();
7955 SDValue CFP = DAG.getConstantFP(Val: V, DL, VT);
7956
7957 // If we already have the use of the negated floating constant, it is free
7958 // to negate it even it has multiple uses.
7959 if (!Op.hasOneUse() && CFP.use_empty())
7960 break;
7961 Cost = NegatibleCost::Neutral;
7962 return CFP;
7963 }
7964 case ISD::SPLAT_VECTOR: {
7965 // fold splat_vector(fneg(X)) -> splat_vector(-X)
7966 SDValue X = Op.getOperand(i: 0);
7967 if (!isOperationLegal(Op: ISD::SPLAT_VECTOR, VT))
7968 break;
7969
7970 SDValue NegX = getCheaperNegatedExpression(Op: X, DAG, LegalOps, OptForSize);
7971 if (!NegX)
7972 break;
7973 Cost = NegatibleCost::Cheaper;
7974 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: NegX);
7975 }
7976 case ISD::BUILD_VECTOR: {
7977 // Only permit BUILD_VECTOR of constants.
7978 if (llvm::any_of(Range: Op->op_values(), P: [&](SDValue N) {
7979 return !N.isUndef() && !isa<ConstantFPSDNode>(Val: N);
7980 }))
7981 break;
7982
7983 bool IsOpLegal =
7984 (isOperationLegal(Op: ISD::ConstantFP, VT) &&
7985 isOperationLegal(Op: ISD::BUILD_VECTOR, VT)) ||
7986 llvm::all_of(Range: Op->op_values(), P: [&](SDValue N) {
7987 return N.isUndef() ||
7988 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: N)->getValueAPF()), VT,
7989 ForCodeSize: OptForSize);
7990 });
7991
7992 if (LegalOps && !IsOpLegal)
7993 break;
7994
7995 SmallVector<SDValue, 4> Ops;
7996 for (SDValue C : Op->op_values()) {
7997 if (C.isUndef()) {
7998 Ops.push_back(Elt: C);
7999 continue;
8000 }
8001 APFloat V = cast<ConstantFPSDNode>(Val&: C)->getValueAPF();
8002 V.changeSign();
8003 Ops.push_back(Elt: DAG.getConstantFP(Val: V, DL, VT: C.getValueType()));
8004 }
8005 Cost = NegatibleCost::Neutral;
8006 return DAG.getBuildVector(VT, DL, Ops);
8007 }
8008 case ISD::FADD: {
8009 if (!Flags.hasNoSignedZeros())
8010 break;
8011
8012 // After operation legalization, it might not be legal to create new FSUBs.
8013 if (LegalOps && !isOperationLegalOrCustom(Op: ISD::FSUB, VT))
8014 break;
8015 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
8016
8017 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
8018 NegatibleCost CostX = NegatibleCost::Expensive;
8019 SDValue NegX =
8020 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8021 // Prevent this node from being deleted by the next call.
8022 if (NegX)
8023 Handles.emplace_back(args&: NegX);
8024
8025 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
8026 NegatibleCost CostY = NegatibleCost::Expensive;
8027 SDValue NegY =
8028 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8029
8030 // We're done with the handles.
8031 Handles.clear();
8032
8033 // Negate the X if its cost is less or equal than Y.
8034 if (NegX && (CostX <= CostY)) {
8035 Cost = CostX;
8036 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegX, N2: Y, Flags);
8037 if (NegY != N)
8038 RemoveDeadNode(NegY);
8039 return N;
8040 }
8041
8042 // Negate the Y if it is not expensive.
8043 if (NegY) {
8044 Cost = CostY;
8045 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegY, N2: X, Flags);
8046 if (NegX != N)
8047 RemoveDeadNode(NegX);
8048 return N;
8049 }
8050 break;
8051 }
8052 case ISD::FSUB: {
8053 // We can't turn -(A-B) into B-A when we honor signed zeros.
8054 if (!Flags.hasNoSignedZeros())
8055 break;
8056
8057 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
8058 // fold (fneg (fsub 0, Y)) -> Y
8059 if (ConstantFPSDNode *C = isConstOrConstSplatFP(N: X, /*AllowUndefs*/ true))
8060 if (C->isZero()) {
8061 Cost = NegatibleCost::Cheaper;
8062 return Y;
8063 }
8064
8065 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
8066 Cost = NegatibleCost::Neutral;
8067 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: X, Flags);
8068 }
8069 case ISD::FMUL:
8070 case ISD::FDIV: {
8071 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
8072
8073 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
8074 NegatibleCost CostX = NegatibleCost::Expensive;
8075 SDValue NegX =
8076 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8077 // Prevent this node from being deleted by the next call.
8078 if (NegX)
8079 Handles.emplace_back(args&: NegX);
8080
8081 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
8082 NegatibleCost CostY = NegatibleCost::Expensive;
8083 SDValue NegY =
8084 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8085
8086 // We're done with the handles.
8087 Handles.clear();
8088
8089 // Negate the X if its cost is less or equal than Y.
8090 if (NegX && (CostX <= CostY)) {
8091 Cost = CostX;
8092 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, Flags);
8093 if (NegY != N)
8094 RemoveDeadNode(NegY);
8095 return N;
8096 }
8097
8098 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
8099 if (auto *C = isConstOrConstSplatFP(N: Op.getOperand(i: 1)))
8100 if (C->isExactlyValue(V: 2.0) && Op.getOpcode() == ISD::FMUL)
8101 break;
8102
8103 // Negate the Y if it is not expensive.
8104 if (NegY) {
8105 Cost = CostY;
8106 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, Flags);
8107 if (NegX != N)
8108 RemoveDeadNode(NegX);
8109 return N;
8110 }
8111 break;
8112 }
8113 case ISD::FMA:
8114 case ISD::FMULADD:
8115 case ISD::FMAD: {
8116 if (!Flags.hasNoSignedZeros())
8117 break;
8118
8119 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), Z = Op.getOperand(i: 2);
8120 NegatibleCost CostZ = NegatibleCost::Expensive;
8121 SDValue NegZ =
8122 getNegatedExpression(Op: Z, DAG, LegalOps, OptForSize, Cost&: CostZ, Depth);
8123 // Give up if fail to negate the Z.
8124 if (!NegZ)
8125 break;
8126
8127 // Prevent this node from being deleted by the next two calls.
8128 Handles.emplace_back(args&: NegZ);
8129
8130 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
8131 NegatibleCost CostX = NegatibleCost::Expensive;
8132 SDValue NegX =
8133 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8134 // Prevent this node from being deleted by the next call.
8135 if (NegX)
8136 Handles.emplace_back(args&: NegX);
8137
8138 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
8139 NegatibleCost CostY = NegatibleCost::Expensive;
8140 SDValue NegY =
8141 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8142
8143 // We're done with the handles.
8144 Handles.clear();
8145
8146 // Negate the X if its cost is less or equal than Y.
8147 if (NegX && (CostX <= CostY)) {
8148 Cost = std::min(a: CostX, b: CostZ);
8149 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, N3: NegZ, Flags);
8150 if (NegY != N)
8151 RemoveDeadNode(NegY);
8152 return N;
8153 }
8154
8155 // Negate the Y if it is not expensive.
8156 if (NegY) {
8157 Cost = std::min(a: CostY, b: CostZ);
8158 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, N3: NegZ, Flags);
8159 if (NegX != N)
8160 RemoveDeadNode(NegX);
8161 return N;
8162 }
8163 break;
8164 }
8165
8166 case ISD::FP_EXTEND:
8167 case ISD::FSIN:
8168 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
8169 OptForSize, Cost, Depth))
8170 return DAG.getNode(Opcode, DL, VT, Operand: NegV);
8171 break;
8172 case ISD::FP_ROUND:
8173 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
8174 OptForSize, Cost, Depth))
8175 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NegV, N2: Op.getOperand(i: 1));
8176 break;
8177 case ISD::SELECT:
8178 case ISD::VSELECT: {
8179 // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
8180 // iff at least one cost is cheaper and the other is neutral/cheaper
8181 SDValue LHS = Op.getOperand(i: 1);
8182 NegatibleCost CostLHS = NegatibleCost::Expensive;
8183 SDValue NegLHS =
8184 getNegatedExpression(Op: LHS, DAG, LegalOps, OptForSize, Cost&: CostLHS, Depth);
8185 if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
8186 RemoveDeadNode(NegLHS);
8187 break;
8188 }
8189
8190 // Prevent this node from being deleted by the next call.
8191 Handles.emplace_back(args&: NegLHS);
8192
8193 SDValue RHS = Op.getOperand(i: 2);
8194 NegatibleCost CostRHS = NegatibleCost::Expensive;
8195 SDValue NegRHS =
8196 getNegatedExpression(Op: RHS, DAG, LegalOps, OptForSize, Cost&: CostRHS, Depth);
8197
8198 // We're done with the handles.
8199 Handles.clear();
8200
8201 if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
8202 (CostLHS != NegatibleCost::Cheaper &&
8203 CostRHS != NegatibleCost::Cheaper)) {
8204 RemoveDeadNode(NegLHS);
8205 RemoveDeadNode(NegRHS);
8206 break;
8207 }
8208
8209 Cost = std::min(a: CostLHS, b: CostRHS);
8210 return DAG.getSelect(DL, VT, Cond: Op.getOperand(i: 0), LHS: NegLHS, RHS: NegRHS);
8211 }
8212 }
8213
8214 return SDValue();
8215}
8216
8217//===----------------------------------------------------------------------===//
8218// Legalization Utilities
8219//===----------------------------------------------------------------------===//
8220
8221bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
8222 SDValue LHS, SDValue RHS,
8223 SmallVectorImpl<SDValue> &Result,
8224 EVT HiLoVT, SelectionDAG &DAG,
8225 MulExpansionKind Kind, SDValue LL,
8226 SDValue LH, SDValue RL, SDValue RH) const {
8227 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
8228 Opcode == ISD::SMUL_LOHI);
8229
8230 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
8231 isOperationLegalOrCustom(Op: ISD::MULHS, VT: HiLoVT);
8232 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
8233 isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT);
8234 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8235 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: HiLoVT);
8236 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8237 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT);
8238
8239 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
8240 return false;
8241
8242 unsigned OuterBitSize = VT.getScalarSizeInBits();
8243 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
8244
8245 // LL, LH, RL, and RH must be either all NULL or all set to a value.
8246 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
8247 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
8248
8249 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
8250 bool Signed) -> bool {
8251 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
8252 SDVTList VTs = DAG.getVTList(VT1: HiLoVT, VT2: HiLoVT);
8253 Lo = DAG.getNode(Opcode: Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, DL: dl, VTList: VTs, N1: L, N2: R);
8254 Hi = Lo.getValue(R: 1);
8255 return true;
8256 }
8257 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
8258 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: L, N2: R);
8259 Hi = DAG.getNode(Opcode: Signed ? ISD::MULHS : ISD::MULHU, DL: dl, VT: HiLoVT, N1: L, N2: R);
8260 return true;
8261 }
8262 return false;
8263 };
8264
8265 SDValue Lo, Hi;
8266
8267 if (!LL.getNode() && !RL.getNode() &&
8268 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8269 LL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LHS);
8270 RL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RHS);
8271 }
8272
8273 if (!LL.getNode())
8274 return false;
8275
8276 APInt HighMask = APInt::getHighBitsSet(numBits: OuterBitSize, hiBitsSet: InnerBitSize);
8277 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask) &&
8278 DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask)) {
8279 // The inputs are both zero-extended.
8280 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
8281 Result.push_back(Elt: Lo);
8282 Result.push_back(Elt: Hi);
8283 if (Opcode != ISD::MUL) {
8284 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8285 Result.push_back(Elt: Zero);
8286 Result.push_back(Elt: Zero);
8287 }
8288 return true;
8289 }
8290 }
8291
8292 if (!VT.isVector() && Opcode == ISD::MUL &&
8293 DAG.ComputeMaxSignificantBits(Op: LHS) <= InnerBitSize &&
8294 DAG.ComputeMaxSignificantBits(Op: RHS) <= InnerBitSize) {
8295 // The input values are both sign-extended.
8296 // TODO non-MUL case?
8297 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
8298 Result.push_back(Elt: Lo);
8299 Result.push_back(Elt: Hi);
8300 return true;
8301 }
8302 }
8303
8304 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
8305 SDValue Shift = DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
8306
8307 if (!LH.getNode() && !RH.getNode() &&
8308 isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
8309 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8310 LH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS, N2: Shift);
8311 LH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LH);
8312 RH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: RHS, N2: Shift);
8313 RH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RH);
8314 }
8315
8316 if (!LH.getNode())
8317 return false;
8318
8319 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
8320 return false;
8321
8322 Result.push_back(Elt: Lo);
8323
8324 if (Opcode == ISD::MUL) {
8325 RH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LL, N2: RH);
8326 LH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LH, N2: RL);
8327 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: RH);
8328 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: LH);
8329 Result.push_back(Elt: Hi);
8330 return true;
8331 }
8332
8333 // Compute the full width result.
8334 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
8335 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
8336 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8337 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
8338 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi);
8339 };
8340
8341 SDValue Next = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8342 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
8343 return false;
8344
8345 // This is effectively the add part of a multiply-add of half-sized operands,
8346 // so it cannot overflow.
8347 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8348
8349 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
8350 return false;
8351
8352 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8353 EVT BoolType = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
8354
8355 bool UseGlue = (isOperationLegalOrCustom(Op: ISD::ADDC, VT) &&
8356 isOperationLegalOrCustom(Op: ISD::ADDE, VT));
8357 if (UseGlue)
8358 Next = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Glue), N1: Next,
8359 N2: Merge(Lo, Hi));
8360 else
8361 Next = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolType), N1: Next,
8362 N2: Merge(Lo, Hi), N3: DAG.getConstant(Val: 0, DL: dl, VT: BoolType));
8363
8364 SDValue Carry = Next.getValue(R: 1);
8365 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8366 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8367
8368 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
8369 return false;
8370
8371 if (UseGlue)
8372 Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::Glue), N1: Hi, N2: Zero,
8373 N3: Carry);
8374 else
8375 Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: BoolType), N1: Hi,
8376 N2: Zero, N3: Carry);
8377
8378 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8379
8380 if (Opcode == ISD::SMUL_LOHI) {
8381 SDValue NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8382 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RL));
8383 Next = DAG.getSelectCC(DL: dl, LHS: LH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8384
8385 NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8386 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LL));
8387 Next = DAG.getSelectCC(DL: dl, LHS: RH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8388 }
8389
8390 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8391 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8392 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8393 return true;
8394}
8395
8396bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
8397 SelectionDAG &DAG, MulExpansionKind Kind,
8398 SDValue LL, SDValue LH, SDValue RL,
8399 SDValue RH) const {
8400 SmallVector<SDValue, 2> Result;
8401 bool Ok = expandMUL_LOHI(Opcode: N->getOpcode(), VT: N->getValueType(ResNo: 0), dl: SDLoc(N),
8402 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Result, HiLoVT,
8403 DAG, Kind, LL, LH, RL, RH);
8404 if (Ok) {
8405 assert(Result.size() == 2);
8406 Lo = Result[0];
8407 Hi = Result[1];
8408 }
8409 return Ok;
8410}
8411
8412// Optimize unsigned division or remainder by constants for types twice as large
8413// as a legal VT.
8414//
8415// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
8416// can be computed
8417// as:
8418// Sum = __builtin_uadd_overflow(Lo, High, &Sum);
8419// Remainder = Sum % Constant;
8420//
8421// If (1 << (BitWidth / 2)) % Constant != 1, we can search for a smaller value
8422// W such that W != (BitWidth / 2) and (1 << W) % Constant == 1. We can break
8423// High:Low into 3 chunks of W bits and compute remainder as
8424// Sum = Chunk0 + Chunk1 + Chunk2;
8425// Remainder = Sum % Constant;
8426//
8427// This is based on "Remainder by Summing Digits" from Hacker's Delight.
8428//
8429// For division, we can compute the remainder using the algorithm described
8430// above, subtract it from the dividend to get an exact multiple of Constant.
8431// Then multiply that exact multiply by the multiplicative inverse modulo
8432// (1 << (BitWidth / 2)) to get the quotient.
8433
8434// If Constant is even, we can shift right the dividend and the divisor by the
8435// number of trailing zeros in Constant before applying the remainder algorithm.
8436// If we're after the quotient, we can subtract this value from the shifted
8437// dividend and multiply by the multiplicative inverse of the shifted divisor.
8438// If we want the remainder, we shift the value left by the number of trailing
8439// zeros and add the bits that were shifted out of the dividend.
8440bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
8441 SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
8442 SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8443 unsigned Opcode = N->getOpcode();
8444 EVT VT = N->getValueType(ResNo: 0);
8445
8446 unsigned BitWidth = Divisor.getBitWidth();
8447 unsigned HBitWidth = BitWidth / 2;
8448 assert(VT.getScalarSizeInBits() == BitWidth &&
8449 HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");
8450
8451 // If the divisor is even, shift it until it becomes odd.
8452 unsigned TrailingZeros = 0;
8453 if (!Divisor[0]) {
8454 TrailingZeros = Divisor.countr_zero();
8455 Divisor.lshrInPlace(ShiftAmt: TrailingZeros);
8456 }
8457
8458 // After removing trailing zeros, the divisor needs to be less than
8459 // (1 << HBitWidth).
8460 APInt HalfMaxPlus1 = APInt::getOneBitSet(numBits: BitWidth, BitNo: HBitWidth);
8461 if (Divisor.uge(RHS: HalfMaxPlus1))
8462 return false;
8463
8464 // Look for the largest chunk width W such that (1 << W) % Divisor == 1 or
8465 // (1 << W) % Divisor == -1.
8466 unsigned BestChunkWidth = 0, AltChunkWidth = 0;
8467 for (unsigned I = HBitWidth, E = HBitWidth / 2; I > E; --I) {
8468 // Skip HBitWidth-1, it doesn't have enough bits for carries.
8469 if (I == HBitWidth - 1)
8470 continue;
8471
8472 APInt Mod = APInt::getOneBitSet(numBits: Divisor.getBitWidth(), BitNo: I).urem(RHS: Divisor);
8473
8474 if (Mod.isOne()) {
8475 BestChunkWidth = I;
8476 break;
8477 }
8478
8479 // We have an alternate strategy for Remainder == Divisor - 1.
8480 // FIXME: Support HBitWidth.
8481 if (I != HBitWidth && Mod == Divisor - 1)
8482 AltChunkWidth = I;
8483 }
8484
8485 bool Alternate = false;
8486 if (!BestChunkWidth) {
8487 if (!AltChunkWidth)
8488 return false;
8489 Alternate = true;
8490 BestChunkWidth = AltChunkWidth;
8491 }
8492
8493 SDLoc dl(N);
8494
8495 assert(!LL == !LH && "Expected both input halves or no input halves!");
8496 if (!LL)
8497 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8498
8499 bool HasFSHR = isOperationLegal(Op: ISD::FSHR, VT: HiLoVT);
8500
8501 auto GetFSHR = [&](SDValue Lo, SDValue Hi, unsigned ShiftAmt) {
8502 assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
8503 if (HasFSHR)
8504 return DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: HiLoVT, N1: Hi, N2: Lo,
8505 N3: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8506 return DAG.getNode(
8507 Opcode: ISD::OR, DL: dl, VT: HiLoVT,
8508 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Lo,
8509 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl)),
8510 N2: DAG.getNode(
8511 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: Hi,
8512 N2: DAG.getShiftAmountConstant(Val: HBitWidth - ShiftAmt, VT: HiLoVT, DL: dl)));
8513 };
8514
8515 // Helper to perform a right shift on a 128-bit value split into two halves.
8516 // Handles shifts >= HBitWidth by moving Hi to Lo and shifting Hi.
8517 auto ShiftRight = [&](SDValue &Lo, SDValue &Hi, unsigned ShiftAmt) {
8518 if (ShiftAmt == 0)
8519 return;
8520 if (ShiftAmt < HBitWidth) {
8521 Lo = GetFSHR(Lo, Hi, ShiftAmt);
8522 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8523 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8524 } else if (ShiftAmt == HBitWidth) {
8525 Lo = Hi;
8526 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8527 } else {
8528 Lo = DAG.getNode(
8529 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8530 N2: DAG.getShiftAmountConstant(Val: ShiftAmt - HBitWidth, VT: HiLoVT, DL: dl));
8531 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8532 }
8533 };
8534
8535 // Shift the input by the number of TrailingZeros in the divisor. The
8536 // shifted out bits will be added to the remainder later.
8537 SDValue PartialRemL, PartialRemH;
8538 if (TrailingZeros && Opcode != ISD::UDIV) {
8539 // Save the shifted off bits if we need the remainder.
8540 if (TrailingZeros < HBitWidth) {
8541 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros);
8542 PartialRemL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LL,
8543 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8544 } else if (TrailingZeros == HBitWidth) {
8545 // All of LL is part of the remainder.
8546 PartialRemL = LL;
8547 } else {
8548 // TrailingZeros > HBitWidth: LL and part of LH are the remainder.
8549 PartialRemL = LL;
8550 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros - HBitWidth);
8551 PartialRemH = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LH,
8552 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8553 }
8554 }
8555
8556 SDValue Sum;
8557 // If BestChunkWidth is HBitWidth add low and high half. If there is a carry
8558 // out, add that to the final sum.
8559 if (BestChunkWidth == HBitWidth) {
8560 assert(!Alternate);
8561 // Shift LH:LL right if there were trailing zeros in the divisor.
8562 ShiftRight(LL, LH, TrailingZeros);
8563
8564 // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
8565 EVT SetCCType =
8566 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: HiLoVT);
8567 if (isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT: HiLoVT)) {
8568 SDVTList VTList = DAG.getVTList(VT1: HiLoVT, VT2: SetCCType);
8569 Sum = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, N1: LL, N2: LH);
8570 Sum = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, N1: Sum,
8571 N2: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT), N3: Sum.getValue(R: 1));
8572 } else {
8573 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: LL, N2: LH);
8574 SDValue Carry = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LL, Cond: ISD::SETULT);
8575 // If the boolean for the target is 0 or 1, we can add the setcc result
8576 // directly.
8577 if (getBooleanContents(Type: HiLoVT) ==
8578 TargetLoweringBase::ZeroOrOneBooleanContent)
8579 Carry = DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: HiLoVT);
8580 else
8581 Carry = DAG.getSelect(DL: dl, VT: HiLoVT, Cond: Carry, LHS: DAG.getConstant(Val: 1, DL: dl, VT: HiLoVT),
8582 RHS: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT));
8583 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum, N2: Carry);
8584 }
8585 } else {
8586 // Otherwise split into multple chunks and add them together. We chose
8587 // BestChunkWidth so that the sum will not overflow.
8588 SDValue Mask = DAG.getConstant(
8589 Val: APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: BestChunkWidth), DL: dl, VT: HiLoVT);
8590
8591 for (unsigned I = 0; I < BitWidth - TrailingZeros; I += BestChunkWidth) {
8592 // If there were trailing zeros in the divisor, increase the shift amount.
8593 unsigned Shift = I + TrailingZeros;
8594 SDValue Chunk;
8595 if (Shift == 0)
8596 Chunk = LL;
8597 else if (Shift >= HBitWidth)
8598 Chunk = DAG.getNode(
8599 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: LH,
8600 N2: DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL: dl));
8601 else
8602 Chunk = GetFSHR(LL, LH, Shift);
8603 // If we're on the last chunk, we don't need an AND.
8604 if (I + BestChunkWidth < BitWidth - TrailingZeros)
8605 Chunk = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: Chunk, N2: Mask);
8606 if (!Sum) {
8607 Sum = Chunk;
8608 } else {
8609 // For Alternate, we need to subtract odd chunks.
8610 unsigned ChunkNum = I / BestChunkWidth;
8611 unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ? ISD::SUB : ISD::ADD;
8612 Sum = DAG.getNode(Opcode: Opc, DL: dl, VT: HiLoVT, N1: Sum, N2: Chunk);
8613 }
8614 }
8615
8616 // For Alternate, the sum may be negative, but we need a positive sum. We
8617 // can increase it by a multiple of the divisor to make it positive. For 3
8618 // chunks the largest negative value is -(2^BestChunkWidth - 1). For 4
8619 // chunks, it's 2*-(2^BestChunkWidth - 1). We know that 2^BestChunkWidth + 1
8620 // is a multiple of the divisor. Add that 1 or 2 times to make the sum
8621 // positive.
8622 if (Alternate) {
8623 unsigned NumChunks = divideCeil(Numerator: BitWidth - TrailingZeros, Denominator: BestChunkWidth);
8624 assert(NumChunks <= 4);
8625
8626 APInt Adjust = APInt::getOneBitSet(numBits: HBitWidth, BitNo: BestChunkWidth);
8627 Adjust.setBit(0);
8628 // If there are 4 chunks, we need to adjust twice.
8629 if (NumChunks == 4)
8630 Adjust <<= 1;
8631 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum,
8632 N2: DAG.getConstant(Val: Adjust, DL: dl, VT: HiLoVT));
8633 }
8634 }
8635
8636 // Perform a HiLoVT urem on the Sum using truncated divisor.
8637 SDValue RemL =
8638 DAG.getNode(Opcode: ISD::UREM, DL: dl, VT: HiLoVT, N1: Sum,
8639 N2: DAG.getConstant(Val: Divisor.trunc(width: HBitWidth), DL: dl, VT: HiLoVT));
8640 SDValue RemH = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8641
8642 if (Opcode != ISD::UREM) {
8643 // If we didn't shift LH/LR earlier, do it now.
8644 if (BestChunkWidth != HBitWidth)
8645 ShiftRight(LL, LH, TrailingZeros);
8646
8647 // Subtract the remainder from the shifted dividend.
8648 SDValue Dividend = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: LL, N2: LH);
8649 SDValue Rem = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: RemL, N2: RemH);
8650
8651 Dividend = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Rem);
8652
8653 // Multiply by the multiplicative inverse of the divisor modulo
8654 // (1 << BitWidth).
8655 APInt MulFactor = Divisor.multiplicativeInverse();
8656
8657 SDValue Quotient = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Dividend,
8658 N2: DAG.getConstant(Val: MulFactor, DL: dl, VT));
8659
8660 // Split the quotient into low and high parts.
8661 SDValue QuotL, QuotH;
8662 std::tie(args&: QuotL, args&: QuotH) = DAG.SplitScalar(N: Quotient, DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8663 Result.push_back(Elt: QuotL);
8664 Result.push_back(Elt: QuotH);
8665 }
8666
8667 if (Opcode != ISD::UDIV) {
8668 // If we shifted the input, shift the remainder left and add the bits we
8669 // shifted off the input.
8670 if (TrailingZeros) {
8671 if (TrailingZeros < HBitWidth) {
8672 // Shift RemH:RemL left by TrailingZeros.
8673 // RemH gets the high bits shifted out of RemL.
8674 RemH = DAG.getNode(
8675 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: RemL,
8676 N2: DAG.getShiftAmountConstant(Val: HBitWidth - TrailingZeros, VT: HiLoVT, DL: dl));
8677 RemL =
8678 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8679 N2: DAG.getShiftAmountConstant(Val: TrailingZeros, VT: HiLoVT, DL: dl));
8680 // OR in the partial remainder.
8681 RemL = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemL, N2: PartialRemL,
8682 Flags: SDNodeFlags::Disjoint);
8683 } else if (TrailingZeros == HBitWidth) {
8684 // Shift left by exactly HBitWidth: RemH becomes RemL, RemL becomes
8685 // PartialRemL.
8686 RemH = RemL;
8687 RemL = PartialRemL;
8688 } else {
8689 // Shift left by more than HBitWidth.
8690 RemH = DAG.getNode(
8691 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8692 N2: DAG.getShiftAmountConstant(Val: TrailingZeros - HBitWidth, VT: HiLoVT, DL: dl));
8693 RemH = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemH, N2: PartialRemH,
8694 Flags: SDNodeFlags::Disjoint);
8695 RemL = PartialRemL;
8696 }
8697 }
8698 Result.push_back(Elt: RemL);
8699 Result.push_back(Elt: RemH);
8700 }
8701
8702 return true;
8703}
8704
8705bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
8706 SDNode *N, const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
8707 EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8708
8709 SDValue N0 = N->getOperand(Num: 0);
8710 EVT VT = N0->getValueType(ResNo: 0);
8711 SDLoc DL{N};
8712
8713 assert(!Divisor.isOne() && "Magic algorithm does not work for division by 1");
8714
8715 // This helper creates a MUL_LOHI of the pair (LL, LH) by a constant.
8716 auto MakeMUL_LOHIByConst = [&](unsigned Opc, SDValue LL, SDValue LH,
8717 const APInt &Const,
8718 SmallVectorImpl<SDValue> &Result) {
8719 SDValue LHS = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT, N1: LL, N2: LH);
8720 SDValue RHS = DAG.getConstant(Val: Const, DL, VT);
8721 auto [RL, RH] = DAG.SplitScalar(N: RHS, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8722 return expandMUL_LOHI(Opcode: Opc, VT, dl: DL, LHS, RHS, Result, HiLoVT, DAG,
8723 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
8724 LL, LH, RL, RH);
8725 };
8726
8727 // This helper creates an ADD/SUB of the pairs (LL, LH) and (RL, RH).
8728 auto MakeAddSubLong = [&](unsigned Opc, SDValue LL, SDValue LH, SDValue RL,
8729 SDValue RH) {
8730 SDValue AddSubNode =
8731 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO : ISD::USUBO, DL,
8732 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LL, N2: RL);
8733 SDValue OutL = AddSubNode.getValue(R: 0);
8734 SDValue Overflow = AddSubNode.getValue(R: 1);
8735 SDValue AddSubWithOverflow =
8736 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, DL,
8737 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LH, N2: RH, N3: Overflow);
8738 SDValue OutH = AddSubWithOverflow.getValue(R: 0);
8739 return std::make_pair(x&: OutL, y&: OutH);
8740 };
8741
8742 // This helper creates a SRL of the pair (LL, LH) by Shift.
8743 auto MakeSRLLong = [&](SDValue LL, SDValue LH, unsigned Shift) {
8744 unsigned HBitWidth = HiLoVT.getScalarSizeInBits();
8745 if (Shift < HBitWidth) {
8746 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift, VT: HiLoVT, DL);
8747 SDValue ResL = DAG.getNode(Opcode: ISD::FSHR, DL, VT: HiLoVT, N1: LH, N2: LL, N3: ShAmt);
8748 SDValue ResH = DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt);
8749 return std::make_pair(x&: ResL, y&: ResH);
8750 }
8751 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HiLoVT);
8752 if (Shift == HBitWidth)
8753 return std::make_pair(x&: LH, y&: Zero);
8754 assert(Shift - HBitWidth < HBitWidth &&
8755 "We shouldn't generate an undefined shift");
8756 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL);
8757 return std::make_pair(x: DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt), y&: Zero);
8758 };
8759
8760 // Knowledge of leading zeros may help to reduce the multiplier.
8761 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
8762
8763 UnsignedDivisionByConstantInfo Magics = UnsignedDivisionByConstantInfo::get(
8764 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
8765
8766 assert(!LL == !LH && "Expected both input halves or no input halves!");
8767 if (!LL)
8768 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N0, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8769 SDValue QL = LL;
8770 SDValue QH = LH;
8771 if (Magics.PreShift != 0)
8772 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PreShift);
8773
8774 SmallVector<SDValue, 4> UMulResult;
8775 if (!MakeMUL_LOHIByConst(ISD::UMUL_LOHI, QL, QH, Magics.Magic, UMulResult))
8776 return false;
8777
8778 QL = UMulResult[2];
8779 QH = UMulResult[3];
8780
8781 if (Magics.IsAdd) {
8782 auto [NPQL, NPQH] = MakeAddSubLong(ISD::SUB, LL, LH, QL, QH);
8783 std::tie(args&: NPQL, args&: NPQH) = MakeSRLLong(NPQL, NPQH, 1);
8784 std::tie(args&: QL, args&: QH) = MakeAddSubLong(ISD::ADD, NPQL, NPQH, QL, QH);
8785 }
8786
8787 if (Magics.PostShift != 0)
8788 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PostShift);
8789
8790 unsigned Opcode = N->getOpcode();
8791 if (Opcode != ISD::UREM) {
8792 Result.push_back(Elt: QL);
8793 Result.push_back(Elt: QH);
8794 }
8795
8796 if (Opcode != ISD::UDIV) {
8797 SmallVector<SDValue, 2> MulResult;
8798 if (!MakeMUL_LOHIByConst(ISD::MUL, QL, QH, Divisor, MulResult))
8799 return false;
8800
8801 assert(MulResult.size() == 2);
8802
8803 auto [RemL, RemH] =
8804 MakeAddSubLong(ISD::SUB, LL, LH, MulResult[0], MulResult[1]);
8805
8806 Result.push_back(Elt: RemL);
8807 Result.push_back(Elt: RemH);
8808 }
8809
8810 return true;
8811}
8812
8813bool TargetLowering::expandDIVREMByConstant(SDNode *N,
8814 SmallVectorImpl<SDValue> &Result,
8815 EVT HiLoVT, SelectionDAG &DAG,
8816 SDValue LL, SDValue LH) const {
8817 unsigned Opcode = N->getOpcode();
8818
8819 // TODO: Support signed division/remainder.
8820 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
8821 return false;
8822 assert(
8823 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
8824 "Unexpected opcode");
8825
8826 auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8827 if (!CN)
8828 return false;
8829
8830 APInt Divisor = CN->getAPIntValue();
8831
8832 // The generated half-width UREM is normally optimized using high multiply.
8833 // If the wide UREM libcall is unavailable, a legal or custom half-width
8834 // UDIVREM can lower it instead.
8835 bool CanDecomposeUREMWithoutMulHi =
8836 Opcode == ISD::UREM &&
8837 getLibcallImpl(Call: RTLIB::getUREM(VT: N->getValueType(ResNo: 0))) ==
8838 RTLIB::Unsupported &&
8839 isOperationLegalOrCustom(Op: ISD::UDIVREM, VT: HiLoVT);
8840 if (!CanDecomposeUREMWithoutMulHi &&
8841 !isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT) &&
8842 !isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT))
8843 return false;
8844
8845 // Prefer the smaller libcall when one is available.
8846 if (DAG.shouldOptForSize() && !CanDecomposeUREMWithoutMulHi)
8847 return false;
8848
8849 // Early out for 0 or 1 divisors.
8850 if (Divisor.ule(RHS: 1))
8851 return false;
8852
8853 if (expandUDIVREMByConstantViaUREMDecomposition(N, Divisor, Result, HiLoVT,
8854 DAG, LL, LH))
8855 return true;
8856
8857 if (expandUDIVREMByConstantViaUMulHiMagic(N, Divisor, Result, HiLoVT, DAG, LL,
8858 LH))
8859 return true;
8860
8861 return false;
8862}
8863
8864// Check that (every element of) Z is undef or not an exact multiple of BW.
8865static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
8866 return ISD::matchUnaryPredicate(
8867 Op: Z,
8868 Match: [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(RHS: BW) != 0; },
8869 /*AllowUndefs=*/true, /*AllowTruncation=*/true);
8870}
8871
8872SDValue TargetLowering::expandFunnelShift(SDNode *Node,
8873 SelectionDAG &DAG) const {
8874 EVT VT = Node->getValueType(ResNo: 0);
8875
8876 if (VT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8877 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8878 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8879 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
8880 return SDValue();
8881
8882 SDValue X = Node->getOperand(Num: 0);
8883 SDValue Y = Node->getOperand(Num: 1);
8884 SDValue Z = Node->getOperand(Num: 2);
8885
8886 unsigned BW = VT.getScalarSizeInBits();
8887 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
8888 SDLoc DL(SDValue(Node, 0));
8889
8890 EVT ShVT = Z.getValueType();
8891
8892 // If a funnel shift in the other direction is more supported, use it.
8893 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
8894 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8895 isOperationLegalOrCustom(Op: RevOpcode, VT) && isPowerOf2_32(Value: BW)) {
8896 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8897 // fshl X, Y, Z -> fshr X, Y, -Z
8898 // fshr X, Y, Z -> fshl X, Y, -Z
8899 Z = DAG.getNegative(Val: Z, DL, VT: ShVT);
8900 } else {
8901 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8902 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8903 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8904 if (IsFSHL) {
8905 Y = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8906 X = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: One);
8907 } else {
8908 X = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8909 Y = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Y, N2: One);
8910 }
8911 Z = DAG.getNOT(DL, Val: Z, VT: ShVT);
8912 }
8913 return DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: Z);
8914 }
8915
8916 SDValue ShX, ShY;
8917 SDValue ShAmt, InvShAmt;
8918 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8919 // fshl: X << C | Y >> (BW - C)
8920 // fshr: X << (BW - C) | Y >> C
8921 // where C = Z % BW is not zero
8922 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8923 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8924 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthC, N2: ShAmt);
8925 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: IsFSHL ? ShAmt : InvShAmt);
8926 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: IsFSHL ? InvShAmt : ShAmt);
8927 } else {
8928 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8929 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8930 SDValue Mask = DAG.getConstant(Val: BW - 1, DL, VT: ShVT);
8931 if (isPowerOf2_32(Value: BW)) {
8932 // Z % BW -> Z & (BW - 1)
8933 ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Z, N2: Mask);
8934 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8935 InvShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: DAG.getNOT(DL, Val: Z, VT: ShVT), N2: Mask);
8936 } else {
8937 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8938 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8939 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Mask, N2: ShAmt);
8940 }
8941
8942 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8943 if (IsFSHL) {
8944 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShAmt);
8945 SDValue ShY1 = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: One);
8946 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShY1, N2: InvShAmt);
8947 } else {
8948 SDValue ShX1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: One);
8949 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShX1, N2: InvShAmt);
8950 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt);
8951 }
8952 }
8953 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShX, N2: ShY);
8954}
8955
8956// TODO: Merge with expandFunnelShift.
8957SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
8958 SelectionDAG &DAG) const {
8959 EVT VT = Node->getValueType(ResNo: 0);
8960 unsigned EltSizeInBits = VT.getScalarSizeInBits();
8961 bool IsLeft = Node->getOpcode() == ISD::ROTL;
8962 SDValue Op0 = Node->getOperand(Num: 0);
8963 SDValue Op1 = Node->getOperand(Num: 1);
8964 SDLoc DL(SDValue(Node, 0));
8965
8966 EVT ShVT = Op1.getValueType();
8967 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: ShVT);
8968
8969 // If a rotate in the other direction is more supported, use it.
8970 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
8971 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8972 isOperationLegalOrCustom(Op: RevRot, VT) && isPowerOf2_32(Value: EltSizeInBits)) {
8973 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8974 return DAG.getNode(Opcode: RevRot, DL, VT, N1: Op0, N2: Sub);
8975 }
8976
8977 if (!AllowVectorOps && VT.isVector() &&
8978 (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8979 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8980 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8981 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT) ||
8982 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT)))
8983 return SDValue();
8984
8985 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
8986 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
8987 SDValue BitWidthMinusOneC = DAG.getConstant(Val: EltSizeInBits - 1, DL, VT: ShVT);
8988 SDValue ShVal;
8989 SDValue HsVal;
8990 if (isPowerOf2_32(Value: EltSizeInBits)) {
8991 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8992 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8993 SDValue NegOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8994 SDValue ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Op1, N2: BitWidthMinusOneC);
8995 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
8996 SDValue HsAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: NegOp1, N2: BitWidthMinusOneC);
8997 HsVal = DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: HsAmt);
8998 } else {
8999 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
9000 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
9001 SDValue BitWidthC = DAG.getConstant(Val: EltSizeInBits, DL, VT: ShVT);
9002 SDValue ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Op1, N2: BitWidthC);
9003 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
9004 SDValue HsAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthMinusOneC, N2: ShAmt);
9005 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
9006 HsVal =
9007 DAG.getNode(Opcode: HsOpc, DL, VT, N1: DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: One), N2: HsAmt);
9008 }
9009 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShVal, N2: HsVal);
9010}
9011
9012/// Check if CLMUL on VT can eventually reach a type with legal CLMUL through
9013/// a chain of halving decompositions (halving element width) and/or vector
9014/// widening (doubling element count). This guides expansion strategy selection:
9015/// if true, the halving/widening path produces better code than bit-by-bit.
9016///
9017/// HalveDepth tracks halving steps only (each creates ~4x more operations).
9018/// Widening steps are cheap (O(1) pad/extract) and don't count.
9019/// Limiting halvings to 2 prevents exponential blowup:
9020/// 1 halving: ~4 sub-CLMULs (good, e.g. v8i16 -> v8i8)
9021/// 2 halvings: ~16 sub-CLMULs (acceptable, e.g. v4i32 -> v4i16 -> v8i8)
9022/// 3 halvings: ~64 sub-CLMULs (worse than bit-by-bit expansion)
9023static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx,
9024 EVT VT, unsigned HalveDepth = 0,
9025 unsigned TotalDepth = 0) {
9026 if (HalveDepth > 2 || TotalDepth > 8 || !VT.isFixedLengthVector())
9027 return false;
9028 if (TLI.isOperationLegalOrCustom(Op: ISD::CLMUL, VT))
9029 return true;
9030 if (!TLI.isTypeLegal(VT))
9031 return false;
9032
9033 unsigned BW = VT.getScalarSizeInBits();
9034
9035 // Halve: halve element width, same element count.
9036 // This is the expensive step -- each halving creates ~4x more operations.
9037 if (BW % 2 == 0) {
9038 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: BW / 2);
9039 EVT HalfVT = VT.changeVectorElementType(Context&: Ctx, EltVT: HalfEltVT);
9040 if (TLI.isTypeLegal(VT: HalfVT) &&
9041 canNarrowCLMULToLegal(TLI, Ctx, VT: HalfVT, HalveDepth: HalveDepth + 1, TotalDepth: TotalDepth + 1))
9042 return true;
9043 }
9044
9045 // Widen: double element count (fixed-width vectors only).
9046 // This is cheap -- just INSERT_SUBVECTOR + EXTRACT_SUBVECTOR.
9047 EVT WideVT = VT.getDoubleNumVectorElementsVT(Context&: Ctx);
9048 if (TLI.isTypeLegal(VT: WideVT) &&
9049 canNarrowCLMULToLegal(TLI, Ctx, VT: WideVT, HalveDepth, TotalDepth: TotalDepth + 1))
9050 return true;
9051
9052 return false;
9053}
9054
9055SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
9056 SDLoc DL(Node);
9057 EVT VT = Node->getValueType(ResNo: 0);
9058 SDValue X = Node->getOperand(Num: 0);
9059 SDValue Y = Node->getOperand(Num: 1);
9060 unsigned BW = VT.getScalarSizeInBits();
9061 unsigned Opcode = Node->getOpcode();
9062 LLVMContext &Ctx = *DAG.getContext();
9063
9064 switch (Opcode) {
9065 case ISD::CLMUL: {
9066 // For vector types, try decomposition strategies that leverage legal
9067 // CLMUL on narrower or wider element types, avoiding the expensive
9068 // bit-by-bit expansion.
9069 if (VT.isVector()) {
9070 // Strategy 1: Halving decomposition to half-element-width CLMUL.
9071 // Applies ExpandIntRes_CLMUL's identity element-wise:
9072 // CLMUL(X, Y) = (Hi << HalfBW) | Lo
9073 // where:
9074 // Lo = CLMUL(XLo, YLo)
9075 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9076 unsigned HalfBW = BW / 2;
9077 if (BW % 2 == 0) {
9078 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: HalfBW);
9079 EVT HalfVT =
9080 EVT::getVectorVT(Context&: Ctx, VT: HalfEltVT, EC: VT.getVectorElementCount());
9081 if (isTypeLegal(VT: HalfVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: HalfVT,
9082 /*HalveDepth=*/1)) {
9083 SDValue ShAmt = DAG.getShiftAmountConstant(Val: HalfBW, VT, DL);
9084
9085 // Extract low and high halves of each element.
9086 SDValue XLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: X);
9087 SDValue XHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
9088 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: ShAmt));
9089 SDValue YLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: Y);
9090 SDValue YHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
9091 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt));
9092
9093 // Lo = CLMUL(XLo, YLo)
9094 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YLo);
9095
9096 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9097 SDValue LoH = DAG.getNode(Opcode: ISD::CLMULH, DL, VT: HalfVT, N1: XLo, N2: YLo);
9098 SDValue Cross1 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YHi);
9099 SDValue Cross2 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XHi, N2: YLo);
9100 SDValue Cross = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: Cross1, N2: Cross2);
9101 SDValue Hi = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: LoH, N2: Cross);
9102
9103 // Reassemble: Result = ZExt(Lo) | (AnyExt(Hi) << HalfBW)
9104 SDValue LoExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Lo);
9105 SDValue HiExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Hi);
9106 SDValue HiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: HiExt, N2: ShAmt);
9107 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoExt, N2: HiShifted);
9108 }
9109 }
9110
9111 // Strategy 2: Promote to double-element-width CLMUL.
9112 // CLMUL(X, Y) = Trunc(CLMUL(AnyExt(X), AnyExt(Y)))
9113 {
9114 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9115 if (isTypeLegal(VT: ExtVT) && isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT)) {
9116 // If CLMUL on ExtVT is Custom (not Legal), the target may
9117 // scalarize it, costing O(NumElements) scalar ops. The bit-by-bit
9118 // fallback costs O(BW) vectorized iterations. Only widen when
9119 // element count is small enough that scalarization is cheaper.
9120 unsigned NumElts = VT.getVectorMinNumElements();
9121 if (isOperationLegal(Op: ISD::CLMUL, VT: ExtVT) || NumElts < BW) {
9122 SDValue XExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: X);
9123 SDValue YExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: Y);
9124 SDValue Mul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9125 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Mul);
9126 }
9127 }
9128 }
9129
9130 // Strategy 3: Widen element count (pad with undef, do CLMUL on wider
9131 // vector, extract lower result). CLMUL is element-wise, so upper
9132 // (undef) lanes don't affect the lower results.
9133 // e.g. v4i16 => pad to v8i16 => halve to v8i8 PMUL => extract v4i16.
9134 if (auto EC = VT.getVectorElementCount(); EC.isFixed()) {
9135 EVT WideVT = EVT::getVectorVT(Context&: Ctx, VT: VT.getVectorElementType(), EC: EC * 2);
9136 if (isTypeLegal(VT: WideVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: WideVT)) {
9137 SDValue Undef = DAG.getUNDEF(VT: WideVT);
9138 SDValue XWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
9139 N2: X, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9140 SDValue YWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
9141 N2: Y, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9142 SDValue WideRes = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: WideVT, N1: XWide, N2: YWide);
9143 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WideRes,
9144 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9145 }
9146 }
9147 }
9148
9149 // Special case: clmul(X, Y) where Y is a known constant (splat) that forms
9150 // a contiguous block of trailing ones whose length N is a power of two
9151 // (e.g. i8 0xFF, i8 0x0F, ...) or equal to the operand width. In this
9152 // special case, clmul(X, Y) is equivalent to a "parallel prefix XOR" or
9153 // "bitwise parity" operation on X.
9154 //
9155 // Note: This special currently dose NOT apply when the mask is neither a
9156 // power of two nor equal to the operand width because the loop inside
9157 // behaves as if the mask was bit-ceiled, and "undoing" the XOR with parts
9158 // of that CLMUL is a recursive problem (e.g. CLMUL with a 20-bit mask
9159 // requires correction XOR with CLMUL with 12-bit mask).
9160 if (auto *C = isConstOrConstSplat(N: Y, /*AllowUndefs=*/true)) {
9161 const APInt &YVal = C->getAPIntValue();
9162 unsigned N = YVal.countr_one();
9163 if (YVal.isAllOnes() || (YVal.isMask() && isPowerOf2_32(Value: N))) {
9164 SDValue R = X;
9165 for (unsigned I = 1; I < N; I <<= 1) {
9166 SDValue ShAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9167 SDValue Shifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: R, N2: ShAmt);
9168 R = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: R, N2: Shifted);
9169 }
9170 return R;
9171 }
9172 }
9173
9174 // NOTE: If you change this expansion, please update the cost model
9175 // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
9176 // Intrinsic::clmul.
9177
9178 // Strategy 4: multiplication with holes.
9179 //
9180 // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
9181 // do occur, they wind up in a "hole" and are subsequently masked out of the
9182 // result.
9183 //
9184 // https://www.bearssl.org/constanttime.html#ghash-for-gcm describes this
9185 // approach.
9186
9187 // Stride S handles operands up to S·2^S bits using S² multiplies.
9188 //
9189 // * BW <= 8 uses S = 2 (holes of 1 bit)
9190 // * BW <= 24 uses S = 3 (holes of 2 bits)
9191 // * BW <= 64 uses S = 4 (holes of 3 bits)
9192 // * BW <= 160 uses S = 5 (holes of 4 bits)
9193 // * BW <= 384 uses S = 6 (holes of 5 bits)
9194 //
9195 // We distribute the BW bits over S phases:
9196 //
9197 // phase 0 keeps bits: 0, S, 2S, ...
9198 // phase 1 keeps bits: 1, S + 1, 2S + 1, ...
9199 // ...
9200 //
9201 // Each phase has up to n = ceil(BW / S) bits set, and the holes are S-1
9202 // bits wide.
9203 //
9204 // Take BW = 4, S = 2, n = 2. The worst case is a fully populated phase (all
9205 // non-hole bits are set to 1) multiplied by itself, 0b0101 * 0b0101. Each
9206 // set bit of one operand shifts a copy of the other, and we add the copies:
9207 //
9208 // col: 4 3 2 1 0
9209 // 0b0101 << 0: 0 0 1 0 1
9210 // 0b0101 << 2: 1 0 1 0 0
9211 // ----------------- +
9212 // count: 1 0 2 0 1
9213 //
9214 // Counting the number of one-bits in each column gives a triangle: the
9215 // counts climb 1, 2, ..., n and back down (here 1, 2, 1 across the data
9216 // columns). So a column holds at most n one-bits, and that maximum n is
9217 // reached in only one column: the peak. Every other column holds at most n
9218 // - 1 one-bits.
9219 //
9220 // A stack of one-bits in a column turns into carries: column 2 above really
9221 // stores the value 1 + 1 = 2 = n. A column spans S bits, its kept bit
9222 // plus S-1 hole bits, and the count is written from the kept bit upward,
9223 // so any count <= 2^S - 1 stays within the column and never interferes with
9224 // the next data bit S positions up. Every non-peak column holds at most n -
9225 // 1, so they all fit as soon as n - 1 <= 2^S - 1.
9226 //
9227 // That leaves only the peak column. Because both operands set all data
9228 // bits, the triangle peaks at the top of the word at the highest data bit
9229 // still inside BW. Here the count reaches exactly n = 2^S and overflows.
9230 // But its carry lands at bit n*S >= BW, off the top, where it (and the
9231 // whole descending half of the triangle) is truncated.
9232 //
9233 // Hence the holes suffice exactly when n = ceil(BW / S) <= 2^S, i.e. BW <=
9234 // S*2^S.
9235 //
9236 // Here we find the smallest S that satisfies this inequality.
9237 unsigned S = 1;
9238 while (S < 32 && divideCeil(Numerator: BW, Denominator: S) > (1u << S))
9239 ++S;
9240
9241 // The "multiplication with holes" expansion emits S*S MULs, 3*S ANDs,
9242 // S*(S-1) XORs and S-1 ORs.
9243 unsigned HolesCost = S * S + 3 * S + S * (S - 1) + (S - 1);
9244
9245 // Estimate the cost of the naive algorithm.
9246 KnownBits KnownY = DAG.computeKnownBits(Op: Y);
9247 unsigned NaiveCost = 0;
9248 for (unsigned I = 0; I < BW; ++I) {
9249 // The iteration folds away entirely and is free.
9250 if (KnownY.Zero[I])
9251 continue;
9252
9253 // On targets with a fast bit test instruction more instructions are used
9254 // to not need a (potentially expensive) multiplication. See also below.
9255 if (hasBitTest(X: Y, Y: DAG.getShiftAmountConstant(Val: I, VT, DL))) {
9256 // AND + SETCC + SHL + SELECT + XOR.
9257 NaiveCost += 5;
9258 } else {
9259 // AND + MUL + XOR.
9260 NaiveCost += 3;
9261 }
9262 }
9263
9264 // Only use multiplication with holes when it is cheaper, else use the naive
9265 // fallback below.
9266 if (HolesCost < NaiveCost &&
9267 isOperationLegalOrCustom(Op: ISD::MUL, VT: getTypeToTransformTo(Context&: Ctx, VT))) {
9268
9269 // Set a bit every S positions, e.g. for S = 4 this is equivalent to
9270 // 0b...00010001...0001.
9271 APInt MaskVal = APInt::getSplat(NewLen: BW, V: APInt(S, 1));
9272
9273 // Create versions of X and Y that keep only the I-th bit of each S-bit
9274 // slice.
9275 SmallVector<SDValue, 4> M(S), Xp(S), Yp(S);
9276 for (unsigned I = 0; I < S; ++I) {
9277 M[I] = DAG.getConstant(Val: MaskVal.shl(shiftAmt: I), DL, VT);
9278 Xp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: M[I]);
9279 Yp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: M[I]);
9280 }
9281
9282 // Codegens these expressions (S*S multiplications), e.g. for S=4:
9283 //
9284 // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
9285 // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
9286 // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
9287 // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
9288 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9289 for (unsigned I = 0; I < S; ++I) {
9290 SDValue Zi = DAG.getConstant(Val: 0, DL, VT);
9291 for (unsigned J = 0; J < S; ++J) {
9292 unsigned K = (I + S - J) % S;
9293 SDValue P = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Xp[J], N2: Yp[K]);
9294 Zi = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Zi, N2: P);
9295 }
9296
9297 // Keep only the bits belonging to this iteration, and bitwise or it all
9298 // together.
9299 Zi = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Zi, N2: M[I]);
9300 Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Res, N2: Zi, Flags: SDNodeFlags::Disjoint);
9301 }
9302 return Res;
9303 }
9304
9305 // Strategy 5: the naive fallback.
9306 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
9307
9308 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9309 for (unsigned I = 0; I < BW; ++I) {
9310 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9311 SDValue Mask = DAG.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: I), DL, VT);
9312 SDValue YMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: Mask);
9313
9314 // For targets with a fast bit test instruction (e.g., x86 BT) or without
9315 // multiply, use a shift-based expansion to avoid expensive MUL
9316 // instructions.
9317 SDValue Part;
9318 if (!hasBitTest(X: Y, Y: ShiftAmt) &&
9319 isOperationLegalOrCustom(
9320 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
9321 Part = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: YMasked);
9322 } else {
9323 // Canonical bit test: (Y & (1 << I)) != 0
9324 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
9325 SDValue Cond = DAG.getSetCC(DL, VT: SetCCVT, LHS: YMasked, RHS: Zero, Cond: ISD::SETEQ);
9326 SDValue XShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftAmt);
9327 Part = DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: XShifted);
9328 }
9329 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Res, N2: Part);
9330 }
9331 return Res;
9332 }
9333 case ISD::CLMULR:
9334 // If we have CLMUL/CLMULH, merge the shifted results to form CLMULR.
9335 if (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) &&
9336 isOperationLegalOrCustom(Op: ISD::CLMULH, VT)) {
9337 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
9338 SDValue Hi = DAG.getNode(Opcode: ISD::CLMULH, DL, VT, N1: X, N2: Y);
9339 Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo,
9340 N2: DAG.getShiftAmountConstant(Val: BW - 1, VT, DL));
9341 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi,
9342 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9343 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Lo, N2: Hi);
9344 }
9345 [[fallthrough]];
9346 case ISD::CLMULH: {
9347 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9348 // Use bitreverse-based lowering (CLMULR/H = rev(CLMUL(rev,rev)) >> S)
9349 // when any of these hold:
9350 // (a) ZERO_EXTEND to ExtVT or SRL on ExtVT isn't legal.
9351 // (b) CLMUL is legal on VT but not on ExtVT (e.g. v8i8 on AArch64).
9352 // (c) CLMUL on ExtVT isn't legal, but CLMUL on VT can be efficiently
9353 // expanded via halving/widening to reach legal CLMUL. The bitreverse
9354 // path creates CLMUL(VT) which will be expanded efficiently. The
9355 // promote path would create CLMUL(ExtVT) => halving => CLMULH(VT),
9356 // causing a cycle.
9357 // Note: when CLMUL is legal on ExtVT, the zext => CLMUL(ExtVT) => shift
9358 // => trunc path is preferred over the bitreverse path, as it avoids the
9359 // cost of 3 bitreverse operations.
9360 if (!isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT: ExtVT) ||
9361 !isOperationLegalOrCustom(Op: ISD::SRL, VT: ExtVT) ||
9362 (!isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT) &&
9363 (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) ||
9364 canNarrowCLMULToLegal(TLI: *this, Ctx, VT)))) {
9365 SDValue XRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: X);
9366 SDValue YRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: Y);
9367 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: XRev, N2: YRev);
9368 SDValue Res = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: ClMul);
9369 if (Opcode == ISD::CLMULH)
9370 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res,
9371 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9372 return Res;
9373 }
9374 SDValue XExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: X);
9375 SDValue YExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: Y);
9376 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9377 unsigned ShAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
9378 SDValue HiBits = DAG.getNode(Opcode: ISD::SRL, DL, VT: ExtVT, N1: ClMul,
9379 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: ExtVT, DL));
9380 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: HiBits);
9381 }
9382 }
9383 llvm_unreachable("Expected CLMUL, CLMULR, or CLMULH");
9384}
9385
9386SDValue TargetLowering::expandPEXT(SDNode *Node, SelectionDAG &DAG) const {
9387 SDLoc DL(Node);
9388 EVT VT = Node->getValueType(ResNo: 0);
9389 SDValue Val = Node->getOperand(Num: 0);
9390 SDValue Msk = Node->getOperand(Num: 1);
9391 unsigned BW = VT.getScalarSizeInBits();
9392
9393 // Just scalarize if scalar PEXT is legal
9394 if (VT.isVector() && isOperationLegal(Op: ISD::PEXT, VT: VT.getVectorElementType()))
9395 return DAG.UnrollVectorOp(N: Node);
9396
9397 // Hacker's Delight §7-4: Compress, or Generalized Extract
9398 SDValue X = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Val, N2: Msk);
9399 SDValue M = Msk;
9400 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9401 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: M, VT), N2: One);
9402
9403 // Repeatedly compute which bits would shift to the right by an odd amount,
9404 // shift all such bits in parallel using a mask, and double the shift amount.
9405 for (unsigned I = 1; I < BW; I *= 2) {
9406 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9407 SDValue Mp =
9408 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9409 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: M);
9410 SDValue ShiftI = DAG.getShiftAmountConstant(Val: I, VT, DL);
9411 SDValue MvS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mv, N2: ShiftI);
9412 M = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: M, N2: Mv), N2: MvS,
9413 Flags: SDNodeFlags::Disjoint);
9414 SDValue T = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Mv);
9415 SDValue TS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: T, N2: ShiftI);
9416 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: X, N2: T), N2: TS,
9417 Flags: SDNodeFlags::Disjoint);
9418 if (I * 2 < BW)
9419 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9420 }
9421
9422 return X;
9423}
9424
9425SDValue TargetLowering::expandPDEP(SDNode *Node, SelectionDAG &DAG) const {
9426 SDLoc DL(Node);
9427 EVT VT = Node->getValueType(ResNo: 0);
9428 SDValue Val = Node->getOperand(Num: 0);
9429 SDValue Msk = Node->getOperand(Num: 1);
9430 unsigned BW = VT.getScalarSizeInBits();
9431
9432 // Just scalarize if scalar PDEP is legal
9433 if (VT.isVector() && isOperationLegal(Op: ISD::PDEP, VT: VT.getVectorElementType()))
9434 return DAG.UnrollVectorOp(N: Node);
9435
9436 // Hacker's Delight §7-5: Expand, or Generalized Insert.
9437 unsigned LogBW = Log2_32_Ceil(Value: BW);
9438 SmallVector<SDValue, 8> MvArray(LogBW);
9439 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9440 SDValue Mc = Msk;
9441 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: Msk, VT), N2: One);
9442
9443 // First pass: compute move masks for each power of two that a bit moves by.
9444 for (unsigned S = 0; S < LogBW; ++S) {
9445 unsigned ShiftS = 1u << S;
9446 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9447 SDValue Mp =
9448 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9449 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: Mc);
9450 MvArray[S] = Mv;
9451 if (S + 1 < LogBW) {
9452 SDValue McXorMv = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Mc, N2: Mv);
9453 SDValue MvShifted = DAG.getNode(
9454 Opcode: ISD::SRL, DL, VT, N1: Mv, N2: DAG.getShiftAmountConstant(Val: ShiftS, VT, DL));
9455 Mc = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: McXorMv, N2: MvShifted,
9456 Flags: SDNodeFlags::Disjoint);
9457 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9458 }
9459 }
9460
9461 // Second pass: move bits by 32, 16, 8, 4, 2, 1, using masks, in parallel.
9462 // Each pass handles half the shift amount of the previous pass.
9463 SDValue X = Val;
9464 for (int S = (int)LogBW - 1; S >= 0; --S) {
9465 SDValue ShiftSv = DAG.getShiftAmountConstant(Val: 1ull << S, VT, DL);
9466 SDValue T = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftSv);
9467 SDValue UnshiftedBits =
9468 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: DAG.getNOT(DL, Val: MvArray[S], VT));
9469 SDValue ShiftedBits = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T, N2: MvArray[S]);
9470 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: UnshiftedBits, N2: ShiftedBits,
9471 Flags: SDNodeFlags::Disjoint);
9472 }
9473
9474 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Msk);
9475}
9476
9477void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
9478 SelectionDAG &DAG) const {
9479 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
9480 EVT VT = Node->getValueType(ResNo: 0);
9481 unsigned VTBits = VT.getScalarSizeInBits();
9482 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
9483
9484 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
9485 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
9486 SDValue ShOpLo = Node->getOperand(Num: 0);
9487 SDValue ShOpHi = Node->getOperand(Num: 1);
9488 SDValue ShAmt = Node->getOperand(Num: 2);
9489 EVT ShAmtVT = ShAmt.getValueType();
9490 EVT ShAmtCCVT =
9491 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ShAmtVT);
9492 SDLoc dl(Node);
9493
9494 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
9495 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
9496 // away during isel.
9497 SDValue SafeShAmt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9498 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT));
9499 SDValue Tmp1 = IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: ShOpHi,
9500 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT))
9501 : DAG.getConstant(Val: 0, DL: dl, VT);
9502
9503 SDValue Tmp2, Tmp3;
9504 if (IsSHL) {
9505 Tmp2 = DAG.getNode(Opcode: ISD::FSHL, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9506 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: SafeShAmt);
9507 } else {
9508 Tmp2 = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9509 Tmp3 = DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: ShOpHi, N2: SafeShAmt);
9510 }
9511
9512 // If the shift amount is larger or equal than the width of a part we don't
9513 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
9514 // values for large shift amounts.
9515 SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9516 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: ShAmtVT));
9517 SDValue Cond = DAG.getSetCC(DL: dl, VT: ShAmtCCVT, LHS: AndNode,
9518 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShAmtVT), Cond: ISD::SETNE);
9519
9520 if (IsSHL) {
9521 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9522 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9523 } else {
9524 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9525 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9526 }
9527}
9528
9529SDValue TargetLowering::expandFCANONICALIZE(SDNode *Node,
9530 SelectionDAG &DAG) const {
9531 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
9532 // suggested in
9533 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
9534 // It uses strict_fp operations even outside a strict_fp context in order
9535 // to guarantee that the canonicalization is not optimized away by later
9536 // passes. The result chain introduced by that is intentionally ignored
9537 // since no ordering requirement is intended here.
9538 EVT VT = Node->getValueType(ResNo: 0);
9539 SDLoc DL(Node);
9540 SDNodeFlags Flags = Node->getFlags();
9541 Flags.setNoFPExcept(true);
9542 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT);
9543 SDValue Mul =
9544 DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {VT, MVT::Other},
9545 Ops: {DAG.getEntryNode(), Node->getOperand(Num: 0), One}, Flags);
9546 return Mul;
9547}
9548
9549SDValue TargetLowering::expandCONVERT_TO_ARBITRARY_FP(SDNode *Node,
9550 SelectionDAG &DAG) const {
9551 // Expand conversion from a native IEEE float type to an arbitrary FP format
9552 // returning the result as an integer using bit manipulation.
9553 EVT ResVT = Node->getValueType(ResNo: 0);
9554 SDLoc dl(Node);
9555
9556 SDValue FloatVal = Node->getOperand(Num: 0);
9557 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9558 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9559 const auto RoundMode =
9560 static_cast<RoundingMode>(Node->getConstantOperandVal(Num: 2));
9561 const bool Saturate = Node->getConstantOperandVal(Num: 3) != 0;
9562
9563 // Supported destination formats.
9564 switch (Sem) {
9565 case APFloatBase::S_Float8E5M2:
9566 case APFloatBase::S_Float8E4M3FN:
9567 case APFloatBase::S_Float8E5M3FNU:
9568 case APFloatBase::S_Float6E3M2FN:
9569 case APFloatBase::S_Float6E2M3FN:
9570 case APFloatBase::S_Float4E2M1FN:
9571 break;
9572 default:
9573 DAG.getContext()->emitError(ErrorStr: "CONVERT_TO_ARBITRARY_FP: not implemented "
9574 "destination format (semantics enum " +
9575 Twine(SemEnum) + ")");
9576 return SDValue();
9577 }
9578
9579 // Supported rounding modes.
9580 switch (RoundMode) {
9581 case RoundingMode::NearestTiesToEven:
9582 case RoundingMode::TowardZero:
9583 case RoundingMode::TowardPositive:
9584 case RoundingMode::TowardNegative:
9585 case RoundingMode::NearestTiesToAway:
9586 break;
9587 default:
9588 DAG.getContext()->emitError(
9589 ErrorStr: "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
9590 Twine(static_cast<int>(RoundMode)) + ")");
9591 return SDValue();
9592 }
9593
9594 // Destination format parameters.
9595 const fltSemantics &DstSem = APFloatBase::EnumToSemantics(S: Sem);
9596 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9597 const unsigned DstPrecision = APFloat::semanticsPrecision(DstSem);
9598 const unsigned DstMant = DstPrecision - 1;
9599 // Unsigned formats spend no bit on the sign.
9600 const bool DstHasSign = APFloat::semanticsHasSignedRepr(DstSem);
9601 const unsigned DstExpBits = DstBits - (DstHasSign ? 1 : 0) - DstMant;
9602 const int DstBias = 1 - APFloat::semanticsMinExponent(DstSem);
9603 const unsigned DstExpMax = (1U << DstExpBits) - 1;
9604 const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
9605 const fltNonfiniteBehavior DstNFBehavior = DstSem.nonFiniteBehavior;
9606 const fltNanEncoding DstNanEnc = DstSem.nanEncoding;
9607
9608 // Compute the maximum normal exponent for the destination format.
9609 const unsigned DstExpMaxNormal =
9610 DstNFBehavior == fltNonfiniteBehavior::IEEE754 ? DstExpMax - 1
9611 : DstExpMax;
9612
9613 // For NanOnly formats the max exponent field for finite values
9614 // is DstExpMax, but the encoding with exp = DstExpMax and
9615 // mant = all-ones is NaN. So DstExpMaxNormal = DstExpMax, but max
9616 // mantissa at that exponent is DstMantMask - 1 (if NanEnc == AllOnes) to
9617 // avoid the NaN encoding.
9618 uint64_t DstMaxMantAtMaxExp = DstMantMask;
9619 if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9620 DstNanEnc == fltNanEncoding::AllOnes)
9621 DstMaxMantAtMaxExp = DstMantMask - 1;
9622
9623 // Source format parameters.
9624 EVT SrcVT = FloatVal.getValueType();
9625 const fltSemantics &SrcSem = SrcVT.getScalarType().getFltSemantics();
9626 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9627 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9628 const unsigned SrcMant = SrcPrecision - 1;
9629 const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;
9630
9631 // Work in the source integer type. Match the destination shape so the
9632 // expansion stays vector when ResVT is a vector.
9633 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcBits);
9634 EVT IntVT = ResVT.changeElementType(Context&: *DAG.getContext(), EltVT: IntScalarVT);
9635 EVT SetCCVT =
9636 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9637 EVT FPSetCCVT =
9638 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
9639
9640 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9641 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9642
9643 // Bitcast source float to integer to extract the sign bit.
9644 SDValue Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FloatVal);
9645 SDValue SignBit =
9646 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9647 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
9648
9649 // Classify the input.
9650 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT);
9651 SDValue FPInf = DAG.getConstantFP(Val: APFloat::getInf(Sem: SrcSem), DL: dl, VT: SrcVT);
9652 SDValue AbsVal = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: SrcVT, Operand: FloatVal);
9653 SDValue IsNaN = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETUO);
9654 SDValue IsInf = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: AbsVal, RHS: FPInf, Cond: ISD::SETOEQ);
9655 SDValue IsZero = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOEQ);
9656
9657 // Split into a normalized fraction and unbiased exponent. FFREXP normalizes
9658 // source denormals automatically. The result is unspecified for Inf/NaN, but
9659 // those inputs are detected above and override the final result.
9660 EVT FrexpExpScalarVT =
9661 getValueType(DL: DAG.getDataLayout(), Ty: Type::getInt32Ty(C&: *DAG.getContext()));
9662 EVT FrexpExpVT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: FrexpExpScalarVT);
9663 SDValue Frexp =
9664 DAG.getNode(Opcode: ISD::FFREXP, DL: dl, VTList: DAG.getVTList(VT1: SrcVT, VT2: FrexpExpVT), N: FloatVal);
9665 SDValue FrexpFrac = Frexp.getValue(R: 0);
9666 SDValue FrexpExp = Frexp.getValue(R: 1);
9667
9668 SDValue FrexpFracInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FrexpFrac);
9669 SDValue EffSrcMant = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FrexpFracInt,
9670 N2: DAG.getConstant(Val: SrcMantMask, DL: dl, VT: IntVT));
9671
9672 SDValue FrexpExpExt = DAG.getSExtOrTrunc(Op: FrexpExp, DL: dl, VT: IntVT);
9673 SDValue NewExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: FrexpExpExt,
9674 N2: DAG.getConstant(Val: DstBias - 1, DL: dl, VT: IntVT));
9675
9676 // Compute rounding increment given the round bit, sticky bits, and LSB
9677 // of the truncated mantissa.
9678 auto ComputeRoundUp = [&](SDValue RoundBit, SDValue StickyBits,
9679 SDValue LSB) -> SDValue {
9680 switch (RoundMode) {
9681 case RoundingMode::NearestTiesToEven: {
9682 // Round up if round_bit && (sticky || lsb)
9683 SDValue StickyOrLSB = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: StickyBits, N2: LSB);
9684 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyOrLSB);
9685 }
9686 case RoundingMode::TowardZero:
9687 return Zero;
9688 case RoundingMode::TowardPositive: {
9689 // Round up if positive and any truncated bits are set.
9690 SDValue AnyTruncBits =
9691 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9692 SDValue HasTruncBits =
9693 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9694 SDValue IsPositive = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETEQ);
9695 SDValue DoRound =
9696 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsPositive);
9697 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9698 }
9699 case RoundingMode::TowardNegative: {
9700 // Round up if negative and any truncated bits are set (to -Inf).
9701 SDValue AnyTruncBits =
9702 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9703 SDValue HasTruncBits =
9704 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9705 SDValue IsNegative = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETNE);
9706 SDValue DoRound =
9707 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsNegative);
9708 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9709 }
9710 case RoundingMode::NearestTiesToAway:
9711 return RoundBit;
9712 default:
9713 llvm_unreachable("unsupported rounding mode");
9714 }
9715 };
9716
9717 // Round mantissa from SrcMant bits to DstMant bits.
9718 SDValue TruncMant;
9719 SDValue RoundUp;
9720 if (SrcMant > DstMant) {
9721 const unsigned Shift = SrcMant - DstMant;
9722 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: Shift, VT: IntVT, DL: dl);
9723 TruncMant = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ShiftConst);
9724
9725 // Check bit at position Shift - 1 aka the round bit.
9726 SDValue RoundBit;
9727 if (Shift >= 1) {
9728 SDValue RoundBitShift = DAG.getShiftAmountConstant(Val: Shift - 1, VT: IntVT, DL: dl);
9729 SDValue ShiftedMant =
9730 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: RoundBitShift);
9731 RoundBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: ShiftedMant, N2: One);
9732 } else {
9733 RoundBit = Zero;
9734 }
9735
9736 // OR of all bits below the round bit to get sticky bits.
9737 SDValue StickyBits;
9738 if (Shift >= 2) {
9739 uint64_t StickyMask = maskTrailingOnes<uint64_t>(N: Shift - 1);
9740 StickyBits = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: EffSrcMant,
9741 N2: DAG.getConstant(Val: StickyMask, DL: dl, VT: IntVT));
9742 StickyBits = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: StickyBits, RHS: Zero, Cond: ISD::SETNE);
9743 StickyBits = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: StickyBits);
9744 } else {
9745 StickyBits = Zero;
9746 }
9747
9748 // LSB of truncated mantissa.
9749 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: TruncMant, N2: One);
9750
9751 RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
9752 } else {
9753 // If DstMant >= SrcMant, then no rounding needed, just shift left.
9754 SDValue MantShift =
9755 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
9756 TruncMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: MantShift);
9757 RoundUp = Zero;
9758 }
9759
9760 // Apply rounding.
9761 SDValue RoundedMant = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: TruncMant, N2: RoundUp);
9762
9763 // Handle mantissa overflow from rounding.
9764 // If rounded_mant > DstMantMask, carry into exponent.
9765 SDValue MantOverflow =
9766 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: RoundedMant,
9767 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9768 // On overflow: mant = 0, exp += 1.
9769 SDValue AdjMant = DAG.getSelect(DL: dl, VT: IntVT, Cond: MantOverflow, LHS: Zero, RHS: RoundedMant);
9770 SDValue AdjExp =
9771 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: NewExp,
9772 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: MantOverflow));
9773
9774 // Precompute sign shifted to MSB of destination. Unsigned formats have no
9775 // sign bit to merge in.
9776 SDValue SignShifted =
9777 DstHasSign
9778 ? DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
9779 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
9780 : Zero;
9781
9782 // Destination denormal conversion (when new_exp <= 0).
9783 // Shift the mantissa right by 1 - new_exp additional bits and set the
9784 // exponent field to 0.
9785 SDValue ExpIsNeg = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9786 RHS: DAG.getConstant(Val: 1, DL: dl, VT: IntVT), Cond: ISD::SETLT);
9787
9788 SDValue DenormResult;
9789 {
9790 // denorm_shift = 1 - NewExp.
9791 SDValue DenormShift = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: One, N2: NewExp);
9792
9793 // full_src_mant = (1 << SrcMant) | EffSrcMant.
9794 SDValue ImplicitOne =
9795 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One,
9796 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl));
9797 SDValue FullSrcMant =
9798 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ImplicitOne);
9799
9800 // Total right shift = DenormShift + (SrcMant - DstMant).
9801 int64_t MantDelta = static_cast<int64_t>(SrcMant) - DstMant;
9802 SDValue TotalShift =
9803 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormShift,
9804 N2: DAG.getSignedConstant(Val: MantDelta, DL: dl, VT: IntVT));
9805
9806 // Clamp total shift to avoid UB, then truncate denorm mantissa.
9807 EVT ShiftVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
9808 SDValue MaxShift = DAG.getConstant(Val: SrcBits - 1, DL: dl, VT: IntVT);
9809 SDValue ClampedShift =
9810 DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IntVT, N1: TotalShift, N2: MaxShift);
9811 SDValue DenormTruncMant =
9812 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant,
9813 N2: DAG.getZExtOrTrunc(Op: ClampedShift, DL: dl, VT: ShiftVT));
9814
9815 // Rounding for denorm path.
9816 SDValue DenormRoundUp;
9817 {
9818 // Round bit is at position TotalShift - 1 of FullSrcMant.
9819 // Clamp to at least 1 so the subtraction doesn't underflow and create
9820 // shift nodes with invalid shift amounts.
9821 SDValue SafeShift = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT: IntVT, N1: ClampedShift, N2: One);
9822 SDValue RoundBitPos = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: SafeShift, N2: One);
9823 SDValue RoundBitPosAmt = DAG.getZExtOrTrunc(Op: RoundBitPos, DL: dl, VT: ShiftVT);
9824 SDValue DenormRoundBit = DAG.getNode(
9825 Opcode: ISD::AND, DL: dl, VT: IntVT,
9826 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant, N2: RoundBitPosAmt), N2: One);
9827
9828 // Sticky: all bits below round bit.
9829 // sticky_mask = (1 << RoundBitPos) - 1
9830 SDValue StickyMask = DAG.getNode(
9831 Opcode: ISD::SUB, DL: dl, VT: IntVT,
9832 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: RoundBitPosAmt), N2: One);
9833 SDValue DenormStickyBits =
9834 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FullSrcMant, N2: StickyMask);
9835 SDValue HasSticky = DAG.getNode(
9836 Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT,
9837 Operand: DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormStickyBits, RHS: Zero, Cond: ISD::SETNE));
9838
9839 SDValue DenormLSB =
9840 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: One);
9841
9842 DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);
9843
9844 // Only apply rounding if TotalShift >= 1 (i.e., there are bits to round).
9845 SDValue ShiftGEOne =
9846 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ClampedShift, RHS: One, Cond: ISD::SETUGE);
9847 DenormRoundUp = DAG.getSelect(DL: dl, VT: IntVT, Cond: ShiftGEOne, LHS: DenormRoundUp, RHS: Zero);
9848 }
9849
9850 SDValue DenormRoundedMant =
9851 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: DenormRoundUp);
9852
9853 // If rounding caused overflow into the normal range, then we get the
9854 // smallest normal number.
9855 SDValue DenormMantOF =
9856 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormRoundedMant,
9857 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9858 SDValue DenormFinalMant =
9859 DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: Zero, RHS: DenormRoundedMant);
9860 SDValue DenormFinalExp = DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: One, RHS: Zero);
9861
9862 // Assemble: sign | (exp << DstMant) | mant
9863 SDValue DenormExpShifted =
9864 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormFinalExp,
9865 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9866 DenormResult = DAG.getNode(
9867 Opcode: ISD::OR, DL: dl, VT: IntVT,
9868 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
9869 N2: DenormFinalMant);
9870 }
9871
9872 // Exponent overflow detection.
9873 SDValue ExpOF =
9874 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9875 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9876
9877 // Also check if AdjExp == DstExpMaxNormal and mantissa overflow into
9878 // a value that exceeds the max allowed mantissa at that exponent.
9879 SDValue ExpAtMax =
9880 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9881 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETEQ);
9882 SDValue MantExceedsMax =
9883 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjMant,
9884 RHS: DAG.getConstant(Val: DstMaxMantAtMaxExp, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9885 SDValue ExpMantOF =
9886 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: ExpAtMax, N2: MantExceedsMax);
9887 SDValue IsOverflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCVT, N1: ExpOF, N2: ExpMantOF);
9888
9889 // Build overflow result.
9890 SDValue OverflowResult;
9891
9892 if (Saturate) {
9893 // Clamp to max finite value:
9894 // sign | (DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp
9895 uint64_t MaxFinite =
9896 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9897 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9898 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9899 } else if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9900 // Produce infinity.
9901 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9902 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9903 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9904 } else {
9905 // Emit poison if no Inf in format and not saturating.
9906 OverflowResult = DAG.getPOISON(VT: IntVT);
9907 }
9908
9909 // Assemble normal result: sign | (AdjExp << DstMant) | AdjMant
9910 SDValue NormExpShifted =
9911 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: AdjExp,
9912 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9913 SDValue NormResult = DAG.getNode(
9914 Opcode: ISD::OR, DL: dl, VT: IntVT,
9915 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted), N2: AdjMant);
9916
9917 // Build special-value results.
9918 SDValue NaNResult;
9919 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9920 // Produce canonical NaN.
9921 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
9922 NaNResult =
9923 DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | QNaNBit, DL: dl, VT: IntVT);
9924 } else if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9925 DstNanEnc == fltNanEncoding::AllOnes) {
9926 // E4M3FN-style: NaN is exp=all-ones, mant=all-ones.
9927 NaNResult = DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | DstMantMask,
9928 DL: dl, VT: IntVT);
9929 } else {
9930 // NaN -> poison for finite only values.
9931 NaNResult = DAG.getPOISON(VT: IntVT);
9932 }
9933
9934 // Inf handling.
9935 SDValue InfResult;
9936 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9937 // Produce signed infinity.
9938 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9939 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9940 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9941 } else if (Saturate) {
9942 // Inf saturates to max finite.
9943 uint64_t MaxFinite =
9944 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9945 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9946 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9947 } else {
9948 // No Inf and not saturating -> poison.
9949 InfResult = DAG.getPOISON(VT: IntVT);
9950 }
9951
9952 SDValue ZeroResult = SignShifted;
9953
9954 // Final selection in an order: NaN takes priority, then Inf, then Zero.
9955 SDValue FiniteResult =
9956 DAG.getSelect(DL: dl, VT: IntVT, Cond: ExpIsNeg, LHS: DenormResult, RHS: NormResult);
9957 FiniteResult =
9958 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: OverflowResult, RHS: FiniteResult);
9959
9960 SDValue Result = FiniteResult;
9961 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
9962 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
9963
9964 // Negative values are unrepresentable in an unsigned format: clamp to zero
9965 // when saturating, poison otherwise so no select is needed. -0.0 is handled
9966 // by IsZero above. Run before the NaN case so a negative NaN still yields
9967 // NaN.
9968 if (!DstHasSign && Saturate) {
9969 SDValue IsNegative =
9970 DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOLT);
9971 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNegative, LHS: Zero, RHS: Result);
9972 }
9973
9974 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
9975
9976 // Truncate to destination integer type.
9977 return DAG.getZExtOrTrunc(Op: Result, DL: dl, VT: ResVT);
9978}
9979
9980SDValue
9981TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
9982 SelectionDAG &DAG) const {
9983 SDLoc dl(Node);
9984 EVT DstVT = Node->getValueType(ResNo: 0);
9985 EVT DstScalarVT = DstVT.getScalarType();
9986
9987 SDValue IntVal = Node->getOperand(Num: 0);
9988 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9989 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9990
9991 // Supported source formats.
9992 switch (Sem) {
9993 case APFloatBase::S_Float8E5M2:
9994 case APFloatBase::S_Float8E4M3FN:
9995 case APFloatBase::S_Float8E5M3FNU:
9996 case APFloatBase::S_Float6E3M2FN:
9997 case APFloatBase::S_Float6E2M3FN:
9998 case APFloatBase::S_Float4E2M1FN:
9999 break;
10000 default:
10001 DAG.getContext()->emitError(ErrorStr: "CONVERT_FROM_ARBITRARY_FP: not implemented "
10002 "source format (semantics enum " +
10003 Twine(SemEnum) + ")");
10004 return SDValue();
10005 }
10006
10007 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(S: Sem);
10008 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
10009 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
10010 const unsigned SrcMant = SrcPrecision - 1;
10011 // Unsigned formats spend no bit on the sign.
10012 const bool SrcHasSign = APFloat::semanticsHasSignedRepr(SrcSem);
10013 const unsigned SrcExp = SrcBits - (SrcHasSign ? 1 : 0) - SrcMant;
10014 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
10015 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
10016
10017 // Destination format parameters.
10018 const fltSemantics &DstSem = DstScalarVT.getFltSemantics();
10019 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
10020 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
10021 const unsigned DstExpBits = DstBits - DstMant - 1;
10022 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
10023 const int DstBias = 1 - DstMinExp;
10024 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
10025
10026 // Work in an integer type matching the destination float width.
10027 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstBits);
10028 EVT IntVT = IntScalarVT;
10029 if (DstVT.isVector()) {
10030 IntVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IntScalarVT,
10031 EC: DstVT.getVectorElementCount());
10032 } else if (!isTypeLegal(VT: IntScalarVT)) {
10033 // Avoid generating illegal type as there is no other places that'll
10034 // legalize it. Vector types don't have this problem because they
10035 // are subject to LegalizeVectorOps and another type legalization phase
10036 // will follow.
10037 if (getTypeAction(Context&: *DAG.getContext(), VT: IntScalarVT) != TypePromoteInteger) {
10038 // We only know how to handle situations where the legal type is wider.
10039 DAG.getContext()->emitError(
10040 ErrorStr: "CONVERT_FROM_ARBITRARY_FP: the requested integer value type for its "
10041 "legalization is not supported");
10042 return SDValue();
10043 }
10044 IntVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: IntScalarVT);
10045 }
10046
10047 SDValue Src = DAG.getZExtOrTrunc(Op: IntVal, DL: dl, VT: IntVT);
10048
10049 EVT SetCCVT =
10050 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
10051
10052 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
10053 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
10054
10055 // Extract bit fields.
10056 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
10057 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
10058
10059 SDValue MantField = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Src,
10060 N2: DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT));
10061
10062 SDValue ExpField =
10063 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT,
10064 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
10065 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl)),
10066 N2: DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT));
10067
10068 // An unsigned source has no sign bit; bit SrcBits - 1 is part of the
10069 // exponent.
10070 SDValue SignShifted =
10071 SrcHasSign
10072 ? DAG.getNode(
10073 Opcode: ISD::SHL, DL: dl, VT: IntVT,
10074 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
10075 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl)),
10076 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
10077 : Zero;
10078
10079 // Classify the input.
10080 SDValue ExpAllOnes = DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT);
10081 SDValue IsExpAllOnes =
10082 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: ExpAllOnes, Cond: ISD::SETEQ);
10083 SDValue IsExpZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: Zero, Cond: ISD::SETEQ);
10084 SDValue IsMantZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETEQ);
10085 SDValue IsMantNonZero =
10086 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETNE);
10087
10088 SDValue IsNaN;
10089 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
10090 IsNaN = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
10091 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
10092 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantNonZero);
10093 } else {
10094 assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
10095 SDValue MantAllOnes = DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT);
10096 SDValue IsMantAllOnes =
10097 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: MantAllOnes, Cond: ISD::SETEQ);
10098 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantAllOnes);
10099 }
10100
10101 SDValue IsInf;
10102 if (NFBehavior == fltNonfiniteBehavior::IEEE754)
10103 IsInf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantZero);
10104 else
10105 IsInf = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
10106
10107 SDValue IsZero = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantZero);
10108 SDValue IsDenorm =
10109 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantNonZero);
10110
10111 // Normal value conversion.
10112 const int BiasAdjust = DstBias - SrcBias;
10113 SDValue NormDstExp = DAG.getNode(
10114 Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: ExpField,
10115 N2: DAG.getConstant(Val: APInt(IntVT.getScalarSizeInBits(), BiasAdjust, true), DL: dl,
10116 VT: IntVT));
10117
10118 SDValue NormDstMant;
10119 if (DstMant > SrcMant) {
10120 SDValue NormDstMantShift =
10121 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
10122 NormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: MantField, N2: NormDstMantShift);
10123 } else {
10124 NormDstMant = MantField;
10125 }
10126
10127 SDValue DstMantShift = DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl);
10128 SDValue NormExpShifted =
10129 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: NormDstExp, N2: DstMantShift);
10130 SDValue NormResult =
10131 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
10132 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted),
10133 N2: NormDstMant);
10134
10135 // With identical exponent biases, denormal values remain denormal and the
10136 // normal conversion's mantissa shift is sufficient.
10137 SDValue DenormResult = NormResult;
10138 if (BiasAdjust != 0) {
10139 const unsigned IntVTBits = IntVT.getScalarSizeInBits();
10140 SDValue LeadingZeros =
10141 DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT: IntVT, Operand: MantField);
10142
10143 const int DenormExpConst =
10144 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
10145 SDValue DenormDstExp = DAG.getNode(
10146 Opcode: ISD::SUB, DL: dl, VT: IntVT,
10147 N1: DAG.getConstant(Val: APInt(IntVTBits, DenormExpConst, true), DL: dl, VT: IntVT),
10148 N2: LeadingZeros);
10149
10150 SDValue MantMSB =
10151 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT,
10152 N1: DAG.getConstant(Val: IntVTBits - 1, DL: dl, VT: IntVT), N2: LeadingZeros);
10153
10154 SDValue LeadingOne = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: MantMSB);
10155 SDValue Frac = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: IntVT, N1: MantField, N2: LeadingOne);
10156
10157 const unsigned ShiftSub = IntVTBits - 1 - DstMant;
10158 SDValue ShiftAmount = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: LeadingZeros,
10159 N2: DAG.getConstant(Val: ShiftSub, DL: dl, VT: IntVT));
10160
10161 SDValue DenormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: Frac, N2: ShiftAmount);
10162
10163 SDValue DenormExpShifted =
10164 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormDstExp, N2: DstMantShift);
10165 DenormResult = DAG.getNode(
10166 Opcode: ISD::OR, DL: dl, VT: IntVT,
10167 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
10168 N2: DenormDstMant);
10169 }
10170
10171 SDValue FiniteResult =
10172 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsDenorm, LHS: DenormResult, RHS: NormResult);
10173
10174 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10175 SDValue NaNResult =
10176 DAG.getConstant(Val: (DstExpAllOnes << DstMant) | QNaNBit, DL: dl, VT: IntVT);
10177
10178 SDValue InfResult =
10179 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
10180 N2: DAG.getConstant(Val: DstExpAllOnes << DstMant, DL: dl, VT: IntVT));
10181
10182 // A source format may have a larger finite exponent range despite having
10183 // fewer bits, as with Float8E5M3FNU converted to half. Its overflowing finite
10184 // values become infinity. The NaN selection below still takes precedence.
10185 if (APFloat::semanticsMaxExponent(SrcSem) >
10186 APFloat::semanticsMaxExponent(DstSem)) {
10187 SDValue IsOverflow =
10188 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NormDstExp,
10189 RHS: DAG.getConstant(Val: DstExpAllOnes, DL: dl, VT: IntVT), Cond: ISD::SETUGE);
10190 FiniteResult =
10191 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: InfResult, RHS: FiniteResult);
10192 }
10193
10194 SDValue ZeroResult = SignShifted;
10195
10196 SDValue Result = FiniteResult;
10197 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
10198 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
10199 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
10200
10201 if (!DstVT.bitsEq(VT: IntVT)) {
10202 // Store to stack before loading it back.
10203 assert(!IntVT.isVector() && IntVT.bitsGT(DstVT));
10204 // IntScalarVT is the original type that has the same width as DstVT.
10205 Align Alignment = DAG.getReducedAlign(VT: IntScalarVT, /*UseABI=*/false);
10206 SDValue StackPtr =
10207 DAG.CreateStackTemporary(Bytes: IntScalarVT.getStoreSize(), Alignment);
10208 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
10209 MachineFunction &MF = DAG.getMachineFunction();
10210 MachinePointerInfo PtrInfo =
10211 MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
10212 SDValue Store = DAG.getTruncStore(Chain: DAG.getEntryNode(), dl, Val: Result, Ptr: StackPtr,
10213 PtrInfo, SVT: IntScalarVT, Alignment);
10214
10215 SDValue Load = DAG.getLoad(VT: DstVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment);
10216 return DAG.getMergeValues(Ops: {Load, Load.getValue(R: 1)}, dl);
10217 }
10218
10219 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT, Operand: Result);
10220}
10221
10222bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
10223 SelectionDAG &DAG) const {
10224 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10225 SDValue Src = Node->getOperand(Num: OpNo);
10226 EVT SrcVT = Src.getValueType();
10227 EVT DstVT = Node->getValueType(ResNo: 0);
10228 SDLoc dl(SDValue(Node, 0));
10229
10230 // FIXME: Only f32 to i64 conversions are supported.
10231 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
10232 return false;
10233
10234 if (Node->isStrictFPOpcode())
10235 // When a NaN is converted to an integer a trap is allowed. We can't
10236 // use this expansion here because it would eliminate that trap. Other
10237 // traps are also allowed and cannot be eliminated. See
10238 // IEEE 754-2008 sec 5.8.
10239 return false;
10240
10241 // Expand f32 -> i64 conversion
10242 // This algorithm comes from compiler-rt's implementation of fixsfdi:
10243 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
10244 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
10245 EVT IntVT = SrcVT.changeTypeToInteger();
10246 EVT IntShVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
10247
10248 SDValue ExponentMask = DAG.getConstant(Val: 0x7F800000, DL: dl, VT: IntVT);
10249 SDValue ExponentLoBit = DAG.getConstant(Val: 23, DL: dl, VT: IntVT);
10250 SDValue Bias = DAG.getConstant(Val: 127, DL: dl, VT: IntVT);
10251 SDValue SignMask = DAG.getConstant(Val: APInt::getSignMask(BitWidth: SrcEltBits), DL: dl, VT: IntVT);
10252 SDValue SignLowBit = DAG.getConstant(Val: SrcEltBits - 1, DL: dl, VT: IntVT);
10253 SDValue MantissaMask = DAG.getConstant(Val: 0x007FFFFF, DL: dl, VT: IntVT);
10254
10255 SDValue Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: Src);
10256
10257 SDValue ExponentBits = DAG.getNode(
10258 Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: ExponentMask),
10259 N2: DAG.getZExtOrTrunc(Op: ExponentLoBit, DL: dl, VT: IntShVT));
10260 SDValue Exponent = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentBits, N2: Bias);
10261
10262 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: IntVT,
10263 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: SignMask),
10264 N2: DAG.getZExtOrTrunc(Op: SignLowBit, DL: dl, VT: IntShVT));
10265 Sign = DAG.getSExtOrTrunc(Op: Sign, DL: dl, VT: DstVT);
10266
10267 SDValue R = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
10268 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: MantissaMask),
10269 N2: DAG.getConstant(Val: 0x00800000, DL: dl, VT: IntVT));
10270
10271 R = DAG.getZExtOrTrunc(Op: R, DL: dl, VT: DstVT);
10272
10273 R = DAG.getSelectCC(
10274 DL: dl, LHS: Exponent, RHS: ExponentLoBit,
10275 True: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: DstVT, N1: R,
10276 N2: DAG.getZExtOrTrunc(
10277 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: Exponent, N2: ExponentLoBit),
10278 DL: dl, VT: IntShVT)),
10279 False: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: DstVT, N1: R,
10280 N2: DAG.getZExtOrTrunc(
10281 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentLoBit, N2: Exponent),
10282 DL: dl, VT: IntShVT)),
10283 Cond: ISD::SETGT);
10284
10285 SDValue Ret = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: DstVT,
10286 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: R, N2: Sign), N2: Sign);
10287
10288 Result = DAG.getSelectCC(DL: dl, LHS: Exponent, RHS: DAG.getConstant(Val: 0, DL: dl, VT: IntVT),
10289 True: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), False: Ret, Cond: ISD::SETLT);
10290 return true;
10291}
10292
10293bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
10294 SDValue &Chain,
10295 SelectionDAG &DAG) const {
10296 SDLoc dl(SDValue(Node, 0));
10297 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10298 SDValue Src = Node->getOperand(Num: OpNo);
10299
10300 EVT SrcVT = Src.getValueType();
10301 EVT DstVT = Node->getValueType(ResNo: 0);
10302 EVT SetCCVT =
10303 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
10304 EVT DstSetCCVT =
10305 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: DstVT);
10306
10307 // Only expand vector types if we have the appropriate vector bit operations.
10308 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
10309 ISD::FP_TO_SINT;
10310 if (DstVT.isVector() && (!isOperationLegalOrCustom(Op: SIntOpcode, VT: DstVT) ||
10311 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT: SrcVT)))
10312 return false;
10313
10314 // If the maximum float value is smaller then the signed integer range,
10315 // the destination signmask can't be represented by the float, so we can
10316 // just use FP_TO_SINT directly.
10317 const fltSemantics &APFSem = SrcVT.getFltSemantics();
10318 APFloat APF(APFSem, APInt::getZero(numBits: SrcVT.getScalarSizeInBits()));
10319 APInt SignMask = APInt::getSignMask(BitWidth: DstVT.getScalarSizeInBits());
10320 if (APFloat::opOverflow &
10321 APF.convertFromAPInt(Input: SignMask, IsSigned: false, RM: APFloat::rmNearestTiesToEven)) {
10322 if (Node->isStrictFPOpcode()) {
10323 Result = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10324 Ops: { Node->getOperand(Num: 0), Src });
10325 Chain = Result.getValue(R: 1);
10326 } else
10327 Result = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10328 return true;
10329 }
10330
10331 // Don't expand it if there isn't cheap fsub instruction.
10332 if (!isOperationLegalOrCustom(
10333 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, VT: SrcVT))
10334 return false;
10335
10336 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
10337 SDValue Sel;
10338
10339 if (Node->isStrictFPOpcode()) {
10340 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
10341 Chain: Node->getOperand(Num: 0), /*IsSignaling*/ true);
10342 Chain = Sel.getValue(R: 1);
10343 } else {
10344 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT);
10345 }
10346
10347 bool Strict = Node->isStrictFPOpcode() ||
10348 shouldUseStrictFP_TO_INT(FpVT: SrcVT, IntVT: DstVT, /*IsSigned*/ false);
10349
10350 if (Strict) {
10351 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
10352 // signmask then offset (the result of which should be fully representable).
10353 // Sel = Src < 0x8000000000000000
10354 // FltOfs = select Sel, 0, 0x8000000000000000
10355 // IntOfs = select Sel, 0, 0x8000000000000000
10356 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
10357
10358 // TODO: Should any fast-math-flags be set for the FSUB?
10359 SDValue FltOfs = DAG.getSelect(DL: dl, VT: SrcVT, Cond: Sel,
10360 LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
10361 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10362 SDValue IntOfs = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel,
10363 LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT),
10364 RHS: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10365 SDValue SInt;
10366 if (Node->isStrictFPOpcode()) {
10367 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: { SrcVT, MVT::Other },
10368 Ops: { Chain, Src, FltOfs });
10369 SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10370 Ops: { Val.getValue(R: 1), Val });
10371 Chain = SInt.getValue(R: 1);
10372 } else {
10373 SDValue Val = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: FltOfs);
10374 SInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Val);
10375 }
10376 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
10377 } else {
10378 // Expand based on maximum range of FP_TO_SINT:
10379 // True = fp_to_sint(Src)
10380 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
10381 // Result = select (Src < 0x8000000000000000), True, False
10382
10383 SDValue True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10384 // TODO: Should any fast-math-flags be set for the FSUB?
10385 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT,
10386 Operand: DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: Cst));
10387 False = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: False,
10388 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10389 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10390 Result = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel, LHS: True, RHS: False);
10391 }
10392 return true;
10393}
10394
10395bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
10396 SDValue &Chain, SelectionDAG &DAG) const {
10397 // This transform is not correct for converting 0 when rounding mode is set
10398 // to round toward negative infinity which will produce -0.0. So disable
10399 // under strictfp.
10400 if (Node->isStrictFPOpcode())
10401 return false;
10402
10403 SDValue Src = Node->getOperand(Num: 0);
10404 EVT SrcVT = Src.getValueType();
10405 EVT DstVT = Node->getValueType(ResNo: 0);
10406
10407 // If the input is known to be non-negative and SINT_TO_FP is legal then use
10408 // it.
10409 if (Node->getFlags().hasNonNeg() &&
10410 isOperationLegalOrCustom(Op: ISD::SINT_TO_FP, VT: SrcVT)) {
10411 Result =
10412 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Node->getOperand(Num: 0));
10413 return true;
10414 }
10415
10416 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
10417 return false;
10418
10419 // Only expand vector types if we have the appropriate vector bit
10420 // operations.
10421 if (SrcVT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SRL, VT: SrcVT) ||
10422 !isOperationLegalOrCustom(Op: ISD::FADD, VT: DstVT) ||
10423 !isOperationLegalOrCustom(Op: ISD::FSUB, VT: DstVT) ||
10424 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: SrcVT) ||
10425 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: SrcVT)))
10426 return false;
10427
10428 SDLoc dl(SDValue(Node, 0));
10429
10430 // Implementation of unsigned i64 to f64 following the algorithm in
10431 // __floatundidf in compiler_rt. This implementation performs rounding
10432 // correctly in all rounding modes with the exception of converting 0
10433 // when rounding toward negative infinity. In that case the fsub will
10434 // produce -0.0. This will be added to +0.0 and produce -0.0 which is
10435 // incorrect.
10436 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), DL: dl, VT: SrcVT);
10437 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
10438 Val: llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), DL: dl, VT: DstVT);
10439 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), DL: dl, VT: SrcVT);
10440 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), DL: dl, VT: SrcVT);
10441 SDValue HiShift = DAG.getShiftAmountConstant(Val: 32, VT: SrcVT, DL: dl);
10442
10443 SDValue Lo = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Src, N2: LoMask);
10444 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Src, N2: HiShift);
10445 SDValue LoOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Lo, N2: TwoP52);
10446 SDValue HiOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Hi, N2: TwoP84);
10447 SDValue LoFlt = DAG.getBitcast(VT: DstVT, V: LoOr);
10448 SDValue HiFlt = DAG.getBitcast(VT: DstVT, V: HiOr);
10449 SDValue HiSub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: DstVT, N1: HiFlt, N2: TwoP84PlusTwoP52);
10450 Result = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DstVT, N1: LoFlt, N2: HiSub);
10451 return true;
10452}
10453
10454SDValue
10455TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node,
10456 SelectionDAG &DAG) const {
10457 unsigned Opcode = Node->getOpcode();
10458 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
10459 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
10460 "Wrong opcode");
10461
10462 if (Node->getFlags().hasNoNaNs()) {
10463 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
10464 EVT VT = Node->getValueType(ResNo: 0);
10465 if ((!isCondCodeLegal(CC: Pred, VT: VT.getSimpleVT()) ||
10466 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)) &&
10467 VT.isVector())
10468 return SDValue();
10469 SDValue Op1 = Node->getOperand(Num: 0);
10470 SDValue Op2 = Node->getOperand(Num: 1);
10471 return DAG.getSelectCC(DL: SDLoc(Node), LHS: Op1, RHS: Op2, True: Op1, False: Op2, Cond: Pred,
10472 Flags: Node->getFlags());
10473 }
10474
10475 return SDValue();
10476}
10477
10478SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
10479 SelectionDAG &DAG) const {
10480 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node, DAG))
10481 return Expanded;
10482
10483 EVT VT = Node->getValueType(ResNo: 0);
10484 if (VT.isScalableVector())
10485 report_fatal_error(
10486 reason: "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
10487
10488 SDLoc dl(Node);
10489 unsigned NewOp =
10490 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10491
10492 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10493 SDValue Quiet0 = Node->getOperand(Num: 0);
10494 SDValue Quiet1 = Node->getOperand(Num: 1);
10495
10496 if (!Node->getFlags().hasNoNaNs()) {
10497 // Insert canonicalizes if it's possible we need to quiet to get correct
10498 // sNaN behavior.
10499 if (!DAG.isKnownNeverSNaN(Op: Quiet0)) {
10500 Quiet0 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet0,
10501 Flags: Node->getFlags());
10502 }
10503 if (!DAG.isKnownNeverSNaN(Op: Quiet1)) {
10504 Quiet1 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet1,
10505 Flags: Node->getFlags());
10506 }
10507 }
10508
10509 return DAG.getNode(Opcode: NewOp, DL: dl, VT, N1: Quiet0, N2: Quiet1, Flags: Node->getFlags());
10510 }
10511
10512 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
10513 // instead if there are no NaNs.
10514 if (Node->getFlags().hasNoNaNs() ||
10515 (DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 0)) &&
10516 DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 1)))) {
10517 unsigned IEEE2018Op =
10518 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10519 if (isOperationLegalOrCustom(Op: IEEE2018Op, VT))
10520 return DAG.getNode(Opcode: IEEE2018Op, DL: dl, VT, N1: Node->getOperand(Num: 0),
10521 N2: Node->getOperand(Num: 1), Flags: Node->getFlags());
10522 }
10523
10524 if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
10525 return SelCC;
10526
10527 return SDValue();
10528}
10529
10530static SDValue isSpecificZeroAfterMaybeRounding(SelectionDAG &DAG,
10531 const TargetLowering &TLI,
10532 const SDLoc &DL, SDValue Val,
10533 FPClassTest FPClass) {
10534 EVT VT = Val.getValueType();
10535 EVT CCVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10536 EVT IntVT = VT.changeTypeToInteger();
10537 EVT FloatVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
10538 SDValue TestZero = DAG.getTargetConstant(Val: FPClass, DL, VT: MVT::i32);
10539 if (!TLI.isTypeLegal(VT: IntVT) &&
10540 !TLI.isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT))
10541 Val = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: FloatVT, N1: Val,
10542 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
10543 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: CCVT, N1: Val, N2: TestZero);
10544}
10545
10546SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
10547 SelectionDAG &DAG) const {
10548 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node: N, DAG))
10549 return Expanded;
10550
10551 SDLoc DL(N);
10552 SDValue LHS = N->getOperand(Num: 0);
10553 SDValue RHS = N->getOperand(Num: 1);
10554 unsigned Opc = N->getOpcode();
10555 EVT VT = N->getValueType(ResNo: 0);
10556 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10557 bool IsMax = Opc == ISD::FMAXIMUM;
10558 SDNodeFlags Flags = N->getFlags();
10559
10560 // First, implement comparison not propagating NaN. If no native fmin or fmax
10561 // available, use plain select with setcc instead.
10562 SDValue MinMax;
10563 unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
10564 unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;
10565
10566 // FIXME: We should probably define fminnum/fmaxnum variants with correct
10567 // signed zero behavior.
10568 bool MinMaxMustRespectOrderedZero = false;
10569
10570 if (isOperationLegalOrCustom(Op: CompOpcIeee, VT)) {
10571 MinMax = DAG.getNode(Opcode: CompOpcIeee, DL, VT, N1: LHS, N2: RHS, Flags);
10572 MinMaxMustRespectOrderedZero = true;
10573 } else if (isOperationLegalOrCustom(Op: CompOpc, VT)) {
10574 MinMax = DAG.getNode(Opcode: CompOpc, DL, VT, N1: LHS, N2: RHS, Flags);
10575 } else {
10576 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
10577 return DAG.UnrollVectorOp(N);
10578
10579 // NaN (if exists) will be propagated later, so orderness doesn't matter.
10580 SDValue Compare =
10581 DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: IsMax ? ISD::SETOGT : ISD::SETOLT);
10582 MinMax = DAG.getSelect(DL, VT, Cond: Compare, LHS, RHS, Flags);
10583 }
10584
10585 // Propagate any NaN of both operands
10586 if (!N->getFlags().hasNoNaNs() &&
10587 (!DAG.isKnownNeverNaN(Op: RHS) || !DAG.isKnownNeverNaN(Op: LHS))) {
10588 ConstantFP *FPNaN = ConstantFP::get(Context&: *DAG.getContext(),
10589 V: APFloat::getNaN(Sem: VT.getFltSemantics()));
10590 MinMax = DAG.getSelect(DL, VT, Cond: DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETUO),
10591 LHS: DAG.getConstantFP(V: *FPNaN, DL, VT), RHS: MinMax, Flags);
10592 }
10593
10594 // fminimum/fmaximum requires -0.0 less than +0.0
10595 if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
10596 !DAG.isKnownNeverLogicalZero(Op: RHS) && !DAG.isKnownNeverLogicalZero(Op: LHS)) {
10597 SDValue IsEqual = DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETOEQ);
10598 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10599 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10600 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS, Flags);
10601 MinMax = DAG.getSelect(DL, VT, Cond: IsEqual, LHS: RetZero, RHS: MinMax, Flags);
10602 }
10603
10604 return MinMax;
10605}
10606
10607SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
10608 SelectionDAG &DAG) const {
10609 SDLoc DL(Node);
10610 SDValue LHS = Node->getOperand(Num: 0);
10611 SDValue RHS = Node->getOperand(Num: 1);
10612 unsigned Opc = Node->getOpcode();
10613 EVT VT = Node->getValueType(ResNo: 0);
10614 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10615 bool IsMax = Opc == ISD::FMAXIMUMNUM;
10616 SDNodeFlags Flags = Node->getFlags();
10617
10618 unsigned NewOp =
10619 Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10620
10621 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10622 if (!Flags.hasNoNaNs()) {
10623 // Insert canonicalizes if it's possible we need to quiet to get correct
10624 // sNaN behavior.
10625 if (!DAG.isKnownNeverSNaN(Op: LHS)) {
10626 LHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: LHS, Flags);
10627 }
10628 if (!DAG.isKnownNeverSNaN(Op: RHS)) {
10629 RHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: RHS, Flags);
10630 }
10631 }
10632
10633 return DAG.getNode(Opcode: NewOp, DL, VT, N1: LHS, N2: RHS, Flags);
10634 }
10635
10636 // We can use FMINIMUM/FMAXIMUM if there is no NaN, since it has
10637 // same behaviors for all of other cases: +0.0 vs -0.0 included.
10638 if (Flags.hasNoNaNs() ||
10639 (DAG.isKnownNeverNaN(Op: LHS) && DAG.isKnownNeverNaN(Op: RHS))) {
10640 unsigned IEEE2019Op =
10641 Opc == ISD::FMINIMUMNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10642 if (isOperationLegalOrCustom(Op: IEEE2019Op, VT))
10643 return DAG.getNode(Opcode: IEEE2019Op, DL, VT, N1: LHS, N2: RHS, Flags);
10644 }
10645
10646 // FMINNUM/FMAXMUM returns qNaN if either operand is sNaN, and it may return
10647 // either one for +0.0 vs -0.0.
10648 if ((Flags.hasNoNaNs() ||
10649 (DAG.isKnownNeverSNaN(Op: LHS) && DAG.isKnownNeverSNaN(Op: RHS))) &&
10650 (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10651 DAG.isKnownNeverLogicalZero(Op: RHS))) {
10652 unsigned IEEE2008Op = Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM : ISD::FMAXNUM;
10653 if (isOperationLegalOrCustom(Op: IEEE2008Op, VT))
10654 return DAG.getNode(Opcode: IEEE2008Op, DL, VT, N1: LHS, N2: RHS, Flags);
10655 }
10656
10657 if (VT.isVector() &&
10658 (isOperationLegalOrCustomOrPromote(Op: Opc, VT: VT.getVectorElementType()) ||
10659 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)))
10660 return DAG.UnrollVectorOp(N: Node);
10661
10662 // If only one operand is NaN, override it with another operand.
10663 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: LHS)) {
10664 LHS = DAG.getSelectCC(DL, LHS, RHS: LHS, True: RHS, False: LHS, Cond: ISD::SETUO);
10665 }
10666 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: RHS)) {
10667 RHS = DAG.getSelectCC(DL, LHS: RHS, RHS, True: LHS, False: RHS, Cond: ISD::SETUO);
10668 }
10669
10670 // Always prefer RHS if equal.
10671 SDValue MinMax =
10672 DAG.getSelectCC(DL, LHS, RHS, True: LHS, False: RHS, Cond: IsMax ? ISD::SETGT : ISD::SETLT);
10673
10674 // TODO: We need quiet sNaN if strictfp.
10675
10676 // Fixup signed zero behavior.
10677 if (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10678 DAG.isKnownNeverLogicalZero(Op: RHS)) {
10679 return MinMax;
10680 }
10681 SDValue IsZero = DAG.getSetCC(DL, VT: CCVT, LHS: MinMax,
10682 RHS: DAG.getConstantFP(Val: 0.0, DL, VT), Cond: ISD::SETEQ);
10683 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10684 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10685 // It's OK to select from LHS and MinMax, with only one ISD::IS_FPCLASS, as
10686 // we preferred RHS when generate MinMax, if the operands are equal.
10687 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS: MinMax, Flags);
10688 return DAG.getSelect(DL, VT, Cond: IsZero, LHS: RetZero, RHS: MinMax, Flags);
10689}
10690
10691/// Returns a true value if if this FPClassTest can be performed with an ordered
10692/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
10693/// std::nullopt if it cannot be performed as a compare with 0.
10694static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
10695 const fltSemantics &Semantics,
10696 const MachineFunction &MF) {
10697 FPClassTest OrderedMask = Test & ~fcNan;
10698 FPClassTest NanTest = Test & fcNan;
10699 bool IsOrdered = NanTest == fcNone;
10700 bool IsUnordered = NanTest == fcNan;
10701
10702 // Skip cases that are testing for only a qnan or snan.
10703 if (!IsOrdered && !IsUnordered)
10704 return std::nullopt;
10705
10706 if (OrderedMask == fcZero &&
10707 MF.getDenormalMode(FPType: Semantics).Input == DenormalMode::IEEE)
10708 return IsOrdered;
10709 if (OrderedMask == (fcZero | fcSubnormal) &&
10710 MF.getDenormalMode(FPType: Semantics).inputsAreZero())
10711 return IsOrdered;
10712 return std::nullopt;
10713}
10714
10715SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
10716 const FPClassTest OrigTestMask,
10717 SDNodeFlags Flags, const SDLoc &DL,
10718 SelectionDAG &DAG) const {
10719 EVT OperandVT = Op.getValueType();
10720 assert(OperandVT.isFloatingPoint());
10721 FPClassTest Test = OrigTestMask;
10722
10723 // Degenerated cases.
10724 if (Test == fcNone)
10725 return DAG.getBoolConstant(V: false, DL, VT: ResultVT, OpVT: OperandVT);
10726 if (Test == fcAllFlags)
10727 return DAG.getBoolConstant(V: true, DL, VT: ResultVT, OpVT: OperandVT);
10728
10729 // PPC double double is a pair of doubles, of which the higher part determines
10730 // the value class.
10731 if (OperandVT == MVT::ppcf128) {
10732 Op = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::f64, N1: Op,
10733 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
10734 OperandVT = MVT::f64;
10735 }
10736
10737 // Floating-point type properties.
10738 EVT ScalarFloatVT = OperandVT.getScalarType();
10739 const Type *FloatTy = ScalarFloatVT.getTypeForEVT(Context&: *DAG.getContext());
10740 const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
10741 bool IsF80 = (ScalarFloatVT == MVT::f80);
10742
10743 // Some checks can be implemented using float comparisons, if floating point
10744 // exceptions are ignored.
10745 if (Flags.hasNoFPExcept() &&
10746 isOperationLegalOrCustom(Op: ISD::SETCC, VT: OperandVT.getScalarType())) {
10747 FPClassTest FPTestMask = Test;
10748 bool IsInvertedFP = false;
10749
10750 if (FPClassTest InvertedFPCheck =
10751 invertFPClassTestIfSimpler(Test: FPTestMask, UseFCmp: true)) {
10752 FPTestMask = InvertedFPCheck;
10753 IsInvertedFP = true;
10754 }
10755
10756 ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ;
10757 ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ;
10758
10759 // See if we can fold an | fcNan into an unordered compare.
10760 FPClassTest OrderedFPTestMask = FPTestMask & ~fcNan;
10761
10762 // Can't fold the ordered check if we're only testing for snan or qnan
10763 // individually.
10764 if ((FPTestMask & fcNan) != fcNan)
10765 OrderedFPTestMask = FPTestMask;
10766
10767 const bool IsOrdered = FPTestMask == OrderedFPTestMask;
10768
10769 if (std::optional<bool> IsCmp0 =
10770 isFCmpEqualZero(Test: FPTestMask, Semantics, MF: DAG.getMachineFunction());
10771 IsCmp0 && (isCondCodeLegalOrCustom(
10772 CC: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
10773 VT: OperandVT.getScalarType().getSimpleVT()))) {
10774
10775 // If denormals could be implicitly treated as 0, this is not equivalent
10776 // to a compare with 0 since it will also be true for denormals.
10777 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op,
10778 RHS: DAG.getConstantFP(Val: 0.0, DL, VT: OperandVT),
10779 Cond: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
10780 }
10781
10782 if (FPTestMask == fcNan &&
10783 isCondCodeLegalOrCustom(CC: IsInvertedFP ? ISD::SETO : ISD::SETUO,
10784 VT: OperandVT.getScalarType().getSimpleVT()))
10785 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Op,
10786 Cond: IsInvertedFP ? ISD::SETO : ISD::SETUO);
10787
10788 bool IsOrderedInf = FPTestMask == fcInf;
10789 if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) &&
10790 isCondCodeLegalOrCustom(CC: IsOrderedInf ? OrderedCmpOpcode
10791 : UnorderedCmpOpcode,
10792 VT: OperandVT.getScalarType().getSimpleVT()) &&
10793 isOperationLegalOrCustom(Op: ISD::FABS, VT: OperandVT.getScalarType()) &&
10794 (isOperationLegal(Op: ISD::ConstantFP, VT: OperandVT.getScalarType()) ||
10795 (OperandVT.isVector() &&
10796 isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: OperandVT)))) {
10797 // isinf(x) --> fabs(x) == inf
10798 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10799 SDValue Inf =
10800 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10801 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf,
10802 Cond: IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
10803 }
10804
10805 if ((OrderedFPTestMask == fcPosInf || OrderedFPTestMask == fcNegInf) &&
10806 isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedCmpOpcode
10807 : UnorderedCmpOpcode,
10808 VT: OperandVT.getSimpleVT())) {
10809 // isposinf(x) --> x == inf
10810 // isneginf(x) --> x == -inf
10811 // isposinf(x) || nan --> x u== inf
10812 // isneginf(x) || nan --> x u== -inf
10813
10814 SDValue Inf = DAG.getConstantFP(
10815 Val: APFloat::getInf(Sem: Semantics, Negative: OrderedFPTestMask == fcNegInf), DL,
10816 VT: OperandVT);
10817 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Inf,
10818 Cond: IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
10819 }
10820
10821 if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
10822 // TODO: Could handle ordered case, but it produces worse code for
10823 // x86. Maybe handle ordered if fabs is free?
10824
10825 ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10826 ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;
10827
10828 if (isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedOp : UnorderedOp,
10829 VT: OperandVT.getScalarType().getSimpleVT())) {
10830 // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal
10831
10832 // TODO: Maybe only makes sense if fabs is free. Integer test of
10833 // exponent bits seems better for x86.
10834 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10835 SDValue SmallestNormal = DAG.getConstantFP(
10836 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10837 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal,
10838 Cond: IsOrdered ? OrderedOp : UnorderedOp);
10839 }
10840 }
10841
10842 if (FPTestMask == fcNormal) {
10843 // TODO: Handle unordered
10844 ISD::CondCode IsFiniteOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10845 ISD::CondCode IsNormalOp = IsInvertedFP ? ISD::SETOLT : ISD::SETUGE;
10846
10847 if (isCondCodeLegalOrCustom(CC: IsFiniteOp,
10848 VT: OperandVT.getScalarType().getSimpleVT()) &&
10849 isCondCodeLegalOrCustom(CC: IsNormalOp,
10850 VT: OperandVT.getScalarType().getSimpleVT()) &&
10851 isFAbsFree(VT: OperandVT)) {
10852 // isnormal(x) --> fabs(x) < infinity && !(fabs(x) < smallest_normal)
10853 SDValue Inf =
10854 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10855 SDValue SmallestNormal = DAG.getConstantFP(
10856 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10857
10858 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10859 SDValue IsFinite = DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf, Cond: IsFiniteOp);
10860 SDValue IsNormal =
10861 DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal, Cond: IsNormalOp);
10862 unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
10863 return DAG.getNode(Opcode: LogicOp, DL, VT: ResultVT, N1: IsFinite, N2: IsNormal);
10864 }
10865 }
10866 }
10867
10868 // Some checks may be represented as inversion of simpler check, for example
10869 // "inf|normal|subnormal|zero" => !"nan".
10870 bool IsInverted = false;
10871
10872 if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, UseFCmp: false)) {
10873 Test = InvertedCheck;
10874 IsInverted = true;
10875 }
10876
10877 // In the general case use integer operations.
10878 unsigned BitSize = OperandVT.getScalarSizeInBits();
10879 EVT IntVT = OperandVT.changeElementType(
10880 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BitSize));
10881 SDValue OpAsInt = DAG.getBitcast(VT: IntVT, V: Op);
10882
10883 // Various masks.
10884 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10885 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10886 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10887 const unsigned ExplicitIntBitInF80 = 63;
10888 APInt ExpMask = Inf;
10889 if (IsF80)
10890 ExpMask.clearBit(BitPosition: ExplicitIntBitInF80);
10891 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10892 APInt QNaNBitMask =
10893 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10894 APInt InversionMask = APInt::getAllOnes(numBits: ResultVT.getScalarSizeInBits());
10895
10896 SDValue ValueMaskV = DAG.getConstant(Val: ValueMask, DL, VT: IntVT);
10897 SDValue SignBitV = DAG.getConstant(Val: SignBit, DL, VT: IntVT);
10898 SDValue ExpMaskV = DAG.getConstant(Val: ExpMask, DL, VT: IntVT);
10899 SDValue ZeroV = DAG.getConstant(Val: 0, DL, VT: IntVT);
10900 SDValue InfV = DAG.getConstant(Val: Inf, DL, VT: IntVT);
10901 SDValue ResultInversionMask = DAG.getConstant(Val: InversionMask, DL, VT: ResultVT);
10902
10903 SDValue Res;
10904 const auto appendResult = [&](SDValue PartialRes) {
10905 if (PartialRes) {
10906 if (Res)
10907 Res = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: Res, N2: PartialRes);
10908 else
10909 Res = PartialRes;
10910 }
10911 };
10912
10913 SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
10914 const auto getIntBitIsSet = [&]() -> SDValue {
10915 if (!IntBitIsSetV) {
10916 APInt IntBitMask(BitSize, 0);
10917 IntBitMask.setBit(ExplicitIntBitInF80);
10918 SDValue IntBitMaskV = DAG.getConstant(Val: IntBitMask, DL, VT: IntVT);
10919 SDValue IntBitV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: IntBitMaskV);
10920 IntBitIsSetV = DAG.getSetCC(DL, VT: ResultVT, LHS: IntBitV, RHS: ZeroV, Cond: ISD::SETNE);
10921 }
10922 return IntBitIsSetV;
10923 };
10924
10925 // Split the value into sign bit and absolute value.
10926 SDValue AbsV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ValueMaskV);
10927 SDValue SignV = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt,
10928 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETLT);
10929
10930 // Tests that involve more than one class should be processed first.
10931 SDValue PartialRes;
10932
10933 if (IsF80)
10934 ; // Detect finite numbers of f80 by checking individual classes because
10935 // they have different settings of the explicit integer bit.
10936 else if ((Test & fcFinite) == fcFinite) {
10937 // finite(V) ==> (a << 1) < (inf << 1)
10938 //
10939 // See https://github.com/llvm/llvm-project/issues/169270, this is slightly
10940 // shorter than the `finite(V) ==> abs(V) < exp_mask` formula used before.
10941
10942 assert(APFloat::isIEEELikeFP(OperandVT.getFltSemantics()) &&
10943 "finite check requires IEEE-like FP");
10944
10945 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: IntVT, DL);
10946 SDValue TwiceOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: OpAsInt, N2: One);
10947 SDValue TwiceInf = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: ExpMaskV, N2: One);
10948
10949 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: TwiceOp, RHS: TwiceInf, Cond: ISD::SETULT);
10950 Test &= ~fcFinite;
10951 } else if ((Test & fcFinite) == fcPosFinite) {
10952 // finite(V) && V > 0 ==> V < exp_mask
10953 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ExpMaskV, Cond: ISD::SETULT);
10954 Test &= ~fcPosFinite;
10955 } else if ((Test & fcFinite) == fcNegFinite) {
10956 // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
10957 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ExpMaskV, Cond: ISD::SETLT);
10958 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10959 Test &= ~fcNegFinite;
10960 }
10961 appendResult(PartialRes);
10962
10963 if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
10964 // fcZero | fcSubnormal => test all exponent bits are 0
10965 // TODO: Handle sign bit specific cases
10966 if (PartialCheck == (fcZero | fcSubnormal)) {
10967 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ExpMaskV);
10968 SDValue ExpIsZero =
10969 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
10970 appendResult(ExpIsZero);
10971 Test &= ~PartialCheck & fcAllFlags;
10972 }
10973 }
10974
10975 // Check for individual classes.
10976
10977 if (unsigned PartialCheck = Test & fcZero) {
10978 if (PartialCheck == fcPosZero)
10979 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ZeroV, Cond: ISD::SETEQ);
10980 else if (PartialCheck == fcZero)
10981 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ZeroV, Cond: ISD::SETEQ);
10982 else // ISD::fcNegZero
10983 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: SignBitV, Cond: ISD::SETEQ);
10984 appendResult(PartialRes);
10985 }
10986
10987 if (unsigned PartialCheck = Test & fcSubnormal) {
10988 // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
10989 // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
10990 SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
10991 SDValue MantissaV = DAG.getConstant(Val: AllOneMantissa, DL, VT: IntVT);
10992 SDValue VMinusOneV =
10993 DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: V, N2: DAG.getConstant(Val: 1, DL, VT: IntVT));
10994 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: VMinusOneV, RHS: MantissaV, Cond: ISD::SETULT);
10995 if (PartialCheck == fcNegSubnormal)
10996 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10997 appendResult(PartialRes);
10998 }
10999
11000 if (unsigned PartialCheck = Test & fcInf) {
11001 if (PartialCheck == fcPosInf)
11002 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: InfV, Cond: ISD::SETEQ);
11003 else if (PartialCheck == fcInf)
11004 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETEQ);
11005 else { // ISD::fcNegInf
11006 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
11007 SDValue NegInfV = DAG.getConstant(Val: NegInf, DL, VT: IntVT);
11008 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: NegInfV, Cond: ISD::SETEQ);
11009 }
11010 appendResult(PartialRes);
11011 }
11012
11013 if (unsigned PartialCheck = Test & fcNan) {
11014 APInt InfWithQnanBit = Inf | QNaNBitMask;
11015 SDValue InfWithQnanBitV = DAG.getConstant(Val: InfWithQnanBit, DL, VT: IntVT);
11016 if (PartialCheck == fcNan) {
11017 // isnan(V) ==> abs(V) > int(inf)
11018 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
11019 if (IsF80) {
11020 // Recognize unsupported values as NaNs for compatibility with glibc.
11021 // In them (exp(V)==0) == int_bit.
11022 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: AbsV, N2: ExpMaskV);
11023 SDValue ExpIsZero =
11024 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
11025 SDValue IsPseudo =
11026 DAG.getSetCC(DL, VT: ResultVT, LHS: getIntBitIsSet(), RHS: ExpIsZero, Cond: ISD::SETEQ);
11027 PartialRes = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: PartialRes, N2: IsPseudo);
11028 }
11029 } else if (PartialCheck == fcQNan) {
11030 // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
11031 PartialRes =
11032 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETGE);
11033 } else { // ISD::fcSNan
11034 // issignaling(V) ==> abs(V) > unsigned(Inf) &&
11035 // abs(V) < (unsigned(Inf) | quiet_bit)
11036 SDValue IsNan = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
11037 SDValue IsNotQnan =
11038 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETLT);
11039 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: IsNan, N2: IsNotQnan);
11040 }
11041 appendResult(PartialRes);
11042 }
11043
11044 if (unsigned PartialCheck = Test & fcNormal) {
11045 // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
11046 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
11047 SDValue ExpLSBV = DAG.getConstant(Val: ExpLSB, DL, VT: IntVT);
11048 SDValue ExpMinus1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: AbsV, N2: ExpLSBV);
11049 APInt ExpLimit = ExpMask - ExpLSB;
11050 SDValue ExpLimitV = DAG.getConstant(Val: ExpLimit, DL, VT: IntVT);
11051 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: ExpMinus1, RHS: ExpLimitV, Cond: ISD::SETULT);
11052 if (PartialCheck == fcNegNormal)
11053 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
11054 else if (PartialCheck == fcPosNormal) {
11055 SDValue PosSignV =
11056 DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: SignV, N2: ResultInversionMask);
11057 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: PosSignV);
11058 }
11059 if (IsF80)
11060 PartialRes =
11061 DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: getIntBitIsSet());
11062 appendResult(PartialRes);
11063 }
11064
11065 if (!Res)
11066 return DAG.getConstant(Val: IsInverted, DL, VT: ResultVT);
11067 if (IsInverted)
11068 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: Res, N2: ResultInversionMask);
11069 return Res;
11070}
11071
11072// Only expand vector types if we have the appropriate vector bit operations.
11073static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
11074 assert(VT.isVector() && "Expected vector type");
11075 unsigned Len = VT.getScalarSizeInBits();
11076 return TLI.isOperationLegalOrCustom(Op: ISD::ADD, VT) &&
11077 TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT) &&
11078 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
11079 (Len == 8 || TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT)) &&
11080 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT);
11081}
11082
11083SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
11084 SDLoc dl(Node);
11085 EVT VT = Node->getValueType(ResNo: 0);
11086 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11087 SDValue Op = Node->getOperand(Num: 0);
11088 unsigned Len = VT.getScalarSizeInBits();
11089
11090 // Compute effective bit width from known bits, allowing us to shift the
11091 // active bits down if necessary to fit into smaller specialized expansions.
11092 KnownBits Known = DAG.computeKnownBits(Op);
11093 unsigned LZ = Known.countMinLeadingZeros();
11094 unsigned TZ = Known.countMinTrailingZeros();
11095 unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ);
11096
11097 // Round up to 8-bit boundary for byte-oriented SWAR algorithm
11098 unsigned EffectiveLen = Len;
11099 if (ShiftedActiveBits > 0 && ShiftedActiveBits < Len)
11100 EffectiveLen = std::min(a: alignTo(Value: ShiftedActiveBits, Align: 8), b: Len);
11101
11102 assert(VT.isInteger() && "CTPOP not implemented for this type.");
11103
11104 // TODO: Add support for irregular type lengths.
11105 if (!(Len <= 128 && Len % 8 == 0))
11106 return SDValue();
11107
11108 // Only expand vector types if we have the appropriate vector bit operations.
11109 if (VT.isVector() && !canExpandVectorCTPOP(TLI: *this, VT))
11110 return SDValue();
11111
11112 // If the active bits are not at the low end, shift them down
11113 if (EffectiveLen < Len && TZ > 0) {
11114 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11115 N2: DAG.getShiftAmountConstant(Val: TZ, VT, DL: dl));
11116 }
11117
11118 // This is the "best" algorithm from
11119 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
11120 SDValue Mask55 =
11121 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x55)), DL: dl, VT);
11122 SDValue Mask33 =
11123 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x33)), DL: dl, VT);
11124 SDValue Mask0F =
11125 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x0F)), DL: dl, VT);
11126
11127 // v = v - ((v >> 1) & 0x55555555...)
11128 Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op,
11129 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11130 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11131 N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT)),
11132 N2: Mask55));
11133 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
11134 Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: Mask33),
11135 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11136 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11137 N2: DAG.getConstant(Val: 2, DL: dl, VT: ShVT)),
11138 N2: Mask33));
11139 // v = (v + (v >> 4)) & 0x0F0F0F0F...
11140 Op = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11141 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
11142 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11143 N2: DAG.getConstant(Val: 4, DL: dl, VT: ShVT))),
11144 N2: Mask0F);
11145
11146 if (EffectiveLen <= 8)
11147 return Op;
11148
11149 // Avoid the multiply if we only have 2 bytes to add.
11150 // TODO: Only doing this for scalars because vectors weren't as obviously
11151 // improved.
11152 if (EffectiveLen == 16 && !VT.isVector()) {
11153 // v = (v + (v >> 8)) & 0x00FF;
11154 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11155 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
11156 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11157 N2: DAG.getConstant(Val: 8, DL: dl, VT: ShVT))),
11158 N2: DAG.getConstant(Val: 0xFF, DL: dl, VT));
11159 }
11160
11161 // v = (v * 0x01010101...) >> (Len - 8)
11162 SDValue V;
11163 if (isOperationLegalOrCustomOrPromote(
11164 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11165 SDValue Mask01 =
11166 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x01)), DL: dl, VT);
11167 V = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Op, N2: Mask01);
11168 } else {
11169 V = Op;
11170 for (unsigned Shift = 8; Shift < EffectiveLen; Shift *= 2) {
11171 SDValue ShiftC = DAG.getShiftAmountConstant(Val: Shift, VT, DL: dl);
11172 V = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: V,
11173 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: ShiftC));
11174 }
11175 }
11176 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: V, N2: DAG.getConstant(Val: Len - 8, DL: dl, VT: ShVT));
11177}
11178
11179SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
11180 SDLoc dl(Node);
11181 EVT VT = Node->getValueType(ResNo: 0);
11182 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11183 SDValue Op = Node->getOperand(Num: 0);
11184 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11185
11186 // If the non-ZERO_POISON version is supported we can use that instead.
11187 if (Node->getOpcode() == ISD::CTLZ_ZERO_POISON &&
11188 isOperationLegalOrCustom(Op: ISD::CTLZ, VT))
11189 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Op);
11190
11191 // If the ZERO_POISON version is supported use that and handle the zero case.
11192 if (isOperationLegalOrCustom(Op: ISD::CTLZ_ZERO_POISON, VT)) {
11193 EVT SetCCVT =
11194 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11195 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11196 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11197 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11198 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11199 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTLZ);
11200 }
11201
11202 // Only expand vector types if we have the appropriate vector bit operations.
11203 // This includes the operations needed to expand CTPOP if it isn't supported.
11204 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11205 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11206 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11207 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
11208 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
11209 return SDValue();
11210
11211 // for now, we do this:
11212 // x = x | (x >> 1);
11213 // x = x | (x >> 2);
11214 // ...
11215 // x = x | (x >>16);
11216 // x = x | (x >>32); // for 64-bit input
11217 // return popcount(~x);
11218 //
11219 // Ref: "Hacker's Delight" by Henry Warren
11220 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
11221 SDValue Tmp = DAG.getConstant(Val: 1ULL << i, DL: dl, VT: ShVT);
11222 Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op,
11223 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: Tmp));
11224 }
11225 Op = DAG.getNOT(DL: dl, Val: Op, VT);
11226 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
11227}
11228
11229SDValue TargetLowering::expandCTLS(SDNode *Node, SelectionDAG &DAG) const {
11230 SDLoc dl(Node);
11231 EVT VT = Node->getValueType(ResNo: 0);
11232 SDValue Op = DAG.getFreeze(V: Node->getOperand(Num: 0));
11233 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11234
11235 // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
11236 // This transforms the sign bits into leading zeros that can be counted.
11237 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: NumBitsPerElt - 1, VT, DL: dl);
11238 SDValue SignBit = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Op, N2: ShiftAmt);
11239 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: SignBit);
11240 SDValue Shl =
11241 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11242 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Shl, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11243 return DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Or);
11244}
11245
11246SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG,
11247 const SDLoc &DL, EVT VT, SDValue Op,
11248 unsigned BitWidth) const {
11249 if (BitWidth != 32 && BitWidth != 64)
11250 return SDValue();
11251
11252 const DataLayout &TD = DAG.getDataLayout();
11253 if (!isOperationCustom(Op: ISD::ConstantPool, VT: getPointerTy(DL: TD)))
11254 return SDValue();
11255
11256 APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
11257 : APInt(64, 0x0218A392CD3D5DBFULL);
11258 MachinePointerInfo PtrInfo =
11259 MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction());
11260 unsigned ShiftAmt = BitWidth - Log2_32(Value: BitWidth);
11261 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
11262 SDValue Lookup = DAG.getNode(
11263 Opcode: ISD::SRL, DL, VT,
11264 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op, N2: Neg),
11265 N2: DAG.getConstant(Val: DeBruijn, DL, VT)),
11266 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
11267 Lookup = DAG.getSExtOrTrunc(Op: Lookup, DL, VT: getPointerTy(DL: TD));
11268
11269 SmallVector<uint8_t> Table(BitWidth, 0);
11270 for (unsigned i = 0; i < BitWidth; i++) {
11271 APInt Shl = DeBruijn.shl(shiftAmt: i);
11272 APInt Lshr = Shl.lshr(shiftAmt: ShiftAmt);
11273 Table[Lshr.getZExtValue()] = i;
11274 }
11275
11276 // Create a ConstantArray in Constant Pool
11277 auto *CA = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: Table);
11278 SDValue CPIdx = DAG.getConstantPool(C: CA, VT: getPointerTy(DL: TD),
11279 Align: TD.getPrefTypeAlign(Ty: CA->getType()));
11280 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: DL, VT, Chain: DAG.getEntryNode(),
11281 Ptr: DAG.getMemBasePlusOffset(Base: CPIdx, Offset: Lookup, DL),
11282 PtrInfo, MemVT: MVT::i8);
11283 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
11284 return ExtLoad;
11285
11286 EVT SetCCVT =
11287 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11288 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
11289 SDValue SrcIsZero = DAG.getSetCC(DL, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11290 return DAG.getSelect(DL, VT, Cond: SrcIsZero,
11291 LHS: DAG.getConstant(Val: BitWidth, DL, VT), RHS: ExtLoad);
11292}
11293
11294SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
11295 SDLoc dl(Node);
11296 EVT VT = Node->getValueType(ResNo: 0);
11297 SDValue Op = Node->getOperand(Num: 0);
11298 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11299
11300 // If the non-ZERO_POISON version is supported we can use that instead.
11301 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON &&
11302 isOperationLegalOrCustom(Op: ISD::CTTZ, VT))
11303 return DAG.getNode(Opcode: ISD::CTTZ, DL: dl, VT, Operand: Op);
11304
11305 // If the ZERO_POISON version is supported use that and handle the zero case.
11306 if (isOperationLegalOrCustom(Op: ISD::CTTZ_ZERO_POISON, VT)) {
11307 EVT SetCCVT =
11308 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11309 SDValue CTTZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11310 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11311 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11312 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11313 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTTZ);
11314 }
11315
11316 // Only expand vector types if we have the appropriate vector bit operations.
11317 // This includes the operations needed to expand CTPOP if it isn't supported.
11318 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11319 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11320 !isOperationLegalOrCustom(Op: ISD::CTLZ, VT) &&
11321 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11322 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
11323 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) ||
11324 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11325 return SDValue();
11326
11327 // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
11328 // to be expanded or converted to a libcall.
11329 if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT) &&
11330 !isOperationLegal(Op: ISD::CTLZ, VT))
11331 if (SDValue V = CTTZTableLookup(Node, DAG, DL: dl, VT, Op, BitWidth: NumBitsPerElt))
11332 return V;
11333
11334 bool UseCTLZ =
11335 isOperationLegal(Op: ISD::CTLZ, VT) && !isOperationLegal(Op: ISD::CTPOP, VT);
11336
11337 // When only ctlz is available and the operand is nonzero we can use:
11338 // { return nlz(x & -x) ^ 31; }
11339 // which is more efficient than:
11340 // { return 32 - nlz(~x & (x - 1)); }.
11341 if (UseCTLZ && Node->getOpcode() == ISD::CTTZ_ZERO_POISON) {
11342 SDValue LowestBit =
11343 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: DAG.getNegative(Val: Op, DL: dl, VT));
11344 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT,
11345 N1: DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: LowestBit),
11346 N2: DAG.getConstant(Val: NumBitsPerElt - 1, DL: dl, VT));
11347 }
11348
11349 // If ctpop is available, we use:
11350 // { return popcount(~x & (x-1)); }
11351 // If the target has ctlz but not ctpop, we use:
11352 // { return 32 - nlz(~x & (x-1)); }
11353 // Ref: "Hacker's Delight" by Henry Warren
11354 SDValue Tmp = DAG.getNode(
11355 Opcode: ISD::AND, DL: dl, VT, N1: DAG.getNOT(DL: dl, Val: Op, VT),
11356 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 1, DL: dl, VT)));
11357
11358 if (UseCTLZ)
11359 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT),
11360 N2: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Tmp));
11361
11362 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Tmp);
11363}
11364
11365SDValue TargetLowering::expandVPCTTZElements(SDNode *N,
11366 SelectionDAG &DAG) const {
11367 // %cond = to_bool_vec %source
11368 // %splat = splat /*val=*/VL
11369 // %tz = step_vector
11370 // %v = select %cond, /*true=*/tz, /*false=*/%splat
11371 // %r = vp.reduce.umin %v
11372 SDLoc DL(N);
11373 SDValue Source = N->getOperand(Num: 0);
11374 SDValue Mask = N->getOperand(Num: 1);
11375 SDValue EVL = N->getOperand(Num: 2);
11376 EVT SrcVT = Source.getValueType();
11377 EVT ResVT = N->getValueType(ResNo: 0);
11378 EVT ResVecVT =
11379 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT, EC: SrcVT.getVectorElementCount());
11380
11381 // Convert to boolean vector.
11382 if (SrcVT.getScalarType() != MVT::i1) {
11383 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
11384 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
11385 EC: SrcVT.getVectorElementCount());
11386 Source = DAG.getSetCC(DL, VT: SrcVT, LHS: Source, RHS: AllZero, Cond: ISD::SETNE);
11387 }
11388
11389 SDValue ExtEVL = DAG.getZExtOrTrunc(Op: EVL, DL, VT: ResVT);
11390 SDValue Splat = DAG.getSplat(VT: ResVecVT, DL, Op: ExtEVL);
11391 SDValue StepVec = DAG.getStepVector(DL, ResVT: ResVecVT);
11392 SDValue Select = DAG.getSelect(DL, VT: ResVecVT, Cond: Source, LHS: StepVec, RHS: Splat);
11393 return DAG.getNode(Opcode: ISD::VP_REDUCE_UMIN, DL, VT: ResVT, N1: ExtEVL, N2: Select, N3: Mask, N4: EVL);
11394}
11395
11396/// Returns a type-legalized version of \p Mask as the first item in the
11397/// pair. The second item contains a type-legalized step vector that's
11398/// guaranteed to fit the number of elements in \p Mask.
11399/// If the stepvector would require splitting, returns an empty SDValue
11400/// as the second item to signal that the operation should be split instead.
11401static std::pair<SDValue, SDValue>
11402getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL,
11403 SelectionDAG &DAG) {
11404 EVT MaskVT = Mask.getValueType();
11405 EVT BoolVT = MaskVT.getScalarType();
11406
11407 // Find a suitable type for a stepvector.
11408 // If zero is poison, we can assume the upper limit of the result is VF-1.
11409 ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
11410 if (MaskVT.isScalableVector())
11411 VScaleRange = getVScaleRange(F: &DAG.getMachineFunction().getFunction(), BitWidth: 64);
11412 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11413 uint64_t EltWidth = TLI.getBitWidthForCttzElements(
11414 RetVT: EVT(TLI.getVectorIdxTy(DL: DAG.getDataLayout())),
11415 EC: MaskVT.getVectorElementCount(), ZeroIsPoison, VScaleRange: &VScaleRange);
11416 // If the step vector element type is smaller than the mask element type,
11417 // use the mask type directly to avoid widening issues.
11418 EltWidth = std::max(a: EltWidth, b: BoolVT.getFixedSizeInBits());
11419 EVT StepVT = MVT::getIntegerVT(BitWidth: EltWidth);
11420 EVT StepVecVT = MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: StepVT);
11421
11422 // If promotion or widening is required to make the type legal, do it here.
11423 // Promotion of integers within LegalizeVectorOps is looking for types of
11424 // the same size but with a smaller number of larger elements, not the usual
11425 // larger size with the same number of larger elements.
11426 TargetLowering::LegalizeTypeAction TypeAction =
11427 TLI.getTypeAction(Context&: *DAG.getContext(), VT: StepVecVT);
11428 SDValue StepVec;
11429 if (TypeAction == TargetLowering::TypePromoteInteger) {
11430 StepVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11431 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11432 } else if (TypeAction == TargetLowering::TypeWidenVector) {
11433 // For widening, the element count changes. Create a step vector with only
11434 // the original elements valid and zeros for padding. Also widen the mask.
11435 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11436 unsigned WideNumElts = WideVecVT.getVectorNumElements();
11437
11438 // Build widened step vector: <0, 1, ..., OrigNumElts-1, poison, poison, ..>
11439 SDValue OrigStepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11440 SDValue UndefStep = DAG.getPOISON(VT: WideVecVT);
11441 StepVec = DAG.getInsertSubvector(DL, Vec: UndefStep, SubVec: OrigStepVec, Idx: 0);
11442
11443 // Widen mask: pad with zeros.
11444 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: BoolVT, NumElements: WideNumElts);
11445 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
11446 Mask = DAG.getInsertSubvector(DL, Vec: ZeroMask, SubVec: Mask, Idx: 0);
11447 } else if (TypeAction == TargetLowering::TypeSplitVector) {
11448 // The stepvector type would require splitting. Signal to the caller
11449 // that the operation should be split instead of expanded.
11450 return {Mask, SDValue()};
11451 } else {
11452 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11453 }
11454
11455 return {Mask, StepVec};
11456}
11457
11458SDValue TargetLowering::expandVectorFindLastActive(SDNode *N,
11459 SelectionDAG &DAG) const {
11460 SDLoc DL(N);
11461 auto [Mask, StepVec] = getLegalMaskAndStepVector(
11462 Mask: N->getOperand(Num: 0), /*ZeroIsPoison=*/true, DL, DAG);
11463
11464 // If StepVec is empty, the stepvector would require splitting.
11465 // Split the operation instead and let it be recursively legalized.
11466 if (!StepVec) {
11467 EVT MaskVT = N->getOperand(Num: 0).getValueType();
11468 EVT ResVT = N->getValueType(ResNo: 0);
11469
11470 // Split the mask
11471 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: MaskVT);
11472 auto [MaskLo, MaskHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
11473
11474 // Create split VECTOR_FIND_LAST_ACTIVE operations
11475 SDValue LoResult =
11476 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskLo);
11477 SDValue HiResult =
11478 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskHi);
11479
11480 // Check if any lane is active in the high mask.
11481 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: MaskHi);
11482 SDValue Cond = DAG.getBoolExtOrTrunc(
11483 Op: AnyHiActive, SL: DL,
11484 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i1),
11485 OpVT: MVT::i1);
11486
11487 // Adjust HiResult by adding the number of elements in Lo
11488 SDValue LoNumElts =
11489 DAG.getElementCount(DL, VT: ResVT, EC: LoVT.getVectorElementCount());
11490 SDValue AdjustedHiResult =
11491 DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: HiResult, N2: LoNumElts);
11492
11493 // Return: AnyHiActive ? AdjustedHiResult : LoResult;
11494 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: ResVT, N1: Cond, N2: AdjustedHiResult,
11495 N3: LoResult);
11496 }
11497
11498 EVT StepVecVT = StepVec.getValueType();
11499 EVT StepVT = StepVec.getValueType().getVectorElementType();
11500
11501 // Zero out lanes with inactive elements, then find the highest remaining
11502 // value from the stepvector.
11503 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
11504 SDValue ActiveElts = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
11505 SDValue HighestIdx = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL, VT: StepVT, Operand: ActiveElts);
11506 return DAG.getZExtOrTrunc(Op: HighestIdx, DL, VT: N->getValueType(ResNo: 0));
11507}
11508
11509SDValue TargetLowering::expandGetActiveLaneMask(SDNode *N,
11510 SelectionDAG &DAG) const {
11511 SDLoc DL(N);
11512
11513 SDValue Start = N->getOperand(Num: 0);
11514 SDValue End = N->getOperand(Num: 1);
11515 EVT VT = N->getValueType(ResNo: 0);
11516 EVT OpVT = Start.getValueType();
11517
11518 auto [Mask, StepVector] = getLegalMaskAndStepVector(
11519 Mask: DAG.getPOISON(VT), /*ZeroIsPoison=*/false, DL, DAG);
11520
11521 // TODO: Handle the step vector needing to be split + widened.
11522 assert(StepVector && "Step vector needs splitting");
11523 assert(Mask.getValueType() == VT && "Step vector was widened");
11524
11525 // Rebase and saturate the termination value.
11526 SDValue Max = DAG.getConstant(Val: maxUIntN(N: StepVector.getScalarValueSizeInBits()),
11527 DL, VT: OpVT);
11528 End = DAG.getNode(Opcode: ISD::USUBSAT, DL, VT: OpVT, N1: End, N2: Start);
11529 End = DAG.getNode(Opcode: ISD::UMIN, DL, VT: OpVT, N1: End, N2: Max);
11530
11531 // cmp <0, 1, 2, 3...>, End
11532 SDValue EndV = DAG.getSplat(VT: StepVector.getValueType(), DL, Op: End);
11533 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
11534 VT: StepVector.getValueType());
11535 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: StepVector, RHS: EndV, Cond: ISD::SETULT);
11536 return DAG.getBoolExtOrTrunc(Op: Cmp, SL: DL, VT, OpVT: StepVector.getValueType());
11537}
11538
11539SDValue TargetLowering::expandLoopDependenceMask(SDNode *N,
11540 SelectionDAG &DAG) const {
11541 SDLoc DL(N);
11542 EVT VT = N->getValueType(ResNo: 0);
11543 SDValue SourceValue = N->getOperand(Num: 0);
11544 SDValue SinkValue = N->getOperand(Num: 1);
11545 SDValue EltSizeInBytes = N->getOperand(Num: 2);
11546
11547 // Note: The lane offset is scalable if the mask is scalable.
11548 ElementCount LaneOffsetEC =
11549 ElementCount::get(MinVal: N->getConstantOperandVal(Num: 3), Scalable: VT.isScalableVT());
11550
11551 EVT AddrVT = SourceValue->getValueType(ResNo: 0);
11552 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
11553
11554 EVT CmpVT =
11555 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: AddrVT);
11556
11557 // Unsigned compare: Source >= Sink.
11558 SDValue SourceAheadOfOrEqualToSink =
11559 DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETUGE);
11560
11561 // Take the difference between the pointers and divided by the element size,
11562 // to see how many lanes separate them.
11563 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: AddrVT, N1: SinkValue, N2: SourceValue);
11564
11565 // RAW_MASK: Diff = Source >= Sink ? (Source - Sink) : (Sink - Source)
11566 if (IsReadAfterWrite)
11567 Diff = DAG.getSelect(DL, VT: AddrVT, Cond: SourceAheadOfOrEqualToSink,
11568 LHS: DAG.getNegative(Val: Diff, DL, VT: AddrVT), RHS: Diff);
11569
11570 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: AddrVT, N1: Diff, N2: EltSizeInBytes);
11571
11572 // The pointers do not alias if:
11573 // - Source >= Sink (WAR_MASK)
11574 // - Source == Sink (RAW_MASK)
11575 SDValue NoAlias = SourceAheadOfOrEqualToSink;
11576 if (IsReadAfterWrite)
11577 NoAlias = DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETEQ);
11578
11579 // The pointers do not alias if:
11580 // Lane + LaneOffset < Diff (WAR/RAW_MASK)
11581 SDValue LaneOffset = DAG.getElementCount(DL, VT: AddrVT, EC: LaneOffsetEC);
11582 SDValue MaskN = DAG.getSelect(
11583 DL, VT: AddrVT, Cond: NoAlias,
11584 LHS: DAG.getConstant(Val: APInt::getMaxValue(numBits: AddrVT.getScalarSizeInBits()), DL,
11585 VT: AddrVT),
11586 RHS: Diff);
11587
11588 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT, N1: LaneOffset, N2: MaskN);
11589}
11590
11591SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
11592 bool IsNegative) const {
11593 SDLoc dl(N);
11594 EVT VT = N->getValueType(ResNo: 0);
11595 SDValue Op = N->getOperand(Num: 0);
11596
11597 // If expanding ABS_MIN_POISON, fall back to ABS if the target supports it.
11598 if (N->getOpcode() == ISD::ABS_MIN_POISON &&
11599 isOperationLegalOrCustom(Op: ISD::ABS, VT)) {
11600 SDValue AbsVal = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT, Operand: Op);
11601 if (IsNegative)
11602 return DAG.getNegative(Val: AbsVal, DL: dl, VT);
11603 return AbsVal;
11604 }
11605
11606 // abs(x) -> smax(x,sub(0,x))
11607 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11608 isOperationLegal(Op: ISD::SMAX, VT)) {
11609 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11610 Op = DAG.getFreeze(V: Op);
11611 return DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: Op,
11612 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11613 }
11614
11615 // abs(x) -> umin(x,sub(0,x))
11616 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11617 isOperationLegal(Op: ISD::UMIN, VT)) {
11618 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11619 Op = DAG.getFreeze(V: Op);
11620 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: Op,
11621 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11622 }
11623
11624 // 0 - abs(x) -> smin(x, sub(0,x))
11625 if (IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11626 isOperationLegal(Op: ISD::SMIN, VT)) {
11627 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11628 Op = DAG.getFreeze(V: Op);
11629 return DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: Op,
11630 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11631 }
11632
11633 // Only expand vector types if we have the appropriate vector operations.
11634 if (VT.isVector() &&
11635 (!isOperationLegalOrCustom(Op: ISD::SRA, VT) ||
11636 (!IsNegative && !isOperationLegalOrCustom(Op: ISD::ADD, VT)) ||
11637 (IsNegative && !isOperationLegalOrCustom(Op: ISD::SUB, VT)) ||
11638 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11639 return SDValue();
11640
11641 Op = DAG.getFreeze(V: Op);
11642 SDValue Shift = DAG.getNode(
11643 Opcode: ISD::SRA, DL: dl, VT, N1: Op,
11644 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11645 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Shift);
11646
11647 // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
11648 if (!IsNegative)
11649 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Shift);
11650
11651 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
11652 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Shift, N2: Xor);
11653}
11654
11655SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
11656 SDLoc dl(N);
11657 EVT VT = N->getValueType(ResNo: 0);
11658 SDValue LHS = N->getOperand(Num: 0);
11659 SDValue RHS = N->getOperand(Num: 1);
11660 bool IsSigned = N->getOpcode() == ISD::ABDS;
11661
11662 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
11663 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
11664 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
11665 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
11666 if (isOperationLegal(Op: MaxOpc, VT) && isOperationLegal(Op: MinOpc, VT)) {
11667 LHS = DAG.getFreeze(V: LHS);
11668 RHS = DAG.getFreeze(V: RHS);
11669 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
11670 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
11671 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
11672 }
11673
11674 // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
11675 if (!IsSigned && isOperationLegal(Op: ISD::USUBSAT, VT)) {
11676 LHS = DAG.getFreeze(V: LHS);
11677 RHS = DAG.getFreeze(V: RHS);
11678 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT,
11679 N1: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: LHS, N2: RHS),
11680 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: RHS, N2: LHS));
11681 }
11682
11683 // If the subtract doesn't overflow then just use abs(sub())
11684 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
11685
11686 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: LHS, N1: RHS))
11687 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11688 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS));
11689
11690 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: RHS, N1: LHS))
11691 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11692 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11693
11694 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11695 ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
11696 LHS = DAG.getFreeze(V: LHS);
11697 RHS = DAG.getFreeze(V: RHS);
11698 SDValue Cmp = DAG.getSetCC(DL: dl, VT: CCVT, LHS, RHS, Cond: CC);
11699
11700 // Branchless expansion iff cmp result is allbits:
11701 // abds(lhs, rhs) -> sub(sgt(lhs, rhs), xor(sgt(lhs, rhs), sub(lhs, rhs)))
11702 // abdu(lhs, rhs) -> sub(ugt(lhs, rhs), xor(ugt(lhs, rhs), sub(lhs, rhs)))
11703 if (CCVT == VT && getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
11704 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
11705 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Diff, N2: Cmp);
11706 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Cmp, N2: Xor);
11707 }
11708
11709 // Similar to the branchless expansion, if we don't prefer selects, use the
11710 // (sign-extended) usubo overflow flag if the (scalar) type is illegal as this
11711 // is more likely to legalize cleanly: abdu(lhs, rhs) -> sub(xor(sub(lhs,
11712 // rhs), uof(lhs, rhs)), uof(lhs, rhs))
11713 if (!IsSigned && VT.isScalarInteger() && !isTypeLegal(VT) &&
11714 !preferSelectsOverBooleanArithmetic(VT)) {
11715 SDValue USubO =
11716 DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {LHS, RHS});
11717 SDValue Cmp = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT, Operand: USubO.getValue(R: 1));
11718 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: USubO.getValue(R: 0), N2: Cmp);
11719 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Cmp);
11720 }
11721
11722 // FIXME: Should really try to split the vector in case it's legal on a
11723 // subvector.
11724 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
11725 return DAG.UnrollVectorOp(N);
11726
11727 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11728 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11729 return DAG.getSelect(DL: dl, VT, Cond: Cmp, LHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS),
11730 RHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11731}
11732
11733SDValue TargetLowering::expandAVG(SDNode *N, SelectionDAG &DAG) const {
11734 SDLoc dl(N);
11735 EVT VT = N->getValueType(ResNo: 0);
11736 SDValue LHS = N->getOperand(Num: 0);
11737 SDValue RHS = N->getOperand(Num: 1);
11738
11739 unsigned Opc = N->getOpcode();
11740 bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
11741 bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
11742 unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
11743 unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
11744 unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
11745 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
11746 assert((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS ||
11747 Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
11748 "Unknown AVG node");
11749
11750 // If the operands are already extended, we can add+shift.
11751 bool IsExt =
11752 (IsSigned && DAG.ComputeNumSignBits(Op: LHS) >= 2 &&
11753 DAG.ComputeNumSignBits(Op: RHS) >= 2) ||
11754 (!IsSigned && DAG.computeKnownBits(Op: LHS).countMinLeadingZeros() >= 1 &&
11755 DAG.computeKnownBits(Op: RHS).countMinLeadingZeros() >= 1);
11756 if (IsExt) {
11757 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
11758 if (!IsFloor)
11759 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Sum, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11760 return DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Sum,
11761 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11762 }
11763
11764 // For scalars, see if we can efficiently extend/truncate to use add+shift.
11765 if (VT.isScalarInteger()) {
11766 EVT ExtVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
11767 if (isTypeLegal(VT: ExtVT) && isTruncateFree(FromVT: ExtVT, ToVT: VT)) {
11768 LHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: LHS);
11769 RHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: RHS);
11770 SDValue Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: LHS, N2: RHS);
11771 if (!IsFloor)
11772 Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: Avg,
11773 N2: DAG.getConstant(Val: 1, DL: dl, VT: ExtVT));
11774 // Just use SRL as we will be truncating away the extended sign bits.
11775 Avg = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ExtVT, N1: Avg,
11776 N2: DAG.getShiftAmountConstant(Val: 1, VT: ExtVT, DL: dl));
11777 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Avg);
11778 }
11779 }
11780
11781 // avgflooru(lhs, rhs) -> or(lshr(add(lhs, rhs),1),shl(overflow, typesize-1))
11782 if (Opc == ISD::AVGFLOORU && VT.isScalarInteger() && !isTypeLegal(VT) &&
11783 isOperationLegalOrCustom(
11784 Op: ISD::UADDO, VT: getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11785 SDValue UAddWithOverflow =
11786 DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {RHS, LHS});
11787
11788 SDValue Sum = UAddWithOverflow.getValue(R: 0);
11789 SDValue Overflow = UAddWithOverflow.getValue(R: 1);
11790
11791 // Right shift the sum by 1
11792 SDValue LShrVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Sum,
11793 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11794
11795 SDValue ZeroExtOverflow = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Overflow);
11796 SDValue OverflowShl = DAG.getNode(
11797 Opcode: ISD::SHL, DL: dl, VT, N1: ZeroExtOverflow,
11798 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11799
11800 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: LShrVal, N2: OverflowShl);
11801 }
11802
11803 // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
11804 // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
11805 // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
11806 // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
11807 LHS = DAG.getFreeze(V: LHS);
11808 RHS = DAG.getFreeze(V: RHS);
11809 SDValue Sign = DAG.getNode(Opcode: SignOpc, DL: dl, VT, N1: LHS, N2: RHS);
11810 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
11811 SDValue Shift =
11812 DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11813 return DAG.getNode(Opcode: SumOpc, DL: dl, VT, N1: Sign, N2: Shift);
11814}
11815
11816SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
11817 SDLoc dl(N);
11818 EVT VT = N->getValueType(ResNo: 0);
11819 SDValue Op = N->getOperand(Num: 0);
11820
11821 if (!VT.isSimple())
11822 return SDValue();
11823
11824 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11825 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
11826 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
11827 default:
11828 return SDValue();
11829 case MVT::i16:
11830 // Use a rotate by 8. This can be further expanded if necessary.
11831 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11832 case MVT::i32:
11833 // This is meant for ARM specifically, which has ROTR but no ROTL.
11834 // t = x ^ rotr(x, 16)
11835 // t = bic(t, 0x00ff0000)
11836 // t = lshr(t, 8)
11837 // x = t ^ rotr(x, 8)
11838 if (isOperationLegalOrCustom(Op: ISD::ROTR, VT)) {
11839 SDValue Rotr16 =
11840 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 16, DL: dl, VT: SHVT));
11841 SDValue Tmp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Rotr16);
11842 Tmp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp,
11843 N2: DAG.getConstant(Val: 0xFF00FFFF, DL: dl, VT));
11844 Tmp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11845 SDValue Rotr8 =
11846 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11847 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Tmp, N2: Rotr8);
11848 }
11849 Tmp4 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11850 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11851 N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11852 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11853 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11854 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11855 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11856 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11857 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11858 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11859 case MVT::i64:
11860 Tmp8 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11861 Tmp7 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11862 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11863 Tmp7 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp7, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11864 Tmp6 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11865 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11866 Tmp6 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp6, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11867 Tmp5 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11868 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11869 Tmp5 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp5, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11870 Tmp4 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11871 Tmp4 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp4,
11872 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11873 Tmp3 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11874 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp3,
11875 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11876 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11877 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2,
11878 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11879 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11880 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp7);
11881 Tmp6 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp6, N2: Tmp5);
11882 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11883 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11884 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp6);
11885 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11886 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp4);
11887 }
11888}
11889
11890SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
11891 SDLoc dl(N);
11892 EVT VT = N->getValueType(ResNo: 0);
11893 SDValue Op = N->getOperand(Num: 0);
11894 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11895 unsigned Sz = VT.getScalarSizeInBits();
11896
11897 SDValue Tmp, Tmp2, Tmp3;
11898
11899 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
11900 // and finally the i1 pairs.
11901 // TODO: We can easily support i4/i2 legal types if any target ever does.
11902 if (Sz >= 8 && isPowerOf2_32(Value: Sz)) {
11903 // Create the masks - repeating the pattern every byte.
11904 APInt Mask4 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x0F));
11905 APInt Mask2 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x33));
11906 APInt Mask1 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x55));
11907
11908 // BSWAP if the type is wider than a single byte.
11909 Tmp = (Sz > 8 ? DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT, Operand: Op) : Op);
11910
11911 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
11912 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11913 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11914 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11915 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11916 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11917
11918 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
11919 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11920 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11921 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11922 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11923 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11924
11925 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
11926 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11927 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11928 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11929 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11930 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11931 return Tmp;
11932 }
11933
11934 Tmp = DAG.getConstant(Val: 0, DL: dl, VT);
11935 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
11936 if (I < J)
11937 Tmp2 =
11938 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: J - I, DL: dl, VT: SHVT));
11939 else
11940 Tmp2 =
11941 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: I - J, DL: dl, VT: SHVT));
11942
11943 APInt Shift = APInt::getOneBitSet(numBits: Sz, BitNo: J);
11944 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Shift, DL: dl, VT));
11945 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp, N2: Tmp2);
11946 }
11947
11948 return Tmp;
11949}
11950
11951std::pair<SDValue, SDValue>
11952TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
11953 SelectionDAG &DAG) const {
11954 SDLoc SL(LD);
11955 SDValue Chain = LD->getChain();
11956 SDValue BasePTR = LD->getBasePtr();
11957 EVT SrcVT = LD->getMemoryVT();
11958 EVT DstVT = LD->getValueType(ResNo: 0);
11959 ISD::LoadExtType ExtType = LD->getExtensionType();
11960
11961 if (SrcVT.isScalableVector())
11962 report_fatal_error(reason: "Cannot scalarize scalable vector loads");
11963
11964 unsigned NumElem = SrcVT.getVectorNumElements();
11965
11966 EVT SrcEltVT = SrcVT.getScalarType();
11967 EVT DstEltVT = DstVT.getScalarType();
11968
11969 // A vector must always be stored in memory as-is, i.e. without any padding
11970 // between the elements, since various code depend on it, e.g. in the
11971 // handling of a bitcast of a vector type to int, which may be done with a
11972 // vector store followed by an integer load. A vector that does not have
11973 // elements that are byte-sized must therefore be stored as an integer
11974 // built out of the extracted vector elements.
11975 if (!SrcEltVT.isByteSized()) {
11976 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
11977 EVT LoadVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumLoadBits);
11978
11979 unsigned NumSrcBits = SrcVT.getSizeInBits();
11980 EVT SrcIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumSrcBits);
11981
11982 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
11983 SDValue SrcEltBitMask = DAG.getConstant(
11984 Val: APInt::getLowBitsSet(numBits: NumLoadBits, loBitsSet: SrcEltBits), DL: SL, VT: LoadVT);
11985
11986 // Load the whole vector and avoid masking off the top bits as it makes
11987 // the codegen worse.
11988 SDValue Load = DAG.getExtLoad(
11989 ExtType: ISD::EXTLOAD, dl: SL, VT: LoadVT, Chain, Ptr: BasePTR, PtrInfo: LD->getPointerInfo(),
11990 MemVT: SrcIntVT, Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
11991 Metadata: LD->getMMOMetadataForSubAccess());
11992
11993 SmallVector<SDValue, 8> Vals;
11994 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11995 unsigned ShiftIntoIdx =
11996 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
11997 SDValue ShiftAmount = DAG.getShiftAmountConstant(
11998 Val: ShiftIntoIdx * SrcEltVT.getSizeInBits(), VT: LoadVT, DL: SL);
11999 SDValue ShiftedElt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: LoadVT, N1: Load, N2: ShiftAmount);
12000 SDValue Elt =
12001 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: LoadVT, N1: ShiftedElt, N2: SrcEltBitMask);
12002 SDValue Scalar = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: SrcEltVT, Operand: Elt);
12003
12004 if (ExtType != ISD::NON_EXTLOAD) {
12005 unsigned ExtendOp = ISD::getExtForLoadExtType(IsFP: false, ExtType);
12006 Scalar = DAG.getNode(Opcode: ExtendOp, DL: SL, VT: DstEltVT, Operand: Scalar);
12007 }
12008
12009 Vals.push_back(Elt: Scalar);
12010 }
12011
12012 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
12013 return std::make_pair(x&: Value, y: Load.getValue(R: 1));
12014 }
12015
12016 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
12017 assert(SrcEltVT.isByteSized());
12018
12019 SmallVector<SDValue, 8> Vals;
12020 SmallVector<SDValue, 8> LoadChains;
12021
12022 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12023 SDValue ScalarLoad = DAG.getExtLoad(
12024 ExtType, dl: SL, VT: DstEltVT, Chain, Ptr: BasePTR,
12025 PtrInfo: LD->getPointerInfo().getWithOffset(O: Idx * Stride), MemVT: SrcEltVT,
12026 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
12027 Metadata: LD->getMMOMetadataForSubAccess());
12028
12029 BasePTR = DAG.getObjectPtrOffset(SL, Ptr: BasePTR, Offset: TypeSize::getFixed(ExactSize: Stride));
12030
12031 Vals.push_back(Elt: ScalarLoad.getValue(R: 0));
12032 LoadChains.push_back(Elt: ScalarLoad.getValue(R: 1));
12033 }
12034
12035 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: LoadChains);
12036 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
12037
12038 return std::make_pair(x&: Value, y&: NewChain);
12039}
12040
12041SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
12042 SelectionDAG &DAG) const {
12043 SDLoc SL(ST);
12044
12045 SDValue Chain = ST->getChain();
12046 SDValue BasePtr = ST->getBasePtr();
12047 SDValue Value = ST->getValue();
12048 EVT StVT = ST->getMemoryVT();
12049
12050 if (StVT.isScalableVector())
12051 report_fatal_error(reason: "Cannot scalarize scalable vector stores");
12052
12053 // The type of the data we want to save
12054 EVT RegVT = Value.getValueType();
12055 EVT RegSclVT = RegVT.getScalarType();
12056
12057 // The type of data as saved in memory.
12058 EVT MemSclVT = StVT.getScalarType();
12059
12060 unsigned NumElem = StVT.getVectorNumElements();
12061
12062 // A vector must always be stored in memory as-is, i.e. without any padding
12063 // between the elements, since various code depend on it, e.g. in the
12064 // handling of a bitcast of a vector type to int, which may be done with a
12065 // vector store followed by an integer load. A vector that does not have
12066 // elements that are byte-sized must therefore be stored as an integer
12067 // built out of the extracted vector elements.
12068 if (!MemSclVT.isByteSized()) {
12069 unsigned NumBits = StVT.getSizeInBits();
12070 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
12071
12072 SDValue CurrVal = DAG.getConstant(Val: 0, DL: SL, VT: IntVT);
12073
12074 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12075 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
12076 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MemSclVT, Operand: Elt);
12077 SDValue ExtElt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: IntVT, Operand: Trunc);
12078 unsigned ShiftIntoIdx =
12079 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
12080 SDValue ShiftAmount =
12081 DAG.getConstant(Val: ShiftIntoIdx * MemSclVT.getSizeInBits(), DL: SL, VT: IntVT);
12082 SDValue ShiftedElt =
12083 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT, N1: ExtElt, N2: ShiftAmount);
12084 CurrVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: CurrVal, N2: ShiftedElt);
12085 }
12086
12087 return DAG.getStore(Chain, dl: SL, Val: CurrVal, Ptr: BasePtr, PtrInfo: ST->getPointerInfo(),
12088 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12089 Metadata: ST->getMMOMetadataForSubAccess());
12090 }
12091
12092 // Store Stride in bytes
12093 unsigned Stride = MemSclVT.getSizeInBits() / 8;
12094 assert(Stride && "Zero stride!");
12095 // Extract each of the elements from the original vector and save them into
12096 // memory individually.
12097 SmallVector<SDValue, 8> Stores;
12098 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12099 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
12100
12101 SDValue Ptr =
12102 DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Idx * Stride));
12103
12104 // This scalar TruncStore may be illegal, but we legalize it later.
12105 SDValue Store = DAG.getTruncStore(
12106 Chain, dl: SL, Val: Elt, Ptr, PtrInfo: ST->getPointerInfo().getWithOffset(O: Idx * Stride),
12107 SVT: MemSclVT, Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12108 Metadata: ST->getMMOMetadataForSubAccess());
12109
12110 Stores.push_back(Elt: Store);
12111 }
12112
12113 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: Stores);
12114}
12115
12116std::pair<SDValue, SDValue>
12117TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
12118 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
12119 "unaligned indexed loads not implemented!");
12120 SDValue Chain = LD->getChain();
12121 SDValue Ptr = LD->getBasePtr();
12122 EVT VT = LD->getValueType(ResNo: 0);
12123 EVT LoadedVT = LD->getMemoryVT();
12124 SDLoc dl(LD);
12125 auto &MF = DAG.getMachineFunction();
12126
12127 if (VT.isFloatingPoint() || VT.isVector()) {
12128 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadedVT.getSizeInBits());
12129 if (isTypeLegal(VT: intVT) && isTypeLegal(VT: LoadedVT)) {
12130 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: intVT) &&
12131 LoadedVT.isVector()) {
12132 // Scalarize the load and let the individual components be handled.
12133 return scalarizeVectorLoad(LD, DAG);
12134 }
12135
12136 // Expand to a (misaligned) integer load of the same size,
12137 // then bitconvert to floating point or vector.
12138 SDValue newLoad = DAG.getLoad(VT: intVT, dl, Chain, Ptr,
12139 MMO: LD->getMemOperand());
12140 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoadedVT, Operand: newLoad);
12141 if (LoadedVT != VT)
12142 Result = DAG.getNode(Opcode: VT.isFloatingPoint() ? ISD::FP_EXTEND :
12143 ISD::ANY_EXTEND, DL: dl, VT, Operand: Result);
12144
12145 return std::make_pair(x&: Result, y: newLoad.getValue(R: 1));
12146 }
12147
12148 // Copy the value to a (aligned) stack slot using (unaligned) integer
12149 // loads and stores, then do a (aligned) load from the stack slot.
12150 MVT RegVT = getRegisterType(Context&: *DAG.getContext(), VT: intVT);
12151 unsigned LoadedBytes = LoadedVT.getStoreSize();
12152 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12153 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
12154
12155 // Make sure the stack slot is also aligned for the register type.
12156 SDValue StackBase = DAG.CreateStackTemporary(VT1: LoadedVT, VT2: RegVT);
12157 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackBase.getNode())->getIndex();
12158 SmallVector<SDValue, 8> Stores;
12159 SDValue StackPtr = StackBase;
12160 unsigned Offset = 0;
12161
12162 EVT PtrVT = Ptr.getValueType();
12163 EVT StackPtrVT = StackPtr.getValueType();
12164
12165 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12166 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12167
12168 // Do all but one copies using the full register width.
12169 for (unsigned i = 1; i < NumRegs; i++) {
12170 // Load one integer register's worth from the original location.
12171 SDValue Load = DAG.getLoad(
12172 VT: RegVT, dl, Chain, Ptr, PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset),
12173 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
12174 Metadata: LD->getMMOMetadataForSubAccess());
12175 // Follow the load with a store to the stack slot. Remember the store.
12176 Stores.push_back(Elt: DAG.getStore(
12177 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12178 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset)));
12179 // Increment the pointers.
12180 Offset += RegBytes;
12181
12182 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12183 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12184 }
12185
12186 // The last copy may be partial. Do an extending load.
12187 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12188 BitWidth: 8 * (LoadedBytes - Offset));
12189 SDValue Load = DAG.getExtLoad(
12190 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain, Ptr,
12191 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT, Alignment: LD->getBaseAlign(),
12192 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getMMOMetadataForSubAccess());
12193 // Follow the load with a store to the stack slot. Remember the store.
12194 // On big-endian machines this requires a truncating store to ensure
12195 // that the bits end up in the right place.
12196 Stores.push_back(Elt: DAG.getTruncStore(
12197 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12198 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), SVT: MemVT));
12199
12200 // The order of the stores doesn't matter - say it with a TokenFactor.
12201 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12202
12203 // Finally, perform the original load only redirected to the stack slot.
12204 Load = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT, Chain: TF, Ptr: StackBase,
12205 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0),
12206 MemVT: LoadedVT);
12207
12208 // Callers expect a MERGE_VALUES node.
12209 return std::make_pair(x&: Load, y&: TF);
12210 }
12211
12212 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
12213 "Unaligned load of unsupported type.");
12214
12215 // Compute the new VT that is half the size of the old one. This is an
12216 // integer MVT.
12217 unsigned NumBits = LoadedVT.getSizeInBits();
12218 EVT NewLoadedVT;
12219 NewLoadedVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits/2);
12220 NumBits >>= 1;
12221
12222 Align Alignment = LD->getBaseAlign();
12223 unsigned IncrementSize = NumBits / 8;
12224 ISD::LoadExtType HiExtType = LD->getExtensionType();
12225
12226 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
12227 if (HiExtType == ISD::NON_EXTLOAD)
12228 HiExtType = ISD::ZEXTLOAD;
12229
12230 // Load the value in two parts
12231 SDValue Lo, Hi;
12232 if (DAG.getDataLayout().isLittleEndian()) {
12233 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12234 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12235 Metadata: LD->getMMOMetadataForSubAccess());
12236
12237 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12238 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr,
12239 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12240 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12241 Metadata: LD->getMMOMetadataForSubAccess());
12242 } else {
12243 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12244 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12245 Metadata: LD->getMMOMetadataForSubAccess());
12246
12247 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12248 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
12249 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12250 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12251 Metadata: LD->getMMOMetadataForSubAccess());
12252 }
12253
12254 // aggregate the two parts
12255 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: NumBits, VT, DL: dl);
12256 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: ShiftAmount);
12257 Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Result, N2: Lo);
12258
12259 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
12260 N2: Hi.getValue(R: 1));
12261
12262 return std::make_pair(x&: Result, y&: TF);
12263}
12264
12265SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
12266 SelectionDAG &DAG) const {
12267 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
12268 "unaligned indexed stores not implemented!");
12269 SDValue Chain = ST->getChain();
12270 SDValue Ptr = ST->getBasePtr();
12271 SDValue Val = ST->getValue();
12272 EVT VT = Val.getValueType();
12273 Align Alignment = ST->getBaseAlign();
12274 auto &MF = DAG.getMachineFunction();
12275 EVT StoreMemVT = ST->getMemoryVT();
12276
12277 SDLoc dl(ST);
12278 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
12279 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
12280 if (isTypeLegal(VT: intVT)) {
12281 if (!isOperationLegalOrCustom(Op: ISD::STORE, VT: intVT) &&
12282 StoreMemVT.isVector()) {
12283 // Scalarize the store and let the individual components be handled.
12284 SDValue Result = scalarizeVectorStore(ST, DAG);
12285 return Result;
12286 }
12287 // Expand to a bitconvert of the value to the integer type of the
12288 // same size, then a (misaligned) int store.
12289 // FIXME: Does not handle truncating floating point stores!
12290 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: intVT, Operand: Val);
12291 Result = DAG.getStore(Chain, dl, Val: Result, Ptr, PtrInfo: ST->getPointerInfo(),
12292 Alignment, MMOFlags: ST->getMemOperand()->getFlags(),
12293 Metadata: ST->getMMOMetadataForSubAccess());
12294 return Result;
12295 }
12296 // Do a (aligned) store to a stack slot, then copy from the stack slot
12297 // to the final destination using (unaligned) integer loads and stores.
12298 MVT RegVT = getRegisterType(
12299 Context&: *DAG.getContext(),
12300 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreMemVT.getSizeInBits()));
12301 EVT PtrVT = Ptr.getValueType();
12302 unsigned StoredBytes = StoreMemVT.getStoreSize();
12303 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12304 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
12305
12306 // Make sure the stack slot is also aligned for the register type.
12307 SDValue StackPtr = DAG.CreateStackTemporary(VT1: StoreMemVT, VT2: RegVT);
12308 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
12309
12310 // Perform the original store, only redirected to the stack slot.
12311 SDValue Store = DAG.getTruncStore(
12312 Chain, dl, Val, Ptr: StackPtr,
12313 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0), SVT: StoreMemVT);
12314
12315 EVT StackPtrVT = StackPtr.getValueType();
12316
12317 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12318 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12319 SmallVector<SDValue, 8> Stores;
12320 unsigned Offset = 0;
12321
12322 // Do all but one copies using the full register width.
12323 for (unsigned i = 1; i < NumRegs; i++) {
12324 // Load one integer register's worth from the stack slot.
12325 SDValue Load = DAG.getLoad(
12326 VT: RegVT, dl, Chain: Store, Ptr: StackPtr,
12327 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset));
12328 // Store it to the final location. Remember the store.
12329 Stores.push_back(Elt: DAG.getStore(
12330 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12331 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), Alignment: ST->getBaseAlign(),
12332 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess()));
12333 // Increment the pointers.
12334 Offset += RegBytes;
12335 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12336 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12337 }
12338
12339 // The last store may be partial. Do a truncating store. On big-endian
12340 // machines this requires an extending load from the stack slot to ensure
12341 // that the bits are in the right place.
12342 EVT LoadMemVT =
12343 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: 8 * (StoredBytes - Offset));
12344
12345 // Load from the stack slot.
12346 SDValue Load = DAG.getExtLoad(
12347 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain: Store, Ptr: StackPtr,
12348 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), MemVT: LoadMemVT);
12349
12350 Stores.push_back(
12351 Elt: DAG.getTruncStore(Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12352 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), SVT: LoadMemVT,
12353 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12354 Metadata: ST->getMMOMetadataForSubAccess()));
12355 // The order of the stores doesn't matter - say it with a TokenFactor.
12356 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12357 return Result;
12358 }
12359
12360 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
12361 "Unaligned store of unknown type.");
12362 // Get the half-size VT
12363 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
12364 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
12365 unsigned IncrementSize = NumBits / 8;
12366
12367 // Divide the stored value in two parts.
12368 SDValue ShiftAmount =
12369 DAG.getShiftAmountConstant(Val: NumBits, VT: Val.getValueType(), DL: dl);
12370 SDValue Lo = Val;
12371 // If Val is a constant, replace the upper bits with 0. The SRL will constant
12372 // fold and not use the upper bits. A smaller constant may be easier to
12373 // materialize.
12374 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Lo); C && !C->isOpaque())
12375 Lo = DAG.getNode(
12376 Opcode: ISD::AND, DL: dl, VT, N1: Lo,
12377 N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: NumBits), DL: dl,
12378 VT));
12379 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Val, N2: ShiftAmount);
12380
12381 // Store the two parts
12382 SDValue Store1, Store2;
12383 Store1 = DAG.getTruncStore(
12384 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Lo : Hi, Ptr,
12385 PtrInfo: ST->getPointerInfo(), SVT: NewStoredVT, Alignment,
12386 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess());
12387
12388 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12389 Store2 = DAG.getTruncStore(
12390 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
12391 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize), SVT: NewStoredVT, Alignment,
12392 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess());
12393
12394 SDValue Result =
12395 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
12396 return Result;
12397}
12398
12399SDValue
12400TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
12401 const SDLoc &DL, EVT DataVT,
12402 SelectionDAG &DAG,
12403 bool IsCompressedMemory) const {
12404 SDValue Increment;
12405 EVT AddrVT = Addr.getValueType();
12406 EVT MaskVT = Mask.getValueType();
12407 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
12408 "Incompatible types of Data and Mask");
12409 if (IsCompressedMemory) {
12410 // Incrementing the pointer according to number of '1's in the mask.
12411 if (DataVT.isScalableVector()) {
12412 EVT MaskExtVT = MaskVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
12413 SDValue MaskExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MaskExtVT, Operand: Mask);
12414 Increment = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: MaskExt);
12415 } else {
12416 EVT MaskIntVT =
12417 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MaskVT.getSizeInBits());
12418 SDValue MaskInIntReg = DAG.getBitcast(VT: MaskIntVT, V: Mask);
12419 if (MaskIntVT.getSizeInBits() < 32) {
12420 MaskInIntReg =
12421 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: MaskInIntReg);
12422 MaskIntVT = MVT::i32;
12423 }
12424 Increment = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MaskIntVT, Operand: MaskInIntReg);
12425 }
12426 // Scale is an element size in bytes.
12427 SDValue Scale = DAG.getConstant(Val: DataVT.getScalarSizeInBits() / 8, DL,
12428 VT: AddrVT);
12429 Increment = DAG.getZExtOrTrunc(Op: Increment, DL, VT: AddrVT);
12430 Increment = DAG.getNode(Opcode: ISD::MUL, DL, VT: AddrVT, N1: Increment, N2: Scale);
12431 } else
12432 Increment = DAG.getTypeSize(DL, VT: AddrVT, TS: DataVT.getStoreSize());
12433
12434 return DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: Addr, N2: Increment);
12435}
12436
12437static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
12438 EVT VecVT, const SDLoc &dl,
12439 ElementCount SubEC) {
12440 assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
12441 "Cannot index a scalable vector within a fixed-width vector");
12442
12443 unsigned NElts = VecVT.getVectorMinNumElements();
12444 unsigned NumSubElts = SubEC.getKnownMinValue();
12445 EVT IdxVT = Idx.getValueType();
12446
12447 if (VecVT.isScalableVector() && !SubEC.isScalable()) {
12448 // If this is a constant index and we know the value plus the number of the
12449 // elements in the subvector minus one is less than the minimum number of
12450 // elements then it's safe to return Idx.
12451 if (auto *IdxCst = dyn_cast<ConstantSDNode>(Val&: Idx))
12452 if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
12453 return Idx;
12454 SDValue VS =
12455 DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getFixedSizeInBits(), NElts));
12456 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
12457 SDValue Sub = DAG.getNode(Opcode: SubOpcode, DL: dl, VT: IdxVT, N1: VS,
12458 N2: DAG.getConstant(Val: NumSubElts, DL: dl, VT: IdxVT));
12459 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx, N2: Sub);
12460 }
12461 if (isPowerOf2_32(Value: NElts) && NumSubElts == 1) {
12462 APInt Imm = APInt::getLowBitsSet(numBits: IdxVT.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
12463 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IdxVT, N1: Idx,
12464 N2: DAG.getConstant(Val: Imm, DL: dl, VT: IdxVT));
12465 }
12466 unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
12467 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx,
12468 N2: DAG.getConstant(Val: MaxIndex, DL: dl, VT: IdxVT));
12469}
12470
12471SDValue
12472TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr,
12473 EVT VecVT, SDValue Index,
12474 const SDNodeFlags PtrArithFlags) const {
12475 return getVectorSubVecPointer(
12476 DAG, VecPtr, VecVT,
12477 SubVecVT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(), NumElements: 1),
12478 Index, PtrArithFlags);
12479}
12480
12481SDValue
12482TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr,
12483 EVT VecVT, EVT SubVecVT, SDValue Index,
12484 const SDNodeFlags PtrArithFlags) const {
12485 SDLoc dl(Index);
12486 // Make sure the index type is big enough to compute in.
12487 Index = DAG.getZExtOrTrunc(Op: Index, DL: dl, VT: VecPtr.getValueType());
12488
12489 EVT EltVT = VecVT.getVectorElementType();
12490
12491 // Calculate the element offset and add it to the pointer.
12492 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
12493 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
12494 "Converting bits to bytes lost precision");
12495 assert(SubVecVT.getVectorElementType() == EltVT &&
12496 "Sub-vector must be a vector with matching element type");
12497
12498 // An out-of-range index only makes the vector operation return poison, but
12499 // a load/store through the pointer computed below would be immediate UB, so
12500 // freeze the index before clamping it into range.
12501 if (!DAG.isGuaranteedNotToBePoison(Op: Index))
12502 Index = DAG.getFreeze(V: Index);
12503
12504 Index = clampDynamicVectorIndex(DAG, Idx: Index, VecVT, dl,
12505 SubEC: SubVecVT.getVectorElementCount());
12506
12507 EVT IdxVT = Index.getValueType();
12508 if (SubVecVT.isScalableVector())
12509 Index =
12510 DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12511 N2: DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getSizeInBits(), 1)));
12512
12513 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12514 N2: DAG.getConstant(Val: EltSize, DL: dl, VT: IdxVT));
12515 return DAG.getMemBasePlusOffset(Base: VecPtr, Offset: Index, DL: dl, Flags: PtrArithFlags);
12516}
12517
12518//===----------------------------------------------------------------------===//
12519// Implementation of Emulated TLS Model
12520//===----------------------------------------------------------------------===//
12521
12522SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
12523 SelectionDAG &DAG) const {
12524 // Access to address of TLS varialbe xyz is lowered to a function call:
12525 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
12526 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12527 PointerType *VoidPtrType = PointerType::get(C&: *DAG.getContext(), AddressSpace: 0);
12528 SDLoc dl(GA);
12529
12530 ArgListTy Args;
12531 const GlobalValue *GV =
12532 cast<GlobalValue>(Val: GA->getGlobal()->stripPointerCastsAndAliases());
12533 SmallString<32> NameString("__emutls_v.");
12534 NameString += GV->getName();
12535 StringRef EmuTlsVarName(NameString);
12536 const GlobalVariable *EmuTlsVar =
12537 GV->getParent()->getNamedGlobal(Name: EmuTlsVarName);
12538 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
12539 Args.emplace_back(args: DAG.getGlobalAddress(GV: EmuTlsVar, DL: dl, VT: PtrVT), args&: VoidPtrType);
12540
12541 SDValue EmuTlsGetAddr = DAG.getExternalSymbol(Sym: "__emutls_get_address", VT: PtrVT);
12542
12543 TargetLowering::CallLoweringInfo CLI(DAG);
12544 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
12545 CLI.setLibCallee(CC: CallingConv::C, ResultType: VoidPtrType, Target: EmuTlsGetAddr, ArgsList: std::move(Args));
12546 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
12547
12548 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
12549 // At last for X86 targets, maybe good for other targets too?
12550 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
12551 MFI.setAdjustsStack(true); // Is this only for X86 target?
12552 MFI.setHasCalls(true);
12553
12554 assert((GA->getOffset() == 0) &&
12555 "Emulated TLS must have zero offset in GlobalAddressSDNode");
12556 return CallResult.first;
12557}
12558
12559SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
12560 SelectionDAG &DAG) const {
12561 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
12562 if (!isCtlzFast())
12563 return SDValue();
12564 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
12565 SDLoc dl(Op);
12566 if (isNullConstant(V: Op.getOperand(i: 1)) && CC == ISD::SETEQ) {
12567 EVT VT = Op.getOperand(i: 0).getValueType();
12568 SDValue Zext = Op.getOperand(i: 0);
12569 if (VT.bitsLT(VT: MVT::i32)) {
12570 VT = MVT::i32;
12571 Zext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Op.getOperand(i: 0));
12572 }
12573 unsigned Log2b = Log2_32(Value: VT.getSizeInBits());
12574 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Zext);
12575 SDValue Scc = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Clz,
12576 N2: DAG.getConstant(Val: Log2b, DL: dl, VT: MVT::i32));
12577 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Scc);
12578 }
12579 return SDValue();
12580}
12581
12582SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
12583 SDValue Op0 = Node->getOperand(Num: 0);
12584 SDValue Op1 = Node->getOperand(Num: 1);
12585 EVT VT = Op0.getValueType();
12586 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12587 unsigned Opcode = Node->getOpcode();
12588 SDLoc DL(Node);
12589
12590 // If both sign bits are zero, flip UMIN/UMAX <-> SMIN/SMAX if legal.
12591 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(MinMaxOpc: Opcode);
12592 if (isOperationLegal(Op: AltOpcode, VT) && DAG.SignBitIsZero(Op: Op0) &&
12593 DAG.SignBitIsZero(Op: Op1))
12594 return DAG.getNode(Opcode: AltOpcode, DL, VT, N1: Op0, N2: Op1);
12595
12596 // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
12597 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(V: Op1, AllowUndefs: true) && BoolVT == VT &&
12598 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12599 Op0 = DAG.getFreeze(V: Op0);
12600 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
12601 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12602 N2: DAG.getSetCC(DL, VT, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ));
12603 }
12604
12605 // umin(x,y) -> sub(x,usubsat(x,y))
12606 // TODO: Missing freeze(Op0)?
12607 if (Opcode == ISD::UMIN && isOperationLegal(Op: ISD::SUB, VT) &&
12608 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12609 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12610 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op0, N2: Op1));
12611 }
12612
12613 // umax(x,y) -> add(x,usubsat(y,x))
12614 // TODO: Missing freeze(Op0)?
12615 if (Opcode == ISD::UMAX && isOperationLegal(Op: ISD::ADD, VT) &&
12616 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12617 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0,
12618 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op1, N2: Op0));
12619 }
12620
12621 // FIXME: Should really try to split the vector in case it's legal on a
12622 // subvector.
12623 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12624 return DAG.UnrollVectorOp(N: Node);
12625
12626 // Attempt to find an existing SETCC node that we can reuse.
12627 // TODO: Do we need a generic doesSETCCNodeExist?
12628 // TODO: Missing freeze(Op0)/freeze(Op1)?
12629 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
12630 ISD::CondCode PrefCommuteCC,
12631 ISD::CondCode AltCommuteCC) {
12632 SDVTList BoolVTList = DAG.getVTList(VT: BoolVT);
12633 for (ISD::CondCode CC : {PrefCC, AltCC}) {
12634 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12635 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12636 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12637 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12638 }
12639 }
12640 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
12641 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12642 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12643 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12644 return DAG.getSelect(DL, VT, Cond, LHS: Op1, RHS: Op0);
12645 }
12646 }
12647 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: PrefCC);
12648 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12649 };
12650
12651 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
12652 // -> Y = (A < B) ? B : A
12653 // -> Y = (A >= B) ? A : B
12654 // -> Y = (A <= B) ? B : A
12655 switch (Opcode) {
12656 case ISD::SMAX:
12657 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
12658 case ISD::SMIN:
12659 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
12660 case ISD::UMAX:
12661 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
12662 case ISD::UMIN:
12663 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
12664 }
12665
12666 llvm_unreachable("How did we get here?");
12667}
12668
12669SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
12670 unsigned Opcode = Node->getOpcode();
12671 SDValue LHS = Node->getOperand(Num: 0);
12672 SDValue RHS = Node->getOperand(Num: 1);
12673 EVT VT = LHS.getValueType();
12674 SDLoc dl(Node);
12675
12676 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
12677 assert(VT.isInteger() && "Expected operands to be integers");
12678
12679 // usub.sat(a, b) -> umax(a, b) - b
12680 if (Opcode == ISD::USUBSAT && isOperationLegal(Op: ISD::UMAX, VT)) {
12681 SDValue Max = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT, N1: LHS, N2: RHS);
12682 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: RHS);
12683 }
12684
12685 // usub.sat(a, 1) -> sub(a, zext(a != 0))
12686 // Prefer this on targets without legal/cost-effective overflow-carry nodes.
12687 if (Opcode == ISD::USUBSAT && isOneOrOneSplat(V: RHS) &&
12688 !isOperationLegalOrCustom(Op: ISD::USUBO_CARRY, VT)) {
12689 LHS = DAG.getFreeze(V: LHS);
12690 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12691 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12692 SDValue IsNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETNE);
12693 SDValue Subtrahend = DAG.getBoolExtOrTrunc(Op: IsNonZero, SL: dl, VT, OpVT: BoolVT);
12694 Subtrahend =
12695 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Subtrahend, N2: DAG.getConstant(Val: 1, DL: dl, VT));
12696 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: Subtrahend);
12697 }
12698
12699 // uadd.sat(a, b) -> umin(a, ~b) + b
12700 if (Opcode == ISD::UADDSAT && isOperationLegal(Op: ISD::UMIN, VT)) {
12701 SDValue InvRHS = DAG.getNOT(DL: dl, Val: RHS, VT);
12702 SDValue Min = DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: LHS, N2: InvRHS);
12703 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Min, N2: RHS);
12704 }
12705
12706 unsigned OverflowOp;
12707 switch (Opcode) {
12708 case ISD::SADDSAT:
12709 OverflowOp = ISD::SADDO;
12710 break;
12711 case ISD::UADDSAT:
12712 OverflowOp = ISD::UADDO;
12713 break;
12714 case ISD::SSUBSAT:
12715 OverflowOp = ISD::SSUBO;
12716 break;
12717 case ISD::USUBSAT:
12718 OverflowOp = ISD::USUBO;
12719 break;
12720 default:
12721 llvm_unreachable("Expected method to receive signed or unsigned saturation "
12722 "addition or subtraction node.");
12723 }
12724
12725 // FIXME: Should really try to split the vector in case it's legal on a
12726 // subvector.
12727 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12728 return DAG.UnrollVectorOp(N: Node);
12729
12730 unsigned BitWidth = LHS.getScalarValueSizeInBits();
12731 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12732 SDValue Result = DAG.getNode(Opcode: OverflowOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12733 SDValue SumDiff = Result.getValue(R: 0);
12734 SDValue Overflow = Result.getValue(R: 1);
12735 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12736 SDValue AllOnes = DAG.getAllOnesConstant(DL: dl, VT);
12737
12738 if (Opcode == ISD::UADDSAT) {
12739 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12740 // (LHS + RHS) | OverflowMask
12741 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12742 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SumDiff, N2: OverflowMask);
12743 }
12744 // Overflow ? 0xffff.... : (LHS + RHS)
12745 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: AllOnes, RHS: SumDiff);
12746 }
12747
12748 if (Opcode == ISD::USUBSAT) {
12749 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12750 // (LHS - RHS) & ~OverflowMask
12751 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12752 SDValue Not = DAG.getNOT(DL: dl, Val: OverflowMask, VT);
12753 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SumDiff, N2: Not);
12754 }
12755 // Overflow ? 0 : (LHS - RHS)
12756 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Zero, RHS: SumDiff);
12757 }
12758
12759 assert((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
12760 "Expected signed saturating add/sub opcode");
12761
12762 const APInt MinVal = APInt::getSignedMinValue(numBits: BitWidth);
12763 const APInt MaxVal = APInt::getSignedMaxValue(numBits: BitWidth);
12764
12765 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
12766 KnownBits KnownRHS = DAG.computeKnownBits(Op: RHS);
12767
12768 // If either of the operand signs are known, then they are guaranteed to
12769 // only saturate in one direction. If non-negative they will saturate
12770 // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
12771 //
12772 // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
12773 // sign of 'y' has to be flipped.
12774
12775 bool LHSIsNonNegative = KnownLHS.isNonNegative();
12776 bool RHSIsNonNegative =
12777 Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() : KnownRHS.isNegative();
12778 if (LHSIsNonNegative || RHSIsNonNegative) {
12779 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12780 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: SumDiff);
12781 }
12782
12783 bool LHSIsNegative = KnownLHS.isNegative();
12784 bool RHSIsNegative =
12785 Opcode == ISD::SADDSAT ? KnownRHS.isNegative() : KnownRHS.isNonNegative();
12786 if (LHSIsNegative || RHSIsNegative) {
12787 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12788 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMin, RHS: SumDiff);
12789 }
12790
12791 // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
12792 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12793 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SumDiff,
12794 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT));
12795 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Shift, N2: SatMin);
12796 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: SumDiff);
12797}
12798
12799SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
12800 unsigned Opcode = Node->getOpcode();
12801 SDValue LHS = Node->getOperand(Num: 0);
12802 SDValue RHS = Node->getOperand(Num: 1);
12803 EVT VT = LHS.getValueType();
12804 EVT ResVT = Node->getValueType(ResNo: 0);
12805 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12806 SDLoc dl(Node);
12807
12808 auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT);
12809 auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT);
12810 SDValue IsLT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: LTPredicate);
12811 SDValue IsGT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: GTPredicate);
12812
12813 // We can't perform arithmetic on i1 values. Extending them would
12814 // probably result in worse codegen, so let's just use two selects instead.
12815 // Some targets are also just better off using selects rather than subtraction
12816 // because one of the conditions can be merged with one of the selects.
12817 // And finally, if we don't know the contents of high bits of a boolean value
12818 // we can't perform any arithmetic either.
12819 if (preferSelectsOverBooleanArithmetic(VT) ||
12820 BoolVT.getScalarSizeInBits() == 1 ||
12821 getBooleanContents(Type: BoolVT) == UndefinedBooleanContent) {
12822 SDValue SelectZeroOrOne =
12823 DAG.getSelect(DL: dl, VT: ResVT, Cond: IsGT, LHS: DAG.getConstant(Val: 1, DL: dl, VT: ResVT),
12824 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ResVT));
12825 return DAG.getSelect(DL: dl, VT: ResVT, Cond: IsLT, LHS: DAG.getAllOnesConstant(DL: dl, VT: ResVT),
12826 RHS: SelectZeroOrOne);
12827 }
12828
12829 if (getBooleanContents(Type: BoolVT) == ZeroOrNegativeOneBooleanContent)
12830 std::swap(a&: IsGT, b&: IsLT);
12831 return DAG.getSExtOrTrunc(Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: BoolVT, N1: IsGT, N2: IsLT), DL: dl,
12832 VT: ResVT);
12833}
12834
12835SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
12836 unsigned Opcode = Node->getOpcode();
12837 bool IsSigned = Opcode == ISD::SSHLSAT;
12838 SDValue LHS = Node->getOperand(Num: 0);
12839 SDValue RHS = Node->getOperand(Num: 1);
12840 EVT VT = LHS.getValueType();
12841 SDLoc dl(Node);
12842
12843 assert((Node->getOpcode() == ISD::SSHLSAT ||
12844 Node->getOpcode() == ISD::USHLSAT) &&
12845 "Expected a SHLSAT opcode");
12846 assert(VT.isInteger() && "Expected operands to be integers");
12847
12848 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12849 return DAG.UnrollVectorOp(N: Node);
12850
12851 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
12852
12853 unsigned BW = VT.getScalarSizeInBits();
12854 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12855 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: RHS);
12856 SDValue Orig =
12857 DAG.getNode(Opcode: IsSigned ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: Result, N2: RHS);
12858
12859 SDValue SatVal;
12860 if (IsSigned) {
12861 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: BW), DL: dl, VT);
12862 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: BW), DL: dl, VT);
12863 SDValue Cond =
12864 DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT);
12865 SatVal = DAG.getSelect(DL: dl, VT, Cond, LHS: SatMin, RHS: SatMax);
12866 } else {
12867 SatVal = DAG.getConstant(Val: APInt::getMaxValue(numBits: BW), DL: dl, VT);
12868 }
12869 SDValue Cond = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Orig, Cond: ISD::SETNE);
12870 return DAG.getSelect(DL: dl, VT, Cond, LHS: SatVal, RHS: Result);
12871}
12872
12873void TargetLowering::forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl,
12874 bool Signed, SDValue &Lo, SDValue &Hi,
12875 SDValue LHS, SDValue RHS,
12876 SDValue HiLHS, SDValue HiRHS) const {
12877 EVT VT = LHS.getValueType();
12878 assert(RHS.getValueType() == VT && "Mismatching operand types");
12879
12880 assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
12881 assert((!Signed || !HiLHS) &&
12882 "Signed flag should only be set when HiLHS and RiRHS are null");
12883
12884 // We'll expand the multiplication by brute force because we have no other
12885 // options. This is a trivially-generalized version of the code from
12886 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
12887 // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
12888 // sign bits while calculating the Hi half.
12889 unsigned Bits = VT.getScalarSizeInBits();
12890 unsigned HalfBits = Bits / 2;
12891 SDValue Mask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: Bits, loBitsSet: HalfBits), DL: dl, VT);
12892 SDValue LL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Mask);
12893 SDValue RL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: RHS, N2: Mask);
12894
12895 SDValue T = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RL);
12896 SDValue TL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: Mask);
12897
12898 SDValue Shift = DAG.getShiftAmountConstant(Val: HalfBits, VT, DL: dl);
12899 // This is always an unsigned shift.
12900 SDValue TH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: T, N2: Shift);
12901
12902 unsigned ShiftOpc = Signed ? ISD::SRA : ISD::SRL;
12903 SDValue LH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: LHS, N2: Shift);
12904 SDValue RH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: RHS, N2: Shift);
12905
12906 SDValue U =
12907 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RL), N2: TH);
12908 SDValue UL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: U, N2: Mask);
12909 SDValue UH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: U, N2: Shift);
12910
12911 SDValue V =
12912 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RH), N2: UL);
12913 SDValue VH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: V, N2: Shift);
12914
12915 Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: TL,
12916 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: Shift));
12917
12918 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RH),
12919 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: UH, N2: VH));
12920
12921 // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
12922 // the products to Hi.
12923 if (HiLHS) {
12924 SDValue RHLL = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: HiRHS, N2: LHS);
12925 SDValue RLLH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: RHS, N2: HiLHS);
12926 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Hi,
12927 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: RHLL, N2: RLLH));
12928 }
12929}
12930
12931void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
12932 bool Signed, const SDValue LHS,
12933 const SDValue RHS, SDValue &Lo,
12934 SDValue &Hi) const {
12935 EVT VT = LHS.getValueType();
12936 assert(RHS.getValueType() == VT && "Mismatching operand types");
12937 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
12938 // We can fall back to a libcall with an illegal type for the MUL if we
12939 // have a libcall big enough.
12940 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
12941 if (WideVT == MVT::i16)
12942 LC = RTLIB::MUL_I16;
12943 else if (WideVT == MVT::i32)
12944 LC = RTLIB::MUL_I32;
12945 else if (WideVT == MVT::i64)
12946 LC = RTLIB::MUL_I64;
12947 else if (WideVT == MVT::i128)
12948 LC = RTLIB::MUL_I128;
12949
12950 RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(Call: LC);
12951 if (LibcallImpl == RTLIB::Unsupported) {
12952 forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
12953 return;
12954 }
12955
12956 // The libcall takes and returns WideVT values as packed VT halves. This only
12957 // works if VT is legal and WideVT isn't. Otherwise, emit a wide multiply and
12958 // let it legalise normally, possibly into a wider libcall.
12959 if (isTypeLegal(VT: WideVT) || !isTypeLegal(VT)) {
12960 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT,
12961 N1: DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL: dl, VT: WideVT),
12962 N2: DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL: dl, VT: WideVT));
12963 SDValue Shift =
12964 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits(), VT: WideVT, DL: dl);
12965 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Mul);
12966 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
12967 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT, N1: Mul, N2: Shift));
12968 return;
12969 }
12970
12971 SDValue HiLHS, HiRHS;
12972 if (Signed) {
12973 // The high part is obtained by SRA'ing all but one of the bits of low
12974 // part.
12975 unsigned LoSize = VT.getFixedSizeInBits();
12976 SDValue Shift = DAG.getShiftAmountConstant(Val: LoSize - 1, VT, DL: dl);
12977 HiLHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS, N2: Shift);
12978 HiRHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: RHS, N2: Shift);
12979 } else {
12980 HiLHS = DAG.getConstant(Val: 0, DL: dl, VT);
12981 HiRHS = DAG.getConstant(Val: 0, DL: dl, VT);
12982 }
12983
12984 // Attempt a libcall.
12985 SDValue Ret;
12986 TargetLowering::MakeLibCallOptions CallOptions;
12987 CallOptions.setIsSigned(Signed);
12988 CallOptions.setIsPostTypeLegalization(true);
12989 if (shouldSplitFunctionArgumentsAsLittleEndian(DL: DAG.getDataLayout())) {
12990 // Halves of WideVT are packed into registers in different order
12991 // depending on platform endianness. This is usually handled by
12992 // the C calling convention, but we can't defer to it in
12993 // the legalizer.
12994 SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
12995 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12996 } else {
12997 SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
12998 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12999 }
13000 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
13001 "Ret value is a collection of constituent nodes holding result.");
13002 if (DAG.getDataLayout().isLittleEndian()) {
13003 // Same as above.
13004 Lo = Ret.getOperand(i: 0);
13005 Hi = Ret.getOperand(i: 1);
13006 } else {
13007 Lo = Ret.getOperand(i: 1);
13008 Hi = Ret.getOperand(i: 0);
13009 }
13010}
13011
13012SDValue
13013TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
13014 assert((Node->getOpcode() == ISD::SMULFIX ||
13015 Node->getOpcode() == ISD::UMULFIX ||
13016 Node->getOpcode() == ISD::SMULFIXSAT ||
13017 Node->getOpcode() == ISD::UMULFIXSAT) &&
13018 "Expected a fixed point multiplication opcode");
13019
13020 SDLoc dl(Node);
13021 SDValue LHS = Node->getOperand(Num: 0);
13022 SDValue RHS = Node->getOperand(Num: 1);
13023 EVT VT = LHS.getValueType();
13024 unsigned Scale = Node->getConstantOperandVal(Num: 2);
13025 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
13026 Node->getOpcode() == ISD::UMULFIXSAT);
13027 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
13028 Node->getOpcode() == ISD::SMULFIXSAT);
13029 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13030 unsigned VTSize = VT.getScalarSizeInBits();
13031
13032 if (!Scale) {
13033 // [us]mul.fix(a, b, 0) -> mul(a, b)
13034 if (!Saturating) {
13035 if (isOperationLegalOrCustom(Op: ISD::MUL, VT))
13036 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13037 } else if (Signed && isOperationLegalOrCustom(Op: ISD::SMULO, VT)) {
13038 SDValue Result =
13039 DAG.getNode(Opcode: ISD::SMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
13040 SDValue Product = Result.getValue(R: 0);
13041 SDValue Overflow = Result.getValue(R: 1);
13042 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13043
13044 APInt MinVal = APInt::getSignedMinValue(numBits: VTSize);
13045 APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize);
13046 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
13047 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
13048 // Xor the inputs, if resulting sign bit is 0 the product will be
13049 // positive, else negative.
13050 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
13051 SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT);
13052 Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax);
13053 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product);
13054 } else if (!Signed && isOperationLegalOrCustom(Op: ISD::UMULO, VT)) {
13055 SDValue Result =
13056 DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
13057 SDValue Product = Result.getValue(R: 0);
13058 SDValue Overflow = Result.getValue(R: 1);
13059
13060 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
13061 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
13062 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product);
13063 }
13064 }
13065
13066 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
13067 "Expected scale to be less than the number of bits if signed or at "
13068 "most the number of bits if unsigned.");
13069 assert(LHS.getValueType() == RHS.getValueType() &&
13070 "Expected both operands to be the same type");
13071
13072 // Select the saturated value when Cond0 <CC> Cond1, keeping it vectorized:
13073 // SELECT_CC is scalarized for vector types, so build SETCC + VSELECT there.
13074 auto getSaturatingSelect = [&](SDValue Cond0, SDValue Cond1, SDValue Sat,
13075 SDValue Val, ISD::CondCode CC) {
13076 if (VT.isVector())
13077 return DAG.getSelect(DL: dl, VT, Cond: DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Cond0, RHS: Cond1, Cond: CC),
13078 LHS: Sat, RHS: Val);
13079 return DAG.getSelectCC(DL: dl, LHS: Cond0, RHS: Cond1, True: Sat, False: Val, Cond: CC);
13080 };
13081
13082 // Get the upper and lower bits of the result.
13083 SDValue Lo, Hi;
13084 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13085 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
13086 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13087 if (isOperationLegalOrCustom(Op: LoHiOp, VT)) {
13088 SDValue Result = DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
13089 Lo = Result.getValue(R: 0);
13090 Hi = Result.getValue(R: 1);
13091 } else if (isOperationLegalOrCustom(Op: HiOp, VT)) {
13092 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13093 Hi = DAG.getNode(Opcode: HiOp, DL: dl, VT, N1: LHS, N2: RHS);
13094 } else if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
13095 // Try for a multiplication using a wider type.
13096 unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
13097 SDValue LHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: LHS);
13098 SDValue RHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: RHS);
13099 SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHSExt, N2: RHSExt);
13100 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Res);
13101 SDValue Shifted =
13102 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: WideVT, N1: Res,
13103 N2: DAG.getShiftAmountConstant(Val: VTSize, VT: WideVT, DL: dl));
13104 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Shifted);
13105 } else if (VT.isVector()) {
13106 return SDValue();
13107 } else {
13108 forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
13109 }
13110
13111 if (Scale == VTSize)
13112 // Result is just the top half since we'd be shifting by the width of the
13113 // operand. Overflow impossible so this works for both UMULFIX and
13114 // UMULFIXSAT.
13115 return Hi;
13116
13117 // The result will need to be shifted right by the scale since both operands
13118 // are scaled. The result is given to us in 2 halves, so we only want part of
13119 // both in the result.
13120 SDValue Result = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: Hi, N2: Lo,
13121 N3: DAG.getShiftAmountConstant(Val: Scale, VT, DL: dl));
13122 if (!Saturating)
13123 return Result;
13124
13125 if (!Signed) {
13126 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
13127 // widened multiplication) aren't all zeroes.
13128
13129 // Saturate to max if ((Hi >> Scale) != 0),
13130 // which is the same as if (Hi > ((1 << Scale) - 1))
13131 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
13132 SDValue LowMask =
13133 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale), DL: dl, VT);
13134 return getSaturatingSelect(Hi, LowMask, DAG.getConstant(Val: MaxVal, DL: dl, VT),
13135 Result, ISD::SETUGT);
13136 }
13137
13138 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
13139 // widened multiplication) aren't all ones or all zeroes.
13140
13141 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: VTSize), DL: dl, VT);
13142 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: VTSize), DL: dl, VT);
13143
13144 if (Scale == 0) {
13145 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Lo,
13146 N2: DAG.getShiftAmountConstant(Val: VTSize - 1, VT, DL: dl));
13147 SDValue Overflow = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Hi, RHS: Sign, Cond: ISD::SETNE);
13148 // Saturated to SatMin if wide product is negative, and SatMax if wide
13149 // product is positive ...
13150 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13151 SDValue ResultIfOverflow =
13152 getSaturatingSelect(Hi, Zero, SatMin, SatMax, ISD::SETLT);
13153 // ... but only if we overflowed.
13154 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: ResultIfOverflow, RHS: Result);
13155 }
13156
13157 // We handled Scale==0 above so all the bits to examine is in Hi.
13158
13159 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
13160 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
13161 SDValue LowMask =
13162 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale - 1), DL: dl, VT);
13163 // Saturate to min if (Hi >> (Scale - 1)) < -1),
13164 // which is the same as if (HI < (-1 << (Scale - 1))
13165 SDValue HighMask = DAG.getConstant(
13166 Val: APInt::getHighBitsSet(numBits: VTSize, hiBitsSet: VTSize - Scale + 1), DL: dl, VT);
13167 Result = getSaturatingSelect(Hi, LowMask, SatMax, Result, ISD::SETGT);
13168 Result = getSaturatingSelect(Hi, HighMask, SatMin, Result, ISD::SETLT);
13169 return Result;
13170}
13171
13172SDValue
13173TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
13174 SDValue LHS, SDValue RHS,
13175 unsigned Scale, SelectionDAG &DAG) const {
13176 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
13177 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
13178 "Expected a fixed point division opcode");
13179
13180 EVT VT = LHS.getValueType();
13181 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
13182 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
13183 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13184
13185 // If there is enough room in the type to upscale the LHS or downscale the
13186 // RHS before the division, we can perform it in this type without having to
13187 // resize. For signed operations, the LHS headroom is the number of
13188 // redundant sign bits, and for unsigned ones it is the number of zeroes.
13189 // The headroom for the RHS is the number of trailing zeroes.
13190 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(Op: LHS) - 1
13191 : DAG.computeKnownBits(Op: LHS).countMinLeadingZeros();
13192 unsigned RHSTrail = DAG.computeKnownBits(Op: RHS).countMinTrailingZeros();
13193
13194 // For signed saturating operations, we need to be able to detect true integer
13195 // division overflow; that is, when you have MIN / -EPS. However, this
13196 // is undefined behavior and if we emit divisions that could take such
13197 // values it may cause undesired behavior (arithmetic exceptions on x86, for
13198 // example).
13199 // Avoid this by requiring an extra bit so that we never get this case.
13200 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
13201 // signed saturating division, we need to emit a whopping 32-bit division.
13202 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
13203 return SDValue();
13204
13205 unsigned LHSShift = std::min(a: LHSLead, b: Scale);
13206 unsigned RHSShift = Scale - LHSShift;
13207
13208 // At this point, we know that if we shift the LHS up by LHSShift and the
13209 // RHS down by RHSShift, we can emit a regular division with a final scaling
13210 // factor of Scale.
13211
13212 if (LHSShift)
13213 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS,
13214 N2: DAG.getShiftAmountConstant(Val: LHSShift, VT, DL: dl));
13215 if (RHSShift)
13216 RHS = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: RHS,
13217 N2: DAG.getShiftAmountConstant(Val: RHSShift, VT, DL: dl));
13218
13219 SDValue Quot;
13220 if (Signed) {
13221 // For signed operations, if the resulting quotient is negative and the
13222 // remainder is nonzero, subtract 1 from the quotient to round towards
13223 // negative infinity.
13224 SDValue Rem;
13225 // FIXME: Ideally we would always produce an SDIVREM here, but if the
13226 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
13227 // we couldn't just form a libcall, but the type legalizer doesn't do it.
13228 if (isTypeLegal(VT) &&
13229 isOperationLegalOrCustom(Op: ISD::SDIVREM, VT)) {
13230 Quot = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl,
13231 VTList: DAG.getVTList(VT1: VT, VT2: VT),
13232 N1: LHS, N2: RHS);
13233 Rem = Quot.getValue(R: 1);
13234 Quot = Quot.getValue(R: 0);
13235 } else {
13236 Quot = DAG.getNode(Opcode: ISD::SDIV, DL: dl, VT,
13237 N1: LHS, N2: RHS);
13238 Rem = DAG.getNode(Opcode: ISD::SREM, DL: dl, VT,
13239 N1: LHS, N2: RHS);
13240 }
13241 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13242 SDValue RemNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Rem, RHS: Zero, Cond: ISD::SETNE);
13243 SDValue LHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETLT);
13244 SDValue RHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13245 SDValue QuotNeg = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: BoolVT, N1: LHSNeg, N2: RHSNeg);
13246 SDValue Sub1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Quot,
13247 N2: DAG.getConstant(Val: 1, DL: dl, VT));
13248 Quot = DAG.getSelect(DL: dl, VT,
13249 Cond: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolVT, N1: RemNonZero, N2: QuotNeg),
13250 LHS: Sub1, RHS: Quot);
13251 } else
13252 Quot = DAG.getNode(Opcode: ISD::UDIV, DL: dl, VT,
13253 N1: LHS, N2: RHS);
13254
13255 return Quot;
13256}
13257
13258void TargetLowering::expandUADDSUBO(
13259 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13260 SDLoc dl(Node);
13261 SDValue LHS = Node->getOperand(Num: 0);
13262 SDValue RHS = Node->getOperand(Num: 1);
13263 bool IsAdd = Node->getOpcode() == ISD::UADDO;
13264
13265 // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
13266 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
13267 if (isOperationLegalOrCustom(Op: OpcCarry, VT: Node->getValueType(ResNo: 0))) {
13268 SDValue CarryIn = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 1));
13269 SDValue NodeCarry = DAG.getNode(Opcode: OpcCarry, DL: dl, VTList: Node->getVTList(),
13270 Ops: { LHS, RHS, CarryIn });
13271 Result = SDValue(NodeCarry.getNode(), 0);
13272 Overflow = SDValue(NodeCarry.getNode(), 1);
13273 return;
13274 }
13275
13276 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13277 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13278
13279 EVT ResultType = Node->getValueType(ResNo: 1);
13280 EVT SetCCType = getSetCCResultType(
13281 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13282 SDValue SetCC;
13283 if (IsAdd && isOneConstant(V: RHS)) {
13284 // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
13285 // the live range of X. We assume comparing with 0 is cheap.
13286 // The general case (X + C) < C is not necessarily beneficial. Although we
13287 // reduce the live range of X, we may introduce the materialization of
13288 // constant C.
13289 SetCC =
13290 DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result,
13291 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETEQ);
13292 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
13293 // Special case: uaddo X, -1 overflows if X != 0.
13294 SetCC =
13295 DAG.getSetCC(DL: dl, VT: SetCCType, LHS,
13296 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETNE);
13297 } else {
13298 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
13299 SetCC = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result, RHS: LHS, Cond: CC);
13300 }
13301 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13302}
13303
13304void TargetLowering::expandSADDSUBO(
13305 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13306 SDLoc dl(Node);
13307 SDValue LHS = Node->getOperand(Num: 0);
13308 SDValue RHS = Node->getOperand(Num: 1);
13309 bool IsAdd = Node->getOpcode() == ISD::SADDO;
13310
13311 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13312 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13313
13314 EVT ResultType = Node->getValueType(ResNo: 1);
13315 EVT OType = getSetCCResultType(
13316 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13317
13318 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
13319 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
13320 if (isOperationLegal(Op: OpcSat, VT: LHS.getValueType())) {
13321 SDValue Sat = DAG.getNode(Opcode: OpcSat, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS);
13322 SDValue SetCC = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Sat, Cond: ISD::SETNE);
13323 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13324 return;
13325 }
13326
13327 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
13328
13329 if (IsAdd) {
13330 // For an addition, the result should be less than one of the operands (LHS)
13331 // if and only if the other operand (RHS) is negative, otherwise there will
13332 // be overflow.
13333 SDValue ResultLowerThanLHS =
13334 DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: LHS, Cond: ISD::SETLT);
13335 SDValue RHSNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13336 Overflow = DAG.getBoolExtOrTrunc(
13337 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: RHSNegative, N2: ResultLowerThanLHS), SL: dl,
13338 VT: ResultType, OpVT: ResultType);
13339 } else {
13340 // For subtraction, overflow occurs when the signed comparison of operands
13341 // doesn't match the sign of the result.
13342 SDValue LHSLessThanRHS = DAG.getSetCC(DL: dl, VT: OType, LHS, RHS, Cond: ISD::SETLT);
13343 SDValue ResultNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Zero, Cond: ISD::SETLT);
13344 Overflow = DAG.getBoolExtOrTrunc(
13345 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: LHSLessThanRHS, N2: ResultNegative), SL: dl,
13346 VT: ResultType, OpVT: ResultType);
13347 }
13348}
13349
13350bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
13351 SDValue &Overflow, SelectionDAG &DAG) const {
13352 SDLoc dl(Node);
13353 EVT VT = Node->getValueType(ResNo: 0);
13354 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13355 SDValue LHS = Node->getOperand(Num: 0);
13356 SDValue RHS = Node->getOperand(Num: 1);
13357 bool isSigned = Node->getOpcode() == ISD::SMULO;
13358
13359 // For power-of-two multiplications we can use a simpler shift expansion.
13360 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
13361 const APInt &C = RHSC->getAPIntValue();
13362 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
13363 if (C.isPowerOf2()) {
13364 // smulo(x, signed_min) is same as umulo(x, signed_min).
13365 bool UseArithShift = isSigned && !C.isMinSignedValue();
13366 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: C.logBase2(), VT, DL: dl);
13367 Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: ShiftAmt);
13368 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT,
13369 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL,
13370 DL: dl, VT, N1: Result, N2: ShiftAmt),
13371 RHS: LHS, Cond: ISD::SETNE);
13372 return true;
13373 }
13374 }
13375
13376 SDValue BottomHalf;
13377 SDValue TopHalf;
13378 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13379
13380 static const unsigned Ops[2][3] =
13381 { { ISD::UMUL_LOHI, ISD::MULHU, ISD::ZERO_EXTEND },
13382 { ISD::SMUL_LOHI, ISD::MULHS, ISD::SIGN_EXTEND }};
13383 if (isOperationLegalOrCustom(Op: Ops[isSigned][0], VT)) {
13384 BottomHalf = DAG.getNode(Opcode: Ops[isSigned][0], DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS,
13385 N2: RHS);
13386 TopHalf = BottomHalf.getValue(R: 1);
13387 } else if (isOperationLegalOrCustom(Op: Ops[isSigned][1], VT)) {
13388 BottomHalf = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13389 TopHalf = DAG.getNode(Opcode: Ops[isSigned][1], DL: dl, VT, N1: LHS, N2: RHS);
13390 } else if (isTypeLegal(VT: WideVT)) {
13391 LHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: LHS);
13392 RHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: RHS);
13393 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS);
13394 BottomHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Mul);
13395 SDValue ShiftAmt =
13396 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits(), VT: WideVT, DL: dl);
13397 TopHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13398 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT, N1: Mul, N2: ShiftAmt));
13399 } else {
13400 if (VT.isVector())
13401 return false;
13402
13403 forceExpandWideMUL(DAG, dl, Signed: isSigned, LHS, RHS, Lo&: BottomHalf, Hi&: TopHalf);
13404 }
13405
13406 Result = BottomHalf;
13407 if (isSigned) {
13408 SDValue ShiftAmt = DAG.getShiftAmountConstant(
13409 Val: VT.getScalarSizeInBits() - 1, VT: BottomHalf.getValueType(), DL: dl);
13410 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: BottomHalf, N2: ShiftAmt);
13411 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf, RHS: Sign, Cond: ISD::SETNE);
13412 } else {
13413 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf,
13414 RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
13415 }
13416
13417 // Truncate the result if SetCC returns a larger type than needed.
13418 EVT RType = Node->getValueType(ResNo: 1);
13419 if (RType.bitsLT(VT: Overflow.getValueType()))
13420 Overflow = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RType, Operand: Overflow);
13421
13422 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
13423 "Unexpected result type for S/UMULO legalization");
13424 return true;
13425}
13426
13427SDValue TargetLowering::expandMULH(SDNode *Node, SelectionDAG &DAG) const {
13428 SDLoc dl(Node);
13429 EVT VT = Node->getValueType(ResNo: 0);
13430 SDValue LHS = Node->getOperand(Num: 0);
13431 SDValue RHS = Node->getOperand(Num: 1);
13432 bool IsSigned = Node->getOpcode() == ISD::MULHS;
13433
13434 // Use MUL_LOHI if legal/custom for the original type.
13435 unsigned LoHiOp = IsSigned ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13436 if (isOperationLegalOrCustom(Op: LoHiOp, VT))
13437 return DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS).getValue(R: 1);
13438
13439 // Use a wide multiply if available.
13440 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13441 if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
13442 unsigned BW = VT.getScalarSizeInBits();
13443 LHS = DAG.getExtOrTrunc(IsSigned, Op: LHS, DL: dl, VT: WideVT);
13444 RHS = DAG.getExtOrTrunc(IsSigned, Op: RHS, DL: dl, VT: WideVT);
13445 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13446 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT,
13447 N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS),
13448 N2: DAG.getShiftAmountConstant(Val: BW, VT: WideVT, DL: dl)));
13449 }
13450
13451 // Let fixed-length vectors be scalarised by the caller.
13452 // Expand everything else with a wide multiply.
13453 if (!VT.isFixedLengthVector()) {
13454 SDValue Lo, Hi;
13455 forceExpandWideMUL(DAG, dl, Signed: IsSigned, LHS, RHS, Lo, Hi);
13456 return Hi;
13457 }
13458
13459 return SDValue();
13460}
13461
13462SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
13463 SDLoc dl(Node);
13464 ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13465 SDValue Op = Node->getOperand(Num: 0);
13466 SDNodeFlags Flags = Node->getFlags();
13467 EVT VT = Op.getValueType();
13468
13469 // Try to use a shuffle reduction for power of two vectors.
13470 if (VT.isPow2VectorType()) {
13471 // See if the reduction opcode is safe to use with widened types.
13472 bool WidenSrc = false;
13473 switch (Node->getOpcode()) {
13474 case ISD::VECREDUCE_FADD:
13475 case ISD::VECREDUCE_FMUL:
13476 case ISD::VECREDUCE_ADD:
13477 case ISD::VECREDUCE_MUL:
13478 case ISD::VECREDUCE_AND:
13479 case ISD::VECREDUCE_OR:
13480 case ISD::VECREDUCE_XOR:
13481 case ISD::VECREDUCE_SMAX:
13482 case ISD::VECREDUCE_SMIN:
13483 case ISD::VECREDUCE_UMAX:
13484 case ISD::VECREDUCE_UMIN:
13485 WidenSrc = VT.isFixedLengthVector();
13486 break;
13487 }
13488
13489 while (VT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
13490 EVT HalfVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
13491 if (!isOperationLegalOrCustom(Op: BaseOpcode, VT: HalfVT)) {
13492 if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
13493 // Attempt to widen the source vectors to a legal op.
13494 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: HalfVT);
13495 if (WideVT.isVector() &&
13496 WideVT.getScalarType() == HalfVT.getScalarType() &&
13497 WideVT.getVectorNumElements() >= HalfVT.getVectorNumElements() &&
13498 isOperationLegalOrCustom(Op: BaseOpcode, VT: WideVT)) {
13499 SDValue Lo, Hi;
13500 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13501 Lo = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Lo, Idx: 0);
13502 Hi = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Hi, Idx: 0);
13503 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: WideVT, N1: Lo, N2: Hi, Flags);
13504 Op = DAG.getExtractSubvector(DL: dl, VT: HalfVT, Vec: Op, Idx: 0);
13505 VT = HalfVT;
13506 continue;
13507 }
13508 }
13509 break;
13510 }
13511
13512 SDValue Lo, Hi;
13513 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13514 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: HalfVT, N1: Lo, N2: Hi, Flags);
13515 VT = HalfVT;
13516
13517 // Stop if splitting is enough to make the reduction legal.
13518 if (isOperationLegalOrCustom(Op: Node->getOpcode(), VT: HalfVT))
13519 return DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op,
13520 Flags);
13521 }
13522 }
13523
13524 if (VT.isScalableVector())
13525 reportFatalInternalError(
13526 reason: "Expanding reductions for scalable vectors is undefined.");
13527
13528 EVT EltVT = VT.getVectorElementType();
13529 unsigned NumElts = VT.getVectorNumElements();
13530
13531 SmallVector<SDValue, 8> Ops;
13532 DAG.ExtractVectorElements(Op, Args&: Ops, Start: 0, Count: NumElts);
13533
13534 SDValue Res = Ops[0];
13535 for (unsigned i = 1; i < NumElts; i++)
13536 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13537
13538 // Result type may be wider than element type.
13539 if (EltVT != Node->getValueType(ResNo: 0))
13540 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res);
13541 return Res;
13542}
13543
13544SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
13545 SDLoc dl(Node);
13546 SDValue AccOp = Node->getOperand(Num: 0);
13547 SDValue VecOp = Node->getOperand(Num: 1);
13548 SDNodeFlags Flags = Node->getFlags();
13549
13550 EVT VT = VecOp.getValueType();
13551 EVT EltVT = VT.getVectorElementType();
13552
13553 if (VT.isScalableVector())
13554 report_fatal_error(
13555 reason: "Expanding reductions for scalable vectors is undefined.");
13556
13557 unsigned NumElts = VT.getVectorNumElements();
13558
13559 SmallVector<SDValue, 8> Ops;
13560 DAG.ExtractVectorElements(Op: VecOp, Args&: Ops, Start: 0, Count: NumElts);
13561
13562 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13563
13564 SDValue Res = AccOp;
13565 for (unsigned i = 0; i < NumElts; i++)
13566 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13567
13568 return Res;
13569}
13570
13571bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
13572 SelectionDAG &DAG) const {
13573 EVT VT = Node->getValueType(ResNo: 0);
13574 SDLoc dl(Node);
13575 bool isSigned = Node->getOpcode() == ISD::SREM;
13576 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
13577 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
13578 SDValue Dividend = Node->getOperand(Num: 0);
13579 SDValue Divisor = Node->getOperand(Num: 1);
13580 if (isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
13581 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
13582 Result = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Dividend, N2: Divisor).getValue(R: 1);
13583 return true;
13584 }
13585 if (isOperationLegalOrCustom(Op: DivOpc, VT)) {
13586 // X % Y -> X-X/Y*Y
13587 SDValue Divide = DAG.getNode(Opcode: DivOpc, DL: dl, VT, N1: Dividend, N2: Divisor);
13588 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Divide, N2: Divisor);
13589 Result = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
13590 return true;
13591 }
13592 return false;
13593}
13594
13595SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
13596 SelectionDAG &DAG) const {
13597 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
13598 SDLoc dl(SDValue(Node, 0));
13599 SDValue Src = Node->getOperand(Num: 0);
13600
13601 // DstVT is the result type, while SatVT is the size to which we saturate
13602 EVT SrcVT = Src.getValueType();
13603 EVT DstVT = Node->getValueType(ResNo: 0);
13604
13605 EVT SatVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
13606 unsigned SatWidth = SatVT.getScalarSizeInBits();
13607 unsigned DstWidth = DstVT.getScalarSizeInBits();
13608 assert(SatWidth <= DstWidth &&
13609 "Expected saturation width smaller than result width");
13610
13611 // Determine minimum and maximum integer values and their corresponding
13612 // floating-point values.
13613 APInt MinInt, MaxInt;
13614 if (IsSigned) {
13615 MinInt = APInt::getSignedMinValue(numBits: SatWidth).sext(width: DstWidth);
13616 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth).sext(width: DstWidth);
13617 } else {
13618 MinInt = APInt::getMinValue(numBits: SatWidth).zext(width: DstWidth);
13619 MaxInt = APInt::getMaxValue(numBits: SatWidth).zext(width: DstWidth);
13620 }
13621
13622 // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
13623 // libcall emission cannot handle this. Large result types will fail.
13624 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
13625 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f32, Operand: Src);
13626 SrcVT = Src.getValueType();
13627 }
13628
13629 const fltSemantics &Sem = SrcVT.getFltSemantics();
13630 APFloat MinFloat(Sem);
13631 APFloat MaxFloat(Sem);
13632
13633 APFloat::opStatus MinStatus =
13634 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
13635 APFloat::opStatus MaxStatus =
13636 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
13637 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
13638 !(MaxStatus & APFloat::opStatus::opInexact);
13639
13640 SDValue MinFloatNode = DAG.getConstantFP(Val: MinFloat, DL: dl, VT: SrcVT);
13641 SDValue MaxFloatNode = DAG.getConstantFP(Val: MaxFloat, DL: dl, VT: SrcVT);
13642
13643 // If the integer bounds are exactly representable as floats and min/max are
13644 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
13645 // of comparisons and selects.
13646 auto EmitMinMax = [&](unsigned MinOpcode, unsigned MaxOpcode,
13647 bool MayPropagateNaN) {
13648 bool MinMaxLegal = isOperationLegalOrCustom(Op: MinOpcode, VT: SrcVT) &&
13649 isOperationLegalOrCustom(Op: MaxOpcode, VT: SrcVT);
13650 if (!MinMaxLegal)
13651 return SDValue();
13652
13653 SDValue Clamped = Src;
13654
13655 // Clamp Src by MinFloat from below. If !MayPropagateNaN and Src is NaN
13656 // then the result is MinFloat.
13657 Clamped = DAG.getNode(Opcode: MaxOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MinFloatNode);
13658 // Clamp by MaxFloat from above. If !MayPropagateNaN then NaN cannot occur.
13659 Clamped = DAG.getNode(Opcode: MinOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MaxFloatNode);
13660 // Convert clamped value to integer.
13661 SDValue FpToInt = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
13662 DL: dl, VT: DstVT, Operand: Clamped);
13663
13664 // If !MayPropagateNan and the conversion is unsigned case we're done,
13665 // because we mapped NaN to MinFloat, which will cast to zero.
13666 if (!MayPropagateNaN && !IsSigned)
13667 return FpToInt;
13668
13669 // Otherwise, select 0 if Src is NaN.
13670 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13671 EVT SetCCVT =
13672 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13673 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13674 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: FpToInt);
13675 };
13676 if (AreExactFloatBounds) {
13677 if (SDValue Res = EmitMinMax(ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
13678 /*MayPropagateNaN=*/false))
13679 return Res;
13680 // These may propagate NaN for sNaN operands.
13681 if (SDValue Res =
13682 EmitMinMax(ISD::FMINNUM, ISD::FMAXNUM, /*MayPropagateNaN=*/true))
13683 return Res;
13684 // These always propagate NaN.
13685 if (SDValue Res =
13686 EmitMinMax(ISD::FMINIMUM, ISD::FMAXIMUM, /*MayPropagateNaN=*/true))
13687 return Res;
13688 }
13689
13690 SDValue MinIntNode = DAG.getConstant(Val: MinInt, DL: dl, VT: DstVT);
13691 SDValue MaxIntNode = DAG.getConstant(Val: MaxInt, DL: dl, VT: DstVT);
13692
13693 // Result of direct conversion. The assumption here is that the operation is
13694 // non-trapping and it's fine to apply it to an out-of-range value if we
13695 // select it away later.
13696 SDValue FpToInt =
13697 DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL: dl, VT: DstVT, Operand: Src);
13698
13699 SDValue Select = FpToInt;
13700
13701 EVT SetCCVT =
13702 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13703
13704 // If Src ULT MinFloat, select MinInt. In particular, this also selects
13705 // MinInt if Src is NaN.
13706 SDValue ULT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MinFloatNode, Cond: ISD::SETULT);
13707 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: ULT, LHS: MinIntNode, RHS: Select);
13708 // If Src OGT MaxFloat, select MaxInt.
13709 SDValue OGT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MaxFloatNode, Cond: ISD::SETOGT);
13710 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: OGT, LHS: MaxIntNode, RHS: Select);
13711
13712 // In the unsigned case we are done, because we mapped NaN to MinInt, which
13713 // is already zero.
13714 if (!IsSigned)
13715 return Select;
13716
13717 // Otherwise, select 0 if Src is NaN.
13718 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13719 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13720 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: Select);
13721}
13722
13723SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op,
13724 const SDLoc &dl,
13725 SelectionDAG &DAG) const {
13726 EVT OperandVT = Op.getValueType();
13727 if (OperandVT.getScalarType() == ResultVT.getScalarType())
13728 return Op;
13729 EVT ResultIntVT = ResultVT.changeTypeToInteger();
13730 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13731 // can induce double-rounding which may alter the results. We can
13732 // correct for this using a trick explained in: Boldo, Sylvie, and
13733 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13734 // World Congress. 2005.
13735 SDValue Narrow = DAG.getFPExtendOrRound(Op, DL: dl, VT: ResultVT);
13736 SDValue NarrowAsWide = DAG.getFPExtendOrRound(Op: Narrow, DL: dl, VT: OperandVT);
13737
13738 // We can keep the narrow value as-is if narrowing was exact (no
13739 // rounding error), the wide value was NaN (the narrow value is also
13740 // NaN and should be preserved) or if we rounded to the odd value.
13741 SDValue NarrowBits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultIntVT, Operand: Narrow);
13742 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResultIntVT);
13743 SDValue NegativeOne = DAG.getAllOnesConstant(DL: dl, VT: ResultIntVT);
13744 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: One);
13745 EVT ResultIntVTCCVT = getSetCCResultType(
13746 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: And.getValueType());
13747 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ResultIntVT);
13748 // The result is already odd so we don't need to do anything.
13749 SDValue AlreadyOdd = DAG.getSetCC(DL: dl, VT: ResultIntVTCCVT, LHS: And, RHS: Zero, Cond: ISD::SETNE);
13750
13751 EVT WideSetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
13752 VT: Op.getValueType());
13753 // We keep results which are exact, odd or NaN.
13754 SDValue KeepNarrow =
13755 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: Op, RHS: NarrowAsWide, Cond: ISD::SETUEQ);
13756 KeepNarrow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: WideSetCCVT, N1: KeepNarrow, N2: AlreadyOdd);
13757 // We morally performed a round-down if AbsNarrow is smaller than
13758 // AbsWide.
13759 SDValue AbsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: Op);
13760 SDValue AbsNarrowAsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: NarrowAsWide);
13761 SDValue NarrowIsRd =
13762 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: AbsWide, RHS: AbsNarrowAsWide, Cond: ISD::SETOGT);
13763 // If the narrow value is odd or exact, pick it.
13764 // Otherwise, narrow is even and corresponds to either the rounded-up
13765 // or rounded-down value. If narrow is the rounded-down value, we want
13766 // the rounded-up value as it will be odd.
13767 SDValue Adjust = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: NarrowIsRd, LHS: One, RHS: NegativeOne);
13768 SDValue Adjusted = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: Adjust);
13769 Op = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: KeepNarrow, LHS: NarrowBits, RHS: Adjusted);
13770 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultVT, Operand: Op);
13771}
13772
13773SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const {
13774 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
13775 SDValue Op = Node->getOperand(Num: 0);
13776 EVT VT = Node->getValueType(ResNo: 0);
13777 SDLoc dl(Node);
13778 if (VT.getScalarType() == MVT::bf16) {
13779 if (Node->getConstantOperandVal(Num: 1) == 1) {
13780 return DAG.getNode(Opcode: ISD::FP_TO_BF16, DL: dl, VT, Operand: Node->getOperand(Num: 0));
13781 }
13782 EVT OperandVT = Op.getValueType();
13783 SDValue IsNaN = DAG.getSetCC(
13784 DL: dl,
13785 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT),
13786 LHS: Op, RHS: Op, Cond: ISD::SETUO);
13787
13788 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13789 // can induce double-rounding which may alter the results. We can
13790 // correct for this using a trick explained in: Boldo, Sylvie, and
13791 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13792 // World Congress. 2005.
13793 EVT F32 = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
13794 EVT I32 = F32.changeTypeToInteger();
13795 Op = expandRoundInexactToOdd(ResultVT: F32, Op, dl, DAG);
13796 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: I32, Operand: Op);
13797
13798 // Conversions should set NaN's quiet bit. This also prevents NaNs from
13799 // turning into infinities.
13800 SDValue NaN =
13801 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: I32, N1: Op, N2: DAG.getConstant(Val: 0x400000, DL: dl, VT: I32));
13802
13803 // Factor in the contribution of the low 16 bits.
13804 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: I32);
13805 SDValue Lsb = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13806 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13807 Lsb = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: I32, N1: Lsb, N2: One);
13808 SDValue RoundingBias =
13809 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Lsb, N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: I32));
13810 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Op, N2: RoundingBias);
13811
13812 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
13813 // 0x80000000.
13814 Op = DAG.getSelect(DL: dl, VT: I32, Cond: IsNaN, LHS: NaN, RHS: Add);
13815
13816 // Now that we have rounded, shift the bits into position.
13817 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13818 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13819 EVT I16 = I32.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i16);
13820 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: I16, Operand: Op);
13821 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
13822 }
13823 return SDValue();
13824}
13825
13826void TargetLowering::expandVectorInterleave(SDNode *Node,
13827 SmallVectorImpl<SDValue> &Results,
13828 SelectionDAG &DAG) const {
13829 assert(Node->getOpcode() == ISD::VECTOR_INTERLEAVE && "Unexpected opcode");
13830 EVT VT = Node->getValueType(ResNo: 0);
13831 unsigned Factor = Node->getNumOperands();
13832 SDLoc DL(Node);
13833
13834 if (Factor > 2 && Factor % 2 == 0) {
13835 SmallVector<EVT> HalfVTs(Factor / 2, VT);
13836 SmallVector<SDValue, 8> LOps, ROps;
13837 // Interleave so we have 2 factors per result:
13838 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
13839 for (unsigned I = 0; I < Factor / 2; I++) {
13840 SDValue Interleave =
13841 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: {VT, VT},
13842 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
13843 LOps.push_back(Elt: Interleave.getValue(R: 0));
13844 ROps.push_back(Elt: Interleave.getValue(R: 1));
13845 }
13846 // Interleave at Factor/2:
13847 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
13848 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: HalfVTs, Ops: LOps);
13849 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: HalfVTs, Ops: ROps);
13850 for (unsigned I = 0; I < Factor / 2; I++)
13851 Results.push_back(Elt: L.getValue(R: I));
13852 for (unsigned I = 0; I < Factor / 2; I++)
13853 Results.push_back(Elt: R.getValue(R: I));
13854 return;
13855 }
13856
13857 if (!VT.isFixedLengthVector())
13858 return;
13859
13860 unsigned NumElts = VT.getVectorNumElements();
13861 EVT EltVT = VT.getVectorElementType();
13862
13863 // A factor of two needs only same-width, two-input shuffles.
13864 if (Factor == 2) {
13865 SmallVector<int, 8> Mask = createInterleaveMask(VF: NumElts, NumVecs: Factor);
13866 for (unsigned I = 0; I != Factor; ++I)
13867 Results.push_back(Elt: DAG.getVectorShuffle(
13868 VT, dl: DL, N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
13869 Mask: ArrayRef<int>(Mask).slice(N: I * NumElts, M: NumElts)));
13870 return;
13871 }
13872
13873 SmallVector<SDValue> InterleavedElts;
13874 InterleavedElts.reserve(N: Factor * NumElts);
13875 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
13876 for (unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
13877 InterleavedElts.push_back(Elt: DAG.getExtractVectorElt(
13878 DL, VT: EltVT, Vec: Node->getOperand(Num: OperandIdx), Idx: EltIdx));
13879
13880 for (unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx)
13881 Results.push_back(Elt: DAG.getBuildVector(
13882 VT, DL, Ops: ArrayRef(InterleavedElts).slice(N: ResultIdx * NumElts, M: NumElts)));
13883}
13884
13885void TargetLowering::expandVectorDeinterleave(SDNode *Node,
13886 SmallVectorImpl<SDValue> &Results,
13887 SelectionDAG &DAG) const {
13888 assert(Node->getOpcode() == ISD::VECTOR_DEINTERLEAVE && "Unexpected opcode");
13889 EVT VT = Node->getValueType(ResNo: 0);
13890 unsigned Factor = Node->getNumOperands();
13891 SDLoc DL(Node);
13892
13893 if (Factor > 2 && Factor % 2 == 0) {
13894 SmallVector<SDValue, 8> Ops(Node->ops());
13895 SmallVector<EVT> HalfVTs(Factor / 2, VT);
13896 // Deinterleave at Factor/2 so each result contains two factors interleaved:
13897 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
13898 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: HalfVTs,
13899 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
13900 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: HalfVTs,
13901 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
13902 Results.resize(N: Factor);
13903 // Deinterleave the 2 factors out:
13904 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
13905 for (unsigned I = 0; I < Factor / 2; ++I) {
13906 SDValue Deinterleave = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: {VT, VT},
13907 Ops: {L.getValue(R: I), R.getValue(R: I)});
13908 Results[I] = Deinterleave.getValue(R: 0);
13909 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
13910 }
13911 return;
13912 }
13913
13914 if (!VT.isFixedLengthVector())
13915 return;
13916
13917 unsigned NumElts = VT.getVectorNumElements();
13918 EVT EltVT = VT.getVectorElementType();
13919
13920 // A factor of two needs only same-width, two-input shuffles.
13921 if (Factor == 2) {
13922 for (unsigned I = 0; I != Factor; ++I)
13923 Results.push_back(
13924 Elt: DAG.getVectorShuffle(VT, dl: DL, N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
13925 Mask: createStrideMask(Start: I, Stride: Factor, VF: NumElts)));
13926 return;
13927 }
13928
13929 SmallVector<SDValue> ConcatElts;
13930 ConcatElts.reserve(N: Factor * NumElts);
13931 for (unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
13932 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
13933 ConcatElts.push_back(Elt: DAG.getExtractVectorElt(
13934 DL, VT: EltVT, Vec: Node->getOperand(Num: OperandIdx), Idx: EltIdx));
13935
13936 SmallVector<SDValue> DeinterleavedElts;
13937 DeinterleavedElts.reserve(N: Factor * NumElts);
13938 for (unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx) {
13939 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
13940 DeinterleavedElts.push_back(Elt: ConcatElts[ResultIdx + EltIdx * Factor]);
13941 Results.push_back(Elt: DAG.getBuildVector(
13942 VT, DL,
13943 Ops: ArrayRef(DeinterleavedElts).slice(N: ResultIdx * NumElts, M: NumElts)));
13944 }
13945}
13946
13947SDValue TargetLowering::expandVectorSplice(SDNode *Node,
13948 SelectionDAG &DAG) const {
13949 assert((Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT ||
13950 Node->getOpcode() == ISD::VECTOR_SPLICE_RIGHT) &&
13951 "Unexpected opcode!");
13952 assert((Node->getValueType(0).isScalableVector() ||
13953 !isa<ConstantSDNode>(Node->getOperand(2))) &&
13954 "Fixed length vector types with constant offsets expected to use "
13955 "SHUFFLE_VECTOR!");
13956
13957 EVT VT = Node->getValueType(ResNo: 0);
13958 SDValue V1 = Node->getOperand(Num: 0);
13959 SDValue V2 = Node->getOperand(Num: 1);
13960 SDValue Offset = Node->getOperand(Num: 2);
13961 SDLoc DL(Node);
13962
13963 // Expand through memory thusly:
13964 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
13965 // Store V1, Ptr
13966 // Store V2, Ptr + sizeof(V1)
13967 // if (VECTOR_SPLICE_LEFT)
13968 // Ptr = Ptr + (Offset * sizeof(VT.Elt))
13969 // else
13970 // Ptr = Ptr + sizeof(V1) - (Offset * size(VT.Elt))
13971 // Res = Load Ptr
13972
13973 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
13974
13975 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
13976 EC: VT.getVectorElementCount() * 2);
13977 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
13978 EVT PtrVT = StackPtr.getValueType();
13979 auto &MF = DAG.getMachineFunction();
13980 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13981 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13982
13983 // Store the lo part of CONCAT_VECTORS(V1, V2)
13984 SDValue StoreV1 =
13985 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, PtrInfo, Alignment);
13986 // Store the hi part of CONCAT_VECTORS(V1, V2)
13987 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
13988 SDValue StackPtr2 = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: VTBytes);
13989 SDValue StoreV2 =
13990 DAG.getStore(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, PtrInfo, Alignment);
13991
13992 // NOTE: TrailingBytes must be clamped so as not to read outside of V1:V2.
13993 SDValue EltByteSize =
13994 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
13995 Offset = DAG.getZExtOrTrunc(Op: Offset, DL, VT: PtrVT);
13996 SDValue TrailingBytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: Offset, N2: EltByteSize);
13997
13998 TrailingBytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: VTBytes);
13999
14000 if (Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT)
14001 StackPtr = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TrailingBytes, DL);
14002 else
14003 StackPtr = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
14004
14005 // Load the spliced result
14006 return DAG.getLoad(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr,
14007 PtrInfo: MachinePointerInfo::getUnknownStack(MF), Alignment);
14008}
14009
14010SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
14011 SelectionDAG &DAG) const {
14012 SDLoc DL(Node);
14013 SDValue Vec = Node->getOperand(Num: 0);
14014 SDValue Mask = Node->getOperand(Num: 1);
14015 SDValue Passthru = Node->getOperand(Num: 2);
14016
14017 EVT VecVT = Vec.getValueType();
14018 EVT ScalarVT = VecVT.getScalarType();
14019 EVT MaskVT = Mask.getValueType();
14020 EVT MaskScalarVT = MaskVT.getScalarType();
14021
14022 // Needs to be handled by targets that have scalable vector types.
14023 if (VecVT.isScalableVector())
14024 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
14025
14026 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
14027 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment);
14028 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
14029 MachinePointerInfo PtrInfo =
14030 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
14031
14032 MVT PositionVT = getVectorIdxTy(DL: DAG.getDataLayout());
14033 SDValue Chain = DAG.getEntryNode();
14034 SDValue OutPos = DAG.getConstant(Val: 0, DL, VT: PositionVT);
14035
14036 bool HasPassthru = !Passthru.isUndef();
14037
14038 // If we have a passthru vector, store it on the stack, overwrite the matching
14039 // positions and then re-write the last element that was potentially
14040 // overwritten even though mask[i] = false.
14041 if (HasPassthru)
14042 Chain = DAG.getStore(Chain, dl: DL, Val: Passthru, Ptr: StackPtr, PtrInfo, Alignment);
14043
14044 SDValue LastWriteVal;
14045 APInt PassthruSplatVal;
14046 bool IsSplatPassthru =
14047 ISD::isConstantSplatVector(N: Passthru.getNode(), SplatValue&: PassthruSplatVal);
14048
14049 if (IsSplatPassthru) {
14050 // As we do not know which position we wrote to last, we cannot simply
14051 // access that index from the passthru vector. So we first check if passthru
14052 // is a splat vector, to use any element ...
14053 LastWriteVal = DAG.getConstant(Val: PassthruSplatVal, DL, VT: ScalarVT);
14054 } else if (HasPassthru) {
14055 // ... if it is not a splat vector, we need to get the passthru value at
14056 // position = popcount(mask) and re-load it from the stack before it is
14057 // overwritten in the loop below.
14058 EVT PopcountVT = ScalarVT.changeTypeToInteger();
14059 SDValue Popcount = DAG.getNode(
14060 Opcode: ISD::TRUNCATE, DL,
14061 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1), Operand: Mask);
14062 Popcount = DAG.getNode(
14063 Opcode: ISD::ZERO_EXTEND, DL,
14064 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: PopcountVT),
14065 Operand: Popcount);
14066 Popcount = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: PopcountVT, Operand: Popcount);
14067 SDValue LastElmtPtr =
14068 getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Popcount);
14069 LastWriteVal = DAG.getLoad(
14070 VT: ScalarVT, dl: DL, Chain, Ptr: LastElmtPtr,
14071 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
14072 Chain = LastWriteVal.getValue(R: 1);
14073 }
14074
14075 unsigned NumElms = VecVT.getVectorNumElements();
14076 for (unsigned I = 0; I < NumElms; I++) {
14077 SDValue ValI = DAG.getExtractVectorElt(DL, VT: ScalarVT, Vec, Idx: I);
14078 SDValue OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
14079 Chain = DAG.getStore(
14080 Chain, dl: DL, Val: ValI, Ptr: OutPtr,
14081 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
14082
14083 // Get the mask value and add it to the current output position. This
14084 // either increments by 1 if MaskI is true or adds 0 otherwise.
14085 // Freeze in case we have poison/undef mask entries.
14086 SDValue MaskI = DAG.getExtractVectorElt(DL, VT: MaskScalarVT, Vec: Mask, Idx: I);
14087 MaskI = DAG.getFreeze(V: MaskI);
14088 MaskI = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: MaskI);
14089 MaskI = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PositionVT, Operand: MaskI);
14090 OutPos = DAG.getNode(Opcode: ISD::ADD, DL, VT: PositionVT, N1: OutPos, N2: MaskI);
14091
14092 if (HasPassthru && I == NumElms - 1) {
14093 SDValue EndOfVector =
14094 DAG.getConstant(Val: VecVT.getVectorNumElements() - 1, DL, VT: PositionVT);
14095 SDValue AllLanesSelected =
14096 DAG.getSetCC(DL, VT: MVT::i1, LHS: OutPos, RHS: EndOfVector, Cond: ISD::CondCode::SETUGT);
14097 OutPos = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PositionVT, N1: OutPos, N2: EndOfVector);
14098 OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
14099
14100 // Re-write the last ValI if all lanes were selected. Otherwise,
14101 // overwrite the last write it with the passthru value.
14102 LastWriteVal = DAG.getSelect(DL, VT: ScalarVT, Cond: AllLanesSelected, LHS: ValI,
14103 RHS: LastWriteVal, Flags: SDNodeFlags::Unpredictable);
14104 Chain = DAG.getStore(
14105 Chain, dl: DL, Val: LastWriteVal, Ptr: OutPtr,
14106 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
14107 }
14108 }
14109
14110 return DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo, Alignment);
14111}
14112
14113SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
14114 SDLoc DL(Node);
14115 EVT VT = Node->getValueType(ResNo: 0);
14116 SDValue Op = Node->getOperand(Num: 0);
14117 ElementCount EC = Op.getValueType().getVectorElementCount();
14118
14119 bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
14120 auto [Mask, StepVec] = getLegalMaskAndStepVector(Mask: Op, ZeroIsPoison, DL, DAG);
14121
14122 // No legal step vector: split mask in half and recombine results.
14123 // LoNumElts uses the non-poison CTTZ_ELTS so its result is well-defined
14124 // (== LoNumElts when no active lane), allowing the SETNE comparison.
14125 // Result: (ResLo != LoNumElts) ? ResLo : (LoNumElts + ResHi)
14126 if (!StepVec) {
14127 EVT ResVT = Node->getValueType(ResNo: 0);
14128 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Op, DL);
14129 SDValue LoNumElts = DAG.getElementCount(
14130 DL, VT: ResVT, EC: MaskLo.getValueType().getVectorElementCount());
14131 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: MaskLo);
14132 SDValue ResHi = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Operand: MaskHi);
14133 SDValue ResLoNotNumElts = DAG.getSetCC(
14134 DL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ResVT),
14135 LHS: ResLo, RHS: LoNumElts, Cond: ISD::SETNE);
14136 // Per LangRef, ResVT must be wide enough to hold the total element count,
14137 // so the sum cannot wrap as an unsigned add. NSW is not guaranteed since
14138 // the count is only required to fit unsigned.
14139 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: LoNumElts, N2: ResHi,
14140 Flags: SDNodeFlags::NoUnsignedWrap);
14141 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotNumElts, LHS: ResLo, RHS: Sum);
14142 }
14143
14144 EVT StepVecVT = StepVec.getValueType();
14145 EVT StepVT = StepVecVT.getVectorElementType();
14146
14147 // Promote the scalar result type early to avoid redundant zexts.
14148 if (getTypeAction(VT: StepVT.getSimpleVT()) == TypePromoteInteger)
14149 StepVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVT);
14150
14151 SDValue VL = DAG.getElementCount(DL, VT: StepVT, EC);
14152 SDValue SplatVL = DAG.getSplat(VT: StepVecVT, DL, Op: VL);
14153 StepVec = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVecVT, N1: SplatVL, N2: StepVec);
14154 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
14155 SDValue Select = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
14156 SDValue Max = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL,
14157 VT: StepVecVT.getVectorElementType(), Operand: Select);
14158 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVT, N1: VL,
14159 N2: DAG.getZExtOrTrunc(Op: Max, DL, VT: StepVT));
14160
14161 return DAG.getZExtOrTrunc(Op: Sub, DL, VT);
14162}
14163
14164SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
14165 SDLoc DL(N);
14166 SDValue Source = N->getOperand(Num: 0);
14167 SDValue Needle = N->getOperand(Num: 1);
14168 SDValue Mask = N->getOperand(Num: 2);
14169 EVT SourceVT = Source.getValueType();
14170 EVT NeedleVT = Needle.getValueType();
14171 EVT ResVT = N->getValueType(ResNo: 0);
14172 EVT CmpVT =
14173 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SourceVT);
14174
14175 assert(NeedleVT.isFixedLengthVector() && "Needle must be a fixed vector");
14176
14177 SDValue Ret = DAG.getConstant(Val: 0, DL, VT: CmpVT);
14178 EVT NeedleEltVT = NeedleVT.getVectorElementType();
14179 for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
14180 SDValue Splat;
14181 if (NeedleVT == SourceVT) {
14182 // Prefer a shuffle over scalar extracts + splat for fixed vectors.
14183 Splat = DAG.getVectorShuffle(
14184 VT: SourceVT, dl: DL, N1: Needle, N2: DAG.getUNDEF(VT: SourceVT),
14185 Mask: SmallVector<int>(NeedleVT.getVectorNumElements(), I));
14186 } else {
14187 SDValue NeedleElt = DAG.getExtractVectorElt(DL, VT: NeedleEltVT, Vec: Needle, Idx: I);
14188 Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: SourceVT, Operand: NeedleElt);
14189 }
14190 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Source, RHS: Splat, Cond: ISD::SETEQ);
14191 Ret = DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Ret, N2: Cmp);
14192 }
14193
14194 EVT UseVT = ResVT;
14195 // If the result is immediately truncated, only extend to that type (to avoid
14196 // unnecessary sign/zero extends).
14197 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TRUNCATE)
14198 UseVT = N->user_begin()->getValueType(ResNo: 0);
14199
14200 Mask = DAG.getBoolExtOrTrunc(Op: Mask, SL: DL, VT: UseVT, OpVT: Mask.getValueType());
14201 Ret = DAG.getBoolExtOrTrunc(Op: Ret, SL: DL, VT: UseVT, OpVT: Ret.getValueType());
14202
14203 Ret = DAG.getNode(Opcode: ISD::AND, DL, VT: UseVT, N1: Ret, N2: Mask);
14204 if (UseVT != ResVT)
14205 Ret = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ResVT, Operand: Ret);
14206 return Ret;
14207}
14208
14209SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
14210 SelectionDAG &DAG) const {
14211 SDLoc DL(N);
14212 SDValue Acc = N->getOperand(Num: 0);
14213 SDValue MulLHS = N->getOperand(Num: 1);
14214 SDValue MulRHS = N->getOperand(Num: 2);
14215 EVT AccVT = Acc.getValueType();
14216 EVT MulOpVT = MulLHS.getValueType();
14217
14218 EVT ExtMulOpVT =
14219 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccVT.getVectorElementType(),
14220 EC: MulOpVT.getVectorElementCount());
14221
14222 unsigned ExtOpcLHS, ExtOpcRHS;
14223 switch (N->getOpcode()) {
14224 default:
14225 llvm_unreachable("Unexpected opcode");
14226 case ISD::PARTIAL_REDUCE_UMLA:
14227 ExtOpcLHS = ExtOpcRHS = ISD::ZERO_EXTEND;
14228 break;
14229 case ISD::PARTIAL_REDUCE_SMLA:
14230 ExtOpcLHS = ExtOpcRHS = ISD::SIGN_EXTEND;
14231 break;
14232 case ISD::PARTIAL_REDUCE_SUMLA:
14233 ExtOpcLHS = ISD::SIGN_EXTEND;
14234 ExtOpcRHS = ISD::ZERO_EXTEND;
14235 break;
14236 case ISD::PARTIAL_REDUCE_FMLA:
14237 ExtOpcLHS = ExtOpcRHS = ISD::FP_EXTEND;
14238 break;
14239 }
14240
14241 // A wide partial reduction is built from a ladder of narrower ones, a rung
14242 // at a time, each halving the element count and doubling the width.
14243 unsigned Opc = N->getOpcode();
14244 ElementCount MulEC = MulOpVT.getVectorElementCount();
14245 ElementCount AccEC = AccVT.getVectorElementCount();
14246 unsigned CountRatio =
14247 MulEC.hasKnownScalarFactor(RHS: AccEC) ? MulEC.getKnownScalarFactor(RHS: AccEC) : 0;
14248 unsigned WidthRatio =
14249 AccVT.getScalarSizeInBits() / MulOpVT.getScalarSizeInBits();
14250 if (Opc != ISD::PARTIAL_REDUCE_FMLA && CountRatio > 2 && WidthRatio >= 2) {
14251 LLVMContext &Ctx = *DAG.getContext();
14252 EVT ProdVT = MulOpVT.widenIntegerVectorElementType(Context&: Ctx);
14253
14254 // A pure reduction peels one rung and re-enters.
14255 if (llvm::isOneOrOneSplat(V: MulRHS)) {
14256 EVT RungVT = ProdVT.getHalfNumVectorElementsVT(Context&: Ctx);
14257 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc,
14258 N2: DAG.getNode(Opcode: Opc, DL, VT: RungVT,
14259 N1: DAG.getConstant(Val: 0, DL, VT: RungVT), N2: MulLHS,
14260 N3: MulRHS),
14261 N3: DAG.getConstant(Val: 1, DL, VT: RungVT));
14262 }
14263
14264 // A multiply widens the products by one rung, which legalizes back into a
14265 // widening multiply per half, and the ladder re-enters as a plain sum.
14266 SDValue Prod = DAG.getNode(Opcode: ISD::MUL, DL, VT: ProdVT,
14267 N1: DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ProdVT, Operand: MulLHS),
14268 N2: DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ProdVT, Operand: MulRHS));
14269 auto [Lo, Hi] = DAG.SplitVector(N: Prod, DL);
14270 SDValue One = DAG.getConstant(Val: 1, DL, VT: Lo.getValueType());
14271
14272 // The halves meet at the narrowest rung, so the accumulator is added once.
14273 EVT MidVT = Lo.getValueType()
14274 .widenIntegerVectorElementType(Context&: Ctx)
14275 .getHalfNumVectorElementsVT(Context&: Ctx);
14276 if (ElementCount::isKnownLE(LHS: MidVT.getVectorElementCount(), RHS: AccEC))
14277 return DAG.getNode(Opcode: Opc, DL, VT: AccVT,
14278 N1: DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Lo, N3: One), N2: Hi, N3: One);
14279 SDValue Mid =
14280 DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: DAG.getConstant(Val: 0, DL, VT: MidVT), N2: Lo, N3: One);
14281 Mid = DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: Mid, N2: Hi, N3: One);
14282 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Mid, N3: DAG.getConstant(Val: 1, DL, VT: MidVT));
14283 }
14284
14285 if (ExtMulOpVT != MulOpVT) {
14286 MulLHS = DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ExtMulOpVT, Operand: MulLHS);
14287 MulRHS = DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ExtMulOpVT, Operand: MulRHS);
14288 }
14289 SDValue Input = MulLHS;
14290 // Maintain the FP flags (if any).
14291 SDNodeFlags Flags = N->getFlags();
14292 if (N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA) {
14293 if (!llvm::isOneOrOneSplatFP(V: MulRHS))
14294 Input = DAG.getNode(Opcode: ISD::FMUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS, Flags);
14295 } else if (!llvm::isOneOrOneSplat(V: MulRHS)) {
14296 Input = DAG.getNode(Opcode: ISD::MUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
14297 }
14298
14299 unsigned Stride = AccVT.getVectorMinNumElements();
14300 unsigned ScaleFactor = MulOpVT.getVectorMinNumElements() / Stride;
14301
14302 // Collect all of the subvectors
14303 std::deque<SDValue> Subvectors = {Acc};
14304 for (unsigned I = 0; I < ScaleFactor; I++)
14305 Subvectors.push_back(x: DAG.getExtractSubvector(DL, VT: AccVT, Vec: Input, Idx: I * Stride));
14306
14307 unsigned FlatNode =
14308 N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA ? ISD::FADD : ISD::ADD;
14309
14310 // Flatten the subvector tree
14311 while (Subvectors.size() > 1) {
14312 Subvectors.push_back(x: DAG.getNode(Opcode: FlatNode, DL, VT: AccVT,
14313 Ops: {Subvectors[0], Subvectors[1]}, Flags));
14314 Subvectors.pop_front();
14315 Subvectors.pop_front();
14316 }
14317
14318 assert(Subvectors.size() == 1 &&
14319 "There should only be one subvector after tree flattening");
14320
14321 return Subvectors[0];
14322}
14323
14324/// Given a store node \p StoreNode, return true if it is safe to fold that node
14325/// into \p FPNode, which expands to a library call with output pointers.
14326static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
14327 SDNode *FPNode) {
14328 SmallVector<const SDNode *, 8> Worklist;
14329 SmallVector<const SDNode *, 8> DeferredNodes;
14330 SmallPtrSet<const SDNode *, 16> Visited;
14331
14332 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
14333 for (SDValue Op : StoreNode->ops())
14334 if (Op.getNode() != FPNode)
14335 Worklist.push_back(Elt: Op.getNode());
14336
14337 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
14338 while (!Worklist.empty()) {
14339 const SDNode *Node = Worklist.pop_back_val();
14340 auto [_, Inserted] = Visited.insert(Ptr: Node);
14341 if (!Inserted)
14342 continue;
14343
14344 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
14345 return false;
14346
14347 // Reached the FPNode (would result in a cycle).
14348 // OR Reached CALLSEQ_START (would result in nested call sequences).
14349 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
14350 return false;
14351
14352 if (Node->getOpcode() == ISD::CALLSEQ_END) {
14353 // Defer looking into call sequences (so we can check we're outside one).
14354 // We still need to look through these for the predecessor check.
14355 DeferredNodes.push_back(Elt: Node);
14356 continue;
14357 }
14358
14359 for (SDValue Op : Node->ops())
14360 Worklist.push_back(Elt: Op.getNode());
14361 }
14362
14363 // True if we're outside a call sequence and don't have the FPNode as a
14364 // predecessor. No cycles or nested call sequences possible.
14365 return !SDNode::hasPredecessorHelper(N: FPNode, Visited, Worklist&: DeferredNodes,
14366 MaxSteps);
14367}
14368
14369bool TargetLowering::expandMultipleResultFPLibCall(
14370 SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
14371 SmallVectorImpl<SDValue> &Results,
14372 std::optional<unsigned> CallRetResNo) const {
14373 if (LC == RTLIB::UNKNOWN_LIBCALL)
14374 return false;
14375
14376 RTLIB::LibcallImpl LibcallImpl =
14377 getLibcallLoweringInfo().getVectorLibcallImplOrMasked(Call: LC);
14378 if (LibcallImpl == RTLIB::Unsupported)
14379 return false;
14380
14381 LLVMContext &Ctx = *DAG.getContext();
14382 EVT VT = Node->getValueType(ResNo: 0);
14383 unsigned NumResults = Node->getNumValues();
14384
14385 // Find users of the node that store the results (and share input chains). The
14386 // destination pointers can be used instead of creating stack allocations.
14387 SDValue StoresInChain;
14388 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
14389 for (SDNode *User : Node->users()) {
14390 if (!ISD::isNormalStore(N: User))
14391 continue;
14392 auto *ST = cast<StoreSDNode>(Val: User);
14393 SDValue StoreValue = ST->getValue();
14394 unsigned ResNo = StoreValue.getResNo();
14395 // Ensure the store corresponds to an output pointer.
14396 if (CallRetResNo == ResNo)
14397 continue;
14398 // Ensure the store to the default address space and not atomic or volatile.
14399 if (!ST->isSimple() || ST->getAddressSpace() != 0)
14400 continue;
14401 // Ensure all store chains are the same (so they don't alias).
14402 if (StoresInChain && ST->getChain() != StoresInChain)
14403 continue;
14404 // Ensure the store is properly aligned.
14405 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Context&: Ctx);
14406 if (ST->getAlign() <
14407 DAG.getDataLayout().getABITypeAlign(Ty: StoreType->getScalarType()))
14408 continue;
14409 // Avoid:
14410 // 1. Creating cyclic dependencies.
14411 // 2. Expanding the node to a call within a call sequence.
14412 if (!canFoldStoreIntoLibCallOutputPointers(StoreNode: ST, FPNode: Node))
14413 continue;
14414 ResultStores[ResNo] = ST;
14415 StoresInChain = ST->getChain();
14416 }
14417
14418 ArgListTy Args;
14419
14420 // Pass the arguments.
14421 for (const SDValue &Op : Node->op_values()) {
14422 EVT ArgVT = Op.getValueType();
14423 Type *ArgTy = ArgVT.getTypeForEVT(Context&: Ctx);
14424 Args.emplace_back(args: Op, args&: ArgTy);
14425 }
14426
14427 // Pass the output pointers.
14428 SmallVector<SDValue, 2> ResultPtrs(NumResults);
14429 Type *PointerTy = PointerType::getUnqual(C&: Ctx);
14430 for (auto [ResNo, ST] : llvm::enumerate(First&: ResultStores)) {
14431 if (ResNo == CallRetResNo)
14432 continue;
14433 EVT ResVT = Node->getValueType(ResNo);
14434 SDValue ResultPtr = ST ? ST->getBasePtr() : DAG.CreateStackTemporary(VT: ResVT);
14435 ResultPtrs[ResNo] = ResultPtr;
14436 Args.emplace_back(args&: ResultPtr, args&: PointerTy);
14437 }
14438
14439 SDLoc DL(Node);
14440
14441 if (RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(Impl: LibcallImpl)) {
14442 // Pass the vector mask (if required).
14443 EVT MaskVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
14444 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT);
14445 Args.emplace_back(args&: Mask, args: MaskVT.getTypeForEVT(Context&: Ctx));
14446 }
14447
14448 Type *RetType = CallRetResNo.has_value()
14449 ? Node->getValueType(ResNo: *CallRetResNo).getTypeForEVT(Context&: Ctx)
14450 : Type::getVoidTy(C&: Ctx);
14451 SDValue InChain = StoresInChain ? StoresInChain : DAG.getEntryNode();
14452 SDValue Callee =
14453 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
14454 TargetLowering::CallLoweringInfo CLI(DAG);
14455 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
14456 CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetType, Target: Callee, ArgsList: std::move(Args));
14457
14458 auto [Call, CallChain] = LowerCallTo(CLI);
14459
14460 for (auto [ResNo, ResultPtr] : llvm::enumerate(First&: ResultPtrs)) {
14461 if (ResNo == CallRetResNo) {
14462 Results.push_back(Elt: Call);
14463 continue;
14464 }
14465 MachinePointerInfo PtrInfo;
14466 SDValue LoadResult = DAG.getLoad(VT: Node->getValueType(ResNo), dl: DL, Chain: CallChain,
14467 Ptr: ResultPtr, PtrInfo);
14468 SDValue OutChain = LoadResult.getValue(R: 1);
14469
14470 if (StoreSDNode *ST = ResultStores[ResNo]) {
14471 // Replace store with the library call.
14472 DAG.ReplaceAllUsesOfValueWith(From: SDValue(ST, 0), To: OutChain);
14473 PtrInfo = ST->getPointerInfo();
14474 } else {
14475 PtrInfo = MachinePointerInfo::getFixedStack(
14476 MF&: DAG.getMachineFunction(),
14477 FI: cast<FrameIndexSDNode>(Val&: ResultPtr)->getIndex());
14478 }
14479
14480 Results.push_back(Elt: LoadResult);
14481 }
14482
14483 return true;
14484}
14485
14486bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
14487 SDValue &LHS, SDValue &RHS,
14488 SDValue &CC, bool &NeedInvert,
14489 const SDLoc &dl, SDValue &Chain,
14490 bool IsSignaling) const {
14491 MVT OpVT = LHS.getSimpleValueType();
14492 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
14493 NeedInvert = false;
14494 switch (getCondCodeAction(CC: CCCode, VT: OpVT)) {
14495 default:
14496 llvm_unreachable("Unknown condition code action!");
14497 case TargetLowering::Legal:
14498 // Nothing to do.
14499 break;
14500 case TargetLowering::Expand: {
14501 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(Operation: CCCode);
14502 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14503 std::swap(a&: LHS, b&: RHS);
14504 CC = DAG.getCondCode(Cond: InvCC);
14505 return true;
14506 }
14507 // Swapping operands didn't work. Try inverting the condition.
14508 bool NeedSwap = false;
14509 InvCC = getSetCCInverse(Operation: CCCode, Type: OpVT);
14510 if (!isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14511 // If inverting the condition is not enough, try swapping operands
14512 // on top of it.
14513 InvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
14514 NeedSwap = true;
14515 }
14516 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14517 CC = DAG.getCondCode(Cond: InvCC);
14518 NeedInvert = true;
14519 if (NeedSwap)
14520 std::swap(a&: LHS, b&: RHS);
14521 return true;
14522 }
14523
14524 // Special case: expand i1 comparisons using logical operations.
14525 if (OpVT == MVT::i1) {
14526 SDValue Ret;
14527 switch (CCCode) {
14528 default:
14529 llvm_unreachable("Unknown integer setcc!");
14530 case ISD::SETEQ: // X == Y --> ~(X ^ Y)
14531 Ret = DAG.getNOT(DL: dl, Val: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS),
14532 VT: MVT::i1);
14533 break;
14534 case ISD::SETNE: // X != Y --> (X ^ Y)
14535 Ret = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS);
14536 break;
14537 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
14538 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
14539 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: RHS,
14540 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14541 break;
14542 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
14543 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
14544 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: LHS,
14545 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14546 break;
14547 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
14548 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
14549 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: RHS,
14550 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14551 break;
14552 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
14553 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
14554 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: LHS,
14555 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14556 break;
14557 }
14558
14559 LHS = DAG.getZExtOrTrunc(Op: Ret, DL: dl, VT);
14560 RHS = SDValue();
14561 CC = SDValue();
14562 return true;
14563 }
14564
14565 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
14566 unsigned Opc = 0;
14567 switch (CCCode) {
14568 default:
14569 llvm_unreachable("Don't know how to expand this condition!");
14570 case ISD::SETUO:
14571 if (isCondCodeLegal(CC: ISD::SETUNE, VT: OpVT)) {
14572 CC1 = ISD::SETUNE;
14573 CC2 = ISD::SETUNE;
14574 Opc = ISD::OR;
14575 break;
14576 }
14577 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14578 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
14579 NeedInvert = true;
14580 [[fallthrough]];
14581 case ISD::SETO:
14582 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14583 "If SETO is expanded, SETOEQ must be legal!");
14584 CC1 = ISD::SETOEQ;
14585 CC2 = ISD::SETOEQ;
14586 Opc = ISD::AND;
14587 break;
14588 case ISD::SETONE:
14589 case ISD::SETUEQ:
14590 // If the SETUO or SETO CC isn't legal, we might be able to use
14591 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
14592 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
14593 // the operands.
14594 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14595 if (!isCondCodeLegal(CC: CC2, VT: OpVT) && (isCondCodeLegal(CC: ISD::SETOGT, VT: OpVT) ||
14596 isCondCodeLegal(CC: ISD::SETOLT, VT: OpVT))) {
14597 CC1 = ISD::SETOGT;
14598 CC2 = ISD::SETOLT;
14599 Opc = ISD::OR;
14600 NeedInvert = ((unsigned)CCCode & 0x8U);
14601 break;
14602 }
14603 [[fallthrough]];
14604 case ISD::SETOEQ:
14605 case ISD::SETOGT:
14606 case ISD::SETOGE:
14607 case ISD::SETOLT:
14608 case ISD::SETOLE:
14609 case ISD::SETUNE:
14610 case ISD::SETUGT:
14611 case ISD::SETUGE:
14612 case ISD::SETULT:
14613 case ISD::SETULE:
14614 // If we are floating point, assign and break, otherwise fall through.
14615 if (!OpVT.isInteger()) {
14616 // We can use the 4th bit to tell if we are the unordered
14617 // or ordered version of the opcode.
14618 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14619 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
14620 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
14621 break;
14622 }
14623 // Fallthrough if we are unsigned integer.
14624 [[fallthrough]];
14625 case ISD::SETLE:
14626 case ISD::SETGT:
14627 case ISD::SETGE:
14628 case ISD::SETLT:
14629 case ISD::SETNE:
14630 case ISD::SETEQ:
14631 // If all combinations of inverting the condition and swapping operands
14632 // didn't work then we have no means to expand the condition.
14633 llvm_unreachable("Don't know how to expand this condition!");
14634 }
14635
14636 SDValue SetCC1, SetCC2;
14637 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
14638 // If we aren't the ordered or unorder operation,
14639 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
14640 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC1, Chain, IsSignaling);
14641 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC2, Chain, IsSignaling);
14642 } else {
14643 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
14644 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS: LHS, Cond: CC1, Chain, IsSignaling);
14645 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS: RHS, RHS, Cond: CC2, Chain, IsSignaling);
14646 }
14647 if (Chain)
14648 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: SetCC1.getValue(R: 1),
14649 N2: SetCC2.getValue(R: 1));
14650 LHS = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SetCC1, N2: SetCC2);
14651 RHS = SDValue();
14652 CC = SDValue();
14653 return true;
14654 }
14655 }
14656 return false;
14657}
14658
14659SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
14660 SelectionDAG &DAG) const {
14661 EVT VT = Node->getValueType(ResNo: 0);
14662 // Despite its documentation, GetSplitDestVTs will assert if VT cannot be
14663 // split into two equal parts.
14664 if (!VT.isVector() || !VT.getVectorElementCount().isKnownMultipleOf(RHS: 2))
14665 return SDValue();
14666
14667 // Restrict expansion to cases where both parts can be concatenated.
14668 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
14669 if (LoVT != HiVT || !isTypeLegal(VT: LoVT))
14670 return SDValue();
14671
14672 SDLoc DL(Node);
14673 unsigned Opcode = Node->getOpcode();
14674
14675 // Don't expand if the result is likely to be unrolled anyway.
14676 if (!isOperationLegalOrCustomOrPromote(Op: Opcode, VT: LoVT))
14677 return SDValue();
14678
14679 SmallVector<SDValue, 4> LoOps, HiOps;
14680 for (const SDValue &V : Node->op_values()) {
14681 if (!V.getValueType().isVector()) {
14682 // Scalar operands pass through to both halves unchanged.
14683 LoOps.push_back(Elt: V);
14684 HiOps.push_back(Elt: V);
14685 continue;
14686 }
14687 auto [Lo, Hi] = DAG.SplitVector(N: V, DL, LoVT, HiVT);
14688 LoOps.push_back(Elt: Lo);
14689 HiOps.push_back(Elt: Hi);
14690 }
14691
14692 SDValue SplitOpLo = DAG.getNode(Opcode, DL, VT: LoVT, Ops: LoOps, Flags: Node->getFlags());
14693 SDValue SplitOpHi = DAG.getNode(Opcode, DL, VT: HiVT, Ops: HiOps, Flags: Node->getFlags());
14694 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: SplitOpLo, N2: SplitOpHi);
14695}
14696
14697SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
14698 const SDLoc &DL,
14699 EVT InVecVT, SDValue EltNo,
14700 LoadSDNode *OriginalLoad,
14701 SelectionDAG &DAG) const {
14702 assert(OriginalLoad->isSimple());
14703
14704 EVT VecEltVT = InVecVT.getVectorElementType();
14705
14706 // If the vector element type is not a multiple of a byte then we are unable
14707 // to correctly compute an address to load only the extracted element as a
14708 // scalar.
14709 if (!VecEltVT.isByteSized())
14710 return SDValue();
14711
14712 ISD::LoadExtType ExtTy =
14713 ResultVT.bitsGT(VT: VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
14714 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: VecEltVT))
14715 return SDValue();
14716
14717 std::optional<unsigned> ByteOffset;
14718 Align Alignment = OriginalLoad->getAlign();
14719 MachinePointerInfo MPI;
14720 if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo)) {
14721 int Elt = ConstEltNo->getZExtValue();
14722 ByteOffset = VecEltVT.getSizeInBits() * Elt / 8;
14723 MPI = OriginalLoad->getPointerInfo().getWithOffset(O: *ByteOffset);
14724 Alignment = commonAlignment(A: Alignment, Offset: *ByteOffset);
14725 } else {
14726 // Discard the pointer info except the address space because the memory
14727 // operand can't represent this new access since the offset is variable.
14728 MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
14729 Alignment = commonAlignment(A: Alignment, Offset: VecEltVT.getSizeInBits() / 8);
14730 }
14731
14732 if (!shouldReduceLoadWidth(Load: OriginalLoad, ExtTy, NewVT: VecEltVT, ByteOffset))
14733 return SDValue();
14734
14735 unsigned IsFast = 0;
14736 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: VecEltVT,
14737 AddrSpace: OriginalLoad->getAddressSpace(), Alignment,
14738 Flags: OriginalLoad->getMemOperand()->getFlags(), Fast: &IsFast) ||
14739 !IsFast)
14740 return SDValue();
14741
14742 // The original DAG loaded the entire vector from memory, so arithmetic
14743 // within it must be inbounds.
14744 SDValue NewPtr = getInboundsVectorElementPointer(
14745 DAG, VecPtr: OriginalLoad->getBasePtr(), VecVT: InVecVT, Index: EltNo);
14746
14747 // We are replacing a vector load with a scalar load. The new load must have
14748 // identical memory op ordering to the original.
14749 SDValue Load;
14750 if (ResultVT.bitsGT(VT: VecEltVT)) {
14751 // If the result type of vextract is wider than the load, then issue an
14752 // extending load instead.
14753 ISD::LoadExtType ExtType =
14754 isLoadLegal(ValVT: ResultVT, MemVT: VecEltVT, Alignment,
14755 AddrSpace: OriginalLoad->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false)
14756 ? ISD::ZEXTLOAD
14757 : ISD::EXTLOAD;
14758 Load = DAG.getExtLoad(ExtType, dl: DL, VT: ResultVT, Chain: OriginalLoad->getChain(),
14759 Ptr: NewPtr, PtrInfo: MPI, MemVT: VecEltVT, Alignment,
14760 MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14761 Metadata: OriginalLoad->getMMOMetadataForSubAccess());
14762 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14763 } else {
14764 // The result type is narrower or the same width as the vector element
14765 Load = DAG.getLoad(VT: VecEltVT, dl: DL, Chain: OriginalLoad->getChain(), Ptr: NewPtr, PtrInfo: MPI,
14766 Alignment, MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14767 Metadata: OriginalLoad->getMMOMetadataForSubAccess());
14768 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14769 if (ResultVT.bitsLT(VT: VecEltVT))
14770 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: Load);
14771 else
14772 Load = DAG.getBitcast(VT: ResultVT, V: Load);
14773 }
14774
14775 return Load;
14776}
14777
14778// Set type id for call site info and metadata 'call_target'.
14779// We are filtering for:
14780// a) The call-graph-section use case that wants to know about indirect
14781// calls, or
14782// b) We want to annotate indirect calls.
14783void TargetLowering::setTypeIdForCallsiteInfo(
14784 const CallBase *CB, MachineFunction &MF,
14785 MachineFunction::CallSiteInfo &CSInfo) const {
14786 if (CB && CB->isIndirectCall() &&
14787 (MF.getTarget().Options.EmitCallGraphSection ||
14788 MF.getTarget().Options.EmitCallSiteInfo))
14789 CSInfo = MachineFunction::CallSiteInfo(*CB);
14790}
14791