1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/Analysis/VectorUtils.h"
17#include "llvm/CodeGen/Analysis.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/CodeGenCommonISel.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/SDPatternMatch.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/GlobalVariable.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCExpr.h"
33#include "llvm/Support/DivisionByConstantInfo.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Target/TargetMachine.h"
38#include <cctype>
39#include <deque>
40#include <list>
41using namespace llvm;
42using namespace llvm::SDPatternMatch;
43
44/// NOTE: The TargetMachine owns TLOF.
45TargetLowering::TargetLowering(const TargetMachine &tm,
46 const TargetSubtargetInfo &STI)
47 : TargetLoweringBase(tm, STI) {}
48
49// Define the virtual destructor out-of-line for build efficiency.
50TargetLowering::~TargetLowering() = default;
51
52const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
53 return nullptr;
54}
55
56bool TargetLowering::isPositionIndependent() const {
57 return getTargetMachine().isPositionIndependent();
58}
59
60/// Check whether a given call node is in tail position within its function. If
61/// so, it sets Chain to the input chain of the tail call.
62bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
63 SDValue &Chain) const {
64 const Function &F = DAG.getMachineFunction().getFunction();
65
66 // First, check if tail calls have been disabled in this function.
67 if (F.getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
68 return false;
69
70 // Conservatively require the attributes of the call to match those of
71 // the return. Ignore following attributes because they don't affect the
72 // call sequence.
73 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
74 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
75 Attribute::DereferenceableOrNull, Attribute::NoAlias,
76 Attribute::NonNull, Attribute::NoUndef,
77 Attribute::Range, Attribute::NoFPClass})
78 CallerAttrs.removeAttribute(Val: Attr);
79
80 if (CallerAttrs.hasAttributes())
81 return false;
82
83 // It's not safe to eliminate the sign / zero extension of the return value.
84 if (CallerAttrs.contains(A: Attribute::ZExt) ||
85 CallerAttrs.contains(A: Attribute::SExt))
86 return false;
87
88 // Check if the only use is a function return node.
89 return isUsedByReturnOnly(Node, Chain);
90}
91
92bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
93 const uint32_t *CallerPreservedMask,
94 const SmallVectorImpl<CCValAssign> &ArgLocs,
95 const SmallVectorImpl<SDValue> &OutVals) const {
96 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
97 const CCValAssign &ArgLoc = ArgLocs[I];
98 if (!ArgLoc.isRegLoc())
99 continue;
100 MCRegister Reg = ArgLoc.getLocReg();
101 // Only look at callee saved registers.
102 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg: Reg))
103 continue;
104 // Check that we pass the value used for the caller.
105 // (We look for a CopyFromReg reading a virtual register that is used
106 // for the function live-in value of register Reg)
107 SDValue Value = OutVals[I];
108 if (Value->getOpcode() == ISD::AssertZext)
109 Value = Value.getOperand(i: 0);
110 if (Value->getOpcode() != ISD::CopyFromReg)
111 return false;
112 Register ArgReg = cast<RegisterSDNode>(Val: Value->getOperand(Num: 1))->getReg();
113 if (MRI.getLiveInPhysReg(VReg: ArgReg) != Reg)
114 return false;
115 }
116 return true;
117}
118
119static bool paramHasAttr(const CallBase &Call, unsigned ArgIdx,
120 Attribute::AttrKind Kind) {
121 return Call.paramHasAttr(ArgNo: ArgIdx, Kind);
122}
123
124static bool paramHasAttr(const AttributeList &Attrs, unsigned ArgIdx,
125 Attribute::AttrKind Kind) {
126 return Attrs.hasParamAttr(ArgNo: ArgIdx, Kind);
127}
128
129static MaybeAlign getParamStackAlign(const CallBase &Call, unsigned ArgIdx) {
130 return Call.getParamStackAlign(ArgNo: ArgIdx);
131}
132
133static MaybeAlign getParamStackAlign(const AttributeList &Attrs,
134 unsigned ArgIdx) {
135 return Attrs.getParamStackAlignment(ArgNo: ArgIdx);
136}
137
138static MaybeAlign getParamAlign(const CallBase &Call, unsigned ArgIdx) {
139 return Call.getParamAlign(ArgNo: ArgIdx);
140}
141
142static MaybeAlign getParamAlign(const AttributeList &Attrs, unsigned ArgIdx) {
143 return Attrs.getParamAlignment(ArgNo: ArgIdx);
144}
145
146/// Set CallLoweringInfo attribute flags based on a call instruction
147/// and called function attributes.
148template <typename SourceT>
149static void setArgListEntryAttributes(TargetLoweringBase::ArgListEntry &Entry,
150 const SourceT &Src, unsigned ArgIdx) {
151 Entry.IsSExt = paramHasAttr(Src, ArgIdx, Attribute::SExt);
152 Entry.IsZExt = paramHasAttr(Src, ArgIdx, Attribute::ZExt);
153 Entry.IsNoExt = paramHasAttr(Src, ArgIdx, Attribute::NoExt);
154 Entry.IsInReg = paramHasAttr(Src, ArgIdx, Attribute::InReg);
155 Entry.IsSRet = paramHasAttr(Src, ArgIdx, Attribute::StructRet);
156 Entry.IsNest = paramHasAttr(Src, ArgIdx, Attribute::Nest);
157 Entry.IsByVal = paramHasAttr(Src, ArgIdx, Attribute::ByVal);
158 Entry.IsPreallocated = paramHasAttr(Src, ArgIdx, Attribute::Preallocated);
159 Entry.IsInAlloca = paramHasAttr(Src, ArgIdx, Attribute::InAlloca);
160 Entry.IsReturned = paramHasAttr(Src, ArgIdx, Attribute::Returned);
161 Entry.IsSwiftSelf = paramHasAttr(Src, ArgIdx, Attribute::SwiftSelf);
162 Entry.IsSwiftAsync = paramHasAttr(Src, ArgIdx, Attribute::SwiftAsync);
163 Entry.IsSwiftError = paramHasAttr(Src, ArgIdx, Attribute::SwiftError);
164 Entry.Alignment = getParamStackAlign(Src, ArgIdx);
165 Entry.IndirectType = nullptr;
166 assert(Entry.IsByVal + Entry.IsPreallocated + Entry.IsInAlloca +
167 Entry.IsSRet <=
168 1 &&
169 "multiple ABI attributes?");
170 if (Entry.IsByVal) {
171 Entry.IndirectType = Src.getParamByValType(ArgIdx);
172 if (!Entry.Alignment)
173 Entry.Alignment = getParamAlign(Src, ArgIdx);
174 }
175 if (Entry.IsPreallocated)
176 Entry.IndirectType = Src.getParamPreallocatedType(ArgIdx);
177 if (Entry.IsInAlloca)
178 Entry.IndirectType = Src.getParamInAllocaType(ArgIdx);
179 if (Entry.IsSRet)
180 Entry.IndirectType = Src.getParamStructRetType(ArgIdx);
181}
182
183void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
184 unsigned ArgIdx) {
185 setArgListEntryAttributes(Entry&: *this, Src: *Call, ArgIdx);
186}
187
188void TargetLoweringBase::ArgListEntry::setAttributes(const AttributeList &Attrs,
189 unsigned ArgIdx) {
190 setArgListEntryAttributes(Entry&: *this, Src: Attrs, ArgIdx);
191}
192
193void TargetLoweringBase::ArgListEntry::setExtAttribute(
194 Attribute::AttrKind Attr) {
195 if (Attr != Attribute::None) {
196 if (Attr == Attribute::SExt)
197 IsSExt = true;
198 else if (Attr == Attribute::ZExt)
199 IsZExt = true;
200 else if (Attr == Attribute::NoExt)
201 IsNoExt = true;
202 else
203 llvm_unreachable("Unexpected extension attribute!");
204 }
205}
206
207TargetLowering::ArgListTy
208TargetLowering::getArgListForFunctionType(FunctionType *FuncTy,
209 const AttributeList &FuncAttrs,
210 ArrayRef<SDValue> Ops) {
211 // TODO: This assumes each parameter maps to exactly one operand node, which
212 // does not hold when an argument requires type splitting.
213 assert(Ops.size() == FuncTy->getNumParams() &&
214 "argument count does not match the function type");
215 ArgListTy Args;
216 Args.reserve(n: Ops.size());
217 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
218 ArgListEntry Entry(Ops[I], FuncTy->getParamType(i: I));
219 Entry.setAttributes(Attrs: FuncAttrs, ArgIdx: I);
220 Args.push_back(x: Entry);
221 }
222 return Args;
223}
224
225std::pair<SDValue, SDValue> TargetLowering::makeLibCall(
226 SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef<SDValue> Ops,
227 MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain) const {
228 return makeLibCall(DAG, LibcallImpl: DAG.getLibcalls().getLibcallImpl(Call: LC), RetVT, Ops,
229 CallOptions, dl, Chain);
230}
231
232/// Generate a libcall taking the given operands as arguments and returning a
233/// result of type RetVT.
234std::pair<SDValue, SDValue>
235TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl,
236 EVT RetVT, ArrayRef<SDValue> Ops,
237 MakeLibCallOptions CallOptions, const SDLoc &dl,
238 SDValue InChain) const {
239 if (LibcallImpl == RTLIB::Unsupported)
240 reportFatalInternalError(reason: "unsupported library call operation");
241
242 if (!InChain)
243 InChain = DAG.getEntryNode();
244
245 TargetLowering::ArgListTy Args;
246 Args.reserve(n: Ops.size());
247
248 ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
249 for (unsigned i = 0; i < Ops.size(); ++i) {
250 SDValue NewOp = Ops[i];
251 Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
252 ? OpsTypeOverrides[i]
253 : NewOp.getValueType().getTypeForEVT(Context&: *DAG.getContext());
254 TargetLowering::ArgListEntry Entry(NewOp, Ty);
255 if (CallOptions.IsSoften)
256 Entry.OrigTy =
257 CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(Context&: *DAG.getContext());
258
259 Entry.IsSExt =
260 shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned: CallOptions.IsSigned);
261 Entry.IsZExt = !Entry.IsSExt;
262
263 if (CallOptions.IsSoften &&
264 !shouldExtendTypeInLibCall(Type: CallOptions.OpsVTBeforeSoften[i])) {
265 Entry.IsSExt = Entry.IsZExt = false;
266 }
267 Args.push_back(x: Entry);
268 }
269
270 SDValue Callee =
271 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
272
273 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
274 Type *OrigRetTy = RetTy;
275 TargetLowering::CallLoweringInfo CLI(DAG);
276 bool signExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: CallOptions.IsSigned);
277 bool zeroExtend = !signExtend;
278
279 if (CallOptions.IsSoften) {
280 OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(Context&: *DAG.getContext());
281 if (!shouldExtendTypeInLibCall(Type: CallOptions.RetVTBeforeSoften))
282 signExtend = zeroExtend = false;
283 }
284
285 CLI.setDebugLoc(dl)
286 .setChain(InChain)
287 .setLibCallee(CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl),
288 ResultType: RetTy, OrigResultType: OrigRetTy, Target: Callee, ArgsList: std::move(Args))
289 .setNoReturn(CallOptions.DoesNotReturn)
290 .setDiscardResult(!CallOptions.IsReturnValueUsed)
291 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
292 .setSExtResult(signExtend)
293 .setZExtResult(zeroExtend);
294 return LowerCallTo(CLI);
295}
296
297bool TargetLowering::findOptimalMemOpLowering(
298 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
299 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
300 const AttributeList &FuncAttributes, EVT *LargestVT) const {
301 EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);
302
303 if (VT == MVT::Other) {
304 // Use the largest integer type whose alignment constraints are satisfied.
305 VT = MVT::LAST_INTEGER_VALUETYPE;
306 if (Op.isFixedDstAlign()) {
307 bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
308 while (VT != MVT::i8) {
309 unsigned VTSize = VT.getSizeInBits() / 8;
310 bool DstOk =
311 Op.getDstAlign() >= VTSize ||
312 allowsMisalignedMemoryAccesses(VT, AddrSpace: DstAS, Alignment: Op.getDstAlign());
313 bool SrcOk =
314 !LoadsFromSrc || Op.getSrcAlign() >= VTSize ||
315 allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcAS, Alignment: Op.getSrcAlign());
316 if (DstOk && SrcOk)
317 break;
318 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
319 }
320 }
321 assert(VT.isInteger());
322
323 // Find the largest legal integer type.
324 MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
325 while (!isTypeLegal(VT: LVT))
326 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
327 assert(LVT.isInteger());
328
329 // If the type we've chosen is larger than the largest legal integer type
330 // then use the largest legal type.
331 if (VT.bitsGT(VT: LVT))
332 VT = LVT;
333 }
334
335 unsigned NumMemOps = 0;
336 uint64_t Size = Op.size();
337 while (Size) {
338 unsigned VTSize = VT.getSizeInBits() / 8;
339 while (VTSize > Size) {
340 // For now, only use non-vector load / store's for the left-over pieces.
341 EVT NewVT = VT;
342 unsigned NewVTSize;
343
344 bool Found = false;
345 if (VT.isVector() || VT.isFloatingPoint()) {
346 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
347 if (isOperationLegalOrCustom(Op: ISD::STORE, VT: NewVT) &&
348 isSafeMemOpType(NewVT.getSimpleVT()))
349 Found = true;
350 else if (NewVT == MVT::i64 &&
351 isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::f64) &&
352 isSafeMemOpType(MVT::f64)) {
353 // i64 is usually not legal on 32-bit targets, but f64 may be.
354 NewVT = MVT::f64;
355 Found = true;
356 }
357 }
358
359 if (!Found) {
360 do {
361 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
362 if (NewVT == MVT::i8)
363 break;
364 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
365 }
366 NewVTSize = NewVT.getSizeInBits() / 8;
367
368 // If the new VT cannot cover all of the remaining bits, then consider
369 // issuing a (or a pair of) unaligned and overlapping load / store.
370 unsigned Fast;
371 if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
372 allowsMisalignedMemoryAccesses(
373 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
374 Flags: MachineMemOperand::MONone, &Fast) &&
375 Fast)
376 VTSize = Size;
377 else {
378 VT = NewVT;
379 VTSize = NewVTSize;
380 }
381 }
382
383 if (++NumMemOps > Limit)
384 return false;
385
386 MemOps.push_back(x: VT);
387 Size -= VTSize;
388 }
389
390 return true;
391}
392
393/// Soften the operands of a comparison. This code is shared among BR_CC,
394/// SELECT_CC, and SETCC handlers.
395void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
396 SDValue &NewLHS, SDValue &NewRHS,
397 ISD::CondCode &CCCode,
398 const SDLoc &dl, const SDValue OldLHS,
399 const SDValue OldRHS) const {
400 SDValue Chain;
401 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: dl, OldLHS,
402 OldRHS, Chain);
403}
404
405/// Select the libcall and the condition code to test its result against 0 for
406/// an ordered floating-point compare. \p BoolLC is the boolean helper (result
407/// is 0/1). \p TriStateLC is the per-predicate three-way helper and \p
408/// GenericLC the generic single-symbol three-way helper (both return -1/0/1,
409/// tested against 0 with \p TriStateCC). The boolean form is preferred, then
410/// the per-predicate three-way, then the generic three-way.
411static std::pair<RTLIB::Libcall, ISD::CondCode>
412selectFPCmpLibcall(const LibcallLoweringInfo &Libcalls, RTLIB::Libcall BoolLC,
413 RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC,
414 ISD::CondCode TriStateCC) {
415 if (Libcalls.getLibcallImpl(Call: BoolLC) != RTLIB::Unsupported)
416 return {BoolLC, ISD::SETNE};
417 if (Libcalls.getLibcallImpl(Call: TriStateLC) != RTLIB::Unsupported)
418 return {TriStateLC, TriStateCC};
419 return {GenericLC, TriStateCC};
420}
421
422void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
423 SDValue &NewLHS, SDValue &NewRHS,
424 ISD::CondCode &CCCode,
425 const SDLoc &dl, const SDValue OldLHS,
426 const SDValue OldRHS,
427 SDValue &Chain,
428 bool IsSignaling) const {
429 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
430 // not supporting it. We can update this code when libgcc provides such
431 // functions.
432
433 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
434 && "Unsupported setcc type!");
435
436 // Expand into one or more soft-fp libcall(s).
437 ISD::CondCode OrigCCCode = CCCode;
438 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
439 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
440 bool ShouldInvertCC = false;
441
442 // Expand a compare libcall family name (e.g. OEQ, FCMP3_PRED_OEQ) to the
443 // RTLIB::Libcall for VT.
444#define FP_CMP_LIBCALL(BASE) \
445 RTLIB::getFPLibCall(VT, RTLIB::BASE##_F32, RTLIB::BASE##_F64, \
446 RTLIB::UNKNOWN_LIBCALL, RTLIB::BASE##_F128, \
447 RTLIB::BASE##_PPCF128)
448
449 switch (CCCode) {
450 case ISD::SETEQ:
451 case ISD::SETOEQ:
452 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
453 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
454 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
455 break;
456 case ISD::SETNE:
457 case ISD::SETUNE:
458 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
459 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(UNE), FP_CMP_LIBCALL(FCMP3_PRED_UNE),
460 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETNE);
461 // Some ABIs (e.g. AEABI) provide neither a not-equal nor a three-way
462 // compare; obtain not-equal (UNE = !OEQ) by inverting ordered-equal.
463 if (DAG.getLibcalls().getLibcallImpl(Call: LC1) == RTLIB::Unsupported) {
464 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
465 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ),
466 FP_CMP_LIBCALL(FCMP3_PRED_OEQ), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
467 ShouldInvertCC = true;
468 }
469 break;
470 case ISD::SETGE:
471 case ISD::SETOGE:
472 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
473 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE), FP_CMP_LIBCALL(FCMP3_PRED_OGE),
474 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
475 break;
476 case ISD::SETLT:
477 case ISD::SETOLT:
478 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
479 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT), FP_CMP_LIBCALL(FCMP3_PRED_OLT),
480 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
481 break;
482 case ISD::SETLE:
483 case ISD::SETOLE:
484 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
485 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE), FP_CMP_LIBCALL(FCMP3_PRED_OLE),
486 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
487 break;
488 case ISD::SETGT:
489 case ISD::SETOGT:
490 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
491 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT), FP_CMP_LIBCALL(FCMP3_PRED_OGT),
492 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
493 break;
494 case ISD::SETO:
495 ShouldInvertCC = true;
496 [[fallthrough]];
497 case ISD::SETUO:
498 // Unordered is a boolean everywhere (__unordXf2 returns 0/1).
499 LC1 = FP_CMP_LIBCALL(UO);
500 CC1 = ISD::SETNE;
501 break;
502 case ISD::SETONE:
503 // SETONE = O && UNE
504 ShouldInvertCC = true;
505 [[fallthrough]];
506 case ISD::SETUEQ:
507 LC1 = FP_CMP_LIBCALL(UO);
508 CC1 = ISD::SETNE;
509 std::tie(args&: LC2, args&: CC2) = selectFPCmpLibcall(
510 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
511 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
512 break;
513 default:
514 // Invert CC for unordered comparisons, handled by the ordered inverse.
515 ShouldInvertCC = true;
516 switch (CCCode) {
517 case ISD::SETULT:
518 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
519 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE),
520 FP_CMP_LIBCALL(FCMP3_PRED_OGE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
521 break;
522 case ISD::SETULE:
523 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
524 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT),
525 FP_CMP_LIBCALL(FCMP3_PRED_OGT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
526 break;
527 case ISD::SETUGT:
528 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
529 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE),
530 FP_CMP_LIBCALL(FCMP3_PRED_OLE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
531 break;
532 case ISD::SETUGE:
533 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
534 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT),
535 FP_CMP_LIBCALL(FCMP3_PRED_OLT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
536 break;
537 default:
538 llvm_unreachable("Do not know how to soften this setcc!");
539 }
540 }
541
542#undef FP_CMP_LIBCALL
543
544 // Use the target specific return value for comparison lib calls.
545 EVT RetVT = EVT::getEVT(Ty: getCmpLibcallReturnType(Ctx&: *DAG.getContext()));
546 SDValue Ops[2] = {NewLHS, NewRHS};
547 TargetLowering::MakeLibCallOptions CallOptions;
548 EVT OpsVT[2] = { OldLHS.getValueType(),
549 OldRHS.getValueType() };
550 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
551
552 auto ReportNoLibcall = [&]() {
553 DAG.getContext()->emitError(
554 ErrorStr: Twine("no libcall available to soften floating-point ") +
555 ISD::getCondCodeName(Operation: OrigCCCode) + " compare with type " +
556 VT.getEVTString());
557 NewLHS = DAG.getPOISON(VT: RetVT);
558 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
559 CCCode = ISD::SETNE;
560 };
561
562 // Check availability before makeLibCall, which fatally errors otherwise.
563 RTLIB::LibcallImpl LC1Impl = DAG.getLibcalls().getLibcallImpl(Call: LC1);
564 if (LC1Impl == RTLIB::Unsupported) {
565 ReportNoLibcall();
566 return;
567 }
568
569 auto Call = makeLibCall(DAG, LibcallImpl: LC1Impl, RetVT, Ops, CallOptions, dl, InChain: Chain);
570 NewLHS = Call.first;
571 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
572
573 CCCode = CC1;
574 if (ShouldInvertCC) {
575 assert(RetVT.isInteger());
576 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
577 }
578
579 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
580 // Update Chain.
581 Chain = Call.second;
582 } else {
583 RTLIB::LibcallImpl LC2Impl = DAG.getLibcalls().getLibcallImpl(Call: LC2);
584 if (LC2Impl == RTLIB::Unsupported) {
585 ReportNoLibcall();
586 return;
587 }
588
589 assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
590 "unordered call should be simple boolean");
591
592 EVT SetCCVT =
593 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: RetVT);
594 if (getBooleanContents(Type: RetVT) == ZeroOrOneBooleanContent) {
595 NewLHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: RetVT, N1: Call.first,
596 N2: DAG.getValueType(MVT::i1));
597 }
598
599 SDValue Tmp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NewLHS, RHS: NewRHS, Cond: CCCode);
600 auto Call2 = makeLibCall(DAG, LibcallImpl: LC2Impl, RetVT, Ops, CallOptions, dl, InChain: Chain);
601 CCCode = CC2;
602 if (ShouldInvertCC)
603 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
604 NewLHS = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Call2.first, RHS: NewRHS, Cond: CCCode);
605 if (Chain)
606 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Call.second,
607 N2: Call2.second);
608 NewLHS = DAG.getNode(Opcode: ShouldInvertCC ? ISD::AND : ISD::OR, DL: dl,
609 VT: Tmp.getValueType(), N1: Tmp, N2: NewLHS);
610 NewRHS = SDValue();
611 }
612}
613
614/// Return the entry encoding for a jump table in the current function. The
615/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
616unsigned TargetLowering::getJumpTableEncoding() const {
617 // In non-pic modes, just use the address of a block.
618 if (!isPositionIndependent())
619 return MachineJumpTableInfo::EK_BlockAddress;
620
621 // Otherwise, use a label difference.
622 return MachineJumpTableInfo::EK_LabelDifference32;
623}
624
625SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
626 SelectionDAG &DAG) const {
627 return Table;
628}
629
630/// This returns the relocation base for the given PIC jumptable, the same as
631/// getPICJumpTableRelocBase, but as an MCExpr.
632const MCExpr *
633TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
634 unsigned JTI,MCContext &Ctx) const{
635 // The normal PIC reloc base is the label at the start of the jump table.
636 return MCSymbolRefExpr::create(Symbol: MF->getJTISymbol(JTI, Ctx), Ctx);
637}
638
639SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
640 SDValue Addr, int JTI,
641 SelectionDAG &DAG) const {
642 SDValue Chain = Value;
643 // Jump table debug info is only needed if CodeView is enabled.
644 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF()) {
645 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
646 }
647 return DAG.getNode(Opcode: ISD::BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
648}
649
650bool
651TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
652 const TargetMachine &TM = getTargetMachine();
653 const GlobalValue *GV = GA->getGlobal();
654
655 // If the address is not even local to this DSO we will have to load it from
656 // a got and then add the offset.
657 if (!TM.shouldAssumeDSOLocal(GV))
658 return false;
659
660 // If the code is position independent we will have to add a base register.
661 if (isPositionIndependent())
662 return false;
663
664 // Otherwise we can do it.
665 return true;
666}
667
668//===----------------------------------------------------------------------===//
669// Optimization Methods
670//===----------------------------------------------------------------------===//
671
672/// If the specified instruction has a constant integer operand and there are
673/// bits set in that constant that are not demanded, then clear those bits and
674/// return true.
675bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
676 const APInt &DemandedBits,
677 const APInt &DemandedElts,
678 TargetLoweringOpt &TLO) const {
679 SDLoc DL(Op);
680 unsigned Opcode = Op.getOpcode();
681
682 // Early-out if we've ended up calling an undemanded node, leave this to
683 // constant folding.
684 if (DemandedBits.isZero() || DemandedElts.isZero())
685 return false;
686
687 // Do target-specific constant optimization.
688 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
689 return TLO.New.getNode();
690
691 // FIXME: ISD::SELECT, ISD::SELECT_CC
692 switch (Opcode) {
693 default:
694 break;
695 case ISD::XOR:
696 case ISD::AND:
697 case ISD::OR: {
698 auto *Op1C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
699 if (!Op1C || Op1C->isOpaque())
700 return false;
701
702 // If this is a 'not' op, don't touch it because that's a canonical form.
703 const APInt &C = Op1C->getAPIntValue();
704 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(RHS: C))
705 return false;
706
707 if (!C.isSubsetOf(RHS: DemandedBits)) {
708 EVT VT = Op.getValueType();
709 SDValue NewC = TLO.DAG.getConstant(Val: DemandedBits & C, DL, VT);
710 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 0), N2: NewC,
711 Flags: Op->getFlags());
712 return TLO.CombineTo(O: Op, N: NewOp);
713 }
714
715 break;
716 }
717 }
718
719 return false;
720}
721
722bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
723 const APInt &DemandedBits,
724 TargetLoweringOpt &TLO) const {
725 EVT VT = Op.getValueType();
726 APInt DemandedElts = VT.isVector()
727 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
728 : APInt(1, 1);
729 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
730}
731
732/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
733/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
734/// but it could be generalized for targets with other types of implicit
735/// widening casts.
736bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
737 const APInt &DemandedBits,
738 TargetLoweringOpt &TLO) const {
739 assert(Op.getNumOperands() == 2 &&
740 "ShrinkDemandedOp only supports binary operators!");
741 assert(Op.getNode()->getNumValues() == 1 &&
742 "ShrinkDemandedOp only supports nodes with one result!");
743
744 EVT VT = Op.getValueType();
745 SelectionDAG &DAG = TLO.DAG;
746 SDLoc dl(Op);
747
748 // Early return, as this function cannot handle vector types.
749 if (VT.isVector())
750 return false;
751
752 assert(Op.getOperand(0).getValueType().getScalarSizeInBits() == BitWidth &&
753 Op.getOperand(1).getValueType().getScalarSizeInBits() == BitWidth &&
754 "ShrinkDemandedOp only supports operands that have the same size!");
755
756 // Don't do this if the node has another user, which may require the
757 // full value.
758 if (!Op.getNode()->hasOneUse())
759 return false;
760
761 // Search for the smallest integer type with free casts to and from
762 // Op's type. For expedience, just check power-of-2 integer types.
763 unsigned DemandedSize = DemandedBits.getActiveBits();
764 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
765 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
766 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SmallVTBits);
767 if (isTruncateFree(Val: Op, VT2: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT)) {
768 // We found a type with free casts.
769
770 // If the operation has the 'disjoint' flag, then the
771 // operands on the new node are also disjoint.
772 SDNodeFlags Flags(Op->getFlags().hasDisjoint() ? SDNodeFlags::Disjoint
773 : SDNodeFlags::None);
774 unsigned Opcode = Op.getOpcode();
775 if (Opcode == ISD::PTRADD) {
776 // It isn't a ptradd anymore if it doesn't operate on the entire
777 // pointer.
778 Opcode = ISD::ADD;
779 }
780 SDValue X = DAG.getNode(
781 Opcode, DL: dl, VT: SmallVT,
782 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
783 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 1)), Flags);
784 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
785 SDValue Z = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: X);
786 return TLO.CombineTo(O: Op, N: Z);
787 }
788 }
789 return false;
790}
791
792bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
793 DAGCombinerInfo &DCI) const {
794 SelectionDAG &DAG = DCI.DAG;
795 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
796 !DCI.isBeforeLegalizeOps());
797 KnownBits Known;
798
799 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
800 if (Simplified) {
801 DCI.AddToWorklist(N: Op.getNode());
802 DCI.CommitTargetLoweringOpt(TLO);
803 }
804 return Simplified;
805}
806
807bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
808 const APInt &DemandedElts,
809 DAGCombinerInfo &DCI) const {
810 SelectionDAG &DAG = DCI.DAG;
811 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
812 !DCI.isBeforeLegalizeOps());
813 KnownBits Known;
814
815 bool Simplified =
816 SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
817 if (Simplified) {
818 DCI.AddToWorklist(N: Op.getNode());
819 DCI.CommitTargetLoweringOpt(TLO);
820 }
821 return Simplified;
822}
823
824bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
825 KnownBits &Known,
826 TargetLoweringOpt &TLO,
827 unsigned Depth,
828 bool AssumeSingleUse) const {
829 EVT VT = Op.getValueType();
830
831 // Since the number of lanes in a scalable vector is unknown at compile time,
832 // we track one bit which is implicitly broadcast to all lanes. This means
833 // that all lanes in a scalable vector are considered demanded.
834 APInt DemandedElts = VT.isFixedLengthVector()
835 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
836 : APInt(1, 1);
837 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
838 AssumeSingleUse);
839}
840
841// TODO: Under what circumstances can we create nodes? Constant folding?
842SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
843 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
844 SelectionDAG &DAG, unsigned Depth) const {
845 EVT VT = Op.getValueType();
846
847 // Limit search depth.
848 if (Depth >= SelectionDAG::MaxRecursionDepth)
849 return SDValue();
850
851 // Ignore UNDEFs.
852 if (Op.isUndef())
853 return SDValue();
854
855 // Not demanding any bits/elts from Op.
856 if (DemandedBits == 0 || DemandedElts == 0)
857 return DAG.getUNDEF(VT);
858
859 bool IsLE = DAG.getDataLayout().isLittleEndian();
860 unsigned NumElts = DemandedElts.getBitWidth();
861 unsigned BitWidth = DemandedBits.getBitWidth();
862 KnownBits LHSKnown, RHSKnown;
863 switch (Op.getOpcode()) {
864 case ISD::Constant: {
865 const APInt &Value = Op->getAsAPIntVal();
866 if (!Value.isZero() && (Value & DemandedBits).isZero())
867 return DAG.getConstant(Val: 0, DL: SDLoc(Op), VT);
868 break;
869 }
870 case ISD::BITCAST: {
871 if (VT.isScalableVector())
872 return SDValue();
873
874 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
875 EVT SrcVT = Src.getValueType();
876 EVT DstVT = Op.getValueType();
877 if (SrcVT == DstVT)
878 return Src;
879
880 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
881 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
882 if (NumSrcEltBits == NumDstEltBits)
883 if (SDValue V = SimplifyMultipleUseDemandedBits(
884 Op: Src, DemandedBits, DemandedElts, DAG, Depth: Depth + 1))
885 return DAG.getBitcast(VT: DstVT, V);
886
887 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
888 unsigned Scale = NumDstEltBits / NumSrcEltBits;
889 unsigned NumSrcElts = SrcVT.getVectorNumElements();
890 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
891 for (unsigned i = 0; i != Scale; ++i) {
892 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
893 unsigned BitOffset = EltOffset * NumSrcEltBits;
894 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
895 }
896 // Recursive calls below may turn not demanded elements into poison, so we
897 // need to demand all smaller source elements that maps to a demanded
898 // destination element.
899 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
900
901 if (SDValue V = SimplifyMultipleUseDemandedBits(
902 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
903 return DAG.getBitcast(VT: DstVT, V);
904 }
905
906 // TODO - bigendian once we have test coverage.
907 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
908 unsigned Scale = NumSrcEltBits / NumDstEltBits;
909 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
910 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
911 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
912 for (unsigned i = 0; i != NumElts; ++i)
913 if (DemandedElts[i]) {
914 unsigned Offset = (i % Scale) * NumDstEltBits;
915 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
916 DemandedSrcElts.setBit(i / Scale);
917 }
918
919 if (SDValue V = SimplifyMultipleUseDemandedBits(
920 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
921 return DAG.getBitcast(VT: DstVT, V);
922 }
923
924 break;
925 }
926 case ISD::SCALAR_TO_VECTOR: {
927 if (!VT.isScalableVector() && !DemandedElts[0])
928 return DAG.getPOISON(VT);
929 break;
930 }
931 case ISD::AND: {
932 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
933 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
934
935 // If all of the demanded bits are known 1 on one side, return the other.
936 // These bits cannot contribute to the result of the 'and' in this
937 // context.
938 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero | RHSKnown.One))
939 return Op.getOperand(i: 0);
940 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.One))
941 return Op.getOperand(i: 1);
942 break;
943 }
944 case ISD::OR: {
945 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
946 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
947
948 // If all of the demanded bits are known zero on one side, return the
949 // other. These bits cannot contribute to the result of the 'or' in this
950 // context.
951 if (DemandedBits.isSubsetOf(RHS: LHSKnown.One | RHSKnown.Zero))
952 return Op.getOperand(i: 0);
953 if (DemandedBits.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
954 return Op.getOperand(i: 1);
955 break;
956 }
957 case ISD::XOR: {
958 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
959 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
960
961 // If all of the demanded bits are known zero on one side, return the
962 // other.
963 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero))
964 return Op.getOperand(i: 0);
965 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero))
966 return Op.getOperand(i: 1);
967 break;
968 }
969 case ISD::ADD:
970 case ISD::MUL:
971 case ISD::SMIN:
972 case ISD::SMAX:
973 case ISD::UMIN:
974 case ISD::UMAX: {
975 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 1),
976 DemandedElts, OperandNo: 1, Depth: Depth + 1))
977 return Op.getOperand(i: 0);
978
979 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 0),
980 DemandedElts, OperandNo: 0, Depth: Depth + 1))
981 return Op.getOperand(i: 1);
982 break;
983 }
984 case ISD::SHL: {
985 // If we are only demanding sign bits then we can use the shift source
986 // directly.
987 if (std::optional<unsigned> MaxSA =
988 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
989 SDValue Op0 = Op.getOperand(i: 0);
990 unsigned ShAmt = *MaxSA;
991 unsigned NumSignBits =
992 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
993 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
994 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
995 return Op0;
996 }
997 break;
998 }
999 case ISD::SRL: {
1000 // If we are only demanding sign bits then we can use the shift source
1001 // directly.
1002 if (std::optional<unsigned> MaxSA =
1003 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
1004 SDValue Op0 = Op.getOperand(i: 0);
1005 unsigned ShAmt = *MaxSA;
1006 // Must already be signbits in DemandedBits bounds, and can't demand any
1007 // shifted in zeroes.
1008 if (DemandedBits.countl_zero() >= ShAmt) {
1009 unsigned NumSignBits =
1010 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
1011 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
1012 return Op0;
1013 }
1014 }
1015 break;
1016 }
1017 case ISD::SETCC: {
1018 SDValue Op0 = Op.getOperand(i: 0);
1019 SDValue Op1 = Op.getOperand(i: 1);
1020 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
1021 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1022 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1023 // -1, we may be able to bypass the setcc.
1024 if (DemandedBits.isSignMask() &&
1025 Op0.getScalarValueSizeInBits() == BitWidth &&
1026 getBooleanContents(Type: Op0.getValueType()) ==
1027 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1028 // If we're testing X < 0, then this compare isn't needed - just use X!
1029 // FIXME: We're limiting to integer types here, but this should also work
1030 // if we don't care about FP signed-zero. The use of SETLT with FP means
1031 // that we don't care about NaNs.
1032 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1033 (isNullConstant(V: Op1) || ISD::isBuildVectorAllZeros(V: Op1)))
1034 return Op0;
1035 }
1036 break;
1037 }
1038 case ISD::SIGN_EXTEND_INREG: {
1039 // If none of the extended bits are demanded, eliminate the sextinreg.
1040 SDValue Op0 = Op.getOperand(i: 0);
1041 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1042 unsigned ExBits = ExVT.getScalarSizeInBits();
1043 if (DemandedBits.getActiveBits() <= ExBits &&
1044 shouldRemoveRedundantExtend(Op))
1045 return Op0;
1046 // If the input is already sign extended, just drop the extension.
1047 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
1048 if (NumSignBits >= (BitWidth - ExBits + 1))
1049 return Op0;
1050 break;
1051 }
1052 case ISD::ANY_EXTEND_VECTOR_INREG:
1053 case ISD::SIGN_EXTEND_VECTOR_INREG:
1054 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1055 if (VT.isScalableVector())
1056 return SDValue();
1057
1058 // If we only want the lowest element and none of extended bits, then we can
1059 // return the bitcasted source vector.
1060 SDValue Src = Op.getOperand(i: 0);
1061 EVT SrcVT = Src.getValueType();
1062 EVT DstVT = Op.getValueType();
1063 if (IsLE && DemandedElts == 1 &&
1064 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
1065 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
1066 return DAG.getBitcast(VT: DstVT, V: Src);
1067 }
1068 break;
1069 }
1070 case ISD::INSERT_VECTOR_ELT: {
1071 if (VT.isScalableVector())
1072 return SDValue();
1073
1074 // If we don't demand the inserted element, return the base vector.
1075 SDValue Vec = Op.getOperand(i: 0);
1076 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1077 EVT VecVT = Vec.getValueType();
1078 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements()) &&
1079 !DemandedElts[CIdx->getZExtValue()])
1080 return Vec;
1081 break;
1082 }
1083 case ISD::INSERT_SUBVECTOR: {
1084 if (VT.isScalableVector())
1085 return SDValue();
1086
1087 SDValue Vec = Op.getOperand(i: 0);
1088 SDValue Sub = Op.getOperand(i: 1);
1089 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1090 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1091 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1092 // If we don't demand the inserted subvector, return the base vector.
1093 if (DemandedSubElts == 0)
1094 return Vec;
1095 break;
1096 }
1097 case ISD::VECTOR_SHUFFLE: {
1098 assert(!VT.isScalableVector());
1099 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1100
1101 // If all the demanded elts are from one operand and are inline,
1102 // then we can use the operand directly.
1103 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
1104 for (unsigned i = 0; i != NumElts; ++i) {
1105 int M = ShuffleMask[i];
1106 if (M < 0 || !DemandedElts[i])
1107 continue;
1108 AllUndef = false;
1109 IdentityLHS &= (M == (int)i);
1110 IdentityRHS &= ((M - NumElts) == i);
1111 }
1112
1113 if (AllUndef)
1114 return DAG.getUNDEF(VT: Op.getValueType());
1115 if (IdentityLHS)
1116 return Op.getOperand(i: 0);
1117 if (IdentityRHS)
1118 return Op.getOperand(i: 1);
1119 break;
1120 }
1121 default:
1122 // TODO: Probably okay to remove after audit; here to reduce change size
1123 // in initial enablement patch for scalable vectors
1124 if (VT.isScalableVector())
1125 return SDValue();
1126
1127 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
1128 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
1129 Op, DemandedBits, DemandedElts, DAG, Depth))
1130 return V;
1131 break;
1132 }
1133 return SDValue();
1134}
1135
1136SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
1137 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
1138 unsigned Depth) const {
1139 EVT VT = Op.getValueType();
1140 // Since the number of lanes in a scalable vector is unknown at compile time,
1141 // we track one bit which is implicitly broadcast to all lanes. This means
1142 // that all lanes in a scalable vector are considered demanded.
1143 APInt DemandedElts = VT.isFixedLengthVector()
1144 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1145 : APInt(1, 1);
1146 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1147 Depth);
1148}
1149
1150SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
1151 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
1152 unsigned Depth) const {
1153 APInt DemandedBits = APInt::getAllOnes(numBits: Op.getScalarValueSizeInBits());
1154 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1155 Depth);
1156}
1157
1158// Find the smallest power-2 type that is at least 8 bits and can hold a value
1159// of type VT that has KnownBits known sign/zero bits. Returns std::nullopt if
1160// that is wider than VT.
1161static std::optional<EVT> getNarrowAVGType(EVT VT, unsigned KnownBits,
1162 LLVMContext &Ctx) {
1163 unsigned MinWidth =
1164 std::max<unsigned>(a: VT.getScalarSizeInBits() - KnownBits, b: 8);
1165 EVT NVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: llvm::bit_ceil(Value: MinWidth));
1166 if (NVT.getScalarSizeInBits() > VT.getScalarSizeInBits())
1167 return std::nullopt;
1168 if (VT.isVector())
1169 NVT = EVT::getVectorVT(Context&: Ctx, VT: NVT, EC: VT.getVectorElementCount());
1170 return NVT;
1171}
1172
1173// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
1174// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
1175static SDValue combineShiftToAVG(SDValue Op,
1176 TargetLowering::TargetLoweringOpt &TLO,
1177 const TargetLowering &TLI,
1178 const APInt &DemandedBits,
1179 const APInt &DemandedElts, unsigned Depth) {
1180 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1181 "SRL or SRA node is required here!");
1182 // Is the right shift using an immediate value of 1?
1183 ConstantSDNode *N1C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
1184 if (!N1C || !N1C->isOne())
1185 return SDValue();
1186
1187 // We are looking for an avgfloor
1188 // add(ext, ext)
1189 // or one of these as a avgceil
1190 // add(add(ext, ext), 1)
1191 // add(add(ext, 1), ext)
1192 // add(ext, add(ext, 1))
1193 SDValue Add = Op.getOperand(i: 0);
1194 if (Add.getOpcode() != ISD::ADD)
1195 return SDValue();
1196
1197 SDValue ExtOpA = Add.getOperand(i: 0);
1198 SDValue ExtOpB = Add.getOperand(i: 1);
1199 SDValue Add2;
1200 auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
1201 ConstantSDNode *ConstOp;
1202 if ((ConstOp = isConstOrConstSplat(N: Op2, DemandedElts)) &&
1203 ConstOp->isOne()) {
1204 ExtOpA = Op1;
1205 ExtOpB = Op3;
1206 Add2 = A;
1207 return true;
1208 }
1209 if ((ConstOp = isConstOrConstSplat(N: Op3, DemandedElts)) &&
1210 ConstOp->isOne()) {
1211 ExtOpA = Op1;
1212 ExtOpB = Op2;
1213 Add2 = A;
1214 return true;
1215 }
1216 return false;
1217 };
1218 bool IsCeil =
1219 (ExtOpA.getOpcode() == ISD::ADD &&
1220 MatchOperands(ExtOpA.getOperand(i: 0), ExtOpA.getOperand(i: 1), ExtOpB, ExtOpA)) ||
1221 (ExtOpB.getOpcode() == ISD::ADD &&
1222 MatchOperands(ExtOpB.getOperand(i: 0), ExtOpB.getOperand(i: 1), ExtOpA, ExtOpB));
1223
1224 // If the shift is signed (sra):
1225 // - Needs >= 2 sign bit for both operands.
1226 // - Needs >= 2 zero bits.
1227 // If the shift is unsigned (srl):
1228 // - Needs >= 1 zero bit for both operands.
1229 // - Needs 1 demanded bit zero and >= 2 sign bits.
1230 SelectionDAG &DAG = TLO.DAG;
1231 unsigned ShiftOpc = Op.getOpcode();
1232 bool IsSigned = false;
1233 unsigned KnownBits;
1234 unsigned NumSignedA = DAG.ComputeNumSignBits(Op: ExtOpA, DemandedElts, Depth);
1235 unsigned NumSignedB = DAG.ComputeNumSignBits(Op: ExtOpB, DemandedElts, Depth);
1236 unsigned NumSigned = std::min(a: NumSignedA, b: NumSignedB) - 1;
1237 unsigned NumZeroA =
1238 DAG.computeKnownBits(Op: ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
1239 unsigned NumZeroB =
1240 DAG.computeKnownBits(Op: ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
1241 unsigned NumZero = std::min(a: NumZeroA, b: NumZeroB);
1242
1243 switch (ShiftOpc) {
1244 default:
1245 llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
1246 case ISD::SRA: {
1247 if (NumZero >= 2 && NumSigned < NumZero) {
1248 IsSigned = false;
1249 KnownBits = NumZero;
1250 break;
1251 }
1252 if (NumSigned >= 1) {
1253 IsSigned = true;
1254 KnownBits = NumSigned;
1255 break;
1256 }
1257 return SDValue();
1258 }
1259 case ISD::SRL: {
1260 if (NumZero >= 1 && NumSigned < NumZero) {
1261 IsSigned = false;
1262 KnownBits = NumZero;
1263 break;
1264 }
1265 if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
1266 IsSigned = true;
1267 KnownBits = NumSigned;
1268 break;
1269 }
1270 return SDValue();
1271 }
1272 }
1273
1274 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
1275 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);
1276
1277 // Find the smallest power-2 type that is legal for this vector size and
1278 // operation, given the original type size and the number of known sign/zero
1279 // bits.
1280 EVT VT = Op.getValueType();
1281 std::optional<EVT> NarrowVT =
1282 getNarrowAVGType(VT, KnownBits, Ctx&: *DAG.getContext());
1283 if (!NarrowVT)
1284 return SDValue();
1285 EVT NVT = *NarrowVT;
1286 if (TLO.LegalTypes() && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT)) {
1287 // If we could not transform, and (both) adds are nuw/nsw, we can use the
1288 // larger type size to do the transform.
1289 if (TLO.LegalOperations() && !TLI.isOperationLegal(Op: AVGOpc, VT))
1290 return SDValue();
1291 if (DAG.willNotOverflowAdd(IsSigned, N0: Add.getOperand(i: 0),
1292 N1: Add.getOperand(i: 1)) &&
1293 (!Add2 || DAG.willNotOverflowAdd(IsSigned, N0: Add2.getOperand(i: 0),
1294 N1: Add2.getOperand(i: 1))))
1295 NVT = VT;
1296 else
1297 return SDValue();
1298 }
1299
1300 // Don't create a AVGFLOOR node with a scalar constant unless its legal as
1301 // this is likely to stop other folds (reassociation, value tracking etc.)
1302 if (!IsCeil && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT) &&
1303 (isa<ConstantSDNode>(Val: ExtOpA) || isa<ConstantSDNode>(Val: ExtOpB)))
1304 return SDValue();
1305
1306 SDLoc DL(Op);
1307 SDValue ResultAVG =
1308 DAG.getNode(Opcode: AVGOpc, DL, VT: NVT, N1: DAG.getExtOrTrunc(IsSigned, Op: ExtOpA, DL, VT: NVT),
1309 N2: DAG.getExtOrTrunc(IsSigned, Op: ExtOpB, DL, VT: NVT));
1310 return DAG.getExtOrTrunc(IsSigned, Op: ResultAVG, DL, VT);
1311}
1312
1313// Attempt to form ext(avgsubfloor(A, B)) from shr(sub(ext(A), ext(B)), 1).
1314static SDValue combineShiftToAVGSUB(SDValue Op,
1315 TargetLowering::TargetLoweringOpt &TLO,
1316 const TargetLowering &TLI,
1317 const APInt &DemandedBits,
1318 const APInt &DemandedElts, unsigned Depth) {
1319 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1320 "SRL or SRA node is required here!");
1321 // Is the right shift using an immediate value of 1?
1322 ConstantSDNode *N1C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
1323 if (!N1C || !N1C->isOne())
1324 return SDValue();
1325
1326 SDValue Sub = Op.getOperand(i: 0);
1327 if (Sub.getOpcode() != ISD::SUB)
1328 return SDValue();
1329
1330 // The result of the avgsub is sign extended to match the sra. For srl, the
1331 // top bit of the result is always zero so it must not be demanded.
1332 if (Op.getOpcode() == ISD::SRL && !DemandedBits.isSignBitClear())
1333 return SDValue();
1334
1335 SDValue A = Sub.getOperand(i: 0);
1336 SDValue B = Sub.getOperand(i: 1);
1337
1338 // If both operands have at least 2 sign bits or at least 1 leading zero, the
1339 // subtract can't overflow and the operands are extended from a narrower type.
1340 SelectionDAG &DAG = TLO.DAG;
1341 bool IsSigned;
1342 unsigned KnownBits;
1343 unsigned NumSigned =
1344 std::min(a: DAG.ComputeNumSignBits(Op: A, DemandedElts, Depth),
1345 b: DAG.ComputeNumSignBits(Op: B, DemandedElts, Depth)) -
1346 1;
1347 unsigned NumZero = std::min(
1348 a: DAG.computeKnownBits(Op: A, DemandedElts, Depth).countMinLeadingZeros(),
1349 b: DAG.computeKnownBits(Op: B, DemandedElts, Depth).countMinLeadingZeros());
1350 if (NumZero >= 1 && NumSigned < NumZero) {
1351 IsSigned = false;
1352 KnownBits = NumZero;
1353 } else if (NumSigned >= 1) {
1354 IsSigned = true;
1355 KnownBits = NumSigned;
1356 } else {
1357 return SDValue();
1358 }
1359
1360 unsigned AVGOpc = IsSigned ? ISD::AVGSUBFLOORS : ISD::AVGSUBFLOORU;
1361
1362 // Return true if the AVG node is legal or custom on the type VT will be
1363 // legalized to, provided that legalization doesn't change the scalar size.
1364 // That means vectors that are widened or split are accepted, but integers
1365 // that are promoted or expanded are not.
1366 LLVMContext &Ctx = *DAG.getContext();
1367 auto IsLegalAfterTypeLegalization = [&](EVT VT) {
1368 EVT LegalVT = TLI.getLegalTypeToTransformTo(Context&: Ctx, VT);
1369 return LegalVT.getScalarSizeInBits() == VT.getScalarSizeInBits() &&
1370 TLI.isOperationLegal(Op: AVGOpc, VT: LegalVT);
1371 };
1372
1373 // Use the narrow type if the node is legal or custom there. Otherwise fall
1374 // back to the original type, which is still correct since the subtract can't
1375 // overflow. Don't form the node if the target can't do it on either type, as
1376 // expanding it may be worse than the original subtract and shift.
1377 // FIXME: Enable for illegal types/operations like combineShiftToAVG.
1378 EVT VT = Op.getValueType();
1379 std::optional<EVT> NarrowVT = getNarrowAVGType(VT, KnownBits, Ctx);
1380 EVT NVT = VT;
1381 if (NarrowVT && IsLegalAfterTypeLegalization(*NarrowVT))
1382 NVT = *NarrowVT;
1383 else if (!IsLegalAfterTypeLegalization(VT))
1384 return SDValue();
1385
1386 // Don't create an AVGSUBFLOOR node with a scalar constant unless it's legal
1387 // as this is likely to stop other folds (reassociation, value tracking etc.)
1388 if (!TLI.isOperationLegal(Op: AVGOpc, VT: NVT) &&
1389 (isa<ConstantSDNode>(Val: A) || isa<ConstantSDNode>(Val: B)))
1390 return SDValue();
1391
1392 // The difference of two unsigned values can be negative, so the result is
1393 // always sign extended.
1394 SDLoc DL(Op);
1395 SDValue ResultAVG =
1396 DAG.getNode(Opcode: AVGOpc, DL, VT: NVT, N1: DAG.getExtOrTrunc(IsSigned, Op: A, DL, VT: NVT),
1397 N2: DAG.getExtOrTrunc(IsSigned, Op: B, DL, VT: NVT));
1398 return DAG.getExtOrTrunc(/*IsSigned=*/true, Op: ResultAVG, DL, VT);
1399}
1400
1401/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
1402/// result of Op are ever used downstream. If we can use this information to
1403/// simplify Op, create a new simplified DAG node and return true, returning the
1404/// original and new nodes in Old and New. Otherwise, analyze the expression and
1405/// return a mask of Known bits for the expression (used to simplify the
1406/// caller). The Known bits may only be accurate for those bits in the
1407/// OriginalDemandedBits and OriginalDemandedElts.
1408bool TargetLowering::SimplifyDemandedBits(
1409 SDValue Op, const APInt &OriginalDemandedBits,
1410 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
1411 unsigned Depth, bool AssumeSingleUse) const {
1412 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
1413 assert(Op.getScalarValueSizeInBits() == BitWidth &&
1414 "Mask size mismatches value type size!");
1415
1416 // Don't know anything.
1417 Known = KnownBits(BitWidth);
1418
1419 EVT VT = Op.getValueType();
1420 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
1421 unsigned NumElts = OriginalDemandedElts.getBitWidth();
1422 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
1423 "Unexpected vector size");
1424
1425 APInt DemandedBits = OriginalDemandedBits;
1426 APInt DemandedElts = OriginalDemandedElts;
1427 SDLoc dl(Op);
1428
1429 // Undef operand.
1430 if (Op.isUndef())
1431 return false;
1432
1433 // We can't simplify target constants.
1434 if (Op.getOpcode() == ISD::TargetConstant)
1435 return false;
1436
1437 if (Op.getOpcode() == ISD::Constant) {
1438 // We know all of the bits for a constant!
1439 Known = KnownBits::makeConstant(C: Op->getAsAPIntVal());
1440 return false;
1441 }
1442
1443 if (Op.getOpcode() == ISD::ConstantFP) {
1444 // We know all of the bits for a floating point constant!
1445 Known = KnownBits::makeConstant(
1446 C: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF().bitcastToAPInt());
1447 return false;
1448 }
1449
1450 // Other users may use these bits.
1451 bool HasMultiUse = false;
1452 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1453 if (Depth >= SelectionDAG::MaxRecursionDepth) {
1454 // Limit search depth.
1455 return false;
1456 }
1457 // Allow multiple uses, just set the DemandedBits/Elts to all bits.
1458 DemandedBits = APInt::getAllOnes(numBits: BitWidth);
1459 DemandedElts = APInt::getAllOnes(numBits: NumElts);
1460 HasMultiUse = true;
1461 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1462 // Not demanding any bits/elts from Op.
1463 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
1464 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
1465 // Limit search depth.
1466 return false;
1467 }
1468
1469 // Before folding to the LHS, recheck bits excluded from its earlier demand
1470 // using the full result demand. Mask identifies these excluded bits.
1471 // KnownMask selects the Zero/One member of the recomputed LHS known bits
1472 // that must cover Mask & DemandedBits.
1473 auto TryFoldToLHS = [&](const APInt &Mask,
1474 const APInt KnownBits::*KnownMask) {
1475 SDValue LHS = Op.getOperand(i: 0);
1476 APInt MaskedBits = Mask & DemandedBits;
1477 if (MaskedBits.isZero())
1478 return TLO.CombineTo(O: Op, N: LHS);
1479
1480 KnownBits LHSKnown;
1481 if (SimplifyDemandedBits(Op: LHS, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known&: LHSKnown, TLO,
1482 Depth: Depth + 1))
1483 return true;
1484
1485 return MaskedBits.isSubsetOf(RHS: LHSKnown.*KnownMask) && TLO.CombineTo(O: Op, N: LHS);
1486 };
1487
1488 // Try reducing the RHS demand using LHS known bits.
1489 // KnownMask selects the LHS Zero/One bits that make RHS bits irrelevant.
1490 auto TryReduceRHSDemand = [&](const KnownBits &LHSKnown,
1491 const APInt KnownBits::*KnownMask) {
1492 SDValue RHS = Op.getOperand(i: 1);
1493 if (!RHS.hasOneUse() || isConstOrConstSplat(N: RHS, DemandedElts))
1494 return false;
1495
1496 // Reuse LHS facts if the RHS did not reduce its earlier demand. Otherwise,
1497 // query independent facts to avoid using bits outside that demand.
1498 SDValue LHS = Op.getOperand(i: 0);
1499 // Keep only the selected mask, then compute DemandedBits & ~Mask in place.
1500 // This avoids copying the unused KnownBits mask and extra APInt
1501 // temporaries, which can allocate memory for masks wider than one word.
1502 APInt RHSDemandedBits =
1503 DemandedBits.intersects(RHS: Known.*KnownMask)
1504 ? TLO.DAG.computeKnownBits(Op: LHS, DemandedElts, Depth: Depth + 1).*KnownMask
1505 : LHSKnown.*KnownMask;
1506 RHSDemandedBits.flipAllBits();
1507 RHSDemandedBits &= DemandedBits;
1508 if (RHSDemandedBits == DemandedBits)
1509 return false;
1510
1511 // A failed retry must not replace Known with reduced-demand facts.
1512 KnownBits RetryKnown;
1513 return SimplifyDemandedBits(Op: RHS, OriginalDemandedBits: RHSDemandedBits, OriginalDemandedElts: DemandedElts, Known&: RetryKnown,
1514 TLO, Depth: Depth + 1);
1515 };
1516
1517 KnownBits Known2;
1518 switch (Op.getOpcode()) {
1519 case ISD::SCALAR_TO_VECTOR: {
1520 if (VT.isScalableVector())
1521 return false;
1522 if (!DemandedElts[0])
1523 return TLO.CombineTo(O: Op, N: TLO.DAG.getPOISON(VT));
1524
1525 KnownBits SrcKnown;
1526 SDValue Src = Op.getOperand(i: 0);
1527 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1528 APInt SrcDemandedBits = DemandedBits.zext(width: SrcBitWidth);
1529 if (SimplifyDemandedBits(Op: Src, DemandedBits: SrcDemandedBits, Known&: SrcKnown, TLO, Depth: Depth + 1))
1530 return true;
1531
1532 // Upper elements are poison, so only get the knownbits if we just demand
1533 // the bottom element.
1534 if (DemandedElts == 1)
1535 Known = SrcKnown.anyextOrTrunc(BitWidth);
1536 break;
1537 }
1538 case ISD::BUILD_VECTOR:
1539 // Collect the known bits that are shared by every demanded element.
1540 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1541 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1542 return false; // Don't fall through, will infinitely loop.
1543 case ISD::SPLAT_VECTOR: {
1544 SDValue Scl = Op.getOperand(i: 0);
1545 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: Scl.getValueSizeInBits());
1546 KnownBits KnownScl;
1547 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1548 return true;
1549
1550 // Implicitly truncate the bits to match the official semantics of
1551 // SPLAT_VECTOR.
1552 Known = KnownScl.trunc(BitWidth);
1553 break;
1554 }
1555 case ISD::FREEZE: {
1556 SDValue N0 = Op.getOperand(i: 0);
1557 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
1558 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
1559 return TLO.CombineTo(O: Op, N: N0);
1560 break;
1561 }
1562 case ISD::LOAD: {
1563 auto *LD = cast<LoadSDNode>(Val&: Op);
1564 if (getTargetConstantFromLoad(LD)) {
1565 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1566 return false; // Don't fall through, will infinitely loop.
1567 }
1568 if (ISD::isZEXTLoad(N: Op.getNode()) && Op.getResNo() == 0) {
1569 // If this is a ZEXTLoad and we are looking at the loaded value.
1570 EVT MemVT = LD->getMemoryVT();
1571 unsigned MemBits = MemVT.getScalarSizeInBits();
1572 Known.Zero.setBitsFrom(MemBits);
1573 return false; // Don't fall through, will infinitely loop.
1574 }
1575 break;
1576 }
1577 case ISD::INSERT_VECTOR_ELT: {
1578 if (VT.isScalableVector())
1579 return false;
1580 SDValue Vec = Op.getOperand(i: 0);
1581 SDValue Scl = Op.getOperand(i: 1);
1582 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1583 EVT VecVT = Vec.getValueType();
1584
1585 // If index isn't constant, assume we need all vector elements AND the
1586 // inserted element.
1587 APInt DemandedVecElts(DemandedElts);
1588 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements())) {
1589 unsigned Idx = CIdx->getZExtValue();
1590 DemandedVecElts.clearBit(BitPosition: Idx);
1591
1592 // Inserted element is not required.
1593 if (!DemandedElts[Idx])
1594 return TLO.CombineTo(O: Op, N: Vec);
1595 }
1596
1597 KnownBits KnownScl;
1598 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1599 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: NumSclBits);
1600 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1601 return true;
1602
1603 Known = KnownScl.anyextOrTrunc(BitWidth);
1604
1605 KnownBits KnownVec;
1606 if (SimplifyDemandedBits(Op: Vec, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedVecElts, Known&: KnownVec, TLO,
1607 Depth: Depth + 1))
1608 return true;
1609
1610 if (!!DemandedVecElts)
1611 Known = Known.intersectWith(RHS: KnownVec);
1612
1613 return false;
1614 }
1615 case ISD::INSERT_SUBVECTOR: {
1616 if (VT.isScalableVector())
1617 return false;
1618 // Demand any elements from the subvector and the remainder from the src its
1619 // inserted into.
1620 SDValue Src = Op.getOperand(i: 0);
1621 SDValue Sub = Op.getOperand(i: 1);
1622 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1623 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1624 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1625 APInt DemandedSrcElts = DemandedElts;
1626 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
1627
1628 KnownBits KnownSub, KnownSrc;
1629 if (SimplifyDemandedBits(Op: Sub, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts, Known&: KnownSub, TLO,
1630 Depth: Depth + 1))
1631 return true;
1632 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known&: KnownSrc, TLO,
1633 Depth: Depth + 1))
1634 return true;
1635
1636 Known.setAllConflict();
1637 if (!!DemandedSubElts)
1638 Known = Known.intersectWith(RHS: KnownSub);
1639 if (!!DemandedSrcElts)
1640 Known = Known.intersectWith(RHS: KnownSrc);
1641
1642 // Attempt to avoid multi-use src if we don't need anything from it.
1643 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1644 !DemandedSrcElts.isAllOnes()) {
1645 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1646 Op: Sub, DemandedBits, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
1647 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1648 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1649 if (NewSub || NewSrc) {
1650 NewSub = NewSub ? NewSub : Sub;
1651 NewSrc = NewSrc ? NewSrc : Src;
1652 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: NewSrc, N2: NewSub,
1653 N3: Op.getOperand(i: 2));
1654 return TLO.CombineTo(O: Op, N: NewOp);
1655 }
1656 }
1657 break;
1658 }
1659 case ISD::EXTRACT_SUBVECTOR: {
1660 if (VT.isScalableVector())
1661 return false;
1662 // Offset the demanded elts by the subvector index.
1663 SDValue Src = Op.getOperand(i: 0);
1664 if (Src.getValueType().isScalableVector())
1665 break;
1666 uint64_t Idx = Op.getConstantOperandVal(i: 1);
1667 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1668 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
1669
1670 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known, TLO,
1671 Depth: Depth + 1))
1672 return true;
1673
1674 // Attempt to avoid multi-use src if we don't need anything from it.
1675 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1676 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1677 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1678 if (DemandedSrc) {
1679 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc,
1680 N2: Op.getOperand(i: 1));
1681 return TLO.CombineTo(O: Op, N: NewOp);
1682 }
1683 }
1684 break;
1685 }
1686 case ISD::CONCAT_VECTORS: {
1687 if (VT.isScalableVector())
1688 return false;
1689 Known.setAllConflict();
1690 EVT SubVT = Op.getOperand(i: 0).getValueType();
1691 unsigned NumSubVecs = Op.getNumOperands();
1692 unsigned NumSubElts = SubVT.getVectorNumElements();
1693 for (unsigned i = 0; i != NumSubVecs; ++i) {
1694 APInt DemandedSubElts =
1695 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
1696 if (SimplifyDemandedBits(Op: Op.getOperand(i), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts,
1697 Known&: Known2, TLO, Depth: Depth + 1))
1698 return true;
1699 // Known bits are shared by every demanded subvector element.
1700 if (!!DemandedSubElts)
1701 Known = Known.intersectWith(RHS: Known2);
1702 }
1703 break;
1704 }
1705 case ISD::VECTOR_SHUFFLE: {
1706 assert(!VT.isScalableVector());
1707 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1708
1709 // Collect demanded elements from shuffle operands..
1710 APInt DemandedLHS, DemandedRHS;
1711 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: ShuffleMask, DemandedElts, DemandedLHS,
1712 DemandedRHS))
1713 break;
1714
1715 if (!!DemandedLHS || !!DemandedRHS) {
1716 SDValue Op0 = Op.getOperand(i: 0);
1717 SDValue Op1 = Op.getOperand(i: 1);
1718
1719 Known.setAllConflict();
1720 if (!!DemandedLHS) {
1721 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedLHS, Known&: Known2, TLO,
1722 Depth: Depth + 1))
1723 return true;
1724 Known = Known.intersectWith(RHS: Known2);
1725 }
1726 if (!!DemandedRHS) {
1727 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedRHS, Known&: Known2, TLO,
1728 Depth: Depth + 1))
1729 return true;
1730 Known = Known.intersectWith(RHS: Known2);
1731 }
1732
1733 // Attempt to avoid multi-use ops if we don't need anything from them.
1734 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1735 Op: Op0, DemandedBits, DemandedElts: DemandedLHS, DAG&: TLO.DAG, Depth: Depth + 1);
1736 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1737 Op: Op1, DemandedBits, DemandedElts: DemandedRHS, DAG&: TLO.DAG, Depth: Depth + 1);
1738 if (DemandedOp0 || DemandedOp1) {
1739 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1740 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1741 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, N1: Op0, N2: Op1, Mask: ShuffleMask);
1742 return TLO.CombineTo(O: Op, N: NewOp);
1743 }
1744 }
1745 break;
1746 }
1747 case ISD::AND: {
1748 SDValue Op0 = Op.getOperand(i: 0);
1749 SDValue Op1 = Op.getOperand(i: 1);
1750
1751 // If the RHS is a constant, check to see if the LHS would be zero without
1752 // using the bits from the RHS. Below, we use knowledge about the RHS to
1753 // simplify the LHS, here we're using information from the LHS to simplify
1754 // the RHS.
1755 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: Op1, DemandedElts)) {
1756 // Do not increment Depth here; that can cause an infinite loop.
1757 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth);
1758 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1759 if ((LHSKnown.Zero & DemandedBits) ==
1760 (~RHSC->getAPIntValue() & DemandedBits))
1761 return TLO.CombineTo(O: Op, N: Op0);
1762
1763 // If any of the set bits in the RHS are known zero on the LHS, shrink
1764 // the constant.
1765 if (ShrinkDemandedConstant(Op, DemandedBits: ~LHSKnown.Zero & DemandedBits,
1766 DemandedElts, TLO))
1767 return true;
1768
1769 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1770 // constant, but if this 'and' is only clearing bits that were just set by
1771 // the xor, then this 'and' can be eliminated by shrinking the mask of
1772 // the xor. For example, for a 32-bit X:
1773 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1774 if (isBitwiseNot(V: Op0) && Op0.hasOneUse() &&
1775 LHSKnown.One == ~RHSC->getAPIntValue()) {
1776 SDValue Xor = TLO.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1777 return TLO.CombineTo(O: Op, N: Xor);
1778 }
1779 }
1780
1781 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2 (or zero).
1782 SDValue X, Y;
1783 if (sd_match(N: Op,
1784 P: m_And(L: m_Value(N&: Y),
1785 R: m_OneUse(P: m_AnyOf(preds: m_Add(L: m_Value(N&: X), R: m_Deferred(V&: Y)),
1786 preds: m_Sub(L: m_Value(N&: X), R: m_Deferred(V&: Y)))))) &&
1787 TLO.DAG.isKnownToBeAPowerOfTwo(Val: Y, DemandedElts, /*OrZero=*/true)) {
1788 return TLO.CombineTo(
1789 O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TLO.DAG.getNOT(DL: dl, Val: X, VT), N2: Y));
1790 }
1791
1792 // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
1793 // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
1794 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
1795 (Op0.getOperand(i: 0).isUndef() ||
1796 ISD::isBuildVectorOfConstantSDNodes(V: Op0.getOperand(i: 0))) &&
1797 Op0->hasOneUse()) {
1798 unsigned NumSubElts =
1799 Op0.getOperand(i: 1).getValueType().getVectorNumElements();
1800 unsigned SubIdx = Op0.getConstantOperandVal(i: 2);
1801 APInt DemandedSub =
1802 APInt::getBitsSet(numBits: NumElts, loBit: SubIdx, hiBit: SubIdx + NumSubElts);
1803 KnownBits KnownSubMask =
1804 TLO.DAG.computeKnownBits(Op: Op1, DemandedElts: DemandedSub & DemandedElts, Depth: Depth + 1);
1805 if (DemandedBits.isSubsetOf(RHS: KnownSubMask.One)) {
1806 SDValue NewAnd =
1807 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1808 SDValue NewInsert =
1809 TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: NewAnd,
1810 N2: Op0.getOperand(i: 1), N3: Op0.getOperand(i: 2));
1811 return TLO.CombineTo(O: Op, N: NewInsert);
1812 }
1813 }
1814
1815 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1816 Depth: Depth + 1))
1817 return true;
1818 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.Zero & DemandedBits, OriginalDemandedElts: DemandedElts,
1819 Known&: Known2, TLO, Depth: Depth + 1))
1820 return true;
1821
1822 // If all of the demanded bits are known one on one side, return the other.
1823 // These bits cannot contribute to the result of the 'and'.
1824 if (DemandedBits.isSubsetOf(RHS: Known2.Zero | Known.One)) {
1825 if (TryFoldToLHS(Known.Zero, &KnownBits::Zero))
1826 return true;
1827 }
1828 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.One))
1829 return TLO.CombineTo(O: Op, N: Op1);
1830 // If all of the demanded bits in the inputs are known zeros, return zero.
1831 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1832 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: dl, VT));
1833 // If the RHS is a constant, see if we can simplify it.
1834 if (ShrinkDemandedConstant(Op, DemandedBits: ~Known2.Zero & DemandedBits, DemandedElts,
1835 TLO))
1836 return true;
1837 // If the operation can be done in a smaller type, do so.
1838 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1839 return true;
1840
1841 // Attempt to avoid multi-use ops if we don't need anything from them.
1842 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1843 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1844 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1845 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1846 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1847 if (DemandedOp0 || DemandedOp1) {
1848 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1849 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1850 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1851 return TLO.CombineTo(O: Op, N: NewOp);
1852 }
1853 }
1854
1855 if (TryReduceRHSDemand(Known2, &KnownBits::Zero))
1856 return true;
1857
1858 Known &= Known2;
1859 break;
1860 }
1861 case ISD::OR: {
1862 SDValue Op0 = Op.getOperand(i: 0);
1863 SDValue Op1 = Op.getOperand(i: 1);
1864 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1865 Depth: Depth + 1)) {
1866 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1867 return true;
1868 }
1869
1870 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.One & DemandedBits, OriginalDemandedElts: DemandedElts,
1871 Known&: Known2, TLO, Depth: Depth + 1)) {
1872 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1873 return true;
1874 }
1875
1876 // If all of the demanded bits are known zero on one side, return the other.
1877 // These bits cannot contribute to the result of the 'or'.
1878 if (DemandedBits.isSubsetOf(RHS: Known2.One | Known.Zero)) {
1879 if (TryFoldToLHS(Known.One, &KnownBits::One)) {
1880 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1881 return true;
1882 }
1883 }
1884 if (DemandedBits.isSubsetOf(RHS: Known.One | Known2.Zero))
1885 return TLO.CombineTo(O: Op, N: Op1);
1886 // If the RHS is a constant, see if we can simplify it.
1887 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1888 return true;
1889 // If the operation can be done in a smaller type, do so.
1890 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1891 return true;
1892
1893 // Attempt to avoid multi-use ops if we don't need anything from them.
1894 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1895 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1896 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1897 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1898 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1899 if (DemandedOp0 || DemandedOp1) {
1900 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1901 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1902 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1903 return TLO.CombineTo(O: Op, N: NewOp);
1904 }
1905 }
1906
1907 // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
1908 // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
1909 SDValue X, Y, C1, C2;
1910 if (sd_match(N: Op, P: m_Or(L: m_OneUse(P: m_And(L: m_Value(N&: X), R: m_Value(N&: C1))),
1911 R: m_OneUse(P: m_And(L: m_Or(L: m_Deferred(V&: X), R: m_Value(N&: Y)),
1912 R: m_Value(N&: C2)))))) {
1913 if (SDValue C12 =
1914 TLO.DAG.FoldConstantArithmetic(Opcode: ISD::OR, DL: dl, VT, Ops: {C1, C2})) {
1915 SDValue MaskX = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: C12);
1916 SDValue MaskY = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Y, N2: C2);
1917 return TLO.CombineTo(O: Op,
1918 N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: MaskX, N2: MaskY));
1919 }
1920 }
1921
1922 if (TryReduceRHSDemand(Known2, &KnownBits::One)) {
1923 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1924 return true;
1925 }
1926
1927 Known |= Known2;
1928 break;
1929 }
1930 case ISD::XOR: {
1931 SDValue Op0 = Op.getOperand(i: 0);
1932 SDValue Op1 = Op.getOperand(i: 1);
1933
1934 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1935 Depth: Depth + 1))
1936 return true;
1937 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
1938 Depth: Depth + 1))
1939 return true;
1940
1941 // If all of the demanded bits are known zero on one side, return the other.
1942 // These bits cannot contribute to the result of the 'xor'.
1943 if (DemandedBits.isSubsetOf(RHS: Known.Zero))
1944 return TLO.CombineTo(O: Op, N: Op0);
1945 if (DemandedBits.isSubsetOf(RHS: Known2.Zero))
1946 return TLO.CombineTo(O: Op, N: Op1);
1947 // If the operation can be done in a smaller type, do so.
1948 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1949 return true;
1950
1951 // If all of the unknown bits are known to be zero on one side or the other
1952 // turn this into an *inclusive* or.
1953 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1954 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1955 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op0, N2: Op1));
1956
1957 ConstantSDNode *C = isConstOrConstSplat(N: Op1, DemandedElts);
1958 if (C) {
1959 // If one side is a constant, and all of the set bits in the constant are
1960 // also known set on the other side, turn this into an AND, as we know
1961 // the bits will be cleared.
1962 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1963 // NB: it is okay if more bits are known than are requested
1964 if (C->getAPIntValue() == Known2.One) {
1965 SDValue ANDC =
1966 TLO.DAG.getConstant(Val: ~C->getAPIntValue() & DemandedBits, DL: dl, VT);
1967 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0, N2: ANDC));
1968 }
1969
1970 // If the RHS is a constant, see if we can change it. Don't alter a -1
1971 // constant because that's a 'not' op, and that is better for combining
1972 // and codegen.
1973 if (!C->isAllOnes() && DemandedBits.isSubsetOf(RHS: C->getAPIntValue())) {
1974 // We're flipping all demanded bits. Flip the undemanded bits too.
1975 SDValue New = TLO.DAG.getNOT(DL: dl, Val: Op0, VT);
1976 return TLO.CombineTo(O: Op, N: New);
1977 }
1978
1979 unsigned Op0Opcode = Op0.getOpcode();
1980 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
1981 if (ConstantSDNode *ShiftC =
1982 isConstOrConstSplat(N: Op0.getOperand(i: 1), DemandedElts)) {
1983 // Don't crash on an oversized shift. We can not guarantee that a
1984 // bogus shift has been simplified to undef.
1985 if (ShiftC->getAPIntValue().ult(RHS: BitWidth)) {
1986 uint64_t ShiftAmt = ShiftC->getZExtValue();
1987 APInt Ones = APInt::getAllOnes(numBits: BitWidth);
1988 Ones = Op0Opcode == ISD::SHL ? Ones.shl(shiftAmt: ShiftAmt)
1989 : Ones.lshr(shiftAmt: ShiftAmt);
1990 if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
1991 isDesirableToCommuteXorWithShift(N: Op.getNode())) {
1992 // If the xor constant is a demanded mask, do a 'not' before the
1993 // shift:
1994 // xor (X << ShiftC), XorC --> (not X) << ShiftC
1995 // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
1996 SDValue Not = TLO.DAG.getNOT(DL: dl, Val: Op0.getOperand(i: 0), VT);
1997 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op0Opcode, DL: dl, VT, N1: Not,
1998 N2: Op0.getOperand(i: 1)));
1999 }
2000 }
2001 }
2002 }
2003 }
2004
2005 // If we can't turn this into a 'not', try to shrink the constant.
2006 if (!C || !C->isAllOnes())
2007 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
2008 return true;
2009
2010 // Attempt to avoid multi-use ops if we don't need anything from them.
2011 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
2012 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2013 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2014 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2015 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2016 if (DemandedOp0 || DemandedOp1) {
2017 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2018 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2019 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
2020 return TLO.CombineTo(O: Op, N: NewOp);
2021 }
2022 }
2023
2024 Known ^= Known2;
2025 break;
2026 }
2027 case ISD::SELECT:
2028 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2029 Known, TLO, Depth: Depth + 1))
2030 return true;
2031 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2032 Known&: Known2, TLO, Depth: Depth + 1))
2033 return true;
2034
2035 // If the operands are constants, see if we can simplify them.
2036 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
2037 return true;
2038
2039 // Only known if known in both the LHS and RHS.
2040 Known = Known.intersectWith(RHS: Known2);
2041 break;
2042 case ISD::VSELECT:
2043 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2044 Known, TLO, Depth: Depth + 1))
2045 return true;
2046 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2047 Known&: Known2, TLO, Depth: Depth + 1))
2048 return true;
2049
2050 // Only known if known in both the LHS and RHS.
2051 Known = Known.intersectWith(RHS: Known2);
2052 break;
2053 case ISD::SELECT_CC:
2054 if (SimplifyDemandedBits(Op: Op.getOperand(i: 3), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2055 Known, TLO, Depth: Depth + 1))
2056 return true;
2057 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2058 Known&: Known2, TLO, Depth: Depth + 1))
2059 return true;
2060
2061 // If the operands are constants, see if we can simplify them.
2062 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
2063 return true;
2064
2065 // Only known if known in both the LHS and RHS.
2066 Known = Known.intersectWith(RHS: Known2);
2067 break;
2068 case ISD::SETCC: {
2069 SDValue Op0 = Op.getOperand(i: 0);
2070 SDValue Op1 = Op.getOperand(i: 1);
2071 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
2072 // If we're testing X < 0, X >= 0, X <= -1 or X > -1
2073 // (X is of integer type) then we only need the sign mask of the previous
2074 // result
2075 if (Op1.getValueType().isInteger() &&
2076 (((CC == ISD::SETLT || CC == ISD::SETGE) && isNullOrNullSplat(V: Op1)) ||
2077 ((CC == ISD::SETLE || CC == ISD::SETGT) &&
2078 isAllOnesOrAllOnesSplat(V: Op1)))) {
2079 KnownBits KnownOp0;
2080 if (SimplifyDemandedBits(
2081 Op: Op0, OriginalDemandedBits: APInt::getSignMask(BitWidth: Op0.getScalarValueSizeInBits()),
2082 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1))
2083 return true;
2084 // If (1) we only need the sign-bit, (2) the setcc operands are the same
2085 // width as the setcc result, and (3) the result of a setcc conforms to 0
2086 // or -1, we may be able to bypass the setcc.
2087 if (DemandedBits.isSignMask() &&
2088 Op0.getScalarValueSizeInBits() == BitWidth &&
2089 getBooleanContents(Type: Op0.getValueType()) ==
2090 BooleanContent::ZeroOrNegativeOneBooleanContent) {
2091 // If we remove a >= 0 or > -1 (for integers), we need to introduce a
2092 // NOT Operation
2093 if (CC == ISD::SETGE || CC == ISD::SETGT) {
2094 SDLoc DL(Op);
2095 EVT VT = Op0.getValueType();
2096 SDValue NotOp0 = TLO.DAG.getNOT(DL, Val: Op0, VT);
2097 return TLO.CombineTo(O: Op, N: NotOp0);
2098 }
2099 return TLO.CombineTo(O: Op, N: Op0);
2100 }
2101 }
2102 if (getBooleanContents(Type: Op0.getValueType()) ==
2103 TargetLowering::ZeroOrOneBooleanContent &&
2104 BitWidth > 1)
2105 Known.Zero.setBitsFrom(1);
2106 break;
2107 }
2108 case ISD::SHL: {
2109 SDValue Op0 = Op.getOperand(i: 0);
2110 SDValue Op1 = Op.getOperand(i: 1);
2111 EVT ShiftVT = Op1.getValueType();
2112
2113 if (std::optional<unsigned> KnownSA =
2114 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2115 unsigned ShAmt = *KnownSA;
2116 if (ShAmt == 0)
2117 return TLO.CombineTo(O: Op, N: Op0);
2118
2119 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
2120 // single shift. We can do this if the bottom bits (which are shifted
2121 // out) are never demanded.
2122 // TODO - support non-uniform vector amounts.
2123 if (Op0.getOpcode() == ISD::SRL) {
2124 if (!DemandedBits.intersects(RHS: APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ShAmt))) {
2125 if (std::optional<unsigned> InnerSA =
2126 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2127 unsigned C1 = *InnerSA;
2128 unsigned Opc = ISD::SHL;
2129 int Diff = ShAmt - C1;
2130 if (Diff < 0) {
2131 Diff = -Diff;
2132 Opc = ISD::SRL;
2133 }
2134 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
2135 return TLO.CombineTo(
2136 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
2137 }
2138 }
2139 }
2140
2141 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
2142 // are not demanded. This will likely allow the anyext to be folded away.
2143 // TODO - support non-uniform vector amounts.
2144 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
2145 SDValue InnerOp = Op0.getOperand(i: 0);
2146 EVT InnerVT = InnerOp.getValueType();
2147 unsigned InnerBits = InnerVT.getScalarSizeInBits();
2148 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
2149 isTypeDesirableForOp(ISD::SHL, VT: InnerVT)) {
2150 SDValue NarrowShl = TLO.DAG.getNode(
2151 Opcode: ISD::SHL, DL: dl, VT: InnerVT, N1: InnerOp,
2152 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: InnerVT, DL: dl));
2153 return TLO.CombineTo(
2154 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
2155 }
2156
2157 // Repeat the SHL optimization above in cases where an extension
2158 // intervenes: (shl (anyext (shr x, c1)), c2) to
2159 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
2160 // aren't demanded (as above) and that the shifted upper c1 bits of
2161 // x aren't demanded.
2162 // TODO - support non-uniform vector amounts.
2163 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
2164 InnerOp.hasOneUse()) {
2165 if (std::optional<unsigned> SA2 = TLO.DAG.getValidShiftAmount(
2166 V: InnerOp, DemandedElts, Depth: Depth + 2)) {
2167 unsigned InnerShAmt = *SA2;
2168 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
2169 DemandedBits.getActiveBits() <=
2170 (InnerBits - InnerShAmt + ShAmt) &&
2171 DemandedBits.countr_zero() >= ShAmt) {
2172 SDValue NewSA =
2173 TLO.DAG.getConstant(Val: ShAmt - InnerShAmt, DL: dl, VT: ShiftVT);
2174 SDValue NewExt = TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT,
2175 Operand: InnerOp.getOperand(i: 0));
2176 return TLO.CombineTo(
2177 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: NewExt, N2: NewSA));
2178 }
2179 }
2180 }
2181 }
2182
2183 APInt InDemandedMask = DemandedBits.lshr(shiftAmt: ShAmt);
2184 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2185 Depth: Depth + 1)) {
2186 // Disable the nsw and nuw flags. We can no longer guarantee that we
2187 // won't wrap after simplification.
2188 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2189 return true;
2190 }
2191 Known <<= ShAmt;
2192 // low bits known zero.
2193 Known.Zero.setLowBits(ShAmt);
2194
2195 // Attempt to avoid multi-use ops if we don't need anything from them.
2196 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2197 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2198 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2199 if (DemandedOp0) {
2200 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2201 return TLO.CombineTo(O: Op, N: NewOp);
2202 }
2203 }
2204
2205 // TODO: Can we merge this fold with the one below?
2206 // Try shrinking the operation as long as the shift amount will still be
2207 // in range.
2208 if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() &&
2209 Op.getNode()->hasOneUse()) {
2210 // Search for the smallest integer type with free casts to and from
2211 // Op's type. For expedience, just check power-of-2 integer types.
2212 unsigned DemandedSize = DemandedBits.getActiveBits();
2213 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
2214 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
2215 EVT SmallVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: SmallVTBits);
2216 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: SmallVT) &&
2217 isTypeDesirableForOp(ISD::SHL, VT: SmallVT) &&
2218 isTruncateFree(FromVT: VT, ToVT: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT) &&
2219 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: SmallVT))) {
2220 assert(DemandedSize <= SmallVTBits &&
2221 "Narrowed below demanded bits?");
2222 // We found a type with free casts.
2223 SDValue NarrowShl = TLO.DAG.getNode(
2224 Opcode: ISD::SHL, DL: dl, VT: SmallVT,
2225 N1: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
2226 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: SmallVT, DL: dl));
2227 return TLO.CombineTo(
2228 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
2229 }
2230 }
2231 }
2232
2233 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2234 // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
2235 // Only do this if we demand the upper half so the knownbits are correct.
2236 unsigned HalfWidth = BitWidth / 2;
2237 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
2238 DemandedBits.countLeadingOnes() >= HalfWidth) {
2239 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: HalfWidth);
2240 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2241 isTypeDesirableForOp(ISD::SHL, VT: HalfVT) &&
2242 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2243 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: HalfVT))) {
2244 // If we're demanding the upper bits at all, we must ensure
2245 // that the upper bits of the shift result are known to be zero,
2246 // which is equivalent to the narrow shift being NUW.
2247 if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
2248 bool IsNSW = Known.countMinSignBits() > HalfWidth;
2249 SDNodeFlags Flags;
2250 Flags.setNoSignedWrap(IsNSW);
2251 Flags.setNoUnsignedWrap(IsNUW);
2252 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2253 SDValue NewShiftAmt =
2254 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2255 SDValue NewShift = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HalfVT, N1: NewOp,
2256 N2: NewShiftAmt, Flags);
2257 SDValue NewExt =
2258 TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift);
2259 return TLO.CombineTo(O: Op, N: NewExt);
2260 }
2261 }
2262 }
2263 } else {
2264 // This is a variable shift, so we can't shift the demand mask by a known
2265 // amount. But if we are not demanding high bits, then we are not
2266 // demanding those bits from the pre-shifted operand either.
2267 if (unsigned CTLZ = DemandedBits.countl_zero()) {
2268 APInt DemandedFromOp(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - CTLZ));
2269 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedFromOp, OriginalDemandedElts: DemandedElts, Known, TLO,
2270 Depth: Depth + 1)) {
2271 // Disable the nsw and nuw flags. We can no longer guarantee that we
2272 // won't wrap after simplification.
2273 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2274 return true;
2275 }
2276 Known.resetAll();
2277 }
2278 }
2279
2280 // If we are only demanding sign bits then we can use the shift source
2281 // directly.
2282 if (std::optional<unsigned> MaxSA =
2283 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2284 unsigned ShAmt = *MaxSA;
2285 unsigned NumSignBits =
2286 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2287 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
2288 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
2289 return TLO.CombineTo(O: Op, N: Op0);
2290 }
2291 break;
2292 }
2293 case ISD::SRL: {
2294 SDValue Op0 = Op.getOperand(i: 0);
2295 SDValue Op1 = Op.getOperand(i: 1);
2296 EVT ShiftVT = Op1.getValueType();
2297
2298 if (std::optional<unsigned> KnownSA =
2299 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2300 unsigned ShAmt = *KnownSA;
2301 if (ShAmt == 0)
2302 return TLO.CombineTo(O: Op, N: Op0);
2303
2304 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
2305 // single shift. We can do this if the top bits (which are shifted out)
2306 // are never demanded.
2307 // TODO - support non-uniform vector amounts.
2308 if (Op0.getOpcode() == ISD::SHL) {
2309 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2310 if (std::optional<unsigned> InnerSA =
2311 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2312 unsigned C1 = *InnerSA;
2313 unsigned Opc = ISD::SRL;
2314 int Diff = ShAmt - C1;
2315 if (Diff < 0) {
2316 Diff = -Diff;
2317 Opc = ISD::SHL;
2318 }
2319 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
2320 return TLO.CombineTo(
2321 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
2322 }
2323 }
2324 }
2325
2326 // If this is (srl (sra X, C1), ShAmt), see if we can combine this into a
2327 // single sra. We can do this if the top bits are never demanded.
2328 if (Op0.getOpcode() == ISD::SRA && Op0.hasOneUse()) {
2329 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2330 if (std::optional<unsigned> InnerSA =
2331 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2332 unsigned C1 = *InnerSA;
2333 // Clamp the combined shift amount if it exceeds the bit width.
2334 unsigned Combined = std::min(a: C1 + ShAmt, b: BitWidth - 1);
2335 SDValue NewSA = TLO.DAG.getConstant(Val: Combined, DL: dl, VT: ShiftVT);
2336 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT,
2337 N1: Op0.getOperand(i: 0), N2: NewSA));
2338 }
2339 }
2340 }
2341
2342 APInt InDemandedMask = (DemandedBits << ShAmt);
2343
2344 // If the shift is exact, then it does demand the low bits (and knows that
2345 // they are zero).
2346 if (Op->getFlags().hasExact())
2347 InDemandedMask.setLowBits(ShAmt);
2348
2349 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2350 // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
2351 if ((BitWidth % 2) == 0 && !VT.isVector()) {
2352 APInt HiBits = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth / 2);
2353 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: BitWidth / 2);
2354 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2355 isTypeDesirableForOp(ISD::SRL, VT: HalfVT) &&
2356 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2357 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT: HalfVT)) &&
2358 ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
2359 TLO.DAG.MaskedValueIsZero(Op: Op0, Mask: HiBits))) {
2360 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2361 SDValue NewShiftAmt =
2362 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2363 SDValue NewShift =
2364 TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HalfVT, N1: NewOp, N2: NewShiftAmt,
2365 Flags: Op->getFlags() & SDNodeFlags::Exact);
2366 return TLO.CombineTo(
2367 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift));
2368 }
2369 }
2370
2371 // Compute the new bits that are at the top now.
2372 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2373 Depth: Depth + 1))
2374 return true;
2375 Known >>= ShAmt;
2376 // High bits known zero.
2377 Known.Zero.setHighBits(ShAmt);
2378
2379 // Attempt to avoid multi-use ops if we don't need anything from them.
2380 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2381 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2382 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2383 if (DemandedOp0) {
2384 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2385 return TLO.CombineTo(O: Op, N: NewOp);
2386 }
2387 }
2388 } else {
2389 // Use generic knownbits computation as it has support for non-uniform
2390 // shift amounts.
2391 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2392 }
2393
2394 // If we are only demanding sign bits then we can use the shift source
2395 // directly.
2396 if (std::optional<unsigned> MaxSA =
2397 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2398 unsigned ShAmt = *MaxSA;
2399 // Must already be signbits in DemandedBits bounds, and can't demand any
2400 // shifted in zeroes.
2401 if (DemandedBits.countl_zero() >= ShAmt) {
2402 unsigned NumSignBits =
2403 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2404 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
2405 return TLO.CombineTo(O: Op, N: Op0);
2406 }
2407 }
2408
2409 // Try to match AVG patterns (after shift simplification).
2410 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2411 DemandedElts, Depth: Depth + 1))
2412 return TLO.CombineTo(O: Op, N: AVG);
2413 if (SDValue AVG = combineShiftToAVGSUB(Op, TLO, TLI: *this, DemandedBits,
2414 DemandedElts, Depth: Depth + 1))
2415 return TLO.CombineTo(O: Op, N: AVG);
2416
2417 break;
2418 }
2419 case ISD::SRA: {
2420 SDValue Op0 = Op.getOperand(i: 0);
2421 SDValue Op1 = Op.getOperand(i: 1);
2422 EVT ShiftVT = Op1.getValueType();
2423
2424 // If we only want bits that already match the signbit then we don't need
2425 // to shift.
2426 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
2427 if (TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1) >=
2428 NumHiDemandedBits)
2429 return TLO.CombineTo(O: Op, N: Op0);
2430
2431 // If this is an arithmetic shift right and only the low-bit is set, we can
2432 // always convert this into a logical shr, even if the shift amount is
2433 // variable. The low bit of the shift cannot be an input sign bit unless
2434 // the shift amount is >= the size of the datatype, which is undefined.
2435 if (DemandedBits.isOne())
2436 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2437
2438 if (std::optional<unsigned> KnownSA =
2439 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2440 unsigned ShAmt = *KnownSA;
2441 if (ShAmt == 0)
2442 return TLO.CombineTo(O: Op, N: Op0);
2443
2444 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
2445 // supports sext_inreg.
2446 if (Op0.getOpcode() == ISD::SHL) {
2447 if (std::optional<unsigned> InnerSA =
2448 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2449 unsigned LowBits = BitWidth - ShAmt;
2450 EVT ExtVT = VT.changeElementType(
2451 Context&: *TLO.DAG.getContext(),
2452 EltVT: EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: LowBits));
2453
2454 if (*InnerSA == ShAmt) {
2455 if (!TLO.LegalOperations() ||
2456 getOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: ExtVT) == Legal)
2457 return TLO.CombineTo(
2458 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT,
2459 N1: Op0.getOperand(i: 0),
2460 N2: TLO.DAG.getValueType(ExtVT)));
2461
2462 // Even if we can't convert to sext_inreg, we might be able to
2463 // remove this shift pair if the input is already sign extended.
2464 unsigned NumSignBits =
2465 TLO.DAG.ComputeNumSignBits(Op: Op0.getOperand(i: 0), DemandedElts);
2466 if (NumSignBits > ShAmt)
2467 return TLO.CombineTo(O: Op, N: Op0.getOperand(i: 0));
2468 }
2469 }
2470 }
2471
2472 APInt InDemandedMask = (DemandedBits << ShAmt);
2473
2474 // If the shift is exact, then it does demand the low bits (and knows that
2475 // they are zero).
2476 if (Op->getFlags().hasExact())
2477 InDemandedMask.setLowBits(ShAmt);
2478
2479 // If any of the demanded bits are produced by the sign extension, we also
2480 // demand the input sign bit.
2481 if (DemandedBits.countl_zero() < ShAmt)
2482 InDemandedMask.setSignBit();
2483
2484 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2485 Depth: Depth + 1))
2486 return true;
2487 Known >>= ShAmt;
2488
2489 // If the input sign bit is known to be zero, or if none of the top bits
2490 // are demanded, turn this into an unsigned shift right.
2491 if (Known.Zero[BitWidth - ShAmt - 1] ||
2492 DemandedBits.countl_zero() >= ShAmt) {
2493 SDNodeFlags Flags;
2494 Flags.setExact(Op->getFlags().hasExact());
2495 return TLO.CombineTo(
2496 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1, Flags));
2497 }
2498
2499 int Log2 = DemandedBits.exactLogBase2();
2500 if (Log2 >= 0) {
2501 // The bit must come from the sign.
2502 SDValue NewSA = TLO.DAG.getConstant(Val: BitWidth - 1 - Log2, DL: dl, VT: ShiftVT);
2503 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: NewSA));
2504 }
2505
2506 if (Known.One[BitWidth - ShAmt - 1])
2507 // New bits are known one.
2508 Known.One.setHighBits(ShAmt);
2509
2510 // Attempt to avoid multi-use ops if we don't need anything from them.
2511 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2512 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2513 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2514 if (DemandedOp0) {
2515 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2516 return TLO.CombineTo(O: Op, N: NewOp);
2517 }
2518 }
2519 }
2520
2521 // Try to match AVG patterns (after shift simplification).
2522 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2523 DemandedElts, Depth: Depth + 1))
2524 return TLO.CombineTo(O: Op, N: AVG);
2525 if (SDValue AVG = combineShiftToAVGSUB(Op, TLO, TLI: *this, DemandedBits,
2526 DemandedElts, Depth: Depth + 1))
2527 return TLO.CombineTo(O: Op, N: AVG);
2528
2529 break;
2530 }
2531 case ISD::FSHL:
2532 case ISD::FSHR: {
2533 SDValue Op0 = Op.getOperand(i: 0);
2534 SDValue Op1 = Op.getOperand(i: 1);
2535 SDValue Op2 = Op.getOperand(i: 2);
2536 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
2537
2538 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op2, DemandedElts)) {
2539 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2540
2541 // For fshl, 0-shift returns the 1st arg.
2542 // For fshr, 0-shift returns the 2nd arg.
2543 if (Amt == 0) {
2544 if (SimplifyDemandedBits(Op: IsFSHL ? Op0 : Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2545 Known, TLO, Depth: Depth + 1))
2546 return true;
2547 break;
2548 }
2549
2550 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
2551 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
2552 APInt Demanded0 = DemandedBits.lshr(shiftAmt: IsFSHL ? Amt : (BitWidth - Amt));
2553 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
2554 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2555 Depth: Depth + 1))
2556 return true;
2557 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: Demanded1, OriginalDemandedElts: DemandedElts, Known, TLO,
2558 Depth: Depth + 1))
2559 return true;
2560
2561 Known2 <<= (IsFSHL ? Amt : (BitWidth - Amt));
2562 Known >>= (IsFSHL ? (BitWidth - Amt) : Amt);
2563 Known = Known.unionWith(RHS: Known2);
2564
2565 // Attempt to avoid multi-use ops if we don't need anything from them.
2566 if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
2567 !DemandedElts.isAllOnes()) {
2568 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2569 Op: Op0, DemandedBits: Demanded0, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2570 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2571 Op: Op1, DemandedBits: Demanded1, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2572 if (DemandedOp0 || DemandedOp1) {
2573 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2574 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2575 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedOp0,
2576 N2: DemandedOp1, N3: Op2);
2577 return TLO.CombineTo(O: Op, N: NewOp);
2578 }
2579 }
2580 }
2581
2582 if (isPowerOf2_32(Value: BitWidth)) {
2583 // Fold FSHR(Op0,Op1,Op2) -> SRL(Op1,Op2)
2584 // iff we're guaranteed not to use Op0.
2585 // TODO: Add FSHL equivalent?
2586 if (!IsFSHL && !DemandedBits.isAllOnes() &&
2587 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT))) {
2588 KnownBits KnownAmt =
2589 TLO.DAG.computeKnownBits(Op: Op2, DemandedElts, Depth: Depth + 1);
2590 unsigned MaxShiftAmt =
2591 KnownAmt.getMaxValue().getLimitedValue(Limit: BitWidth - 1);
2592 // Check we don't demand any shifted bits outside Op1.
2593 if (DemandedBits.countl_zero() >= MaxShiftAmt) {
2594 EVT AmtVT = Op2.getValueType();
2595 SDValue NewAmt =
2596 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Op2,
2597 N2: TLO.DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
2598 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op1, N2: NewAmt);
2599 return TLO.CombineTo(O: Op, N: NewOp);
2600 }
2601 }
2602
2603 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2604 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
2605 if (SimplifyDemandedBits(Op: Op2, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2606 Depth: Depth + 1))
2607 return true;
2608 }
2609 break;
2610 }
2611 case ISD::ROTL:
2612 case ISD::ROTR: {
2613 SDValue Op0 = Op.getOperand(i: 0);
2614 SDValue Op1 = Op.getOperand(i: 1);
2615 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
2616
2617 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
2618 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1))
2619 return TLO.CombineTo(O: Op, N: Op0);
2620
2621 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op1, DemandedElts)) {
2622 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2623 unsigned RevAmt = BitWidth - Amt;
2624
2625 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
2626 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
2627 APInt Demanded0 = DemandedBits.rotr(rotateAmt: IsROTL ? Amt : RevAmt);
2628 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2629 Depth: Depth + 1))
2630 return true;
2631
2632 // rot*(x, 0) --> x
2633 if (Amt == 0)
2634 return TLO.CombineTo(O: Op, N: Op0);
2635
2636 // See if we don't demand either half of the rotated bits.
2637 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT)) &&
2638 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2639 Op1 = TLO.DAG.getConstant(Val: IsROTL ? Amt : RevAmt, DL: dl, VT: Op1.getValueType());
2640 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: Op1));
2641 }
2642 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT)) &&
2643 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2644 Op1 = TLO.DAG.getConstant(Val: IsROTL ? RevAmt : Amt, DL: dl, VT: Op1.getValueType());
2645 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2646 }
2647 }
2648
2649 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2650 if (isPowerOf2_32(Value: BitWidth)) {
2651 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
2652 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2653 Depth: Depth + 1))
2654 return true;
2655 }
2656 break;
2657 }
2658 case ISD::SMIN:
2659 case ISD::SMAX:
2660 case ISD::UMIN:
2661 case ISD::UMAX: {
2662 unsigned Opc = Op.getOpcode();
2663 SDValue Op0 = Op.getOperand(i: 0);
2664 SDValue Op1 = Op.getOperand(i: 1);
2665
2666 // If we're only demanding signbits, then we can simplify to OR/AND node.
2667 unsigned BitOp =
2668 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
2669 unsigned NumSignBits =
2670 std::min(a: TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1),
2671 b: TLO.DAG.ComputeNumSignBits(Op: Op1, DemandedElts, Depth: Depth + 1));
2672 unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
2673 if (NumSignBits >= NumDemandedUpperBits)
2674 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: BitOp, DL: SDLoc(Op), VT, N1: Op0, N2: Op1));
2675
2676 // Check if one arg is always less/greater than (or equal) to the other arg.
2677 KnownBits Known0 = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2678 KnownBits Known1 = TLO.DAG.computeKnownBits(Op: Op1, DemandedElts, Depth: Depth + 1);
2679 switch (Opc) {
2680 case ISD::SMIN:
2681 if (std::optional<bool> IsSLE = KnownBits::sle(LHS: Known0, RHS: Known1))
2682 return TLO.CombineTo(O: Op, N: *IsSLE ? Op0 : Op1);
2683 if (std::optional<bool> IsSLT = KnownBits::slt(LHS: Known0, RHS: Known1))
2684 return TLO.CombineTo(O: Op, N: *IsSLT ? Op0 : Op1);
2685 Known = KnownBits::smin(LHS: Known0, RHS: Known1);
2686 break;
2687 case ISD::SMAX:
2688 if (std::optional<bool> IsSGE = KnownBits::sge(LHS: Known0, RHS: Known1))
2689 return TLO.CombineTo(O: Op, N: *IsSGE ? Op0 : Op1);
2690 if (std::optional<bool> IsSGT = KnownBits::sgt(LHS: Known0, RHS: Known1))
2691 return TLO.CombineTo(O: Op, N: *IsSGT ? Op0 : Op1);
2692 Known = KnownBits::smax(LHS: Known0, RHS: Known1);
2693 break;
2694 case ISD::UMIN:
2695 if (std::optional<bool> IsULE = KnownBits::ule(LHS: Known0, RHS: Known1))
2696 return TLO.CombineTo(O: Op, N: *IsULE ? Op0 : Op1);
2697 if (std::optional<bool> IsULT = KnownBits::ult(LHS: Known0, RHS: Known1))
2698 return TLO.CombineTo(O: Op, N: *IsULT ? Op0 : Op1);
2699 Known = KnownBits::umin(LHS: Known0, RHS: Known1);
2700 break;
2701 case ISD::UMAX:
2702 if (std::optional<bool> IsUGE = KnownBits::uge(LHS: Known0, RHS: Known1))
2703 return TLO.CombineTo(O: Op, N: *IsUGE ? Op0 : Op1);
2704 if (std::optional<bool> IsUGT = KnownBits::ugt(LHS: Known0, RHS: Known1))
2705 return TLO.CombineTo(O: Op, N: *IsUGT ? Op0 : Op1);
2706 Known = KnownBits::umax(LHS: Known0, RHS: Known1);
2707 break;
2708 }
2709 break;
2710 }
2711 case ISD::BITREVERSE: {
2712 SDValue Src = Op.getOperand(i: 0);
2713 APInt DemandedSrcBits = DemandedBits.reverseBits();
2714 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2715 Depth: Depth + 1))
2716 return true;
2717 Known = Known2.reverseBits();
2718 break;
2719 }
2720 case ISD::BSWAP: {
2721 SDValue Src = Op.getOperand(i: 0);
2722
2723 // If the only bits demanded come from one byte of the bswap result,
2724 // just shift the input byte into position to eliminate the bswap.
2725 unsigned NLZ = DemandedBits.countl_zero();
2726 unsigned NTZ = DemandedBits.countr_zero();
2727
2728 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
2729 // we need all the bits down to bit 8. Likewise, round NLZ. If we
2730 // have 14 leading zeros, round to 8.
2731 NLZ = alignDown(Value: NLZ, Align: 8);
2732 NTZ = alignDown(Value: NTZ, Align: 8);
2733 // If we need exactly one byte, we can do this transformation.
2734 if (BitWidth - NLZ - NTZ == 8) {
2735 // Replace this with either a left or right shift to get the byte into
2736 // the right place.
2737 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
2738 if (!TLO.LegalOperations() || isOperationLegal(Op: ShiftOpcode, VT)) {
2739 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2740 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
2741 SDValue NewOp = TLO.DAG.getNode(Opcode: ShiftOpcode, DL: dl, VT, N1: Src, N2: ShAmt);
2742 return TLO.CombineTo(O: Op, N: NewOp);
2743 }
2744 }
2745
2746 APInt DemandedSrcBits = DemandedBits.byteSwap();
2747 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2748 Depth: Depth + 1))
2749 return true;
2750 Known = Known2.byteSwap();
2751 break;
2752 }
2753 case ISD::CTPOP: {
2754 // If only 1 bit is demanded, replace with PARITY as long as we're before
2755 // op legalization.
2756 // FIXME: Limit to scalars for now.
2757 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
2758 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT,
2759 Operand: Op.getOperand(i: 0)));
2760
2761 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2762 break;
2763 }
2764 case ISD::PDEP: {
2765 SDValue Op0 = Op.getOperand(i: 0);
2766 SDValue Op1 = Op.getOperand(i: 1);
2767
2768 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2769 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
2770
2771 // If the demanded bits has leading zeroes, we don't demand those from the
2772 // mask.
2773 if (SimplifyDemandedBits(Op: Op1, DemandedBits: LoMask, Known, TLO, Depth: Depth + 1))
2774 return true;
2775
2776 // The number of possible 1s in the mask determines the number of LSBs of
2777 // operand 0 used. Undemanded bits from the mask don't matter so filter
2778 // them before counting.
2779 KnownBits Known2;
2780 uint64_t Count = (~Known.Zero & LoMask).popcount();
2781 APInt DemandedMask(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: Count));
2782 if (SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, Known&: Known2, TLO, Depth: Depth + 1))
2783 return true;
2784
2785 // Zeroes are retained from the mask, but not ones.
2786 Known.One.clearAllBits();
2787 // The result will have at least as many trailing zeros as the non-mask
2788 // operand since bits can only map to the same or higher bit position.
2789 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
2790 break;
2791 }
2792 case ISD::SIGN_EXTEND_INREG: {
2793 SDValue Op0 = Op.getOperand(i: 0);
2794 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2795 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2796
2797 // If we only care about the highest bit, don't bother shifting right.
2798 if (DemandedBits.isSignMask()) {
2799 unsigned MinSignedBits =
2800 TLO.DAG.ComputeMaxSignificantBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2801 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2802 // However if the input is already sign extended we expect the sign
2803 // extension to be dropped altogether later and do not simplify.
2804 if (!AlreadySignExtended) {
2805 // Compute the correct shift amount type, which must be getShiftAmountTy
2806 // for scalar types after legalization.
2807 SDValue ShiftAmt =
2808 TLO.DAG.getShiftAmountConstant(Val: BitWidth - ExVTBits, VT, DL: dl);
2809 return TLO.CombineTo(O: Op,
2810 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: ShiftAmt));
2811 }
2812 }
2813
2814 // If none of the extended bits are demanded, eliminate the sextinreg.
2815 if (DemandedBits.getActiveBits() <= ExVTBits)
2816 return TLO.CombineTo(O: Op, N: Op0);
2817
2818 APInt InputDemandedBits = DemandedBits.getLoBits(numBits: ExVTBits);
2819
2820 // Since the sign extended bits are demanded, we know that the sign
2821 // bit is demanded.
2822 InputDemandedBits.setBit(ExVTBits - 1);
2823
2824 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InputDemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
2825 Depth: Depth + 1))
2826 return true;
2827
2828 // If the sign bit of the input is known set or clear, then we know the
2829 // top bits of the result.
2830
2831 // If the input sign bit is known zero, convert this into a zero extension.
2832 if (Known.Zero[ExVTBits - 1])
2833 return TLO.CombineTo(O: Op, N: TLO.DAG.getZeroExtendInReg(Op: Op0, DL: dl, VT: ExVT));
2834
2835 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ExVTBits);
2836 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
2837 Known.One.setBitsFrom(ExVTBits);
2838 Known.Zero &= Mask;
2839 } else { // Input sign bit unknown
2840 Known.Zero &= Mask;
2841 Known.One &= Mask;
2842 }
2843 break;
2844 }
2845 case ISD::BUILD_PAIR: {
2846 EVT HalfVT = Op.getOperand(i: 0).getValueType();
2847 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
2848
2849 APInt MaskLo = DemandedBits.getLoBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2850 APInt MaskHi = DemandedBits.getHiBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2851
2852 KnownBits KnownLo, KnownHi;
2853
2854 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: MaskLo, Known&: KnownLo, TLO, Depth: Depth + 1))
2855 return true;
2856
2857 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: MaskHi, Known&: KnownHi, TLO, Depth: Depth + 1))
2858 return true;
2859
2860 Known = KnownHi.concat(Lo: KnownLo);
2861 break;
2862 }
2863 case ISD::ZERO_EXTEND_VECTOR_INREG:
2864 if (VT.isScalableVector())
2865 return false;
2866 [[fallthrough]];
2867 case ISD::ZERO_EXTEND: {
2868 SDValue Src = Op.getOperand(i: 0);
2869 EVT SrcVT = Src.getValueType();
2870 unsigned InBits = SrcVT.getScalarSizeInBits();
2871 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2872 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
2873
2874 // If none of the top bits are demanded, convert this into an any_extend.
2875 if (DemandedBits.getActiveBits() <= InBits) {
2876 // If we only need the non-extended bits of the bottom element
2877 // then we can just bitcast to the result.
2878 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2879 VT.getSizeInBits() == SrcVT.getSizeInBits())
2880 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2881
2882 unsigned Opc =
2883 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2884 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2885 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2886 }
2887
2888 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2889 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2890 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2891 Depth: Depth + 1)) {
2892 Op->dropFlags(Mask: SDNodeFlags::NonNeg);
2893 return true;
2894 }
2895 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2896 Known = Known.zext(BitWidth);
2897
2898 // Attempt to avoid multi-use ops if we don't need anything from them.
2899 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2900 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2901 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2902 break;
2903 }
2904 case ISD::SIGN_EXTEND_VECTOR_INREG:
2905 if (VT.isScalableVector())
2906 return false;
2907 [[fallthrough]];
2908 case ISD::SIGN_EXTEND: {
2909 SDValue Src = Op.getOperand(i: 0);
2910 EVT SrcVT = Src.getValueType();
2911 unsigned InBits = SrcVT.getScalarSizeInBits();
2912 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2913 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
2914
2915 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2916 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2917
2918 // Since some of the sign extended bits are demanded, we know that the sign
2919 // bit is demanded.
2920 InDemandedBits.setBit(InBits - 1);
2921
2922 // If none of the top bits are demanded, convert this into an any_extend.
2923 if (DemandedBits.getActiveBits() <= InBits) {
2924 // If we only need the non-extended bits of the bottom element
2925 // then we can just bitcast to the result.
2926 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2927 VT.getSizeInBits() == SrcVT.getSizeInBits())
2928 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2929
2930 // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
2931 if (getBooleanContents(Type: VT) != ZeroOrNegativeOneBooleanContent ||
2932 TLO.DAG.ComputeNumSignBits(Op: Src, DemandedElts: InDemandedElts, Depth: Depth + 1) !=
2933 InBits) {
2934 unsigned Opc =
2935 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2936 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2937 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2938 }
2939 }
2940
2941 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2942 Depth: Depth + 1))
2943 return true;
2944 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2945
2946 // If the sign bit is known one, the top bits match.
2947 Known = Known.sext(BitWidth);
2948
2949 // If the sign bit is known zero, convert this to a zero extend.
2950 if (Known.isNonNegative()) {
2951 unsigned Opc =
2952 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
2953 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT)) {
2954 SDNodeFlags Flags;
2955 if (!IsVecInReg)
2956 Flags |= SDNodeFlags::NonNeg;
2957 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src, Flags));
2958 }
2959 }
2960
2961 // Attempt to avoid multi-use ops if we don't need anything from them.
2962 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2963 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2964 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2965 break;
2966 }
2967 case ISD::ANY_EXTEND_VECTOR_INREG:
2968 if (VT.isScalableVector())
2969 return false;
2970 [[fallthrough]];
2971 case ISD::ANY_EXTEND: {
2972 SDValue Src = Op.getOperand(i: 0);
2973 EVT SrcVT = Src.getValueType();
2974 unsigned InBits = SrcVT.getScalarSizeInBits();
2975 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2976 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2977
2978 // If we only need the bottom element then we can just bitcast.
2979 // TODO: Handle ANY_EXTEND?
2980 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2981 VT.getSizeInBits() == SrcVT.getSizeInBits())
2982 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2983
2984 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2985 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2986 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2987 Depth: Depth + 1))
2988 return true;
2989 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2990 Known = Known.anyext(BitWidth);
2991
2992 // Attempt to avoid multi-use ops if we don't need anything from them.
2993 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2994 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2995 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2996 break;
2997 }
2998 case ISD::TRUNCATE: {
2999 SDValue Src = Op.getOperand(i: 0);
3000
3001 // Simplify the input, using demanded bit information, and compute the known
3002 // zero/one bits live out.
3003 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
3004 APInt TruncMask = DemandedBits.zext(width: OperandBitWidth);
3005 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: TruncMask, OriginalDemandedElts: DemandedElts, Known, TLO,
3006 Depth: Depth + 1)) {
3007 // Disable the nsw and nuw flags. We can no longer guarantee that we
3008 // won't wrap after simplification.
3009 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
3010 return true;
3011 }
3012 Known = Known.trunc(BitWidth);
3013
3014 // Attempt to avoid multi-use ops if we don't need anything from them.
3015 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
3016 Op: Src, DemandedBits: TruncMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
3017 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: NewSrc));
3018
3019 // If the input is only used by this truncate, see if we can shrink it based
3020 // on the known demanded bits.
3021 switch (Src.getOpcode()) {
3022 default:
3023 break;
3024 case ISD::SRL:
3025 // Shrink SRL by a constant if none of the high bits shifted in are
3026 // demanded.
3027 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
3028 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
3029 // undesirable.
3030 break;
3031
3032 if (Src.getNode()->hasOneUse()) {
3033 if (isTruncateFree(Val: Src, VT2: VT) &&
3034 !isTruncateFree(FromVT: Src.getValueType(), ToVT: VT)) {
3035 // If truncate is only free at trunc(srl), do not turn it into
3036 // srl(trunc). The check is done by first check the truncate is free
3037 // at Src's opcode(srl), then check the truncate is not done by
3038 // referencing sub-register. In test, if both trunc(srl) and
3039 // srl(trunc)'s trunc are free, srl(trunc) performs better. If only
3040 // trunc(srl)'s trunc is free, trunc(srl) is better.
3041 break;
3042 }
3043
3044 std::optional<unsigned> ShAmtC =
3045 TLO.DAG.getValidShiftAmount(V: Src, DemandedElts, Depth: Depth + 2);
3046 if (!ShAmtC || *ShAmtC >= BitWidth)
3047 break;
3048 unsigned ShVal = *ShAmtC;
3049
3050 APInt HighBits =
3051 APInt::getHighBitsSet(numBits: OperandBitWidth, hiBitsSet: OperandBitWidth - BitWidth);
3052 HighBits.lshrInPlace(ShiftAmt: ShVal);
3053 HighBits = HighBits.trunc(width: BitWidth);
3054 if (!(HighBits & DemandedBits)) {
3055 // None of the shifted in bits are needed. Add a truncate of the
3056 // shift input, then shift it.
3057 SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
3058 SDValue NewTrunc =
3059 TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Src.getOperand(i: 0));
3060 return TLO.CombineTo(
3061 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NewTrunc, N2: NewShAmt));
3062 }
3063 }
3064 break;
3065 }
3066
3067 break;
3068 }
3069 case ISD::AssertZext: {
3070 // AssertZext demands all of the high bits, plus any of the low bits
3071 // demanded by its users.
3072 EVT ZVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3073 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ZVT.getSizeInBits());
3074 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: ~InMask | DemandedBits, Known,
3075 TLO, Depth: Depth + 1))
3076 return true;
3077
3078 Known.Zero |= ~InMask;
3079 Known.One &= (~Known.Zero);
3080 break;
3081 }
3082 case ISD::EXTRACT_VECTOR_ELT: {
3083 SDValue Src = Op.getOperand(i: 0);
3084 SDValue Idx = Op.getOperand(i: 1);
3085 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
3086 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
3087
3088 if (SrcEltCnt.isScalable())
3089 return false;
3090
3091 // Demand the bits from every vector element without a constant index.
3092 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
3093 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
3094 if (auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx))
3095 if (CIdx->getAPIntValue().ult(RHS: NumSrcElts))
3096 DemandedSrcElts = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: CIdx->getZExtValue());
3097
3098 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3099 // anything about the extended bits.
3100 APInt DemandedSrcBits = DemandedBits;
3101 if (BitWidth > EltBitWidth)
3102 DemandedSrcBits = DemandedSrcBits.trunc(width: EltBitWidth);
3103
3104 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts, Known&: Known2, TLO,
3105 Depth: Depth + 1))
3106 return true;
3107
3108 // Attempt to avoid multi-use ops if we don't need anything from them.
3109 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
3110 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
3111 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
3112 SDValue NewOp =
3113 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc, N2: Idx);
3114 return TLO.CombineTo(O: Op, N: NewOp);
3115 }
3116 }
3117
3118 Known = Known2;
3119 if (BitWidth > EltBitWidth)
3120 Known = Known.anyext(BitWidth);
3121 break;
3122 }
3123 case ISD::BITCAST: {
3124 if (VT.isScalableVector())
3125 return false;
3126 SDValue Src = Op.getOperand(i: 0);
3127 EVT SrcVT = Src.getValueType();
3128 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
3129
3130 // If this is an FP->Int bitcast and if the sign bit is the only
3131 // thing demanded, turn this into a FGETSIGN.
3132 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
3133 DemandedBits == APInt::getSignMask(BitWidth: Op.getValueSizeInBits()) &&
3134 SrcVT.isFloatingPoint()) {
3135 if (isOperationLegalOrCustom(Op: ISD::FGETSIGN, VT)) {
3136 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
3137 // place. We expect the SHL to be eliminated by other optimizations.
3138 SDValue Sign = TLO.DAG.getNode(Opcode: ISD::FGETSIGN, DL: dl, VT, Operand: Src);
3139 unsigned ShVal = Op.getValueSizeInBits() - 1;
3140 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
3141 return TLO.CombineTo(O: Op,
3142 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Sign, N2: ShAmt));
3143 }
3144 }
3145
3146 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
3147 // Demand the elt/bit if any of the original elts/bits are demanded.
3148 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
3149 unsigned Scale = BitWidth / NumSrcEltBits;
3150 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3151 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
3152 for (unsigned i = 0; i != Scale; ++i) {
3153 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
3154 unsigned BitOffset = EltOffset * NumSrcEltBits;
3155 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
3156 }
3157 // Recursive calls below may turn not demanded elements into poison, so we
3158 // need to demand all smaller source elements that maps to a demanded
3159 // destination element.
3160 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3161
3162 APInt KnownSrcUndef, KnownSrcZero;
3163 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
3164 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
3165 return true;
3166
3167 KnownBits KnownSrcBits;
3168 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
3169 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
3170 return true;
3171 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
3172 // TODO - bigendian once we have test coverage.
3173 unsigned Scale = NumSrcEltBits / BitWidth;
3174 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
3175 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
3176 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
3177 for (unsigned i = 0; i != NumElts; ++i)
3178 if (DemandedElts[i]) {
3179 unsigned Offset = (i % Scale) * BitWidth;
3180 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
3181 DemandedSrcElts.setBit(i / Scale);
3182 }
3183
3184 if (SrcVT.isVector()) {
3185 APInt KnownSrcUndef, KnownSrcZero;
3186 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
3187 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
3188 return true;
3189 }
3190
3191 KnownBits KnownSrcBits;
3192 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
3193 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
3194 return true;
3195
3196 // Attempt to avoid multi-use ops if we don't need anything from them.
3197 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
3198 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
3199 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
3200 SDValue NewOp = TLO.DAG.getBitcast(VT, V: DemandedSrc);
3201 return TLO.CombineTo(O: Op, N: NewOp);
3202 }
3203 }
3204 }
3205
3206 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
3207 // recursive call where Known may be useful to the caller.
3208 if (Depth > 0) {
3209 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3210 return false;
3211 }
3212 break;
3213 }
3214 case ISD::MUL:
3215 if (DemandedBits.isPowerOf2()) {
3216 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
3217 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
3218 // odd (has LSB set), then the left-shifted low bit of X is the answer.
3219 unsigned CTZ = DemandedBits.countr_zero();
3220 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
3221 if (C && C->getAPIntValue().countr_zero() == CTZ) {
3222 SDValue AmtC = TLO.DAG.getShiftAmountConstant(Val: CTZ, VT, DL: dl);
3223 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op.getOperand(i: 0), N2: AmtC);
3224 return TLO.CombineTo(O: Op, N: Shl);
3225 }
3226 }
3227 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
3228 // X * X is odd iff X is odd.
3229 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
3230 if (Op.getOperand(i: 0) == Op.getOperand(i: 1) && DemandedBits.ult(RHS: 4)) {
3231 SDValue One = TLO.DAG.getConstant(Val: 1, DL: dl, VT);
3232 SDValue And1 = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op.getOperand(i: 0), N2: One);
3233 return TLO.CombineTo(O: Op, N: And1);
3234 }
3235 [[fallthrough]];
3236 case ISD::PTRADD:
3237 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
3238 break;
3239 // PTRADD behaves like ADD if pointers are represented as integers.
3240 [[fallthrough]];
3241 case ISD::ADD:
3242 case ISD::SUB: {
3243 // Add, Sub, and Mul don't demand any bits in positions beyond that
3244 // of the highest bit demanded of them.
3245 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
3246 SDNodeFlags Flags = Op.getNode()->getFlags();
3247 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
3248 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
3249 KnownBits KnownOp0, KnownOp1;
3250 auto GetDemandedBitsLHSMask = [&](APInt Demanded,
3251 const KnownBits &KnownRHS) {
3252 if (Op.getOpcode() == ISD::MUL)
3253 Demanded.clearHighBits(hiBits: KnownRHS.countMinTrailingZeros());
3254 return Demanded;
3255 };
3256 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: LoMask, OriginalDemandedElts: DemandedElts, Known&: KnownOp1, TLO,
3257 Depth: Depth + 1) ||
3258 SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: GetDemandedBitsLHSMask(LoMask, KnownOp1),
3259 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1) ||
3260 // See if the operation should be performed at a smaller bit width.
3261 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
3262 // Disable the nsw and nuw flags. We can no longer guarantee that we
3263 // won't wrap after simplification.
3264 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
3265 return true;
3266 }
3267
3268 // neg x with only low bit demanded is simply x.
3269 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
3270 isNullConstant(V: Op0))
3271 return TLO.CombineTo(O: Op, N: Op1);
3272
3273 // Attempt to avoid multi-use ops if we don't need anything from them.
3274 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
3275 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
3276 Op: Op0, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3277 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
3278 Op: Op1, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3279 if (DemandedOp0 || DemandedOp1) {
3280 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
3281 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
3282 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1,
3283 Flags: Flags & ~SDNodeFlags::NoWrap);
3284 return TLO.CombineTo(O: Op, N: NewOp);
3285 }
3286 }
3287
3288 // If we have a constant operand, we may be able to turn it into -1 if we
3289 // do not demand the high bits. This can make the constant smaller to
3290 // encode, allow more general folding, or match specialized instruction
3291 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
3292 // is probably not useful (and could be detrimental).
3293 ConstantSDNode *C = isConstOrConstSplat(N: Op1);
3294 APInt HighMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: DemandedBitsLZ);
3295 if (C && !C->isAllOnes() && !C->isOne() &&
3296 (C->getAPIntValue() | HighMask).isAllOnes()) {
3297 SDValue Neg1 = TLO.DAG.getAllOnesConstant(DL: dl, VT);
3298 // Disable the nsw and nuw flags. We can no longer guarantee that we
3299 // won't wrap after simplification.
3300 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Neg1,
3301 Flags: Flags & ~SDNodeFlags::NoWrap);
3302 return TLO.CombineTo(O: Op, N: NewOp);
3303 }
3304
3305 // Match a multiply with a disguised negated-power-of-2 and convert to a
3306 // an equivalent shift-left amount.
3307 // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3308 auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
3309 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
3310 return 0;
3311
3312 // Don't touch opaque constants. Also, ignore zero and power-of-2
3313 // multiplies. Those will get folded later.
3314 ConstantSDNode *MulC = isConstOrConstSplat(N: Mul.getOperand(i: 1));
3315 if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
3316 !MulC->getAPIntValue().isPowerOf2()) {
3317 APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
3318 if (UnmaskedC.isNegatedPowerOf2())
3319 return (-UnmaskedC).logBase2();
3320 }
3321 return 0;
3322 };
3323
3324 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
3325 unsigned ShlAmt) {
3326 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(Val: ShlAmt, VT, DL: dl);
3327 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: X, N2: ShlAmtC);
3328 SDValue Res = TLO.DAG.getNode(Opcode: NT, DL: dl, VT, N1: Y, N2: Shl);
3329 return TLO.CombineTo(O: Op, N: Res);
3330 };
3331
3332 if (isOperationLegalOrCustom(Op: ISD::SHL, VT)) {
3333 if (Op.getOpcode() == ISD::ADD) {
3334 // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3335 if (unsigned ShAmt = getShiftLeftAmt(Op0))
3336 return foldMul(ISD::SUB, Op0.getOperand(i: 0), Op1, ShAmt);
3337 // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
3338 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3339 return foldMul(ISD::SUB, Op1.getOperand(i: 0), Op0, ShAmt);
3340 }
3341 if (Op.getOpcode() == ISD::SUB) {
3342 // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
3343 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3344 return foldMul(ISD::ADD, Op1.getOperand(i: 0), Op0, ShAmt);
3345 }
3346 }
3347
3348 if (Op.getOpcode() == ISD::MUL) {
3349 Known = KnownBits::mul(LHS: KnownOp0, RHS: KnownOp1);
3350 } else { // Op.getOpcode() is either ISD::ADD, ISD::PTRADD, or ISD::SUB.
3351 Known = KnownBits::computeForAddSub(
3352 Add: Op.getOpcode() != ISD::SUB, NSW: Flags.hasNoSignedWrap(),
3353 NUW: Flags.hasNoUnsignedWrap(), LHS: KnownOp0, RHS: KnownOp1);
3354 }
3355 break;
3356 }
3357 case ISD::FABS: {
3358 SDValue Op0 = Op.getOperand(i: 0);
3359 APInt SignMask = APInt::getSignMask(BitWidth);
3360
3361 if (!DemandedBits.intersects(RHS: SignMask))
3362 return TLO.CombineTo(O: Op, N: Op0);
3363
3364 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3365 Depth: Depth + 1))
3366 return true;
3367
3368 if (Known.isNonNegative())
3369 return TLO.CombineTo(O: Op, N: Op0);
3370 if (Known.isNegative())
3371 return TLO.CombineTo(
3372 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3373
3374 Known.Zero |= SignMask;
3375 Known.One &= ~SignMask;
3376
3377 break;
3378 }
3379 case ISD::FCOPYSIGN: {
3380 SDValue Op0 = Op.getOperand(i: 0);
3381 SDValue Op1 = Op.getOperand(i: 1);
3382
3383 unsigned BitWidth0 = Op0.getScalarValueSizeInBits();
3384 unsigned BitWidth1 = Op1.getScalarValueSizeInBits();
3385 APInt SignMask0 = APInt::getSignMask(BitWidth: BitWidth0);
3386 APInt SignMask1 = APInt::getSignMask(BitWidth: BitWidth1);
3387
3388 if (!DemandedBits.intersects(RHS: SignMask0))
3389 return TLO.CombineTo(O: Op, N: Op0);
3390
3391 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~SignMask0 & DemandedBits, OriginalDemandedElts: DemandedElts,
3392 Known, TLO, Depth: Depth + 1) ||
3393 SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: SignMask1, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
3394 Depth: Depth + 1))
3395 return true;
3396
3397 if (Known2.isNonNegative())
3398 return TLO.CombineTo(
3399 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3400
3401 if (Known2.isNegative())
3402 return TLO.CombineTo(
3403 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT,
3404 Operand: TLO.DAG.getNode(Opcode: ISD::FABS, DL: SDLoc(Op0), VT, Operand: Op0)));
3405
3406 Known.Zero &= ~SignMask0;
3407 Known.One &= ~SignMask0;
3408 break;
3409 }
3410 case ISD::FNEG: {
3411 SDValue Op0 = Op.getOperand(i: 0);
3412 APInt SignMask = APInt::getSignMask(BitWidth);
3413
3414 if (!DemandedBits.intersects(RHS: SignMask))
3415 return TLO.CombineTo(O: Op, N: Op0);
3416
3417 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3418 Depth: Depth + 1))
3419 return true;
3420
3421 if (!Known.isSignUnknown()) {
3422 Known.Zero ^= SignMask;
3423 Known.One ^= SignMask;
3424 }
3425
3426 break;
3427 }
3428 default:
3429 // We also ask the target about intrinsics (which could be specific to it).
3430 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3431 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
3432 // TODO: Probably okay to remove after audit; here to reduce change size
3433 // in initial enablement patch for scalable vectors
3434 if (Op.getValueType().isScalableVector())
3435 break;
3436 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
3437 Known, TLO, Depth))
3438 return true;
3439 break;
3440 }
3441
3442 // Just use computeKnownBits to compute output bits.
3443 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3444 break;
3445 }
3446
3447 // If we know the value of all of the demanded bits, return this as a
3448 // constant.
3449 if (!isTargetCanonicalConstantNode(Op) &&
3450 DemandedBits.isSubsetOf(RHS: Known.Zero | Known.One)) {
3451 // Avoid folding to a constant if any OpaqueConstant is involved.
3452 if (llvm::any_of(Range: Op->ops(), P: [](SDValue V) {
3453 auto *C = dyn_cast<ConstantSDNode>(Val&: V);
3454 return C && C->isOpaque();
3455 }))
3456 return false;
3457 if (VT.isInteger())
3458 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: Known.One, DL: dl, VT));
3459 if (VT.isFloatingPoint())
3460 return TLO.CombineTo(
3461 O: Op, N: TLO.DAG.getConstantFP(Val: APFloat(VT.getFltSemantics(), Known.One),
3462 DL: dl, VT));
3463 }
3464
3465 // A multi use 'all demanded elts' simplify failed to find any knownbits.
3466 // Try again just for the original demanded elts.
3467 // Ensure we do this AFTER constant folding above.
3468 if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
3469 Known = TLO.DAG.computeKnownBits(Op, DemandedElts: OriginalDemandedElts, Depth);
3470
3471 return false;
3472}
3473
3474bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
3475 const APInt &DemandedElts,
3476 DAGCombinerInfo &DCI) const {
3477 SelectionDAG &DAG = DCI.DAG;
3478 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
3479 !DCI.isBeforeLegalizeOps());
3480
3481 APInt KnownUndef, KnownZero;
3482 bool Simplified =
3483 SimplifyDemandedVectorElts(Op, DemandedEltMask: DemandedElts, KnownUndef, KnownZero, TLO);
3484 if (Simplified) {
3485 DCI.AddToWorklist(N: Op.getNode());
3486 DCI.CommitTargetLoweringOpt(TLO);
3487 }
3488
3489 return Simplified;
3490}
3491
3492/// Given a vector binary operation and known undefined elements for each input
3493/// operand, compute whether each element of the output is undefined.
3494static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
3495 const APInt &UndefOp0,
3496 const APInt &UndefOp1) {
3497 EVT VT = BO.getValueType();
3498 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
3499 "Vector binop only");
3500
3501 EVT EltVT = VT.getVectorElementType();
3502 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
3503 assert(UndefOp0.getBitWidth() == NumElts &&
3504 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
3505
3506 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
3507 const APInt &UndefVals) {
3508 if (UndefVals[Index])
3509 return DAG.getUNDEF(VT: EltVT);
3510
3511 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
3512 // Try hard to make sure that the getNode() call is not creating temporary
3513 // nodes. Ignore opaque integers because they do not constant fold.
3514 SDValue Elt = BV->getOperand(Num: Index);
3515 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
3516 if (isa<ConstantFPSDNode>(Val: Elt) || Elt.isUndef() || (C && !C->isOpaque()))
3517 return Elt;
3518 }
3519
3520 return SDValue();
3521 };
3522
3523 APInt KnownUndef = APInt::getZero(numBits: NumElts);
3524 for (unsigned i = 0; i != NumElts; ++i) {
3525 // If both inputs for this element are either constant or undef and match
3526 // the element type, compute the constant/undef result for this element of
3527 // the vector.
3528 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
3529 // not handle FP constants. The code within getNode() should be refactored
3530 // to avoid the danger of creating a bogus temporary node here.
3531 SDValue C0 = getUndefOrConstantElt(BO.getOperand(i: 0), i, UndefOp0);
3532 SDValue C1 = getUndefOrConstantElt(BO.getOperand(i: 1), i, UndefOp1);
3533 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
3534 if (DAG.getNode(Opcode: BO.getOpcode(), DL: SDLoc(BO), VT: EltVT, N1: C0, N2: C1).isUndef())
3535 KnownUndef.setBit(i);
3536 }
3537 return KnownUndef;
3538}
3539
3540bool TargetLowering::SimplifyDemandedVectorElts(
3541 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
3542 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
3543 bool AssumeSingleUse) const {
3544 EVT VT = Op.getValueType();
3545 unsigned Opcode = Op.getOpcode();
3546 APInt DemandedElts = OriginalDemandedElts;
3547 unsigned NumElts = DemandedElts.getBitWidth();
3548 assert(VT.isVector() && "Expected vector op");
3549
3550 KnownUndef = KnownZero = APInt::getZero(numBits: NumElts);
3551
3552 if (!shouldSimplifyDemandedVectorElts(Op, TLO))
3553 return false;
3554
3555 // TODO: For now we assume we know nothing about scalable vectors.
3556 if (VT.isScalableVector())
3557 return false;
3558
3559 assert(VT.getVectorNumElements() == NumElts &&
3560 "Mask size mismatches value type element count!");
3561
3562 // Undef operand.
3563 if (Op.isUndef()) {
3564 KnownUndef.setAllBits();
3565 return false;
3566 }
3567
3568 // If Op has other users, assume that all elements are needed.
3569 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
3570 DemandedElts.setAllBits();
3571
3572 // Not demanding any elements from Op.
3573 if (DemandedElts == 0) {
3574 KnownUndef.setAllBits();
3575 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3576 }
3577
3578 // Limit search depth.
3579 if (Depth >= SelectionDAG::MaxRecursionDepth)
3580 return false;
3581
3582 SDLoc DL(Op);
3583 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3584 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
3585
3586 auto TryShrinkBinOp = [&](SDValue Op0, SDValue Op1) {
3587 unsigned ShrunkSize = getPreferredShrunkVectorSizeInBits(Op, DemandedElts);
3588 if (!ShrunkSize)
3589 return false;
3590
3591 assert(ShrunkSize % EltSizeInBits == 0 &&
3592 "Shrunk size not a multiple of element size");
3593 assert(ShrunkSize < VT.getSizeInBits() &&
3594 "Shrunk size must be < original vector size");
3595 assert(ShrunkSize >= EltSizeInBits * DemandedElts.getActiveBits() &&
3596 "Shrunk size must be >= demanded size");
3597
3598 EVT ShrunkVT = VT.changeVectorElementCount(
3599 Context&: *TLO.DAG.getContext(),
3600 EC: ElementCount::getFixed(MinVal: ShrunkSize / EltSizeInBits));
3601 Op0 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op0, Idx: 0);
3602 Op1 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op1, Idx: 0);
3603 SDValue NewOp =
3604 TLO.DAG.getNode(Opcode, DL, VT: ShrunkVT, N1: Op0, N2: Op1, Flags: Op->getFlags());
3605 return TLO.CombineTo(
3606 O: Op, N: TLO.DAG.getInsertSubvector(DL, Vec: TLO.DAG.getUNDEF(VT), SubVec: NewOp, Idx: 0));
3607 };
3608
3609 // Helper for demanding the specified elements and all the bits of both binary
3610 // operands.
3611 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
3612 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op: Op0, DemandedElts,
3613 DAG&: TLO.DAG, Depth: Depth + 1);
3614 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op: Op1, DemandedElts,
3615 DAG&: TLO.DAG, Depth: Depth + 1);
3616 if (NewOp0 || NewOp1) {
3617 SDValue NewOp =
3618 TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, N1: NewOp0 ? NewOp0 : Op0,
3619 N2: NewOp1 ? NewOp1 : Op1, Flags: Op->getFlags());
3620 return TLO.CombineTo(O: Op, N: NewOp);
3621 }
3622
3623 if (TryShrinkBinOp(Op0, Op1))
3624 return true;
3625
3626 return false;
3627 };
3628
3629 switch (Opcode) {
3630 case ISD::SCALAR_TO_VECTOR: {
3631 if (!DemandedElts[0])
3632 return TLO.CombineTo(O: Op, N: TLO.DAG.getPOISON(VT));
3633 // Upper elements are poison, not undef - don't mark them as KnownUndef.
3634 break;
3635 }
3636 case ISD::BITCAST: {
3637 SDValue Src = Op.getOperand(i: 0);
3638 EVT SrcVT = Src.getValueType();
3639
3640 if (!SrcVT.isVector()) {
3641 // TODO - bigendian once we have test coverage.
3642 if (IsLE) {
3643 APInt DemandedSrcBits = APInt::getZero(numBits: SrcVT.getSizeInBits());
3644 unsigned EltSize = VT.getScalarSizeInBits();
3645 for (unsigned I = 0; I != NumElts; ++I) {
3646 if (DemandedElts[I]) {
3647 unsigned Offset = I * EltSize;
3648 DemandedSrcBits.setBits(loBit: Offset, hiBit: Offset + EltSize);
3649 }
3650 }
3651 KnownBits Known;
3652 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, Known, TLO, Depth: Depth + 1))
3653 return true;
3654 }
3655 break;
3656 }
3657
3658 // Fast handling of 'identity' bitcasts.
3659 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3660 if (NumSrcElts == NumElts)
3661 return SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedElts, KnownUndef,
3662 KnownZero, TLO, Depth: Depth + 1);
3663
3664 APInt SrcDemandedElts, SrcZero, SrcUndef;
3665
3666 // Bitcast from 'large element' src vector to 'small element' vector, we
3667 // must demand a source element if any DemandedElt maps to it.
3668 if ((NumElts % NumSrcElts) == 0) {
3669 unsigned Scale = NumElts / NumSrcElts;
3670 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3671 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3672 TLO, Depth: Depth + 1))
3673 return true;
3674
3675 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
3676 // of the large element.
3677 // TODO - bigendian once we have test coverage.
3678 if (IsLE) {
3679 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
3680 APInt SrcDemandedBits = APInt::getZero(numBits: SrcEltSizeInBits);
3681 for (unsigned i = 0; i != NumElts; ++i)
3682 if (DemandedElts[i]) {
3683 unsigned Ofs = (i % Scale) * EltSizeInBits;
3684 SrcDemandedBits.setBits(loBit: Ofs, hiBit: Ofs + EltSizeInBits);
3685 }
3686
3687 KnownBits Known;
3688 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: SrcDemandedBits, OriginalDemandedElts: SrcDemandedElts, Known,
3689 TLO, Depth: Depth + 1))
3690 return true;
3691
3692 // The bitcast has split each wide element into a number of
3693 // narrow subelements. We have just computed the Known bits
3694 // for wide elements. See if element splitting results in
3695 // some subelements being zero. Only for demanded elements!
3696 for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3697 if (!Known.Zero.extractBits(numBits: EltSizeInBits, bitPosition: SubElt * EltSizeInBits)
3698 .isAllOnes())
3699 continue;
3700 for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3701 unsigned Elt = Scale * SrcElt + SubElt;
3702 // A wholly-undef source lane is reported as undef below; don't also
3703 // flag it as zero, keeping the undef and zero sets disjoint.
3704 if (DemandedElts[Elt] && !SrcUndef[SrcElt])
3705 KnownZero.setBit(Elt);
3706 }
3707 }
3708 }
3709
3710 // If the src element is zero/undef then all the output elements will be -
3711 // only demanded elements are guaranteed to be correct.
3712 for (unsigned i = 0; i != NumSrcElts; ++i) {
3713 if (SrcDemandedElts[i]) {
3714 if (SrcZero[i])
3715 KnownZero.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3716 if (SrcUndef[i])
3717 KnownUndef.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3718 }
3719 }
3720 }
3721
3722 // Bitcast from 'small element' src vector to 'large element' vector, we
3723 // demand all smaller source elements covered by the larger demanded element
3724 // of this vector.
3725 if ((NumSrcElts % NumElts) == 0) {
3726 unsigned Scale = NumSrcElts / NumElts;
3727 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3728 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3729 TLO, Depth: Depth + 1))
3730 return true;
3731
3732 // If all the src elements covering an output element are zero/undef, then
3733 // the output element will be as well, assuming it was demanded.
3734 for (unsigned i = 0; i != NumElts; ++i) {
3735 if (DemandedElts[i]) {
3736 if (SrcZero.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3737 KnownZero.setBit(i);
3738 if (SrcUndef.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3739 KnownUndef.setBit(i);
3740 }
3741 }
3742 }
3743 break;
3744 }
3745 case ISD::FREEZE: {
3746 SDValue N0 = Op.getOperand(i: 0);
3747 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
3748 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
3749 return TLO.CombineTo(O: Op, N: N0);
3750
3751 // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE
3752 // freeze(op(x, ...)) -> op(freeze(x), ...).
3753 // Don't sink the freeze below SCALAR_TO_VECTOR when the scalar is a load
3754 // of a promoted (wider than the element) type: freeze(load) can never be
3755 // folded away (the loaded value may be poison in memory), and the extra
3756 // freeze node then blocks ISel patterns matching scalar_to_vector of a
3757 // load, e.g. the AArch64 scalar_to_vector(extload) -> ldr b/h forms.
3758 // freeze(scalar_to_vector(load)) is equivalent for the demanded element
3759 // zero, and ISel selects the freeze as a plain copy.
3760 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1) {
3761 SDValue Scalar = N0.getOperand(i: 0);
3762 bool IsPromotedLoad = Scalar.getOpcode() == ISD::LOAD &&
3763 Scalar.getValueType() != VT.getVectorElementType();
3764 if (!IsPromotedLoad)
3765 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT,
3766 Operand: TLO.DAG.getFreeze(V: Scalar)));
3767 }
3768 break;
3769 }
3770 case ISD::BUILD_VECTOR: {
3771 // Check all elements and simplify any unused elements with UNDEF.
3772 if (!DemandedElts.isAllOnes()) {
3773 // Don't simplify BROADCASTS.
3774 if (llvm::any_of(Range: Op->op_values(),
3775 P: [&](SDValue Elt) { return Op.getOperand(i: 0) != Elt; })) {
3776 SmallVector<SDValue, 32> Ops(Op->ops());
3777 bool Updated = false;
3778 for (unsigned i = 0; i != NumElts; ++i) {
3779 if (!DemandedElts[i] && !Ops[i].isUndef()) {
3780 Ops[i] = TLO.DAG.getUNDEF(VT: Ops[0].getValueType());
3781 KnownUndef.setBit(i);
3782 Updated = true;
3783 }
3784 }
3785 if (Updated)
3786 return TLO.CombineTo(O: Op, N: TLO.DAG.getBuildVector(VT, DL, Ops));
3787 }
3788 }
3789 for (unsigned i = 0; i != NumElts; ++i) {
3790 SDValue SrcOp = Op.getOperand(i);
3791 if (SrcOp.isUndef()) {
3792 KnownUndef.setBit(i);
3793 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
3794 (isNullConstant(V: SrcOp) || isNullFPConstant(V: SrcOp))) {
3795 KnownZero.setBit(i);
3796 }
3797 }
3798 break;
3799 }
3800 case ISD::CONCAT_VECTORS: {
3801 EVT SubVT = Op.getOperand(i: 0).getValueType();
3802 unsigned NumSubVecs = Op.getNumOperands();
3803 unsigned NumSubElts = SubVT.getVectorNumElements();
3804 for (unsigned i = 0; i != NumSubVecs; ++i) {
3805 SDValue SubOp = Op.getOperand(i);
3806 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3807 APInt SubUndef, SubZero;
3808 if (SimplifyDemandedVectorElts(Op: SubOp, OriginalDemandedElts: SubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3809 Depth: Depth + 1))
3810 return true;
3811 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: i * NumSubElts);
3812 KnownZero.insertBits(SubBits: SubZero, bitPosition: i * NumSubElts);
3813 }
3814
3815 // Attempt to avoid multi-use ops if we don't need anything from them.
3816 if (!DemandedElts.isAllOnes()) {
3817 bool FoundNewSub = false;
3818 SmallVector<SDValue, 2> DemandedSubOps;
3819 for (unsigned i = 0; i != NumSubVecs; ++i) {
3820 SDValue SubOp = Op.getOperand(i);
3821 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3822 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3823 Op: SubOp, DemandedElts: SubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3824 DemandedSubOps.push_back(Elt: NewSubOp ? NewSubOp : SubOp);
3825 FoundNewSub = NewSubOp ? true : FoundNewSub;
3826 }
3827 if (FoundNewSub) {
3828 SDValue NewOp =
3829 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, Ops: DemandedSubOps);
3830 return TLO.CombineTo(O: Op, N: NewOp);
3831 }
3832 }
3833 break;
3834 }
3835 case ISD::INSERT_SUBVECTOR: {
3836 // Demand any elements from the subvector and the remainder from the src it
3837 // is inserted into.
3838 SDValue Src = Op.getOperand(i: 0);
3839 SDValue Sub = Op.getOperand(i: 1);
3840 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3841 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3842 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3843 APInt DemandedSrcElts = DemandedElts;
3844 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3845
3846 // If none of the sub operand elements are demanded, bypass the insert.
3847 if (!DemandedSubElts)
3848 return TLO.CombineTo(O: Op, N: Src);
3849
3850 APInt SubUndef, SubZero;
3851 if (SimplifyDemandedVectorElts(Op: Sub, OriginalDemandedElts: DemandedSubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3852 Depth: Depth + 1))
3853 return true;
3854
3855 // If none of the src operand elements are demanded, replace it with undef.
3856 if (!DemandedSrcElts && !Src.isUndef())
3857 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT,
3858 N1: TLO.DAG.getUNDEF(VT), N2: Sub,
3859 N3: Op.getOperand(i: 2)));
3860
3861 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef, KnownZero,
3862 TLO, Depth: Depth + 1))
3863 return true;
3864 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: Idx);
3865 KnownZero.insertBits(SubBits: SubZero, bitPosition: Idx);
3866
3867 // Attempt to avoid multi-use ops if we don't need anything from them.
3868 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
3869 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3870 Op: Src, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
3871 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3872 Op: Sub, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3873 if (NewSrc || NewSub) {
3874 NewSrc = NewSrc ? NewSrc : Src;
3875 NewSub = NewSub ? NewSub : Sub;
3876 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3877 N2: NewSub, N3: Op.getOperand(i: 2));
3878 return TLO.CombineTo(O: Op, N: NewOp);
3879 }
3880 }
3881 break;
3882 }
3883 case ISD::EXTRACT_SUBVECTOR: {
3884 // Offset the demanded elts by the subvector index.
3885 SDValue Src = Op.getOperand(i: 0);
3886 if (Src.getValueType().isScalableVector())
3887 break;
3888 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3889 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3890 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3891
3892 APInt SrcUndef, SrcZero;
3893 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3894 Depth: Depth + 1))
3895 return true;
3896 KnownUndef = SrcUndef.extractBits(numBits: NumElts, bitPosition: Idx);
3897 KnownZero = SrcZero.extractBits(numBits: NumElts, bitPosition: Idx);
3898
3899 // Attempt to avoid multi-use ops if we don't need anything from them.
3900 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(Op: Src, DemandedElts: DemandedSrcElts,
3901 DAG&: TLO.DAG, Depth: Depth + 1);
3902 if (NewSrc) {
3903 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3904 N2: Op.getOperand(i: 1));
3905 return TLO.CombineTo(O: Op, N: NewOp);
3906 }
3907 break;
3908 }
3909 case ISD::INSERT_VECTOR_ELT: {
3910 SDValue Vec = Op.getOperand(i: 0);
3911 SDValue Scl = Op.getOperand(i: 1);
3912 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
3913
3914 // For a legal, constant insertion index, if we don't need this insertion
3915 // then strip it, else remove it from the demanded elts.
3916 if (CIdx && CIdx->getAPIntValue().ult(RHS: NumElts)) {
3917 unsigned Idx = CIdx->getZExtValue();
3918 if (!DemandedElts[Idx])
3919 return TLO.CombineTo(O: Op, N: Vec);
3920
3921 APInt DemandedVecElts(DemandedElts);
3922 DemandedVecElts.clearBit(BitPosition: Idx);
3923 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedVecElts, KnownUndef,
3924 KnownZero, TLO, Depth: Depth + 1))
3925 return true;
3926
3927 KnownUndef.setBitVal(BitPosition: Idx, BitValue: Scl.isUndef());
3928
3929 KnownZero.setBitVal(BitPosition: Idx, BitValue: isNullConstant(V: Scl) || isNullFPConstant(V: Scl));
3930 break;
3931 }
3932
3933 APInt VecUndef, VecZero;
3934 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedElts, KnownUndef&: VecUndef, KnownZero&: VecZero, TLO,
3935 Depth: Depth + 1))
3936 return true;
3937 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
3938 break;
3939 }
3940 case ISD::VSELECT: {
3941 SDValue Sel = Op.getOperand(i: 0);
3942 SDValue LHS = Op.getOperand(i: 1);
3943 SDValue RHS = Op.getOperand(i: 2);
3944
3945 // Try to transform the select condition based on the current demanded
3946 // elements.
3947 APInt UndefSel, ZeroSel;
3948 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3949 Depth: Depth + 1))
3950 return true;
3951
3952 // See if we can simplify either vselect operand.
3953 APInt DemandedLHS(DemandedElts);
3954 APInt DemandedRHS(DemandedElts);
3955 APInt UndefLHS, ZeroLHS;
3956 APInt UndefRHS, ZeroRHS;
3957 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3958 Depth: Depth + 1))
3959 return true;
3960 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3961 Depth: Depth + 1))
3962 return true;
3963
3964 KnownUndef = UndefLHS & UndefRHS;
3965 KnownZero = ZeroLHS & ZeroRHS;
3966
3967 // If we know that the selected element is always zero, we don't need the
3968 // select value element.
3969 APInt DemandedSel = DemandedElts & ~KnownZero;
3970 if (DemandedSel != DemandedElts)
3971 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedSel, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3972 Depth: Depth + 1))
3973 return true;
3974
3975 break;
3976 }
3977 case ISD::VECTOR_SHUFFLE: {
3978 SDValue LHS = Op.getOperand(i: 0);
3979 SDValue RHS = Op.getOperand(i: 1);
3980 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
3981
3982 // Collect demanded elements from shuffle operands..
3983 APInt DemandedLHS(NumElts, 0);
3984 APInt DemandedRHS(NumElts, 0);
3985 for (unsigned i = 0; i != NumElts; ++i) {
3986 int M = ShuffleMask[i];
3987 if (M < 0 || !DemandedElts[i])
3988 continue;
3989 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
3990 if (M < (int)NumElts)
3991 DemandedLHS.setBit(M);
3992 else
3993 DemandedRHS.setBit(M - NumElts);
3994 }
3995
3996 // If either side isn't demanded, replace it by UNDEF. We handle this
3997 // explicitly here to also simplify in case of multiple uses (on the
3998 // contrary to the SimplifyDemandedVectorElts calls below).
3999 bool FoldLHS = !DemandedLHS && !LHS.isUndef();
4000 bool FoldRHS = !DemandedRHS && !RHS.isUndef();
4001 if (FoldLHS || FoldRHS) {
4002 LHS = FoldLHS ? TLO.DAG.getUNDEF(VT: LHS.getValueType()) : LHS;
4003 RHS = FoldRHS ? TLO.DAG.getUNDEF(VT: RHS.getValueType()) : RHS;
4004 SDValue NewOp =
4005 TLO.DAG.getVectorShuffle(VT, dl: SDLoc(Op), N1: LHS, N2: RHS, Mask: ShuffleMask);
4006 return TLO.CombineTo(O: Op, N: NewOp);
4007 }
4008
4009 // See if we can simplify either shuffle operand.
4010 APInt UndefLHS, ZeroLHS;
4011 APInt UndefRHS, ZeroRHS;
4012 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4013 Depth: Depth + 1))
4014 return true;
4015 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
4016 Depth: Depth + 1))
4017 return true;
4018
4019 // Simplify mask using undef elements from LHS/RHS.
4020 bool Updated = false;
4021 bool IdentityLHS = true, IdentityRHS = true;
4022 SmallVector<int, 32> NewMask(ShuffleMask);
4023 for (unsigned i = 0; i != NumElts; ++i) {
4024 int &M = NewMask[i];
4025 if (M < 0)
4026 continue;
4027 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
4028 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
4029 Updated = true;
4030 M = -1;
4031 }
4032 IdentityLHS &= (M < 0) || (M == (int)i);
4033 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
4034 }
4035
4036 // Update legal shuffle masks based on demanded elements if it won't reduce
4037 // to Identity which can cause premature removal of the shuffle mask.
4038 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
4039 SDValue LegalShuffle =
4040 buildLegalVectorShuffle(VT, DL, N0: LHS, N1: RHS, Mask: NewMask, DAG&: TLO.DAG);
4041 if (LegalShuffle)
4042 return TLO.CombineTo(O: Op, N: LegalShuffle);
4043 }
4044
4045 // Propagate undef/zero elements from LHS/RHS.
4046 for (unsigned i = 0; i != NumElts; ++i) {
4047 int M = ShuffleMask[i];
4048 if (M < 0) {
4049 KnownUndef.setBit(i);
4050 } else if (M < (int)NumElts) {
4051 if (UndefLHS[M])
4052 KnownUndef.setBit(i);
4053 if (ZeroLHS[M])
4054 KnownZero.setBit(i);
4055 } else {
4056 if (UndefRHS[M - NumElts])
4057 KnownUndef.setBit(i);
4058 if (ZeroRHS[M - NumElts])
4059 KnownZero.setBit(i);
4060 }
4061 }
4062 break;
4063 }
4064 case ISD::ANY_EXTEND_VECTOR_INREG:
4065 case ISD::SIGN_EXTEND_VECTOR_INREG:
4066 case ISD::ZERO_EXTEND_VECTOR_INREG: {
4067 APInt SrcUndef, SrcZero;
4068 SDValue Src = Op.getOperand(i: 0);
4069 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4070 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
4071 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
4072 Depth: Depth + 1))
4073 return true;
4074 KnownZero = SrcZero.zextOrTrunc(width: NumElts);
4075 KnownUndef = SrcUndef.zextOrTrunc(width: NumElts);
4076
4077 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
4078 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
4079 DemandedSrcElts == 1) {
4080 // aext - if we just need the bottom element then we can bitcast.
4081 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
4082 }
4083
4084 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
4085 // zext(undef) upper bits are guaranteed to be zero.
4086 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4087 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
4088 KnownUndef.clearAllBits();
4089
4090 // zext - if we just need the bottom element then we can mask:
4091 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
4092 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
4093 Op->isOnlyUserOf(N: Src.getNode()) &&
4094 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
4095 SDLoc DL(Op);
4096 EVT SrcVT = Src.getValueType();
4097 EVT SrcSVT = SrcVT.getScalarType();
4098
4099 // If we're after type legalization and SrcSVT is not legal, use the
4100 // promoted type for creating constants to avoid creating nodes with
4101 // illegal types.
4102 if (TLO.LegalTypes())
4103 SrcSVT = getLegalTypeToTransformTo(Context&: *TLO.DAG.getContext(), VT: SrcSVT);
4104
4105 SmallVector<SDValue> MaskElts;
4106 MaskElts.push_back(Elt: TLO.DAG.getAllOnesConstant(DL, VT: SrcSVT));
4107 MaskElts.append(NumInputs: NumSrcElts - 1, Elt: TLO.DAG.getConstant(Val: 0, DL, VT: SrcSVT));
4108 SDValue Mask = TLO.DAG.getBuildVector(VT: SrcVT, DL, Ops: MaskElts);
4109 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
4110 Opcode: ISD::AND, DL, VT: SrcVT, Ops: {Src.getOperand(i: 1), Mask})) {
4111 Fold = TLO.DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src.getOperand(i: 0), N2: Fold);
4112 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Fold));
4113 }
4114 }
4115 }
4116 break;
4117 }
4118
4119 // TODO: There are more binop opcodes that could be handled here - MIN,
4120 // MAX, saturated math, etc.
4121 case ISD::ADD: {
4122 SDValue Op0 = Op.getOperand(i: 0);
4123 SDValue Op1 = Op.getOperand(i: 1);
4124 if (Op0 == Op1 && Op->isOnlyUserOf(N: Op0.getNode())) {
4125 APInt UndefLHS, ZeroLHS;
4126 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4127 Depth: Depth + 1, /*AssumeSingleUse*/ true))
4128 return true;
4129 }
4130 [[fallthrough]];
4131 }
4132 case ISD::AVGCEILS:
4133 case ISD::AVGCEILU:
4134 case ISD::AVGFLOORS:
4135 case ISD::AVGFLOORU:
4136 case ISD::AVGSUBFLOORS:
4137 case ISD::AVGSUBFLOORU:
4138 case ISD::OR:
4139 case ISD::XOR:
4140 case ISD::SUB:
4141 case ISD::FADD:
4142 case ISD::FSUB:
4143 case ISD::FMUL:
4144 case ISD::FDIV:
4145 case ISD::FREM:
4146 case ISD::PSEUDO_FMIN:
4147 case ISD::PSEUDO_FMAX: {
4148 SDValue Op0 = Op.getOperand(i: 0);
4149 SDValue Op1 = Op.getOperand(i: 1);
4150
4151 APInt UndefRHS, ZeroRHS;
4152 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
4153 Depth: Depth + 1))
4154 return true;
4155 APInt UndefLHS, ZeroLHS;
4156 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4157 Depth: Depth + 1))
4158 return true;
4159
4160 // 0.0 / 0.0 and frem(0.0, 0.0) are NaN, not zero.
4161 if (Opcode != ISD::FDIV && Opcode != ISD::FREM)
4162 KnownZero = ZeroLHS & ZeroRHS;
4163 KnownUndef = getKnownUndefForVectorBinop(BO: Op, DAG&: TLO.DAG, UndefOp0: UndefLHS, UndefOp1: UndefRHS);
4164
4165 // Attempt to avoid multi-use ops if we don't need anything from them.
4166 // TODO - use KnownUndef to relax the demandedelts?
4167 if (!DemandedElts.isAllOnes())
4168 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4169 return true;
4170 break;
4171 }
4172 case ISD::SHL:
4173 case ISD::SRL:
4174 case ISD::SRA:
4175 case ISD::ROTL:
4176 case ISD::ROTR: {
4177 SDValue Op0 = Op.getOperand(i: 0);
4178 SDValue Op1 = Op.getOperand(i: 1);
4179
4180 APInt UndefRHS, ZeroRHS;
4181 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
4182 Depth: Depth + 1))
4183 return true;
4184 APInt UndefLHS, ZeroLHS;
4185 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4186 Depth: Depth + 1))
4187 return true;
4188
4189 KnownZero = ZeroLHS & ~UndefRHS;
4190 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
4191
4192 // Attempt to avoid multi-use ops if we don't need anything from them.
4193 // TODO - use KnownUndef to relax the demandedelts?
4194 if (!DemandedElts.isAllOnes())
4195 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4196 return true;
4197 break;
4198 }
4199 case ISD::MUL:
4200 case ISD::MULHU:
4201 case ISD::MULHS:
4202 case ISD::AND: {
4203 SDValue Op0 = Op.getOperand(i: 0);
4204 SDValue Op1 = Op.getOperand(i: 1);
4205
4206 APInt SrcUndef, SrcZero;
4207 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
4208 Depth: Depth + 1))
4209 return true;
4210 // FIXME: If we know that a demanded element was zero in Op1 we don't need
4211 // to demand it in Op0 - its guaranteed to be zero. There is however a
4212 // restriction, as we must not make any of the originally demanded elements
4213 // more poisonous. We could reduce amount of elements demanded, but then we
4214 // also need a to inform SimplifyDemandedVectorElts that some elements must
4215 // not be made more poisonous.
4216 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef, KnownZero,
4217 TLO, Depth: Depth + 1))
4218 return true;
4219
4220 KnownUndef &= DemandedElts;
4221 KnownZero &= DemandedElts;
4222
4223 // If every element pair has a zero/undef/poison then just fold to zero.
4224 // fold (and x, undef/poison) -> 0 / (and x, 0) -> 0
4225 // fold (mul x, undef/poison) -> 0 / (mul x, 0) -> 0
4226 if (DemandedElts.isSubsetOf(RHS: SrcZero | KnownZero | SrcUndef | KnownUndef))
4227 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
4228
4229 // If either side has a zero element, then the result element is zero, even
4230 // if the other is an UNDEF.
4231 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
4232 // and then handle 'and' nodes with the rest of the binop opcodes.
4233 KnownZero |= SrcZero;
4234 KnownUndef &= SrcUndef;
4235 KnownUndef &= ~KnownZero;
4236
4237 // Attempt to avoid multi-use ops if we don't need anything from them.
4238 if (!DemandedElts.isAllOnes())
4239 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4240 return true;
4241 break;
4242 }
4243 case ISD::TRUNCATE:
4244 case ISD::SIGN_EXTEND:
4245 case ISD::ZERO_EXTEND:
4246 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
4247 KnownZero, TLO, Depth: Depth + 1))
4248 return true;
4249
4250 if (!DemandedElts.isAllOnes())
4251 if (SDValue NewOp = SimplifyMultipleUseDemandedVectorElts(
4252 Op: Op.getOperand(i: 0), DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
4253 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, Operand: NewOp));
4254
4255 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
4256 // zext(undef) upper bits are guaranteed to be zero.
4257 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4258 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
4259 KnownUndef.clearAllBits();
4260 }
4261 break;
4262 case ISD::SINT_TO_FP:
4263 case ISD::UINT_TO_FP:
4264 case ISD::FP_TO_SINT:
4265 case ISD::FP_TO_UINT:
4266 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
4267 KnownZero, TLO, Depth: Depth + 1))
4268 return true;
4269 // Don't fall through to generic undef -> undef handling.
4270 return false;
4271 default: {
4272 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
4273 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
4274 KnownZero, TLO, Depth))
4275 return true;
4276 } else {
4277 KnownBits Known;
4278 APInt DemandedBits = APInt::getAllOnes(numBits: EltSizeInBits);
4279 if (SimplifyDemandedBits(Op, OriginalDemandedBits: DemandedBits, OriginalDemandedElts, Known,
4280 TLO, Depth, AssumeSingleUse))
4281 return true;
4282 }
4283 break;
4284 }
4285 }
4286
4287 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
4288
4289 // Constant fold all undef cases.
4290 // TODO: Handle zero cases as well.
4291 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4292 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
4293
4294 return false;
4295}
4296
4297/// Determine which of the bits specified in Mask are known to be either zero or
4298/// one and return them in the Known.
4299void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
4300 KnownBits &Known,
4301 const APInt &DemandedElts,
4302 const SelectionDAG &DAG,
4303 unsigned Depth) const {
4304 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4305 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4306 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4307 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4308 "Should use MaskedValueIsZero if you don't know whether Op"
4309 " is a target node!");
4310 Known.resetAll();
4311}
4312
4313void TargetLowering::computeKnownBitsForTargetInstr(
4314 GISelValueTracking &Analysis, Register R, KnownBits &Known,
4315 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4316 unsigned Depth) const {
4317 Known.resetAll();
4318}
4319
4320void TargetLowering::computeKnownFPClassForTargetInstr(
4321 GISelValueTracking &Analysis, Register R, KnownFPClass &Known,
4322 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4323 unsigned Depth) const {
4324 Known.resetAll();
4325}
4326
4327void TargetLowering::computeKnownBitsForStackObjectPointer(
4328 KnownBits &Known, const MachineFunction &, Align Alignment) const {
4329 // The low bits are known zero if the pointer is aligned.
4330 Known.Zero.setLowBits(Log2(A: Alignment));
4331}
4332
4333SDValue TargetLowering::annotateStackObjectPointer(SDValue Ptr,
4334 SelectionDAG &DAG,
4335 const SDLoc &DL,
4336 Align Alignment) const {
4337 // Materialize leading-zero stack object pointer facts as AssertZext.
4338 // Alignment-derived low zero bits are not represented on the returned DAG
4339 // value here.
4340 EVT PtrVT = Ptr.getValueType();
4341
4342 unsigned RegSize = PtrVT.getScalarSizeInBits();
4343 KnownBits Known(RegSize);
4344 computeKnownBitsForStackObjectPointer(Known, DAG.getMachineFunction(),
4345 Alignment);
4346
4347 unsigned NumZeroBits = Known.countMinLeadingZeros();
4348 if (!NumZeroBits)
4349 return Ptr;
4350
4351 EVT FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
4352 return DAG.getNode(Opcode: ISD::AssertZext, DL, VT: PtrVT, N1: Ptr, N2: DAG.getValueType(FromVT));
4353}
4354
4355Align TargetLowering::computeKnownAlignForTargetInstr(
4356 GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI,
4357 unsigned Depth) const {
4358 return Align(1);
4359}
4360
4361/// This method can be implemented by targets that want to expose additional
4362/// information about sign bits to the DAG Combiner.
4363unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
4364 const APInt &,
4365 const SelectionDAG &,
4366 unsigned Depth) const {
4367 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4368 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4369 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4370 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4371 "Should use ComputeNumSignBits if you don't know whether Op"
4372 " is a target node!");
4373 return 1;
4374}
4375
4376unsigned TargetLowering::computeNumSignBitsForTargetInstr(
4377 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4378 const MachineRegisterInfo &MRI, unsigned Depth) const {
4379 return 1;
4380}
4381
4382bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
4383 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
4384 TargetLoweringOpt &TLO, unsigned Depth) const {
4385 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4386 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4387 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4388 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4389 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
4390 " is a target node!");
4391 return false;
4392}
4393
4394bool TargetLowering::SimplifyDemandedBitsForTargetNode(
4395 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4396 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
4397 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4398 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4399 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4400 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4401 "Should use SimplifyDemandedBits if you don't know whether Op"
4402 " is a target node!");
4403 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
4404 return false;
4405}
4406
4407SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
4408 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4409 SelectionDAG &DAG, unsigned Depth) const {
4410 assert(
4411 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4412 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4413 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4414 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4415 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
4416 " is a target node!");
4417 return SDValue();
4418}
4419
4420SDValue
4421TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
4422 SDValue N1, MutableArrayRef<int> Mask,
4423 SelectionDAG &DAG) const {
4424 bool LegalMask = isShuffleMaskLegal(Mask, VT);
4425 if (!LegalMask) {
4426 std::swap(a&: N0, b&: N1);
4427 ShuffleVectorSDNode::commuteMask(Mask);
4428 LegalMask = isShuffleMaskLegal(Mask, VT);
4429 }
4430
4431 if (!LegalMask)
4432 return SDValue();
4433
4434 return DAG.getVectorShuffle(VT, dl: DL, N1: N0, N2: N1, Mask);
4435}
4436
4437const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
4438 return nullptr;
4439}
4440
4441bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4442 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4443 UndefPoisonKind Kind, unsigned Depth) const {
4444 assert(
4445 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4446 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4447 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4448 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4449 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
4450 " is a target node!");
4451
4452 // If Op can't create undef/poison and none of its operands are undef/poison
4453 // then Op is never undef/poison.
4454 return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, Kind,
4455 /*ConsiderFlags*/ true, Depth) &&
4456 all_of(Range: Op->ops(), P: [&](SDValue V) {
4457 return DAG.isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
4458 });
4459}
4460
4461bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
4462 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4463 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
4464 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4465 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4466 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4467 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4468 "Should use canCreateUndefOrPoison if you don't know whether Op"
4469 " is a target node!");
4470 // Be conservative and return true.
4471 return true;
4472}
4473
4474void TargetLowering::computeKnownFPClassForTargetNode(const SDValue Op,
4475 KnownFPClass &Known,
4476 const APInt &DemandedElts,
4477 const SelectionDAG &DAG,
4478 unsigned Depth) const {
4479 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4480 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4481 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4482 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4483 "Should use computeKnownFPClass if you don't know whether Op"
4484 " is a target node!");
4485}
4486
4487bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
4488 const APInt &DemandedElts,
4489 const SelectionDAG &DAG,
4490 bool SNaN,
4491 unsigned Depth) const {
4492 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4493 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4494 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4495 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4496 "Should use isKnownNeverNaN if you don't know whether Op"
4497 " is a target node!");
4498 return false;
4499}
4500
4501bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
4502 const APInt &DemandedElts,
4503 APInt &UndefElts,
4504 const SelectionDAG &DAG,
4505 unsigned Depth) const {
4506 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4507 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4508 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4509 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4510 "Should use isSplatValue if you don't know whether Op"
4511 " is a target node!");
4512 return false;
4513}
4514
4515// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
4516// work with truncating build vectors and vectors with elements of less than
4517// 8 bits.
4518bool TargetLowering::isConstTrueVal(SDValue N) const {
4519 if (!N)
4520 return false;
4521
4522 unsigned EltWidth;
4523 APInt CVal;
4524 if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
4525 /*AllowTruncation=*/true)) {
4526 CVal = CN->getAPIntValue();
4527 EltWidth = N.getValueType().getScalarSizeInBits();
4528 } else
4529 return false;
4530
4531 // If this is a truncating splat, truncate the splat value.
4532 // Otherwise, we may fail to match the expected values below.
4533 if (EltWidth < CVal.getBitWidth())
4534 CVal = CVal.trunc(width: EltWidth);
4535
4536 switch (getBooleanContents(Type: N.getValueType())) {
4537 case UndefinedBooleanContent:
4538 return CVal[0];
4539 case ZeroOrOneBooleanContent:
4540 return CVal.isOne();
4541 case ZeroOrNegativeOneBooleanContent:
4542 return CVal.isAllOnes();
4543 }
4544
4545 llvm_unreachable("Invalid boolean contents");
4546}
4547
4548bool TargetLowering::isConstFalseVal(SDValue N) const {
4549 if (!N)
4550 return false;
4551
4552 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N);
4553 if (!CN) {
4554 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
4555 if (!BV)
4556 return false;
4557
4558 // Only interested in constant splats, we don't care about undef
4559 // elements in identifying boolean constants and getConstantSplatNode
4560 // returns NULL if all ops are undef;
4561 CN = BV->getConstantSplatNode();
4562 if (!CN)
4563 return false;
4564 }
4565
4566 if (getBooleanContents(Type: N->getValueType(ResNo: 0)) == UndefinedBooleanContent)
4567 return !CN->getAPIntValue()[0];
4568
4569 return CN->isZero();
4570}
4571
4572bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
4573 bool SExt) const {
4574 if (VT == MVT::i1)
4575 return N->isOne();
4576
4577 TargetLowering::BooleanContent Cnt = getBooleanContents(Type: VT);
4578 switch (Cnt) {
4579 case TargetLowering::ZeroOrOneBooleanContent:
4580 // An extended value of 1 is always true, unless its original type is i1,
4581 // in which case it will be sign extended to -1.
4582 return (N->isOne() && !SExt) || (SExt && (N->getValueType(ResNo: 0) != MVT::i1));
4583 case TargetLowering::UndefinedBooleanContent:
4584 case TargetLowering::ZeroOrNegativeOneBooleanContent:
4585 return N->isAllOnes() && SExt;
4586 }
4587 llvm_unreachable("Unexpected enumeration.");
4588}
4589
4590/// This helper function of SimplifySetCC tries to optimize the comparison when
4591/// either operand of the SetCC node is a bitwise-and instruction.
4592SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
4593 ISD::CondCode Cond, const SDLoc &DL,
4594 DAGCombinerInfo &DCI) const {
4595 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
4596 std::swap(a&: N0, b&: N1);
4597
4598 SelectionDAG &DAG = DCI.DAG;
4599 EVT OpVT = N0.getValueType();
4600 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
4601 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4602 return SDValue();
4603
4604 // (X & Y) != 0 --> zextOrTrunc(X & Y)
4605 // iff everything but LSB is known zero:
4606 if (Cond == ISD::SETNE && isNullConstant(V: N1) &&
4607 (getBooleanContents(Type: OpVT) == TargetLowering::UndefinedBooleanContent ||
4608 getBooleanContents(Type: OpVT) == TargetLowering::ZeroOrOneBooleanContent)) {
4609 unsigned NumEltBits = OpVT.getScalarSizeInBits();
4610 APInt UpperBits = APInt::getHighBitsSet(numBits: NumEltBits, hiBitsSet: NumEltBits - 1);
4611 if (DAG.MaskedValueIsZero(Op: N0, Mask: UpperBits))
4612 return DAG.getBoolExtOrTrunc(Op: N0, SL: DL, VT, OpVT);
4613 }
4614
4615 // Try to eliminate a power-of-2 mask constant by converting to a signbit
4616 // test in a narrow type that we can truncate to with no cost. Examples:
4617 // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
4618 // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
4619 // TODO: This conservatively checks for type legality on the source and
4620 // destination types. That may inhibit optimizations, but it also
4621 // allows setcc->shift transforms that may be more beneficial.
4622 auto *AndC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
4623 if (AndC && isNullConstant(V: N1) && AndC->getAPIntValue().isPowerOf2() &&
4624 isTypeLegal(VT: OpVT) && N0.hasOneUse()) {
4625 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
4626 BitWidth: AndC->getAPIntValue().getActiveBits());
4627 if (isTruncateFree(FromVT: OpVT, ToVT: NarrowVT) && isTypeLegal(VT: NarrowVT)) {
4628 SDValue Trunc = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 0), DL, VT: NarrowVT);
4629 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: NarrowVT);
4630 return DAG.getSetCC(DL, VT, LHS: Trunc, RHS: Zero,
4631 Cond: Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT);
4632 }
4633 }
4634
4635 // Match these patterns in any of their permutations:
4636 // (X & Y) == Y
4637 // (X & Y) != Y
4638 SDValue X, Y;
4639 if (N0.getOperand(i: 0) == N1) {
4640 X = N0.getOperand(i: 1);
4641 Y = N0.getOperand(i: 0);
4642 } else if (N0.getOperand(i: 1) == N1) {
4643 X = N0.getOperand(i: 0);
4644 Y = N0.getOperand(i: 1);
4645 } else {
4646 return SDValue();
4647 }
4648
4649 // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
4650 // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
4651 // its liable to create and infinite loop.
4652 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: OpVT);
4653 if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
4654 DAG.isKnownToBeAPowerOfTwo(Val: Y)) {
4655 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
4656 // Note that where Y is variable and is known to have at most one bit set
4657 // (for example, if it is Z & 1) we cannot do this; the expressions are not
4658 // equivalent when Y == 0.
4659 assert(OpVT.isInteger());
4660 Cond = ISD::getSetCCInverse(Operation: Cond, Type: OpVT);
4661 if (DCI.isBeforeLegalizeOps() ||
4662 isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType()))
4663 return DAG.getSetCC(DL, VT, LHS: N0, RHS: Zero, Cond);
4664 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
4665 // If the target supports an 'and-not' or 'and-complement' logic operation,
4666 // try to use that to make a comparison operation more efficient.
4667 // But don't do this transform if the mask is a single bit because there are
4668 // more efficient ways to deal with that case (for example, 'bt' on x86 or
4669 // 'rlwinm' on PPC).
4670
4671 // Bail out if the compare operand that we want to turn into a zero is
4672 // already a zero (otherwise, infinite loop).
4673 if (isNullConstant(V: Y))
4674 return SDValue();
4675
4676 // Transform this into: ~X & Y == 0.
4677 SDValue NotX = DAG.getNOT(DL: SDLoc(X), Val: X, VT: OpVT);
4678 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: NotX, N2: Y);
4679 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: Zero, Cond);
4680 }
4681
4682 return SDValue();
4683}
4684
4685/// This helper function of SimplifySetCC tries to optimize the comparison when
4686/// either operand of the SetCC node is a bitwise-or instruction.
4687/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
4688SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
4689 ISD::CondCode Cond, const SDLoc &DL,
4690 DAGCombinerInfo &DCI) const {
4691 if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
4692 std::swap(a&: N0, b&: N1);
4693
4694 SelectionDAG &DAG = DCI.DAG;
4695 EVT OpVT = N0.getValueType();
4696 if (!N0.hasOneUse() || !OpVT.isInteger() ||
4697 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4698 return SDValue();
4699
4700 // (X | Y) == Y
4701 // (X | Y) != Y
4702 SDValue X;
4703 if (sd_match(N: N0, P: m_Or(L: m_Value(N&: X), R: m_Specific(N: N1))) && hasAndNotCompare(Y: X)) {
4704 // If the target supports an 'and-not' or 'and-complement' logic operation,
4705 // try to use that to make a comparison operation more efficient.
4706
4707 // Bail out if the compare operand that we want to turn into a zero is
4708 // already a zero (otherwise, infinite loop).
4709 if (isNullConstant(V: N1))
4710 return SDValue();
4711
4712 // Transform this into: X & ~Y ==/!= 0.
4713 SDValue NotY = DAG.getNOT(DL: SDLoc(N1), Val: N1, VT: OpVT);
4714 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: X, N2: NotY);
4715 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4716 }
4717
4718 return SDValue();
4719}
4720
4721/// There are multiple IR patterns that could be checking whether certain
4722/// truncation of a signed number would be lossy or not. The pattern which is
4723/// best at IR level, may not lower optimally. Thus, we want to unfold it.
4724/// We are looking for the following pattern: (KeptBits is a constant)
4725/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
4726/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
4727/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
4728/// We will unfold it into the natural trunc+sext pattern:
4729/// ((%x << C) a>> C) dstcond %x
4730/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
4731SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4732 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
4733 const SDLoc &DL) const {
4734 // We must be comparing with a constant.
4735 ConstantSDNode *C1;
4736 if (!(C1 = dyn_cast<ConstantSDNode>(Val&: N1)))
4737 return SDValue();
4738
4739 // N0 should be: add %x, (1 << (KeptBits-1))
4740 if (N0->getOpcode() != ISD::ADD)
4741 return SDValue();
4742
4743 // And we must be 'add'ing a constant.
4744 ConstantSDNode *C01;
4745 if (!(C01 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1))))
4746 return SDValue();
4747
4748 SDValue X = N0->getOperand(Num: 0);
4749 EVT XVT = X.getValueType();
4750
4751 // Validate constants ...
4752
4753 APInt I1 = C1->getAPIntValue();
4754
4755 ISD::CondCode NewCond;
4756 if (Cond == ISD::CondCode::SETULT) {
4757 NewCond = ISD::CondCode::SETEQ;
4758 } else if (Cond == ISD::CondCode::SETULE) {
4759 NewCond = ISD::CondCode::SETEQ;
4760 // But need to 'canonicalize' the constant.
4761 I1 += 1;
4762 } else if (Cond == ISD::CondCode::SETUGT) {
4763 NewCond = ISD::CondCode::SETNE;
4764 // But need to 'canonicalize' the constant.
4765 I1 += 1;
4766 } else if (Cond == ISD::CondCode::SETUGE) {
4767 NewCond = ISD::CondCode::SETNE;
4768 } else
4769 return SDValue();
4770
4771 APInt I01 = C01->getAPIntValue();
4772
4773 auto checkConstants = [&I1, &I01]() -> bool {
4774 // Both of them must be power-of-two, and the constant from setcc is bigger.
4775 return I1.ugt(RHS: I01) && I1.isPowerOf2() && I01.isPowerOf2();
4776 };
4777
4778 if (checkConstants()) {
4779 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
4780 } else {
4781 // What if we invert constants? (and the target predicate)
4782 I1.negate();
4783 I01.negate();
4784 assert(XVT.isInteger());
4785 NewCond = getSetCCInverse(Operation: NewCond, Type: XVT);
4786 if (!checkConstants())
4787 return SDValue();
4788 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
4789 }
4790
4791 // They are power-of-two, so which bit is set?
4792 const unsigned KeptBits = I1.logBase2();
4793 const unsigned KeptBitsMinusOne = I01.logBase2();
4794
4795 // Magic!
4796 if (KeptBits != (KeptBitsMinusOne + 1))
4797 return SDValue();
4798 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
4799
4800 // We don't want to do this in every single case.
4801 SelectionDAG &DAG = DCI.DAG;
4802 if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
4803 return SDValue();
4804
4805 // Unfold into: sext_inreg(%x) cond %x
4806 // Where 'cond' will be either 'eq' or 'ne'.
4807 SDValue SExtInReg = DAG.getNode(
4808 Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: XVT, N1: X,
4809 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: KeptBits)));
4810 return DAG.getSetCC(DL, VT: SCCVT, LHS: SExtInReg, RHS: X, Cond: NewCond);
4811}
4812
4813// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4814SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4815 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4816 DAGCombinerInfo &DCI, const SDLoc &DL) const {
4817 assert(isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->isZero() &&
4818 "Should be a comparison with 0.");
4819 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4820 "Valid only for [in]equality comparisons.");
4821
4822 unsigned NewShiftOpcode;
4823 SDValue X, C, Y;
4824
4825 SelectionDAG &DAG = DCI.DAG;
4826
4827 // Look for '(C l>>/<< Y)'.
4828 auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
4829 // The shift should be one-use.
4830 if (!V.hasOneUse())
4831 return false;
4832 unsigned OldShiftOpcode = V.getOpcode();
4833 switch (OldShiftOpcode) {
4834 case ISD::SHL:
4835 NewShiftOpcode = ISD::SRL;
4836 break;
4837 case ISD::SRL:
4838 NewShiftOpcode = ISD::SHL;
4839 break;
4840 default:
4841 return false; // must be a logical shift.
4842 }
4843 // We should be shifting a constant.
4844 // FIXME: best to use isConstantOrConstantVector().
4845 C = V.getOperand(i: 0);
4846 ConstantSDNode *CC =
4847 isConstOrConstSplat(N: C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4848 if (!CC)
4849 return false;
4850 Y = V.getOperand(i: 1);
4851
4852 ConstantSDNode *XC =
4853 isConstOrConstSplat(N: X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4854 return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
4855 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
4856 };
4857
4858 // LHS of comparison should be an one-use 'and'.
4859 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
4860 return SDValue();
4861
4862 X = N0.getOperand(i: 0);
4863 SDValue Mask = N0.getOperand(i: 1);
4864
4865 // 'and' is commutative!
4866 if (!Match(Mask)) {
4867 std::swap(a&: X, b&: Mask);
4868 if (!Match(Mask))
4869 return SDValue();
4870 }
4871
4872 EVT VT = X.getValueType();
4873
4874 // Produce:
4875 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
4876 SDValue T0 = DAG.getNode(Opcode: NewShiftOpcode, DL, VT, N1: X, N2: Y);
4877 SDValue T1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T0, N2: C);
4878 SDValue T2 = DAG.getSetCC(DL, VT: SCCVT, LHS: T1, RHS: N1C, Cond);
4879 return T2;
4880}
4881
4882/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
4883/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
4884/// handle the commuted versions of these patterns.
4885SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
4886 ISD::CondCode Cond, const SDLoc &DL,
4887 DAGCombinerInfo &DCI) const {
4888 unsigned BOpcode = N0.getOpcode();
4889 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
4890 "Unexpected binop");
4891 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
4892
4893 // (X + Y) == X --> Y == 0
4894 // (X - Y) == X --> Y == 0
4895 // (X ^ Y) == X --> Y == 0
4896 SelectionDAG &DAG = DCI.DAG;
4897 EVT OpVT = N0.getValueType();
4898 SDValue X = N0.getOperand(i: 0);
4899 SDValue Y = N0.getOperand(i: 1);
4900 if (X == N1)
4901 return DAG.getSetCC(DL, VT, LHS: Y, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4902
4903 if (Y != N1)
4904 return SDValue();
4905
4906 // (X + Y) == Y --> X == 0
4907 // (X ^ Y) == Y --> X == 0
4908 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
4909 return DAG.getSetCC(DL, VT, LHS: X, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4910
4911 // The shift would not be valid if the operands are boolean (i1).
4912 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
4913 return SDValue();
4914
4915 // (X - Y) == Y --> X == Y << 1
4916 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: OpVT, DL);
4917 SDValue YShl1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: N1.getValueType(), N1: Y, N2: One);
4918 if (!DCI.isCalledByLegalizer())
4919 DCI.AddToWorklist(N: YShl1.getNode());
4920 return DAG.getSetCC(DL, VT, LHS: X, RHS: YShl1, Cond);
4921}
4922
4923static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
4924 SDValue N0, const APInt &C1,
4925 ISD::CondCode Cond, const SDLoc &dl,
4926 SelectionDAG &DAG) {
4927 // Look through truncs that don't change the value of a ctpop.
4928 // FIXME: Add vector support? Need to be careful with setcc result type below.
4929 SDValue CTPOP = N0;
4930 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
4931 N0.getScalarValueSizeInBits() > Log2_32(Value: N0.getOperand(i: 0).getScalarValueSizeInBits()))
4932 CTPOP = N0.getOperand(i: 0);
4933
4934 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
4935 return SDValue();
4936
4937 EVT CTVT = CTPOP.getValueType();
4938 SDValue CTOp = CTPOP.getOperand(i: 0);
4939
4940 // Expand a power-of-2-or-zero comparison based on ctpop:
4941 // (ctpop x) u< 2 -> (x & x-1) == 0
4942 // (ctpop x) u> 1 -> (x & x-1) != 0
4943 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
4944 // Keep the CTPOP if it is a cheap vector op.
4945 if (CTVT.isVector() && TLI.isCtpopFast(VT: CTVT))
4946 return SDValue();
4947
4948 unsigned CostLimit = TLI.getCustomCtpopCost(VT: CTVT, Cond);
4949 if (C1.ugt(RHS: CostLimit + (Cond == ISD::SETULT)))
4950 return SDValue();
4951 if (C1 == 0 && (Cond == ISD::SETULT))
4952 return SDValue(); // This is handled elsewhere.
4953
4954 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
4955
4956 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4957 SDValue Result = CTOp;
4958 for (unsigned i = 0; i < Passes; i++) {
4959 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: Result, N2: NegOne);
4960 Result = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: Result, N2: Add);
4961 }
4962 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
4963 return DAG.getSetCC(DL: dl, VT, LHS: Result, RHS: DAG.getConstant(Val: 0, DL: dl, VT: CTVT), Cond: CC);
4964 }
4965
4966 // Expand a power-of-2 comparison based on ctpop
4967 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
4968 // Keep the CTPOP if it is cheap.
4969 if (TLI.isCtpopFast(VT: CTVT))
4970 return SDValue();
4971
4972 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: CTVT);
4973 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4974 assert(CTVT.isInteger());
4975 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: CTOp, N2: NegOne);
4976
4977 // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
4978 // check before emitting a potentially unnecessary op.
4979 if (DAG.isKnownNeverZero(Op: CTOp)) {
4980 // (ctpop x) == 1 --> (x & x-1) == 0
4981 // (ctpop x) != 1 --> (x & x-1) != 0
4982 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4983 SDValue RHS = DAG.getSetCC(DL: dl, VT, LHS: And, RHS: Zero, Cond);
4984 return RHS;
4985 }
4986
4987 // (ctpop x) == 1 --> (x ^ x-1) > x-1
4988 // (ctpop x) != 1 --> (x ^ x-1) <= x-1
4989 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4990 ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE;
4991 return DAG.getSetCC(DL: dl, VT, LHS: Xor, RHS: Add, Cond: CmpCond);
4992 }
4993
4994 return SDValue();
4995}
4996
4997static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1,
4998 ISD::CondCode Cond, const SDLoc &dl,
4999 SelectionDAG &DAG) {
5000 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
5001 return SDValue();
5002
5003 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
5004 if (!C1 || !(C1->isZero() || C1->isAllOnes()))
5005 return SDValue();
5006
5007 auto getRotateSource = [](SDValue X) {
5008 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
5009 return X.getOperand(i: 0);
5010 return SDValue();
5011 };
5012
5013 // Peek through a rotated value compared against 0 or -1:
5014 // (rot X, Y) == 0/-1 --> X == 0/-1
5015 // (rot X, Y) != 0/-1 --> X != 0/-1
5016 if (SDValue R = getRotateSource(N0))
5017 return DAG.getSetCC(DL: dl, VT, LHS: R, RHS: N1, Cond);
5018
5019 // Peek through an 'or' of a rotated value compared against 0:
5020 // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
5021 // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
5022 //
5023 // TODO: Add the 'and' with -1 sibling.
5024 // TODO: Recurse through a series of 'or' ops to find the rotate.
5025 EVT OpVT = N0.getValueType();
5026 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
5027 if (SDValue R = getRotateSource(N0.getOperand(i: 0))) {
5028 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 1));
5029 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
5030 }
5031 if (SDValue R = getRotateSource(N0.getOperand(i: 1))) {
5032 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 0));
5033 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
5034 }
5035 }
5036
5037 return SDValue();
5038}
5039
5040static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1,
5041 ISD::CondCode Cond, const SDLoc &dl,
5042 SelectionDAG &DAG) {
5043 // If we are testing for all-bits-clear, we might be able to do that with
5044 // less shifting since bit-order does not matter.
5045 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
5046 return SDValue();
5047
5048 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
5049 if (!C1 || !C1->isZero())
5050 return SDValue();
5051
5052 if (!N0.hasOneUse() ||
5053 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
5054 return SDValue();
5055
5056 unsigned BitWidth = N0.getScalarValueSizeInBits();
5057 auto *ShAmtC = isConstOrConstSplat(N: N0.getOperand(i: 2));
5058 if (!ShAmtC)
5059 return SDValue();
5060
5061 uint64_t ShAmt = ShAmtC->getAPIntValue().urem(RHS: BitWidth);
5062 if (ShAmt == 0)
5063 return SDValue();
5064
5065 // Canonicalize fshr as fshl to reduce pattern-matching.
5066 if (N0.getOpcode() == ISD::FSHR)
5067 ShAmt = BitWidth - ShAmt;
5068
5069 // Match an 'or' with a specific operand 'Other' in either commuted variant.
5070 SDValue X, Y;
5071 auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
5072 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
5073 return false;
5074 if (Or.getOperand(i: 0) == Other) {
5075 X = Or.getOperand(i: 0);
5076 Y = Or.getOperand(i: 1);
5077 return true;
5078 }
5079 if (Or.getOperand(i: 1) == Other) {
5080 X = Or.getOperand(i: 1);
5081 Y = Or.getOperand(i: 0);
5082 return true;
5083 }
5084 return false;
5085 };
5086
5087 EVT OpVT = N0.getValueType();
5088 EVT ShAmtVT = N0.getOperand(i: 2).getValueType();
5089 SDValue F0 = N0.getOperand(i: 0);
5090 SDValue F1 = N0.getOperand(i: 1);
5091 if (matchOr(F0, F1)) {
5092 // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
5093 SDValue NewShAmt = DAG.getConstant(Val: ShAmt, DL: dl, VT: ShAmtVT);
5094 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
5095 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
5096 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
5097 }
5098 if (matchOr(F1, F0)) {
5099 // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
5100 SDValue NewShAmt = DAG.getConstant(Val: BitWidth - ShAmt, DL: dl, VT: ShAmtVT);
5101 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
5102 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
5103 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
5104 }
5105
5106 return SDValue();
5107}
5108
5109/// Try to simplify a setcc built with the specified operands and cc. If it is
5110/// unable to simplify it, return a null SDValue.
5111SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
5112 ISD::CondCode Cond, bool foldBooleans,
5113 DAGCombinerInfo &DCI,
5114 const SDLoc &dl) const {
5115 SelectionDAG &DAG = DCI.DAG;
5116 const DataLayout &Layout = DAG.getDataLayout();
5117 EVT OpVT = N0.getValueType();
5118 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5119
5120 // Constant fold or commute setcc.
5121 if (SDValue Fold = DAG.FoldSetCC(VT, N1: N0, N2: N1, Cond, dl))
5122 return Fold;
5123
5124 bool N0ConstOrSplat =
5125 isConstOrConstSplat(N: N0, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
5126 bool N1ConstOrSplat =
5127 isConstOrConstSplat(N: N1, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
5128
5129 // Canonicalize toward having the constant on the RHS.
5130 // TODO: Handle non-splat vector constants. All undef causes trouble.
5131 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
5132 // infinite loop here when we encounter one.
5133 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Operation: Cond);
5134 if (N0ConstOrSplat && !N1ConstOrSplat &&
5135 (DCI.isBeforeLegalizeOps() ||
5136 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())))
5137 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
5138
5139 // If we have a subtract with the same 2 non-constant operands as this setcc
5140 // -- but in reverse order -- then try to commute the operands of this setcc
5141 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
5142 // instruction on some targets.
5143 if (!N0ConstOrSplat && !N1ConstOrSplat &&
5144 (DCI.isBeforeLegalizeOps() ||
5145 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())) &&
5146 DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N1, N0}) &&
5147 !DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N0, N1}))
5148 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
5149
5150 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
5151 return V;
5152
5153 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
5154 return V;
5155
5156 if (auto *N1C = isConstOrConstSplat(N: N1)) {
5157 const APInt &C1 = N1C->getAPIntValue();
5158
5159 // Optimize some CTPOP cases.
5160 if (SDValue V = simplifySetCCWithCTPOP(TLI: *this, VT, N0, C1, Cond, dl, DAG))
5161 return V;
5162
5163 // For equality to 0 of a no-wrap multiply, decompose and test each op:
5164 // X * Y == 0 --> (X == 0) || (Y == 0)
5165 // X * Y != 0 --> (X != 0) && (Y != 0)
5166 // TODO: This bails out if minsize is set, but if the target doesn't have a
5167 // single instruction multiply for this type, it would likely be
5168 // smaller to decompose.
5169 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5170 N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
5171 (N0->getFlags().hasNoUnsignedWrap() ||
5172 N0->getFlags().hasNoSignedWrap()) &&
5173 !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
5174 SDValue IsXZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
5175 SDValue IsYZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1, Cond);
5176 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
5177 return DAG.getNode(Opcode: LogicOp, DL: dl, VT, N1: IsXZero, N2: IsYZero);
5178 }
5179
5180 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
5181 // equality comparison, then we're just comparing whether X itself is
5182 // zero.
5183 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
5184 N0.getOperand(i: 0).getOpcode() == ISD::CTLZ &&
5185 llvm::has_single_bit<uint32_t>(Value: N0.getScalarValueSizeInBits())) {
5186 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
5187 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5188 ShAmt->getAPIntValue() == Log2_32(Value: N0.getScalarValueSizeInBits())) {
5189 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
5190 // (srl (ctlz x), 5) == 0 -> X != 0
5191 // (srl (ctlz x), 5) != 1 -> X != 0
5192 Cond = ISD::SETNE;
5193 } else {
5194 // (srl (ctlz x), 5) != 0 -> X == 0
5195 // (srl (ctlz x), 5) == 1 -> X == 0
5196 Cond = ISD::SETEQ;
5197 }
5198 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: N0.getValueType());
5199 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0).getOperand(i: 0), RHS: Zero,
5200 Cond);
5201 }
5202 }
5203 }
5204 }
5205
5206 // setcc X, 0, setlt --> X (when X is all sign bits)
5207 // setcc X, 0, setne --> X (when X is all sign bits)
5208 //
5209 // When we know that X has 0 or -1 in each element (or scalar), this
5210 // comparison will produce X. This is only true when boolean contents are
5211 // represented via 0s and -1s.
5212 if (VT == OpVT &&
5213 // Check that the result of setcc is 0 and -1.
5214 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent &&
5215 // Match only for checks X < 0 and X != 0
5216 (Cond == ISD::SETLT || Cond == ISD::SETNE) && isNullOrNullSplat(V: N1) &&
5217 // The identity holds iff we know all sign bits for all lanes.
5218 DAG.ComputeNumSignBits(Op: N0) == N0.getScalarValueSizeInBits())
5219 return N0;
5220
5221 // FIXME: Support vectors.
5222 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5223 const APInt &C1 = N1C->getAPIntValue();
5224
5225 // (zext x) == C --> x == (trunc C)
5226 // (sext x) == C --> x == (trunc C)
5227 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5228 DCI.isBeforeLegalize() && N0->hasOneUse()) {
5229 unsigned MinBits = N0.getValueSizeInBits();
5230 SDValue PreExt;
5231 bool Signed = false;
5232 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
5233 // ZExt
5234 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
5235 PreExt = N0->getOperand(Num: 0);
5236 } else if (N0->getOpcode() == ISD::AND) {
5237 // DAGCombine turns costly ZExts into ANDs
5238 if (auto *C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
5239 if ((C->getAPIntValue()+1).isPowerOf2()) {
5240 MinBits = C->getAPIntValue().countr_one();
5241 PreExt = N0->getOperand(Num: 0);
5242 }
5243 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
5244 // SExt
5245 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
5246 PreExt = N0->getOperand(Num: 0);
5247 Signed = true;
5248 } else if (auto *LN0 = dyn_cast<LoadSDNode>(Val&: N0)) {
5249 // ZEXTLOAD / SEXTLOAD
5250 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
5251 MinBits = LN0->getMemoryVT().getSizeInBits();
5252 PreExt = N0;
5253 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
5254 Signed = true;
5255 MinBits = LN0->getMemoryVT().getSizeInBits();
5256 PreExt = N0;
5257 }
5258 }
5259
5260 // Figure out how many bits we need to preserve this constant.
5261 unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();
5262
5263 // Make sure we're not losing bits from the constant.
5264 if (MinBits > 0 &&
5265 MinBits < C1.getBitWidth() &&
5266 MinBits >= ReqdBits) {
5267 EVT MinVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MinBits);
5268 if (isTypeDesirableForOp(ISD::SETCC, VT: MinVT)) {
5269 // Will get folded away.
5270 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MinVT, Operand: PreExt);
5271 if (MinBits == 1 && C1 == 1)
5272 // Invert the condition.
5273 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i1),
5274 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5275 SDValue C = DAG.getConstant(Val: C1.trunc(width: MinBits), DL: dl, VT: MinVT);
5276 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: C, Cond);
5277 }
5278
5279 // If truncating the setcc operands is not desirable, we can still
5280 // simplify the expression in some cases:
5281 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
5282 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
5283 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
5284 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
5285 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
5286 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
5287 SDValue TopSetCC = N0->getOperand(Num: 0);
5288 unsigned N0Opc = N0->getOpcode();
5289 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
5290 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
5291 TopSetCC.getOpcode() == ISD::SETCC &&
5292 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
5293 (isConstFalseVal(N: N1) ||
5294 isExtendedTrueVal(N: N1C, VT: N0->getValueType(ResNo: 0), SExt))) {
5295
5296 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
5297 (!N1C->isZero() && Cond == ISD::SETNE);
5298
5299 if (!Inverse)
5300 return TopSetCC;
5301
5302 ISD::CondCode InvCond =
5303 ISD::getSetCCInverse(Operation: TopSetCC.getOperandCondCode(i: 2),
5304 Type: TopSetCC.getOperand(i: 0).getValueType());
5305 return DAG.getSetCC(DL: dl, VT, LHS: TopSetCC.getOperand(i: 0),
5306 RHS: TopSetCC.getOperand(i: 1), Cond: InvCond);
5307 }
5308 }
5309 }
5310
5311 // If the LHS is '(and load, const)', the RHS is 0, the test is for
5312 // equality or unsigned, and all 1 bits of the const are in the same
5313 // partial word, see if we can shorten the load.
5314 if (DCI.isBeforeLegalize() &&
5315 !ISD::isSignedIntSetCC(Code: Cond) &&
5316 N0.getOpcode() == ISD::AND && C1 == 0 &&
5317 N0.getNode()->hasOneUse() &&
5318 isa<LoadSDNode>(Val: N0.getOperand(i: 0)) &&
5319 N0.getOperand(i: 0).getNode()->hasOneUse() &&
5320 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5321 auto *Lod = cast<LoadSDNode>(Val: N0.getOperand(i: 0));
5322 APInt bestMask;
5323 unsigned bestWidth = 0, bestOffset = 0;
5324 if (Lod->isSimple() && Lod->isUnindexed() &&
5325 (Lod->getMemoryVT().isByteSized() ||
5326 isPaddedAtMostSignificantBitsWhenStored(VT: Lod->getMemoryVT()))) {
5327 unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
5328 unsigned origWidth = N0.getValueSizeInBits();
5329 unsigned maskWidth = origWidth;
5330 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
5331 // 8 bits, but have to be careful...
5332 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
5333 origWidth = Lod->getMemoryVT().getSizeInBits();
5334 const APInt &Mask = N0.getConstantOperandAPInt(i: 1);
5335 // Only consider power-of-2 widths (and at least one byte) as candiates
5336 // for the narrowed load.
5337 for (unsigned width = 8; width < origWidth; width *= 2) {
5338 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: width);
5339 APInt newMask = APInt::getLowBitsSet(numBits: maskWidth, loBitsSet: width);
5340 // Avoid accessing any padding here for now (we could use memWidth
5341 // instead of origWidth here otherwise).
5342 unsigned maxOffset = origWidth - width;
5343 for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
5344 if (Mask.isSubsetOf(RHS: newMask)) {
5345 unsigned ptrOffset =
5346 Layout.isLittleEndian() ? offset : memWidth - width - offset;
5347 unsigned IsFast = 0;
5348 assert((ptrOffset % 8) == 0 && "Non-Bytealigned pointer offset");
5349 Align NewAlign = commonAlignment(A: Lod->getAlign(), Offset: ptrOffset / 8);
5350 if (shouldReduceLoadWidth(Load: Lod, ExtTy: ISD::NON_EXTLOAD, NewVT: newVT,
5351 ByteOffset: ptrOffset / 8) &&
5352 allowsMemoryAccess(
5353 Context&: *DAG.getContext(), DL: Layout, VT: newVT, AddrSpace: Lod->getAddressSpace(),
5354 Alignment: NewAlign, Flags: Lod->getMemOperand()->getFlags(), Fast: &IsFast) &&
5355 IsFast) {
5356 bestOffset = ptrOffset / 8;
5357 bestMask = Mask.lshr(shiftAmt: offset);
5358 bestWidth = width;
5359 break;
5360 }
5361 }
5362 newMask <<= 8;
5363 }
5364 if (bestWidth)
5365 break;
5366 }
5367 }
5368 if (bestWidth) {
5369 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: bestWidth);
5370 SDValue Ptr = Lod->getBasePtr();
5371 if (bestOffset != 0)
5372 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: bestOffset));
5373 SDValue NewLoad =
5374 DAG.getLoad(VT: newVT, dl, Chain: Lod->getChain(), Ptr,
5375 PtrInfo: Lod->getPointerInfo().getWithOffset(O: bestOffset),
5376 Alignment: Lod->getBaseAlign(), MMOFlags: MachineMemOperand::MONone,
5377 Metadata: Lod->getMMOMetadataForSubAccess());
5378 SDValue And =
5379 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: newVT, N1: NewLoad,
5380 N2: DAG.getConstant(Val: bestMask.trunc(width: bestWidth), DL: dl, VT: newVT));
5381 return DAG.getSetCC(DL: dl, VT, LHS: And, RHS: DAG.getConstant(Val: 0LL, DL: dl, VT: newVT), Cond);
5382 }
5383 }
5384
5385 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
5386 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
5387 unsigned InSize = N0.getOperand(i: 0).getValueSizeInBits();
5388
5389 // If the comparison constant has bits in the upper part, the
5390 // zero-extended value could never match.
5391 if (C1.intersects(RHS: APInt::getHighBitsSet(numBits: C1.getBitWidth(),
5392 hiBitsSet: C1.getBitWidth() - InSize))) {
5393 switch (Cond) {
5394 case ISD::SETUGT:
5395 case ISD::SETUGE:
5396 case ISD::SETEQ:
5397 return DAG.getConstant(Val: 0, DL: dl, VT);
5398 case ISD::SETULT:
5399 case ISD::SETULE:
5400 case ISD::SETNE:
5401 return DAG.getConstant(Val: 1, DL: dl, VT);
5402 case ISD::SETGT:
5403 case ISD::SETGE:
5404 // True if the sign bit of C1 is set.
5405 return DAG.getConstant(Val: C1.isNegative(), DL: dl, VT);
5406 case ISD::SETLT:
5407 case ISD::SETLE:
5408 // True if the sign bit of C1 isn't set.
5409 return DAG.getConstant(Val: C1.isNonNegative(), DL: dl, VT);
5410 default:
5411 break;
5412 }
5413 }
5414
5415 // Otherwise, we can perform the comparison with the low bits.
5416 switch (Cond) {
5417 case ISD::SETEQ:
5418 case ISD::SETNE:
5419 case ISD::SETUGT:
5420 case ISD::SETUGE:
5421 case ISD::SETULT:
5422 case ISD::SETULE: {
5423 EVT newVT = N0.getOperand(i: 0).getValueType();
5424 // FIXME: Should use isNarrowingProfitable.
5425 if (DCI.isBeforeLegalizeOps() ||
5426 (isOperationLegal(Op: ISD::SETCC, VT: newVT) &&
5427 isCondCodeLegal(CC: Cond, VT: newVT.getSimpleVT()) &&
5428 isTypeDesirableForOp(ISD::SETCC, VT: newVT))) {
5429 EVT NewSetCCVT = getSetCCResultType(DL: Layout, Context&: *DAG.getContext(), VT: newVT);
5430 SDValue NewConst = DAG.getConstant(Val: C1.trunc(width: InSize), DL: dl, VT: newVT);
5431
5432 SDValue NewSetCC = DAG.getSetCC(DL: dl, VT: NewSetCCVT, LHS: N0.getOperand(i: 0),
5433 RHS: NewConst, Cond);
5434 return DAG.getBoolExtOrTrunc(Op: NewSetCC, SL: dl, VT, OpVT: N0.getValueType());
5435 }
5436 break;
5437 }
5438 default:
5439 break; // todo, be more careful with signed comparisons
5440 }
5441 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5442 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5443 !isSExtCheaperThanZExt(FromTy: cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT(),
5444 ToTy: OpVT)) {
5445 EVT ExtSrcTy = cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT();
5446 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
5447 EVT ExtDstTy = N0.getValueType();
5448 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
5449
5450 // If the constant doesn't fit into the number of bits for the source of
5451 // the sign extension, it is impossible for both sides to be equal.
5452 if (C1.getSignificantBits() > ExtSrcTyBits)
5453 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
5454
5455 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
5456 ExtDstTy != ExtSrcTy && "Unexpected types!");
5457 APInt Imm = APInt::getLowBitsSet(numBits: ExtDstTyBits, loBitsSet: ExtSrcTyBits);
5458 SDValue ZextOp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ExtDstTy, N1: N0.getOperand(i: 0),
5459 N2: DAG.getConstant(Val: Imm, DL: dl, VT: ExtDstTy));
5460 if (!DCI.isCalledByLegalizer())
5461 DCI.AddToWorklist(N: ZextOp.getNode());
5462 // Otherwise, make this a use of a zext.
5463 return DAG.getSetCC(DL: dl, VT, LHS: ZextOp,
5464 RHS: DAG.getConstant(Val: C1 & Imm, DL: dl, VT: ExtDstTy), Cond);
5465 } else if ((N1C->isZero() || N1C->isOne()) &&
5466 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5467 // SETCC (X), [0|1], [EQ|NE] -> X if X is known 0/1. i1 types are
5468 // excluded as they are handled below whilst checking for foldBooleans.
5469 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
5470 isTypeLegal(VT) && VT.bitsLE(VT: N0.getValueType()) &&
5471 (N0.getValueType() == MVT::i1 ||
5472 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5473 DAG.MaskedValueIsZero(
5474 Op: N0, Mask: APInt::getBitsSetFrom(numBits: N0.getValueSizeInBits(), loBit: 1))) {
5475 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
5476 if (TrueWhenTrue)
5477 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: N0);
5478 // Invert the condition.
5479 if (N0.getOpcode() == ISD::SETCC) {
5480 ISD::CondCode CC = N0.getOperandCondCode(i: 2);
5481 CC = ISD::getSetCCInverse(Operation: CC, Type: N0.getOperand(i: 0).getValueType());
5482 if (DCI.isBeforeLegalizeOps() ||
5483 isCondCodeLegal(CC, VT: N0.getOperand(i: 0).getSimpleValueType()))
5484 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond: CC);
5485 }
5486 }
5487
5488 if ((N0.getOpcode() == ISD::XOR ||
5489 (N0.getOpcode() == ISD::AND &&
5490 N0.getOperand(i: 0).getOpcode() == ISD::XOR &&
5491 N0.getOperand(i: 1) == N0.getOperand(i: 0).getOperand(i: 1))) &&
5492 isOneConstant(V: N0.getOperand(i: 1))) {
5493 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
5494 // can only do this if the top bits are known zero.
5495 unsigned BitWidth = N0.getValueSizeInBits();
5496 if (DAG.MaskedValueIsZero(Op: N0,
5497 Mask: APInt::getHighBitsSet(numBits: BitWidth,
5498 hiBitsSet: BitWidth-1))) {
5499 // Okay, get the un-inverted input value.
5500 SDValue Val;
5501 if (N0.getOpcode() == ISD::XOR) {
5502 Val = N0.getOperand(i: 0);
5503 } else {
5504 assert(N0.getOpcode() == ISD::AND &&
5505 N0.getOperand(0).getOpcode() == ISD::XOR);
5506 // ((X^1)&1)^1 -> X & 1
5507 Val = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N0.getValueType(),
5508 N1: N0.getOperand(i: 0).getOperand(i: 0),
5509 N2: N0.getOperand(i: 1));
5510 }
5511
5512 return DAG.getSetCC(DL: dl, VT, LHS: Val, RHS: N1,
5513 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5514 }
5515 } else if (N1C->isOne()) {
5516 SDValue Op0 = N0;
5517 if (Op0.getOpcode() == ISD::TRUNCATE)
5518 Op0 = Op0.getOperand(i: 0);
5519
5520 if ((Op0.getOpcode() == ISD::XOR) &&
5521 Op0.getOperand(i: 0).getOpcode() == ISD::SETCC &&
5522 Op0.getOperand(i: 1).getOpcode() == ISD::SETCC) {
5523 SDValue XorLHS = Op0.getOperand(i: 0);
5524 SDValue XorRHS = Op0.getOperand(i: 1);
5525 // Ensure that the input setccs return an i1 type or 0/1 value.
5526 if (Op0.getValueType() == MVT::i1 ||
5527 (getBooleanContents(Type: XorLHS.getOperand(i: 0).getValueType()) ==
5528 ZeroOrOneBooleanContent &&
5529 getBooleanContents(Type: XorRHS.getOperand(i: 0).getValueType()) ==
5530 ZeroOrOneBooleanContent)) {
5531 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
5532 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
5533 return DAG.getSetCC(DL: dl, VT, LHS: XorLHS, RHS: XorRHS, Cond);
5534 }
5535 }
5536 if (Op0.getOpcode() == ISD::AND && isOneConstant(V: Op0.getOperand(i: 1))) {
5537 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
5538 if (Op0.getValueType().bitsGT(VT))
5539 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5540 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5541 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5542 else if (Op0.getValueType().bitsLT(VT))
5543 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5544 N1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5545 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5546
5547 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5548 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5549 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5550 }
5551 if (Op0.getOpcode() == ISD::AssertZext &&
5552 cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT() == MVT::i1)
5553 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5554 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5555 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5556 }
5557 }
5558
5559 // Given:
5560 // icmp eq/ne (urem %x, %y), 0
5561 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
5562 // icmp eq/ne %x, 0
5563 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
5564 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5565 KnownBits XKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 0));
5566 KnownBits YKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 1));
5567 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
5568 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
5569 }
5570
5571 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
5572 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
5573 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5574 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
5575 N0.getConstantOperandAPInt(i: 1) == OpVT.getScalarSizeInBits() - 1 &&
5576 N1C->isAllOnes()) {
5577 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
5578 RHS: DAG.getConstant(Val: 0, DL: dl, VT: OpVT),
5579 Cond: Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
5580 }
5581
5582 // fold (setcc (trunc x) c) -> (setcc x c)
5583 if (N0.getOpcode() == ISD::TRUNCATE &&
5584 ((N0->getFlags().hasNoUnsignedWrap() && !ISD::isSignedIntSetCC(Code: Cond)) ||
5585 (N0->getFlags().hasNoSignedWrap() &&
5586 !ISD::isUnsignedIntSetCC(Code: Cond))) &&
5587 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5588 EVT NewVT = N0.getOperand(i: 0).getValueType();
5589 SDValue NewConst = DAG.getConstant(
5590 Val: (N0->getFlags().hasNoSignedWrap() && !ISD::isUnsignedIntSetCC(Code: Cond))
5591 ? C1.sext(width: NewVT.getSizeInBits())
5592 : C1.zext(width: NewVT.getSizeInBits()),
5593 DL: dl, VT: NewVT);
5594 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NewConst, Cond);
5595 }
5596
5597 if (SDValue V =
5598 optimizeSetCCOfSignedTruncationCheck(SCCVT: VT, N0, N1, Cond, DCI, DL: dl))
5599 return V;
5600 }
5601
5602 // These simplifications apply to splat vectors as well.
5603 // TODO: Handle more splat vector cases.
5604 if (auto *N1C = isConstOrConstSplat(N: N1)) {
5605 const APInt &C1 = N1C->getAPIntValue();
5606
5607 APInt MinVal, MaxVal;
5608 unsigned OperandBitSize = N1C->getValueType(ResNo: 0).getScalarSizeInBits();
5609 if (ISD::isSignedIntSetCC(Code: Cond)) {
5610 MinVal = APInt::getSignedMinValue(numBits: OperandBitSize);
5611 MaxVal = APInt::getSignedMaxValue(numBits: OperandBitSize);
5612 } else {
5613 MinVal = APInt::getMinValue(numBits: OperandBitSize);
5614 MaxVal = APInt::getMaxValue(numBits: OperandBitSize);
5615 }
5616
5617 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
5618 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
5619 // X >= MIN --> true
5620 if (C1 == MinVal)
5621 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5622
5623 if (!VT.isVector()) { // TODO: Support this for vectors.
5624 // X >= C0 --> X > (C0 - 1)
5625 APInt C = C1 - 1;
5626 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
5627 if ((DCI.isBeforeLegalizeOps() ||
5628 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5629 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5630 isLegalICmpImmediate(C.getSExtValue())))) {
5631 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5632 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5633 Cond: NewCC);
5634 }
5635 }
5636 }
5637
5638 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
5639 // X <= MAX --> true
5640 if (C1 == MaxVal)
5641 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5642
5643 // X <= C0 --> X < (C0 + 1)
5644 if (!VT.isVector()) { // TODO: Support this for vectors.
5645 APInt C = C1 + 1;
5646 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
5647 if ((DCI.isBeforeLegalizeOps() ||
5648 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5649 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5650 isLegalICmpImmediate(C.getSExtValue())))) {
5651 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5652 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5653 Cond: NewCC);
5654 }
5655 }
5656 }
5657
5658 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
5659 if (C1 == MinVal)
5660 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X < MIN --> false
5661
5662 // TODO: Support this for vectors after legalize ops.
5663 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5664 // Canonicalize setlt X, Max --> setne X, Max
5665 if (C1 == MaxVal)
5666 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5667
5668 // If we have setult X, 1, turn it into seteq X, 0
5669 if (C1 == MinVal+1)
5670 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5671 RHS: DAG.getConstant(Val: MinVal, DL: dl, VT: N0.getValueType()),
5672 Cond: ISD::SETEQ);
5673 }
5674 }
5675
5676 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
5677 if (C1 == MaxVal)
5678 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X > MAX --> false
5679
5680 // TODO: Support this for vectors after legalize ops.
5681 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5682 // Canonicalize setgt X, Min --> setne X, Min
5683 if (C1 == MinVal)
5684 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5685
5686 // If we have setugt X, Max-1, turn it into seteq X, Max
5687 if (C1 == MaxVal-1)
5688 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5689 RHS: DAG.getConstant(Val: MaxVal, DL: dl, VT: N0.getValueType()),
5690 Cond: ISD::SETEQ);
5691 }
5692 }
5693
5694 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
5695 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
5696 if (C1.isZero())
5697 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
5698 SCCVT: VT, N0, N1C: N1, Cond, DCI, DL: dl))
5699 return CC;
5700
5701 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
5702 // For example, when high 32-bits of i64 X are known clear:
5703 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
5704 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
5705 bool CmpZero = N1C->isZero();
5706 bool CmpNegOne = N1C->isAllOnes();
5707 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
5708 // Match or(lo,shl(hi,bw/2)) pattern.
5709 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
5710 unsigned EltBits = V.getScalarValueSizeInBits();
5711 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
5712 return false;
5713 SDValue LHS = V.getOperand(i: 0);
5714 SDValue RHS = V.getOperand(i: 1);
5715 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: EltBits / 2);
5716 // Unshifted element must have zero upperbits.
5717 if (RHS.getOpcode() == ISD::SHL &&
5718 isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)) &&
5719 RHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5720 DAG.MaskedValueIsZero(Op: LHS, Mask: HiBits)) {
5721 Lo = LHS;
5722 Hi = RHS.getOperand(i: 0);
5723 return true;
5724 }
5725 if (LHS.getOpcode() == ISD::SHL &&
5726 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
5727 LHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5728 DAG.MaskedValueIsZero(Op: RHS, Mask: HiBits)) {
5729 Lo = RHS;
5730 Hi = LHS.getOperand(i: 0);
5731 return true;
5732 }
5733 return false;
5734 };
5735
5736 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
5737 unsigned EltBits = N0.getScalarValueSizeInBits();
5738 unsigned HalfBits = EltBits / 2;
5739 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: HalfBits);
5740 SDValue LoBits = DAG.getConstant(Val: ~HiBits, DL: dl, VT: OpVT);
5741 SDValue HiMask = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Hi, N2: LoBits);
5742 SDValue NewN0 =
5743 DAG.getNode(Opcode: CmpZero ? ISD::OR : ISD::AND, DL: dl, VT: OpVT, N1: Lo, N2: HiMask);
5744 SDValue NewN1 = CmpZero ? DAG.getConstant(Val: 0, DL: dl, VT: OpVT) : LoBits;
5745 return DAG.getSetCC(DL: dl, VT, LHS: NewN0, RHS: NewN1, Cond);
5746 };
5747
5748 SDValue Lo, Hi;
5749 if (IsConcat(N0, Lo, Hi))
5750 return MergeConcat(Lo, Hi);
5751
5752 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
5753 SDValue Lo0, Lo1, Hi0, Hi1;
5754 if (IsConcat(N0.getOperand(i: 0), Lo0, Hi0) &&
5755 IsConcat(N0.getOperand(i: 1), Lo1, Hi1)) {
5756 return MergeConcat(DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Lo0, N2: Lo1),
5757 DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Hi0, N2: Hi1));
5758 }
5759 }
5760 }
5761 }
5762
5763 // If we have "setcc X, C0", check to see if we can shrink the immediate
5764 // by changing cc.
5765 // TODO: Support this for vectors after legalize ops.
5766 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5767 // SETUGT X, SINTMAX -> SETLT X, 0
5768 // SETUGE X, SINTMIN -> SETLT X, 0
5769 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
5770 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
5771 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5772 RHS: DAG.getConstant(Val: 0, DL: dl, VT: N1.getValueType()),
5773 Cond: ISD::SETLT);
5774
5775 // SETULT X, SINTMIN -> SETGT X, -1
5776 // SETULE X, SINTMAX -> SETGT X, -1
5777 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
5778 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
5779 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5780 RHS: DAG.getAllOnesConstant(DL: dl, VT: N1.getValueType()),
5781 Cond: ISD::SETGT);
5782 }
5783 }
5784
5785 // Back to non-vector simplifications.
5786 // TODO: Can we do these for vector splats?
5787 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5788 const APInt &C1 = N1C->getAPIntValue();
5789 EVT ShValTy = N0.getValueType();
5790
5791 // Fold bit comparisons when we can. This will result in an
5792 // incorrect value when boolean false is negative one, unless
5793 // the bitsize is 1 in which case the false value is the same
5794 // in practice regardless of the representation.
5795 if ((VT.getSizeInBits() == 1 ||
5796 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5797 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5798 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(VT: ShValTy))) &&
5799 N0.getOpcode() == ISD::AND) {
5800 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5801 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
5802 // Perform the xform if the AND RHS is a single bit.
5803 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5804 if (AndRHS->getAPIntValue().isPowerOf2() &&
5805 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5806 return DAG.getNode(
5807 Opcode: ISD::TRUNCATE, DL: dl, VT,
5808 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5809 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5810 }
5811 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5812 // (X & 8) == 8 --> (X & 8) >> 3
5813 // Perform the xform if C1 is a single bit.
5814 unsigned ShCt = C1.logBase2();
5815 if (C1.isPowerOf2() && !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5816 return DAG.getNode(
5817 Opcode: ISD::TRUNCATE, DL: dl, VT,
5818 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5819 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5820 }
5821 }
5822 }
5823 }
5824
5825 if (C1.getSignificantBits() <= 64 &&
5826 !isLegalICmpImmediate(C1.getSExtValue())) {
5827 // (X & -256) == 256 -> (X >> 8) == 1
5828 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5829 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
5830 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5831 const APInt &AndRHSC = AndRHS->getAPIntValue();
5832 if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(RHS: AndRHSC)) {
5833 unsigned ShiftBits = AndRHSC.countr_zero();
5834 if (!shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5835 // If using an unsigned shift doesn't yield a legal compare
5836 // immediate, try using sra instead.
5837 APInt NewC = C1.lshr(shiftAmt: ShiftBits);
5838 if (NewC.getSignificantBits() <= 64 &&
5839 !isLegalICmpImmediate(NewC.getSExtValue())) {
5840 APInt SignedC = C1.ashr(ShiftAmt: ShiftBits);
5841 if (SignedC.getSignificantBits() <= 64 &&
5842 isLegalICmpImmediate(SignedC.getSExtValue())) {
5843 SDValue Shift = DAG.getNode(
5844 Opcode: ISD::SRA, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5845 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5846 SDValue CmpRHS = DAG.getConstant(Val: SignedC, DL: dl, VT: ShValTy);
5847 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5848 }
5849 }
5850 SDValue Shift = DAG.getNode(
5851 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5852 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5853 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5854 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5855 }
5856 }
5857 }
5858 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
5859 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
5860 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
5861 // X < 0x100000000 -> (X >> 32) < 1
5862 // X >= 0x100000000 -> (X >> 32) >= 1
5863 // X <= 0x0ffffffff -> (X >> 32) < 1
5864 // X > 0x0ffffffff -> (X >> 32) >= 1
5865 unsigned ShiftBits;
5866 APInt NewC = C1;
5867 ISD::CondCode NewCond = Cond;
5868 if (AdjOne) {
5869 ShiftBits = C1.countr_one();
5870 NewC = NewC + 1;
5871 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
5872 } else {
5873 ShiftBits = C1.countr_zero();
5874 }
5875 APInt RangeWidth = NewC;
5876 NewC.lshrInPlace(ShiftAmt: ShiftBits);
5877 if (ShiftBits && NewC.getSignificantBits() <= 64 &&
5878 isLegalICmpImmediate(NewC.getSExtValue()) &&
5879 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5880 // If this is an offset range check, try to move the offset after the
5881 // shift to avoid preserving the pre-shift add with a mask.
5882 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse()) {
5883 if (auto *AddC = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
5884 const APInt &AddVal = AddC->getAPIntValue();
5885 if (AddVal.countr_zero() >= ShiftBits) {
5886 APInt RangeLower = -AddVal;
5887 bool Overflow;
5888 (void)RangeLower.uadd_ov(RHS: RangeWidth, Overflow);
5889 if (!RangeWidth.isZero() && !Overflow) {
5890 SDValue Shift = DAG.getNode(
5891 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5892 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5893 APInt Offset = -RangeLower.lshr(shiftAmt: ShiftBits);
5894 SDValue ShiftedAdd =
5895 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ShValTy, N1: Shift,
5896 N2: DAG.getConstant(Val: Offset, DL: dl, VT: ShValTy));
5897 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5898 return DAG.getSetCC(DL: dl, VT, LHS: ShiftedAdd, RHS: CmpRHS, Cond: NewCond);
5899 }
5900 }
5901 }
5902 }
5903 SDValue Shift =
5904 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5905 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5906 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5907 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond: NewCond);
5908 }
5909 }
5910 }
5911 }
5912
5913 if (!isa<ConstantFPSDNode>(Val: N0) && isa<ConstantFPSDNode>(Val: N1)) {
5914 auto *CFP = cast<ConstantFPSDNode>(Val&: N1);
5915 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
5916
5917 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
5918 // constant if knowing that the operand is non-nan is enough. We prefer to
5919 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
5920 // materialize 0.0.
5921 if (Cond == ISD::SETO || Cond == ISD::SETUO)
5922 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N0, Cond);
5923
5924 // setcc (fneg x), C -> setcc swap(pred) x, -C
5925 if (N0.getOpcode() == ISD::FNEG) {
5926 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Operation: Cond);
5927 if (DCI.isBeforeLegalizeOps() ||
5928 isCondCodeLegal(CC: SwapCond, VT: N0.getSimpleValueType())) {
5929 SDValue NegN1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: N0.getValueType(), Operand: N1);
5930 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NegN1, Cond: SwapCond);
5931 }
5932 }
5933
5934 // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
5935 if (isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT: N0.getValueType()) &&
5936 !isFPImmLegal(Imm: CFP->getValueAPF(), VT: CFP->getValueType(ResNo: 0))) {
5937 bool IsFabs = N0.getOpcode() == ISD::FABS;
5938 SDValue Op = IsFabs ? N0.getOperand(i: 0) : N0;
5939 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
5940 FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
5941 : (IsFabs ? fcInf : fcPosInf);
5942 if (Cond == ISD::SETUEQ)
5943 Flag |= fcNan;
5944 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: dl, VT, N1: Op,
5945 N2: DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32));
5946 }
5947 }
5948
5949 // If the condition is not legal, see if we can find an equivalent one
5950 // which is legal.
5951 if (!isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType())) {
5952 // If the comparison was an awkward floating-point == or != and one of
5953 // the comparison operands is infinity or negative infinity, convert the
5954 // condition to a less-awkward <= or >=.
5955 if (CFP->getValueAPF().isInfinity()) {
5956 bool IsNegInf = CFP->getValueAPF().isNegative();
5957 ISD::CondCode NewCond = ISD::SETCC_INVALID;
5958 switch (Cond) {
5959 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
5960 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
5961 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
5962 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
5963 default: break;
5964 }
5965 if (NewCond != ISD::SETCC_INVALID &&
5966 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType()))
5967 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5968 }
5969 }
5970 }
5971
5972 if (N0 == N1) {
5973 // The sext(setcc()) => setcc() optimization relies on the appropriate
5974 // constant being emitted.
5975 assert(!N0.getValueType().isInteger() &&
5976 "Integer types should be handled by FoldSetCC");
5977
5978 bool EqTrue = ISD::isTrueWhenEqual(Cond);
5979 unsigned UOF = ISD::getUnorderedFlavor(Cond);
5980 if (UOF == 2) // FP operators that are undefined on NaNs.
5981 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5982 if (UOF == unsigned(EqTrue))
5983 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5984 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
5985 // if it is not already.
5986 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
5987 if (NewCond != Cond &&
5988 (DCI.isBeforeLegalizeOps() ||
5989 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType())))
5990 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5991 }
5992
5993 // ~X > ~Y --> Y > X
5994 // ~X < ~Y --> Y < X
5995 // ~X < C --> X > ~C
5996 // ~X > C --> X < ~C
5997 if ((isSignedIntSetCC(Code: Cond) || isUnsignedIntSetCC(Code: Cond)) &&
5998 N0.getValueType().isInteger()) {
5999 if (isBitwiseNot(V: N0)) {
6000 if (isBitwiseNot(V: N1))
6001 return DAG.getSetCC(DL: dl, VT, LHS: N1.getOperand(i: 0), RHS: N0.getOperand(i: 0), Cond);
6002
6003 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N1) &&
6004 !DAG.isConstantIntBuildVectorOrConstantInt(N: N0.getOperand(i: 0))) {
6005 SDValue Not = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
6006 return DAG.getSetCC(DL: dl, VT, LHS: Not, RHS: N0.getOperand(i: 0), Cond);
6007 }
6008 }
6009 }
6010
6011 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
6012 N0.getValueType().isInteger()) {
6013 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
6014 N0.getOpcode() == ISD::XOR) {
6015 // Simplify (X+Y) == (X+Z) --> Y == Z
6016 if (N0.getOpcode() == N1.getOpcode()) {
6017 if (N0.getOperand(i: 0) == N1.getOperand(i: 0))
6018 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 1), Cond);
6019 if (N0.getOperand(i: 1) == N1.getOperand(i: 1))
6020 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
6021 if (isCommutativeBinOp(Opcode: N0.getOpcode())) {
6022 // If X op Y == Y op X, try other combinations.
6023 if (N0.getOperand(i: 0) == N1.getOperand(i: 1))
6024 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 0),
6025 Cond);
6026 if (N0.getOperand(i: 1) == N1.getOperand(i: 0))
6027 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 1),
6028 Cond);
6029 }
6030 }
6031
6032 // If RHS is a legal immediate value for a compare instruction, we need
6033 // to be careful about increasing register pressure needlessly.
6034 bool LegalRHSImm = false;
6035
6036 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: N1)) {
6037 if (auto *LHSR = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
6038 // Turn (X+C1) == C2 --> X == C2-C1
6039 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
6040 return DAG.getSetCC(
6041 DL: dl, VT, LHS: N0.getOperand(i: 0),
6042 RHS: DAG.getConstant(Val: RHSC->getAPIntValue() - LHSR->getAPIntValue(),
6043 DL: dl, VT: N0.getValueType()),
6044 Cond);
6045
6046 // Turn (X^C1) == C2 --> X == C1^C2
6047 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
6048 return DAG.getSetCC(
6049 DL: dl, VT, LHS: N0.getOperand(i: 0),
6050 RHS: DAG.getConstant(Val: LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
6051 DL: dl, VT: N0.getValueType()),
6052 Cond);
6053 }
6054
6055 // Turn (C1-X) == C2 --> X == C1-C2
6056 if (auto *SUBC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0)))
6057 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
6058 return DAG.getSetCC(
6059 DL: dl, VT, LHS: N0.getOperand(i: 1),
6060 RHS: DAG.getConstant(Val: SUBC->getAPIntValue() - RHSC->getAPIntValue(),
6061 DL: dl, VT: N0.getValueType()),
6062 Cond);
6063
6064 // Could RHSC fold directly into a compare?
6065 if (RHSC->getValueType(ResNo: 0).getSizeInBits() <= 64)
6066 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
6067 }
6068
6069 // (X+Y) == X --> Y == 0 and similar folds.
6070 // Don't do this if X is an immediate that can fold into a cmp
6071 // instruction and X+Y has other uses. It could be an induction variable
6072 // chain, and the transform would increase register pressure.
6073 if (!LegalRHSImm || N0.hasOneUse())
6074 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, DL: dl, DCI))
6075 return V;
6076 }
6077
6078 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
6079 N1.getOpcode() == ISD::XOR)
6080 if (SDValue V = foldSetCCWithBinOp(VT, N0: N1, N1: N0, Cond, DL: dl, DCI))
6081 return V;
6082
6083 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, DL: dl, DCI))
6084 return V;
6085
6086 if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, DL: dl, DCI))
6087 return V;
6088 }
6089
6090 // Fold remainder of division by a constant.
6091 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
6092 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
6093 // When division is cheap or optimizing for minimum size,
6094 // fall through to DIVREM creation by skipping this fold.
6095 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
6096 if (N0.getOpcode() == ISD::UREM) {
6097 if (SDValue Folded = buildUREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
6098 return Folded;
6099 } else if (N0.getOpcode() == ISD::SREM) {
6100 if (SDValue Folded = buildSREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
6101 return Folded;
6102 }
6103 }
6104 }
6105
6106 // Fold away ALL boolean setcc's.
6107 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
6108 SDValue Temp;
6109 switch (Cond) {
6110 default: llvm_unreachable("Unknown integer setcc!");
6111 case ISD::SETEQ: // X == Y -> ~(X^Y)
6112 Temp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
6113 N0 = DAG.getNOT(DL: dl, Val: Temp, VT: OpVT);
6114 if (!DCI.isCalledByLegalizer())
6115 DCI.AddToWorklist(N: Temp.getNode());
6116 break;
6117 case ISD::SETNE: // X != Y --> (X^Y)
6118 N0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
6119 break;
6120 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
6121 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
6122 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
6123 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1, N2: Temp);
6124 if (!DCI.isCalledByLegalizer())
6125 DCI.AddToWorklist(N: Temp.getNode());
6126 break;
6127 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
6128 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
6129 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
6130 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: N0, N2: Temp);
6131 if (!DCI.isCalledByLegalizer())
6132 DCI.AddToWorklist(N: Temp.getNode());
6133 break;
6134 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
6135 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
6136 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
6137 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1, N2: Temp);
6138 if (!DCI.isCalledByLegalizer())
6139 DCI.AddToWorklist(N: Temp.getNode());
6140 break;
6141 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
6142 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
6143 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
6144 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: N0, N2: Temp);
6145 break;
6146 }
6147 if (VT.getScalarType() != MVT::i1) {
6148 if (!DCI.isCalledByLegalizer())
6149 DCI.AddToWorklist(N: N0.getNode());
6150 // FIXME: If running after legalize, we probably can't do this.
6151 ISD::NodeType ExtendCode = getExtendForContent(Content: getBooleanContents(Type: OpVT));
6152 N0 = DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: N0);
6153 }
6154 return N0;
6155 }
6156
6157 // Fold (setcc (trunc x) (trunc y)) -> (setcc x y)
6158 if (N0.getOpcode() == ISD::TRUNCATE && N1.getOpcode() == ISD::TRUNCATE &&
6159 N0.getOperand(i: 0).getValueType() == N1.getOperand(i: 0).getValueType() &&
6160 ((!ISD::isSignedIntSetCC(Code: Cond) && N0->getFlags().hasNoUnsignedWrap() &&
6161 N1->getFlags().hasNoUnsignedWrap()) ||
6162 (!ISD::isUnsignedIntSetCC(Code: Cond) && N0->getFlags().hasNoSignedWrap() &&
6163 N1->getFlags().hasNoSignedWrap())) &&
6164 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
6165 if (VT.getScalarType() == MVT::i1)
6166 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
6167 // For (legal) non vXi1 cases - ensure we adjust the cmp and result types.
6168 EVT OldCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
6169 VT: N0.getValueType());
6170 if (VT == OldCCVT) {
6171 EVT NewCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
6172 VT: N0.getOperand(i: 0).getValueType());
6173 return DAG.getBoolExtOrTrunc(
6174 Op: DAG.getSetCC(DL: dl, VT: NewCCVT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond),
6175 SL: dl, VT, OpVT: N0.getOperand(i: 0).getValueType());
6176 }
6177 }
6178
6179 // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
6180 // TODO: Remove that .isVector() check
6181 if (VT.isVector() && isZeroOrZeroSplat(N: N1) && N0.getOpcode() == ISD::SUB &&
6182 N0->getFlags().hasNoSignedWrap() && ISD::isSignedIntSetCC(Code: Cond)) {
6183 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond);
6184 }
6185
6186 // Could not fold it.
6187 return SDValue();
6188}
6189
6190/// Returns true (and the GlobalValue and the offset) if the node is a
6191/// GlobalAddress + offset.
6192bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
6193 int64_t &Offset) const {
6194
6195 SDNode *N = unwrapAddress(N: SDValue(WN, 0)).getNode();
6196
6197 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(Val: N)) {
6198 GA = GASD->getGlobal();
6199 Offset += GASD->getOffset();
6200 return true;
6201 }
6202
6203 if (N->isAnyAdd()) {
6204 SDValue N1 = N->getOperand(Num: 0);
6205 SDValue N2 = N->getOperand(Num: 1);
6206 if (isGAPlusOffset(WN: N1.getNode(), GA, Offset)) {
6207 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N2)) {
6208 Offset += V->getSExtValue();
6209 return true;
6210 }
6211 } else if (isGAPlusOffset(WN: N2.getNode(), GA, Offset)) {
6212 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N1)) {
6213 Offset += V->getSExtValue();
6214 return true;
6215 }
6216 }
6217 }
6218
6219 return false;
6220}
6221
6222SDValue TargetLowering::PerformDAGCombine(SDNode *N,
6223 DAGCombinerInfo &DCI) const {
6224 // Default implementation: no optimization.
6225 return SDValue();
6226}
6227
6228//===----------------------------------------------------------------------===//
6229// Inline Assembler Implementation Methods
6230//===----------------------------------------------------------------------===//
6231
6232TargetLowering::ConstraintType
6233TargetLowering::getConstraintType(StringRef Constraint) const {
6234 unsigned S = Constraint.size();
6235
6236 if (S == 1) {
6237 switch (Constraint[0]) {
6238 default: break;
6239 case 'r':
6240 return C_RegisterClass;
6241 case 'm': // memory
6242 case 'o': // offsetable
6243 case 'V': // not offsetable
6244 return C_Memory;
6245 case 'p': // Address.
6246 return C_Address;
6247 case 'n': // Simple Integer
6248 case 'E': // Floating Point Constant
6249 case 'F': // Floating Point Constant
6250 return C_Immediate;
6251 case 'i': // Simple Integer or Relocatable Constant
6252 case 's': // Relocatable Constant
6253 case 'X': // Allow ANY value.
6254 case 'I': // Target registers.
6255 case 'J':
6256 case 'K':
6257 case 'L':
6258 case 'M':
6259 case 'N':
6260 case 'O':
6261 case 'P':
6262 case '<':
6263 case '>':
6264 return C_Other;
6265 }
6266 }
6267
6268 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
6269 if (S == 8 && Constraint.substr(Start: 1, N: 6) == "memory") // "{memory}"
6270 return C_Memory;
6271 return C_Register;
6272 }
6273 return C_Unknown;
6274}
6275
6276/// Try to replace an X constraint, which matches anything, with another that
6277/// has more specific requirements based on the type of the corresponding
6278/// operand.
6279const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
6280 if (ConstraintVT.isInteger())
6281 return "r";
6282 if (ConstraintVT.isFloatingPoint())
6283 return "f"; // works for many targets
6284 return nullptr;
6285}
6286
6287SDValue TargetLowering::LowerAsmOutputForConstraint(
6288 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
6289 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
6290 return SDValue();
6291}
6292
6293/// Lower the specified operand into the Ops vector.
6294/// If it is invalid, don't add anything to Ops.
6295void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
6296 StringRef Constraint,
6297 std::vector<SDValue> &Ops,
6298 SelectionDAG &DAG) const {
6299
6300 if (Constraint.size() > 1)
6301 return;
6302
6303 char ConstraintLetter = Constraint[0];
6304 switch (ConstraintLetter) {
6305 default: break;
6306 case 'X': // Allows any operand
6307 case 'i': // Simple Integer or Relocatable Constant
6308 case 'n': // Simple Integer
6309 case 's': { // Relocatable Constant
6310
6311 ConstantSDNode *C;
6312 uint64_t Offset = 0;
6313
6314 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
6315 // etc., since getelementpointer is variadic. We can't use
6316 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
6317 // while in this case the GA may be furthest from the root node which is
6318 // likely an ISD::ADD.
6319 while (true) {
6320 if ((C = dyn_cast<ConstantSDNode>(Val&: Op)) && ConstraintLetter != 's') {
6321 // gcc prints these as sign extended. Sign extend value to 64 bits
6322 // now; without this it would get ZExt'd later in
6323 // ScheduleDAGSDNodes::EmitNode, which is very generic.
6324 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
6325 BooleanContent BCont = getBooleanContents(Type: MVT::i64);
6326 ISD::NodeType ExtOpc =
6327 IsBool ? getExtendForContent(Content: BCont) : ISD::SIGN_EXTEND;
6328 int64_t ExtVal =
6329 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
6330 Ops.push_back(
6331 x: DAG.getTargetConstant(Val: Offset + ExtVal, DL: SDLoc(C), VT: MVT::i64));
6332 return;
6333 }
6334 if (ConstraintLetter != 'n') {
6335 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
6336 Ops.push_back(x: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(Op),
6337 VT: GA->getValueType(ResNo: 0),
6338 offset: Offset + GA->getOffset()));
6339 return;
6340 }
6341 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
6342 Ops.push_back(x: DAG.getTargetBlockAddress(
6343 BA: BA->getBlockAddress(), VT: BA->getValueType(ResNo: 0),
6344 Offset: Offset + BA->getOffset(), TargetFlags: BA->getTargetFlags()));
6345 return;
6346 }
6347 if (isa<BasicBlockSDNode>(Val: Op)) {
6348 Ops.push_back(x: Op);
6349 return;
6350 }
6351 }
6352 const unsigned OpCode = Op.getOpcode();
6353 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
6354 if ((C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 0))))
6355 Op = Op.getOperand(i: 1);
6356 // Subtraction is not commutative.
6357 else if (OpCode == ISD::ADD &&
6358 (C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))))
6359 Op = Op.getOperand(i: 0);
6360 else
6361 return;
6362 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
6363 continue;
6364 }
6365 return;
6366 }
6367 break;
6368 }
6369 }
6370}
6371
6372void TargetLowering::CollectTargetIntrinsicOperands(
6373 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
6374}
6375
6376std::pair<unsigned, const TargetRegisterClass *>
6377TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
6378 StringRef Constraint,
6379 MVT VT) const {
6380 if (!Constraint.starts_with(Prefix: "{"))
6381 return std::make_pair(x: 0u, y: static_cast<TargetRegisterClass *>(nullptr));
6382 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
6383
6384 // Remove the braces from around the name.
6385 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
6386
6387 std::pair<unsigned, const TargetRegisterClass *> R =
6388 std::make_pair(x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
6389
6390 // Figure out which register class contains this reg.
6391 for (const TargetRegisterClass &RC : RI->regclasses()) {
6392 // If none of the value types for this register class are valid, we
6393 // can't use it. For example, 64-bit reg classes on 32-bit targets.
6394 if (!isLegalRC(TRI: *RI, RC))
6395 continue;
6396
6397 for (const MCPhysReg &PR : RC) {
6398 if (RegName.equals_insensitive(RHS: RI->getRegAsmName(Reg: PR))) {
6399 std::pair<unsigned, const TargetRegisterClass *> S =
6400 std::make_pair(x: PR, y: &RC);
6401
6402 // If this register class has the requested value type, return it,
6403 // otherwise keep searching and return the first class found
6404 // if no other is found which explicitly has the requested type.
6405 if (RI->isTypeLegalForClass(RC, T: VT))
6406 return S;
6407 if (!R.second)
6408 R = S;
6409 }
6410 }
6411 }
6412
6413 return R;
6414}
6415
6416//===----------------------------------------------------------------------===//
6417// Constraint Selection.
6418
6419/// Return true of this is an input operand that is a matching constraint like
6420/// "4".
6421bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
6422 assert(!ConstraintCode.empty() && "No known constraint!");
6423 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
6424}
6425
6426/// If this is an input matching constraint, this method returns the output
6427/// operand it matches.
6428unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
6429 assert(!ConstraintCode.empty() && "No known constraint!");
6430 return atoi(nptr: ConstraintCode.c_str());
6431}
6432
6433/// Split up the constraint string from the inline assembly value into the
6434/// specific constraints and their prefixes, and also tie in the associated
6435/// operand values.
6436/// If this returns an empty vector, and if the constraint string itself
6437/// isn't empty, there was an error parsing.
6438TargetLowering::AsmOperandInfoVector
6439TargetLowering::ParseConstraints(const DataLayout &DL,
6440 const TargetRegisterInfo *TRI,
6441 const CallBase &Call) const {
6442 /// Information about all of the constraints.
6443 AsmOperandInfoVector ConstraintOperands;
6444 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
6445 unsigned maCount = 0; // Largest number of multiple alternative constraints.
6446
6447 // Do a prepass over the constraints, canonicalizing them, and building up the
6448 // ConstraintOperands list.
6449 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6450 unsigned ResNo = 0; // ResNo - The result number of the next output.
6451 unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.
6452
6453 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
6454 ConstraintOperands.emplace_back(args: std::move(CI));
6455 AsmOperandInfo &OpInfo = ConstraintOperands.back();
6456
6457 // Update multiple alternative constraint count.
6458 if (OpInfo.multipleAlternatives.size() > maCount)
6459 maCount = OpInfo.multipleAlternatives.size();
6460
6461 OpInfo.ConstraintVT = MVT::Other;
6462
6463 // Compute the value type for each operand.
6464 switch (OpInfo.Type) {
6465 case InlineAsm::isOutput: {
6466 // Indirect outputs just consume an argument.
6467 if (OpInfo.isIndirect) {
6468 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6469 break;
6470 }
6471
6472 // The return value of the call is this value. As such, there is no
6473 // corresponding argument.
6474 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
6475 EVT VT;
6476 if (auto *STy = dyn_cast<StructType>(Val: Call.getType())) {
6477 VT = getAsmOperandValueType(DL, Ty: STy->getElementType(N: ResNo));
6478 } else {
6479 assert(ResNo == 0 && "Asm only has one result!");
6480 VT = getAsmOperandValueType(DL, Ty: Call.getType());
6481 }
6482 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6483 ++ResNo;
6484 break;
6485 }
6486 case InlineAsm::isInput:
6487 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6488 break;
6489 case InlineAsm::isLabel:
6490 OpInfo.CallOperandVal = cast<CallBrInst>(Val: &Call)->getIndirectDest(i: LabelNo);
6491 ++LabelNo;
6492 continue;
6493 case InlineAsm::isClobber:
6494 // Nothing to do.
6495 break;
6496 }
6497
6498 if (OpInfo.CallOperandVal) {
6499 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
6500 if (OpInfo.isIndirect) {
6501 OpTy = Call.getParamElementType(ArgNo);
6502 assert(OpTy && "Indirect operand must have elementtype attribute");
6503 }
6504
6505 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6506 if (StructType *STy = dyn_cast<StructType>(Val: OpTy))
6507 if (STy->getNumElements() == 1)
6508 OpTy = STy->getElementType(N: 0);
6509
6510 // If OpTy is not a single value, it may be a struct/union that we
6511 // can tile with integers.
6512 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6513 unsigned BitSize = DL.getTypeSizeInBits(Ty: OpTy);
6514 switch (BitSize) {
6515 default: break;
6516 case 1:
6517 case 8:
6518 case 16:
6519 case 32:
6520 case 64:
6521 case 128:
6522 OpTy = IntegerType::get(C&: OpTy->getContext(), NumBits: BitSize);
6523 break;
6524 }
6525 }
6526
6527 EVT VT = getAsmOperandValueType(DL, Ty: OpTy, AllowUnknown: true);
6528 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6529 ArgNo++;
6530 }
6531 }
6532
6533 // If we have multiple alternative constraints, select the best alternative.
6534 if (!ConstraintOperands.empty()) {
6535 if (maCount) {
6536 unsigned bestMAIndex = 0;
6537 int bestWeight = -1;
6538 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
6539 int weight = -1;
6540 unsigned maIndex;
6541 // Compute the sums of the weights for each alternative, keeping track
6542 // of the best (highest weight) one so far.
6543 for (maIndex = 0; maIndex < maCount; ++maIndex) {
6544 int weightSum = 0;
6545 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6546 cIndex != eIndex; ++cIndex) {
6547 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6548 if (OpInfo.Type == InlineAsm::isClobber)
6549 continue;
6550
6551 // If this is an output operand with a matching input operand,
6552 // look up the matching input. If their types mismatch, e.g. one
6553 // is an integer, the other is floating point, or their sizes are
6554 // different, flag it as an maCantMatch.
6555 if (OpInfo.hasMatchingInput()) {
6556 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6557 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6558 if ((OpInfo.ConstraintVT.isInteger() !=
6559 Input.ConstraintVT.isInteger()) ||
6560 (OpInfo.ConstraintVT.getSizeInBits() !=
6561 Input.ConstraintVT.getSizeInBits())) {
6562 weightSum = -1; // Can't match.
6563 break;
6564 }
6565 }
6566 }
6567 weight = getMultipleConstraintMatchWeight(info&: OpInfo, maIndex);
6568 if (weight == -1) {
6569 weightSum = -1;
6570 break;
6571 }
6572 weightSum += weight;
6573 }
6574 // Update best.
6575 if (weightSum > bestWeight) {
6576 bestWeight = weightSum;
6577 bestMAIndex = maIndex;
6578 }
6579 }
6580
6581 // Now select chosen alternative in each constraint.
6582 for (AsmOperandInfo &cInfo : ConstraintOperands)
6583 if (cInfo.Type != InlineAsm::isClobber)
6584 cInfo.selectAlternative(index: bestMAIndex);
6585 }
6586 }
6587
6588 // Check and hook up tied operands, choose constraint code to use.
6589 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6590 cIndex != eIndex; ++cIndex) {
6591 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6592
6593 // If this is an output operand with a matching input operand, look up the
6594 // matching input. If their types mismatch, e.g. one is an integer, the
6595 // other is floating point, or their sizes are different, flag it as an
6596 // error.
6597 if (OpInfo.hasMatchingInput()) {
6598 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6599
6600 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6601 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6602 getRegForInlineAsmConstraint(RI: TRI, Constraint: OpInfo.ConstraintCode,
6603 VT: OpInfo.ConstraintVT);
6604 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6605 getRegForInlineAsmConstraint(RI: TRI, Constraint: Input.ConstraintCode,
6606 VT: Input.ConstraintVT);
6607 const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
6608 OpInfo.ConstraintVT.isFloatingPoint();
6609 const bool InOpIsIntOrFP = Input.ConstraintVT.isInteger() ||
6610 Input.ConstraintVT.isFloatingPoint();
6611 if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
6612 (MatchRC.second != InputRC.second)) {
6613 report_fatal_error(reason: "Unsupported asm: input constraint"
6614 " with a matching output constraint of"
6615 " incompatible type!");
6616 }
6617 }
6618 }
6619 }
6620
6621 return ConstraintOperands;
6622}
6623
6624/// Return a number indicating our preference for chosing a type of constraint
6625/// over another, for the purpose of sorting them. Immediates are almost always
6626/// preferrable (when they can be emitted). A higher return value means a
6627/// stronger preference for one constraint type relative to another.
6628/// FIXME: We should prefer registers over memory but doing so may lead to
6629/// unrecoverable register exhaustion later.
6630/// https://github.com/llvm/llvm-project/issues/20571
6631static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) {
6632 switch (CT) {
6633 case TargetLowering::C_Immediate:
6634 case TargetLowering::C_Other:
6635 return 4;
6636 case TargetLowering::C_Memory:
6637 case TargetLowering::C_Address:
6638 return 3;
6639 case TargetLowering::C_RegisterClass:
6640 return 2;
6641 case TargetLowering::C_Register:
6642 return 1;
6643 case TargetLowering::C_Unknown:
6644 return 0;
6645 }
6646 llvm_unreachable("Invalid constraint type");
6647}
6648
6649/// Examine constraint type and operand type and determine a weight value.
6650/// This object must already have been set up with the operand type
6651/// and the current alternative constraint selected.
6652TargetLowering::ConstraintWeight
6653 TargetLowering::getMultipleConstraintMatchWeight(
6654 AsmOperandInfo &info, int maIndex) const {
6655 InlineAsm::ConstraintCodeVector *rCodes;
6656 if (maIndex >= (int)info.multipleAlternatives.size())
6657 rCodes = &info.Codes;
6658 else
6659 rCodes = &info.multipleAlternatives[maIndex].Codes;
6660 ConstraintWeight BestWeight = CW_Invalid;
6661
6662 // Loop over the options, keeping track of the most general one.
6663 for (const std::string &rCode : *rCodes) {
6664 ConstraintWeight weight =
6665 getSingleConstraintMatchWeight(info, constraint: rCode.c_str());
6666 if (weight > BestWeight)
6667 BestWeight = weight;
6668 }
6669
6670 return BestWeight;
6671}
6672
6673/// Examine constraint type and operand type and determine a weight value.
6674/// This object must already have been set up with the operand type
6675/// and the current alternative constraint selected.
6676TargetLowering::ConstraintWeight
6677 TargetLowering::getSingleConstraintMatchWeight(
6678 AsmOperandInfo &info, const char *constraint) const {
6679 ConstraintWeight weight = CW_Invalid;
6680 Value *CallOperandVal = info.CallOperandVal;
6681 // If we don't have a value, we can't do a match,
6682 // but allow it at the lowest weight.
6683 if (!CallOperandVal)
6684 return CW_Default;
6685 // Look at the constraint type.
6686 switch (*constraint) {
6687 case 'i': // immediate integer.
6688 case 'n': // immediate integer with a known value.
6689 if (isa<ConstantInt>(Val: CallOperandVal))
6690 weight = CW_Constant;
6691 break;
6692 case 's': // non-explicit intregal immediate.
6693 if (isa<GlobalValue>(Val: CallOperandVal))
6694 weight = CW_Constant;
6695 break;
6696 case 'E': // immediate float if host format.
6697 case 'F': // immediate float.
6698 if (isa<ConstantFP>(Val: CallOperandVal))
6699 weight = CW_Constant;
6700 break;
6701 case '<': // memory operand with autodecrement.
6702 case '>': // memory operand with autoincrement.
6703 case 'm': // memory operand.
6704 case 'o': // offsettable memory operand
6705 case 'V': // non-offsettable memory operand
6706 weight = CW_Memory;
6707 break;
6708 case 'r': // general register.
6709 case 'g': // general register, memory operand or immediate integer.
6710 // note: Clang converts "g" to "imr".
6711 if (CallOperandVal->getType()->isIntegerTy())
6712 weight = CW_Register;
6713 break;
6714 case 'X': // any operand.
6715 default:
6716 weight = CW_Default;
6717 break;
6718 }
6719 return weight;
6720}
6721
6722/// If there are multiple different constraints that we could pick for this
6723/// operand (e.g. "imr") try to pick the 'best' one.
6724/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
6725/// into seven classes:
6726/// Register -> one specific register
6727/// RegisterClass -> a group of regs
6728/// Memory -> memory
6729/// Address -> a symbolic memory reference
6730/// Immediate -> immediate values
6731/// Other -> magic values (such as "Flag Output Operands")
6732/// Unknown -> something we don't recognize yet and can't handle
6733/// Ideally, we would pick the most specific constraint possible: if we have
6734/// something that fits into a register, we would pick it. The problem here
6735/// is that if we have something that could either be in a register or in
6736/// memory that use of the register could cause selection of *other*
6737/// operands to fail: they might only succeed if we pick memory. Because of
6738/// this the heuristic we use is:
6739///
6740/// 1) If there is an 'other' constraint, and if the operand is valid for
6741/// that constraint, use it. This makes us take advantage of 'i'
6742/// constraints when available.
6743/// 2) Otherwise, pick the most general constraint present. This prefers
6744/// 'm' over 'r', for example.
6745///
6746TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
6747 TargetLowering::AsmOperandInfo &OpInfo) const {
6748 ConstraintGroup Ret;
6749
6750 Ret.reserve(N: OpInfo.Codes.size());
6751 for (StringRef Code : OpInfo.Codes) {
6752 TargetLowering::ConstraintType CType = getConstraintType(Constraint: Code);
6753
6754 // Indirect 'other' or 'immediate' constraints are not allowed.
6755 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6756 CType == TargetLowering::C_Register ||
6757 CType == TargetLowering::C_RegisterClass))
6758 continue;
6759
6760 // Things with matching constraints can only be registers, per gcc
6761 // documentation. This mainly affects "g" constraints.
6762 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
6763 continue;
6764
6765 Ret.emplace_back(Args&: Code, Args&: CType);
6766 }
6767
6768 llvm::stable_sort(Range&: Ret, C: [](ConstraintPair a, ConstraintPair b) {
6769 return getConstraintPiority(CT: a.second) > getConstraintPiority(CT: b.second);
6770 });
6771
6772 return Ret;
6773}
6774
6775/// If we have an immediate, see if we can lower it. Return true if we can,
6776/// false otherwise.
6777static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P,
6778 SDValue Op, SelectionDAG *DAG,
6779 const TargetLowering &TLI) {
6780
6781 assert((P.second == TargetLowering::C_Other ||
6782 P.second == TargetLowering::C_Immediate) &&
6783 "need immediate or other");
6784
6785 if (!Op.getNode())
6786 return false;
6787
6788 std::vector<SDValue> ResultOps;
6789 TLI.LowerAsmOperandForConstraint(Op, Constraint: P.first, Ops&: ResultOps, DAG&: *DAG);
6790 return !ResultOps.empty();
6791}
6792
6793/// Determines the constraint code and constraint type to use for the specific
6794/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
6795void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
6796 SDValue Op,
6797 SelectionDAG *DAG) const {
6798 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
6799
6800 // Single-letter constraints ('r') are very common.
6801 if (OpInfo.Codes.size() == 1) {
6802 OpInfo.ConstraintCode = OpInfo.Codes[0];
6803 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6804 } else {
6805 ConstraintGroup G = getConstraintPreferences(OpInfo);
6806 if (G.empty())
6807 return;
6808
6809 unsigned BestIdx = 0;
6810 for (const unsigned E = G.size();
6811 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
6812 G[BestIdx].second == TargetLowering::C_Immediate);
6813 ++BestIdx) {
6814 if (lowerImmediateIfPossible(P&: G[BestIdx], Op, DAG, TLI: *this))
6815 break;
6816 // If we're out of constraints, just pick the first one.
6817 if (BestIdx + 1 == E) {
6818 BestIdx = 0;
6819 break;
6820 }
6821 }
6822
6823 OpInfo.ConstraintCode = G[BestIdx].first;
6824 OpInfo.ConstraintType = G[BestIdx].second;
6825 }
6826
6827 // 'X' matches anything.
6828 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
6829 // Constants are handled elsewhere. For Functions, the type here is the
6830 // type of the result, which is not what we want to look at; leave them
6831 // alone.
6832 Value *v = OpInfo.CallOperandVal;
6833 if (isa<ConstantInt>(Val: v) || isa<Function>(Val: v)) {
6834 return;
6835 }
6836
6837 if (isa<BasicBlock>(Val: v) || isa<BlockAddress>(Val: v)) {
6838 OpInfo.ConstraintCode = "i";
6839 return;
6840 }
6841
6842 // Otherwise, try to resolve it to something we know about by looking at
6843 // the actual operand type.
6844 if (const char *Repl = LowerXConstraint(ConstraintVT: OpInfo.ConstraintVT)) {
6845 OpInfo.ConstraintCode = Repl;
6846 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6847 }
6848 }
6849}
6850
6851/// Given an exact SDIV by a constant, create a multiplication
6852/// with the multiplicative inverse of the constant.
6853/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6854static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
6855 const SDLoc &dl, SelectionDAG &DAG,
6856 SmallVectorImpl<SDNode *> &Created) {
6857 SDValue Op0 = N->getOperand(Num: 0);
6858 SDValue Op1 = N->getOperand(Num: 1);
6859 EVT VT = N->getValueType(ResNo: 0);
6860 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6861 EVT ShSVT = ShVT.getScalarType();
6862
6863 bool UseSRA = false;
6864 SmallVector<SDValue, 16> Shifts, Factors;
6865
6866 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6867 if (C->isZero())
6868 return false;
6869
6870 EVT CT = C->getValueType(ResNo: 0);
6871 APInt Divisor = C->getAPIntValue();
6872 unsigned Shift = Divisor.countr_zero();
6873 if (Shift) {
6874 Divisor.ashrInPlace(ShiftAmt: Shift);
6875 UseSRA = true;
6876 }
6877 APInt Factor = Divisor.multiplicativeInverse();
6878 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6879 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6880 return true;
6881 };
6882
6883 // Collect all magic values from the build vector.
6884 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildSDIVPattern))
6885 return SDValue();
6886
6887 SDValue Shift, Factor;
6888 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6889 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6890 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6891 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6892 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6893 "Expected matchUnaryPredicate to return one element for scalable "
6894 "vectors");
6895 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6896 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6897 } else {
6898 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6899 Shift = Shifts[0];
6900 Factor = Factors[0];
6901 }
6902
6903 SDValue Res = Op0;
6904 if (UseSRA) {
6905 Res = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6906 Created.push_back(Elt: Res.getNode());
6907 }
6908
6909 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6910}
6911
6912/// Given an exact UDIV by a constant, create a multiplication
6913/// with the multiplicative inverse of the constant.
6914/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6915static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
6916 const SDLoc &dl, SelectionDAG &DAG,
6917 SmallVectorImpl<SDNode *> &Created) {
6918 EVT VT = N->getValueType(ResNo: 0);
6919 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6920 EVT ShSVT = ShVT.getScalarType();
6921
6922 bool UseSRL = false;
6923 SmallVector<SDValue, 16> Shifts, Factors;
6924
6925 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6926 if (C->isZero())
6927 return false;
6928
6929 EVT CT = C->getValueType(ResNo: 0);
6930 APInt Divisor = C->getAPIntValue();
6931 unsigned Shift = Divisor.countr_zero();
6932 if (Shift) {
6933 Divisor.lshrInPlace(ShiftAmt: Shift);
6934 UseSRL = true;
6935 }
6936 // Calculate the multiplicative inverse modulo BW.
6937 APInt Factor = Divisor.multiplicativeInverse();
6938 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6939 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6940 return true;
6941 };
6942
6943 SDValue Op1 = N->getOperand(Num: 1);
6944
6945 // Collect all magic values from the build vector.
6946 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildUDIVPattern))
6947 return SDValue();
6948
6949 SDValue Shift, Factor;
6950 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6951 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6952 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6953 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6954 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6955 "Expected matchUnaryPredicate to return one element for scalable "
6956 "vectors");
6957 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6958 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6959 } else {
6960 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6961 Shift = Shifts[0];
6962 Factor = Factors[0];
6963 }
6964
6965 SDValue Res = N->getOperand(Num: 0);
6966 if (UseSRL) {
6967 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6968 Created.push_back(Elt: Res.getNode());
6969 }
6970
6971 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6972}
6973
6974SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
6975 SelectionDAG &DAG,
6976 SmallVectorImpl<SDNode *> &Created) const {
6977 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6978 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6979 return SDValue(N, 0); // Lower SDIV as SDIV
6980 return SDValue();
6981}
6982
6983SDValue
6984TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
6985 SelectionDAG &DAG,
6986 SmallVectorImpl<SDNode *> &Created) const {
6987 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6988 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6989 return SDValue(N, 0); // Lower SREM as SREM
6990 return SDValue();
6991}
6992
6993/// Build sdiv by power-of-2 with conditional move instructions
6994/// Ref: "Hacker's Delight" by Henry Warren 10-1
6995/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
6996/// bgez x, label
6997/// add x, x, 2**k-1
6998/// label:
6999/// sra res, x, k
7000/// neg res, res (when the divisor is negative)
7001SDValue TargetLowering::buildSDIVPow2WithCMov(
7002 SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
7003 SmallVectorImpl<SDNode *> &Created) const {
7004 unsigned Lg2 = Divisor.countr_zero();
7005 EVT VT = N->getValueType(ResNo: 0);
7006
7007 SDLoc DL(N);
7008 SDValue N0 = N->getOperand(Num: 0);
7009 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
7010 APInt Lg2Mask = APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: Lg2);
7011 SDValue Pow2MinusOne = DAG.getConstant(Val: Lg2Mask, DL, VT);
7012
7013 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
7014 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7015 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: N0, RHS: Zero, Cond: ISD::SETLT);
7016 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: Pow2MinusOne);
7017 SDValue CMov = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cmp, N2: Add, N3: N0);
7018
7019 Created.push_back(Elt: Cmp.getNode());
7020 Created.push_back(Elt: Add.getNode());
7021 Created.push_back(Elt: CMov.getNode());
7022
7023 // Divide by pow2.
7024 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CMov,
7025 N2: DAG.getShiftAmountConstant(Val: Lg2, VT, DL));
7026
7027 // If we're dividing by a positive value, we're done. Otherwise, we must
7028 // negate the result.
7029 if (Divisor.isNonNegative())
7030 return SRA;
7031
7032 Created.push_back(Elt: SRA.getNode());
7033 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: SRA);
7034}
7035
7036/// Given an ISD::SDIV node expressing a divide by constant,
7037/// return a DAG expression to select that will generate the same value by
7038/// multiplying by a magic number.
7039/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
7040SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
7041 bool IsAfterLegalization,
7042 bool IsAfterLegalTypes,
7043 SmallVectorImpl<SDNode *> &Created) const {
7044 SDLoc dl(N);
7045
7046 // If the sdiv has an 'exact' bit we can use a simpler lowering.
7047 if (N->getFlags().hasExact())
7048 return BuildExactSDIV(TLI: *this, N, dl, DAG, Created);
7049
7050 EVT VT = N->getValueType(ResNo: 0);
7051 EVT SVT = VT.getScalarType();
7052 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7053 EVT ShSVT = ShVT.getScalarType();
7054 unsigned EltBits = VT.getScalarSizeInBits();
7055 EVT MulVT;
7056
7057 // Check to see if we can do this.
7058 // FIXME: We should be more aggressive here.
7059 EVT QueryVT = VT;
7060 if (VT.isVector()) {
7061 // If the vector type will be legalized to a vector type with the same
7062 // element type, allow the transform before type legalization if MULHS or
7063 // SMUL_LOHI are supported.
7064 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
7065 if (!QueryVT.isVector() ||
7066 QueryVT.getVectorElementType() != VT.getVectorElementType())
7067 return SDValue();
7068 } else if (!isTypeLegal(VT)) {
7069 // Limit this to simple scalars for now.
7070 if (!VT.isSimple())
7071 return SDValue();
7072
7073 // If this type will be promoted to a large enough type with a legal
7074 // multiply operation, we can go ahead and do this transform.
7075 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
7076 return SDValue();
7077
7078 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7079 if (MulVT.getSizeInBits() < (2 * EltBits) ||
7080 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
7081 return SDValue();
7082 }
7083
7084 bool HasMULHS =
7085 isOperationLegalOrCustom(Op: ISD::MULHS, VT: QueryVT, LegalOnly: IsAfterLegalization);
7086 bool HasSMUL_LOHI =
7087 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
7088
7089 if (isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
7090 // If type twice as wide legal, widen and use a mul plus a shift.
7091 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
7092 // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
7093 // custom lowered. This is very expensive so avoid it at all costs for
7094 // constant divisors.
7095 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::SDIV, VT) &&
7096 isOperationCustom(Op: ISD::SDIVREM, VT: VT.getScalarType())) ||
7097 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
7098 MulVT = WideVT;
7099 }
7100
7101 if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
7102 return SDValue();
7103
7104 // If we're after type legalization and SVT is not legal, use the
7105 // promoted type for creating constants to avoid creating nodes with
7106 // illegal types.
7107 if (IsAfterLegalTypes && VT.isVector()) {
7108 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
7109 if (SVT.bitsLT(VT: VT.getScalarType()))
7110 return SDValue();
7111 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
7112 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
7113 return SDValue();
7114 }
7115 const unsigned SVTBits = SVT.getSizeInBits();
7116
7117 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
7118
7119 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
7120 if (C->isZero())
7121 return false;
7122 // Truncate the divisor to the target scalar type in case it was promoted
7123 // during type legalization.
7124 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
7125 SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(D: Divisor);
7126 int NumeratorFactor = 0;
7127 int ShiftMask = -1;
7128
7129 if (Divisor.isOne() || Divisor.isAllOnes()) {
7130 // If d is +1/-1, we just multiply the numerator by +1/-1.
7131 NumeratorFactor = Divisor.getSExtValue();
7132 magics.Magic = 0;
7133 magics.ShiftAmount = 0;
7134 ShiftMask = 0;
7135 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
7136 // If d > 0 and m < 0, add the numerator.
7137 NumeratorFactor = 1;
7138 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
7139 // If d < 0 and m > 0, subtract the numerator.
7140 NumeratorFactor = -1;
7141 }
7142
7143 MagicFactors.push_back(
7144 Elt: DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT));
7145 Factors.push_back(Elt: DAG.getSignedConstant(Val: NumeratorFactor, DL: dl, VT: SVT));
7146 Shifts.push_back(Elt: DAG.getConstant(Val: magics.ShiftAmount, DL: dl, VT: ShSVT));
7147 ShiftMasks.push_back(Elt: DAG.getSignedConstant(Val: ShiftMask, DL: dl, VT: SVT));
7148 return true;
7149 };
7150
7151 SDValue N0 = N->getOperand(Num: 0);
7152 SDValue N1 = N->getOperand(Num: 1);
7153
7154 // Collect the shifts / magic values from each element.
7155 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildSDIVPattern, /*AllowUndefs=*/false,
7156 /*AllowTruncation=*/true))
7157 return SDValue();
7158
7159 SDValue MagicFactor, Factor, Shift, ShiftMask;
7160 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7161 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
7162 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
7163 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
7164 ShiftMask = DAG.getBuildVector(VT, DL: dl, Ops: ShiftMasks);
7165 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7166 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
7167 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
7168 "Expected matchUnaryPredicate to return one element for scalable "
7169 "vectors");
7170 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
7171 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
7172 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
7173 ShiftMask = DAG.getSplatVector(VT, DL: dl, Op: ShiftMasks[0]);
7174 } else {
7175 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7176 MagicFactor = MagicFactors[0];
7177 Factor = Factors[0];
7178 Shift = Shifts[0];
7179 ShiftMask = ShiftMasks[0];
7180 }
7181
7182 // Multiply the numerator (operand 0) by the magic value.
7183 auto GetMULHS = [&](SDValue X, SDValue Y) {
7184 if (HasMULHS)
7185 return DAG.getNode(Opcode: ISD::MULHS, DL: dl, VT, N1: X, N2: Y);
7186 if (HasSMUL_LOHI) {
7187 SDValue LoHi =
7188 DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7189 return LoHi.getValue(R: 1);
7190 }
7191
7192 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: X);
7193 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7194 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7195 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7196 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7197 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7198 };
7199
7200 SDValue Q = GetMULHS(N0, MagicFactor);
7201 if (!Q)
7202 return SDValue();
7203
7204 Created.push_back(Elt: Q.getNode());
7205
7206 // (Optionally) Add/subtract the numerator using Factor.
7207 Factor = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: N0, N2: Factor);
7208 Created.push_back(Elt: Factor.getNode());
7209 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: Factor);
7210 Created.push_back(Elt: Q.getNode());
7211
7212 // Shift right algebraic by shift value.
7213 Q = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Q, N2: Shift);
7214 Created.push_back(Elt: Q.getNode());
7215
7216 // Extract the sign bit, mask it and add it to the quotient.
7217 SDValue SignShift = DAG.getConstant(Val: EltBits - 1, DL: dl, VT: ShVT);
7218 SDValue T = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: SignShift);
7219 Created.push_back(Elt: T.getNode());
7220 T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: ShiftMask);
7221 Created.push_back(Elt: T.getNode());
7222 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: T);
7223}
7224
7225/// Given an ISD::UDIV node expressing a divide by constant,
7226/// return a DAG expression to select that will generate the same value by
7227/// multiplying by a magic number.
7228/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
7229SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
7230 bool IsAfterLegalization,
7231 bool IsAfterLegalTypes,
7232 SmallVectorImpl<SDNode *> &Created) const {
7233 SDLoc dl(N);
7234
7235 // If the udiv has an 'exact' bit we can use a simpler lowering.
7236 if (N->getFlags().hasExact())
7237 return BuildExactUDIV(TLI: *this, N, dl, DAG, Created);
7238
7239 EVT VT = N->getValueType(ResNo: 0);
7240 EVT SVT = VT.getScalarType();
7241 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7242 EVT ShSVT = ShVT.getScalarType();
7243 unsigned EltBits = VT.getScalarSizeInBits();
7244 EVT MulVT;
7245
7246 // Check to see if we can do this.
7247 // FIXME: We should be more aggressive here.
7248 EVT QueryVT = VT;
7249 if (VT.isVector()) {
7250 // If the vector type will be legalized to a vector type with the same
7251 // element type, allow the transform before type legalization if MULHU or
7252 // UMUL_LOHI are supported.
7253 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
7254 if (!QueryVT.isVector() ||
7255 QueryVT.getVectorElementType() != VT.getVectorElementType())
7256 return SDValue();
7257 } else if (!isTypeLegal(VT)) {
7258 // Limit this to simple scalars for now.
7259 if (!VT.isSimple())
7260 return SDValue();
7261
7262 // If this type will be promoted to a large enough type with a legal
7263 // multiply operation, we can go ahead and do this transform.
7264 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
7265 return SDValue();
7266
7267 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7268 if (MulVT.getSizeInBits() < (2 * EltBits) ||
7269 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
7270 return SDValue();
7271 }
7272
7273 bool HasMULHU =
7274 isOperationLegalOrCustom(Op: ISD::MULHU, VT: QueryVT, LegalOnly: IsAfterLegalization);
7275 bool HasUMUL_LOHI =
7276 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
7277
7278 if (isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
7279 // If type twice as wide legal, widen and use a mul plus a shift.
7280 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
7281 // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
7282 // custom lowered. This is very expensive so avoid it at all costs for
7283 // constant divisors.
7284 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::UDIV, VT) &&
7285 isOperationCustom(Op: ISD::UDIVREM, VT: VT.getScalarType())) ||
7286 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
7287 MulVT = WideVT;
7288 }
7289
7290 if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
7291 return SDValue();
7292
7293 SDValue N0 = N->getOperand(Num: 0);
7294 SDValue N1 = N->getOperand(Num: 1);
7295
7296 // Try to use leading zeros of the dividend to reduce the multiplier and
7297 // avoid expensive fixups.
7298 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
7299
7300 // If we're after type legalization and SVT is not legal, use the
7301 // promoted type for creating constants to avoid creating nodes with
7302 // illegal types.
7303 if (IsAfterLegalTypes && VT.isVector()) {
7304 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
7305 if (SVT.bitsLT(VT: VT.getScalarType()))
7306 return SDValue();
7307 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
7308 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
7309 return SDValue();
7310 }
7311 const unsigned SVTBits = SVT.getSizeInBits();
7312
7313 // Allow i32 to be widened to i64 for uncooperative divisors if i64 MULHU or
7314 // UMUL_LOHI is supported.
7315 const EVT WideSVT = MVT::i64;
7316 const bool HasWideMULHU =
7317 VT == MVT::i32 &&
7318 isOperationLegalOrCustom(Op: ISD::MULHU, VT: WideSVT, LegalOnly: IsAfterLegalization);
7319 const bool HasWideUMUL_LOHI =
7320 VT == MVT::i32 &&
7321 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: WideSVT, LegalOnly: IsAfterLegalization);
7322 const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
7323
7324 // For even divisors with a 33-bit magic number, the widened high-multiply
7325 // path is only worthwhile over the even-divisor rewrite on targets that
7326 // zero-extend i32 to i64 for free (e.g. x86-64 and AArch64). Elsewhere (e.g.
7327 // RISC-V) keep the even-divisor rewrite, which avoids the explicit extension.
7328 const bool AllowEvenToWiden = AllowWiden && isZExtFree(FromTy: VT, ToTy: WideSVT);
7329
7330 bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
7331 bool UseWiden = false;
7332 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
7333
7334 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
7335 if (C->isZero())
7336 return false;
7337 // Truncate the divisor to the target scalar type in case it was promoted
7338 // during type legalization.
7339 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
7340
7341 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
7342
7343 // Magic algorithm doesn't work for division by 1. We need to emit a select
7344 // at the end.
7345 if (Divisor.isOne()) {
7346 PreShift = PostShift = DAG.getUNDEF(VT: ShSVT);
7347 MagicFactor = NPQFactor = DAG.getUNDEF(VT: SVT);
7348 } else {
7349 UnsignedDivisionByConstantInfo magics =
7350 UnsignedDivisionByConstantInfo::get(
7351 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()),
7352 /*AllowEvenDivisorOptimization=*/!AllowEvenToWiden,
7353 /*AllowWidenOptimization=*/AllowWiden);
7354
7355 if (magics.Widen) {
7356 UseWiden = true;
7357 MagicFactor = DAG.getConstant(Val: magics.Magic, DL: dl, VT: WideSVT);
7358 } else {
7359 MagicFactor = DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT);
7360 }
7361
7362 assert(magics.PreShift < Divisor.getBitWidth() &&
7363 "We shouldn't generate an undefined shift!");
7364 assert(magics.PostShift < Divisor.getBitWidth() &&
7365 "We shouldn't generate an undefined shift!");
7366 assert((!magics.IsAdd || magics.PreShift == 0) &&
7367 "Unexpected pre-shift");
7368 PreShift = DAG.getConstant(Val: magics.PreShift, DL: dl, VT: ShSVT);
7369 PostShift = DAG.getConstant(Val: magics.PostShift, DL: dl, VT: ShSVT);
7370 NPQFactor = DAG.getConstant(
7371 Val: magics.IsAdd ? APInt::getOneBitSet(numBits: SVTBits, BitNo: EltBits - 1)
7372 : APInt::getZero(numBits: SVTBits),
7373 DL: dl, VT: SVT);
7374 UseNPQ |= magics.IsAdd;
7375 UsePreShift |= magics.PreShift != 0;
7376 UsePostShift |= magics.PostShift != 0;
7377 }
7378
7379 PreShifts.push_back(Elt: PreShift);
7380 MagicFactors.push_back(Elt: MagicFactor);
7381 NPQFactors.push_back(Elt: NPQFactor);
7382 PostShifts.push_back(Elt: PostShift);
7383 return true;
7384 };
7385
7386 // Collect the shifts/magic values from each element.
7387 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildUDIVPattern, /*AllowUndefs=*/false,
7388 /*AllowTruncation=*/true))
7389 return SDValue();
7390
7391 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
7392 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7393 PreShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PreShifts);
7394 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
7395 NPQFactor = DAG.getBuildVector(VT, DL: dl, Ops: NPQFactors);
7396 PostShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PostShifts);
7397 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7398 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
7399 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
7400 "Expected matchUnaryPredicate to return one for scalable vectors");
7401 PreShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PreShifts[0]);
7402 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
7403 NPQFactor = DAG.getSplatVector(VT, DL: dl, Op: NPQFactors[0]);
7404 PostShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PostShifts[0]);
7405 } else {
7406 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7407 PreShift = PreShifts[0];
7408 MagicFactor = MagicFactors[0];
7409 PostShift = PostShifts[0];
7410 }
7411
7412 if (UseWiden) {
7413 // Compute: (WideSVT(x) * MagicFactor) >> WideSVTBits.
7414 SDValue WideN0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: WideSVT, Operand: N0);
7415
7416 // Perform WideSVTxWideSVT -> 2*WideSVT multiplication and extract high
7417 // WideSVT bits
7418 SDValue High;
7419 if (HasWideMULHU) {
7420 High = DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT: WideSVT, N1: WideN0, N2: MagicFactor);
7421 } else {
7422 assert(HasWideUMUL_LOHI);
7423 SDValue LoHi =
7424 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: WideSVT, VT2: WideSVT),
7425 N1: WideN0, N2: MagicFactor);
7426 High = LoHi.getValue(R: 1);
7427 }
7428
7429 Created.push_back(Elt: High.getNode());
7430 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: High);
7431 }
7432
7433 SDValue Q = N0;
7434 if (UsePreShift) {
7435 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PreShift);
7436 Created.push_back(Elt: Q.getNode());
7437 }
7438
7439 auto GetMULHU = [&](SDValue X, SDValue Y) {
7440 if (HasMULHU)
7441 return DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT, N1: X, N2: Y);
7442 if (HasUMUL_LOHI) {
7443 SDValue LoHi =
7444 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7445 return LoHi.getValue(R: 1);
7446 }
7447
7448 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: X);
7449 Y = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7450 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7451 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7452 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7453 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7454 };
7455
7456 // Multiply the numerator (operand 0) by the magic value.
7457 Q = GetMULHU(Q, MagicFactor);
7458 if (!Q)
7459 return SDValue();
7460
7461 Created.push_back(Elt: Q.getNode());
7462
7463 if (UseNPQ) {
7464 SDValue NPQ = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: N0, N2: Q);
7465 Created.push_back(Elt: NPQ.getNode());
7466
7467 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
7468 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
7469 if (VT.isVector())
7470 NPQ = GetMULHU(NPQ, NPQFactor);
7471 else
7472 NPQ = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NPQ, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT));
7473
7474 Created.push_back(Elt: NPQ.getNode());
7475
7476 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: NPQ, N2: Q);
7477 Created.push_back(Elt: Q.getNode());
7478 }
7479
7480 if (UsePostShift) {
7481 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PostShift);
7482 Created.push_back(Elt: Q.getNode());
7483 }
7484
7485 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7486
7487 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
7488 SDValue IsOne = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N1, RHS: One, Cond: ISD::SETEQ);
7489 return DAG.getSelect(DL: dl, VT, Cond: IsOne, LHS: N0, RHS: Q);
7490}
7491
7492/// If all values in Values that *don't* match the predicate are same 'splat'
7493/// value, then replace all values with that splat value.
7494/// Else, if AlternativeReplacement was provided, then replace all values that
7495/// do match predicate with AlternativeReplacement value.
7496static void
7497turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
7498 std::function<bool(SDValue)> Predicate,
7499 SDValue AlternativeReplacement = SDValue()) {
7500 SDValue Replacement;
7501 // Is there a value for which the Predicate does *NOT* match? What is it?
7502 auto SplatValue = llvm::find_if_not(Range&: Values, P: Predicate);
7503 if (SplatValue != Values.end()) {
7504 // Does Values consist only of SplatValue's and values matching Predicate?
7505 if (llvm::all_of(Range&: Values, P: [Predicate, SplatValue](SDValue Value) {
7506 return Value == *SplatValue || Predicate(Value);
7507 })) // Then we shall replace values matching predicate with SplatValue.
7508 Replacement = *SplatValue;
7509 }
7510 if (!Replacement) {
7511 // Oops, we did not find the "baseline" splat value.
7512 if (!AlternativeReplacement)
7513 return; // Nothing to do.
7514 // Let's replace with provided value then.
7515 Replacement = AlternativeReplacement;
7516 }
7517 std::replace_if(first: Values.begin(), last: Values.end(), pred: Predicate, new_value: Replacement);
7518}
7519
7520/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
7521/// where the divisor and comparison target are constants,
7522/// return a DAG expression that will generate the same comparison result
7523/// using only multiplications, additions and shifts/rotations.
7524/// Ref: "Hacker's Delight" 10-17.
7525SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
7526 SDValue CompTargetNode,
7527 ISD::CondCode Cond,
7528 DAGCombinerInfo &DCI,
7529 const SDLoc &DL) const {
7530 SmallVector<SDNode *, 5> Built;
7531 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7532 DCI, DL, Created&: Built)) {
7533 for (SDNode *N : Built)
7534 DCI.AddToWorklist(N);
7535 return Folded;
7536 }
7537
7538 return SDValue();
7539}
7540
7541SDValue
7542TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
7543 SDValue CompTargetNode, ISD::CondCode Cond,
7544 DAGCombinerInfo &DCI, const SDLoc &DL,
7545 SmallVectorImpl<SDNode *> &Created) const {
7546 // fold (seteq/ne (urem N, D), C) ->
7547 // (setule/ugt (rotr (mul (sub N, C), P), K), Q)
7548 // - D must be constant, with D = D0 * 2^K where D0 is odd
7549 // - P is the multiplicative inverse of D0 modulo 2^W
7550 // - Q = floor(((2^W) - 1) / D)
7551 // where W is the width of the common type of N and D.
7552 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7553 "Only applicable for (in)equality comparisons.");
7554
7555 SelectionDAG &DAG = DCI.DAG;
7556
7557 EVT VT = REMNode.getValueType();
7558 EVT SVT = VT.getScalarType();
7559 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7560 EVT ShSVT = ShVT.getScalarType();
7561
7562 // If MUL is unavailable, we cannot proceed in any case.
7563 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7564 return SDValue();
7565
7566 bool ComparingWithAllZeros = true;
7567 bool AllComparisonsWithNonZerosAreTautological = true;
7568 bool HadTautologicalLanes = false;
7569 bool AllLanesAreTautological = true;
7570 bool HadEvenDivisor = false;
7571 bool AllDivisorsArePowerOfTwo = true;
7572 bool HadTautologicalInvertedLanes = false;
7573 SmallVector<SDValue, 16> PAmts, KAmts, QAmts;
7574
7575 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
7576 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7577 if (CDiv->isZero())
7578 return false;
7579
7580 const APInt &D = CDiv->getAPIntValue();
7581 const APInt &Cmp = CCmp->getAPIntValue();
7582
7583 ComparingWithAllZeros &= Cmp.isZero();
7584
7585 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7586 // if C2 is not less than C1, the comparison is always false.
7587 // But we will only be able to produce the comparison that will give the
7588 // opposive tautological answer. So this lane would need to be fixed up.
7589 bool TautologicalInvertedLane = D.ule(RHS: Cmp);
7590 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
7591
7592 // If all lanes are tautological (either all divisors are ones, or divisor
7593 // is not greater than the constant we are comparing with),
7594 // we will prefer to avoid the fold.
7595 bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
7596 HadTautologicalLanes |= TautologicalLane;
7597 AllLanesAreTautological &= TautologicalLane;
7598
7599 // If we are comparing with non-zero, we need'll need to subtract said
7600 // comparison value from the LHS. But there is no point in doing that if
7601 // every lane where we are comparing with non-zero is tautological..
7602 if (!Cmp.isZero())
7603 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
7604
7605 // Decompose D into D0 * 2^K
7606 unsigned K = D.countr_zero();
7607 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7608 APInt D0 = D.lshr(shiftAmt: K);
7609
7610 // D is even if it has trailing zeros.
7611 HadEvenDivisor |= (K != 0);
7612 // D is a power-of-two if D0 is one.
7613 // If all divisors are power-of-two, we will prefer to avoid the fold.
7614 AllDivisorsArePowerOfTwo &= D0.isOne();
7615
7616 // P = inv(D0, 2^W)
7617 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7618 unsigned W = D.getBitWidth();
7619 APInt P = D0.multiplicativeInverse();
7620 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7621
7622 // Q = floor((2^W - 1) u/ D)
7623 // R = ((2^W - 1) u% D)
7624 APInt Q, R;
7625 APInt::udivrem(LHS: APInt::getAllOnes(numBits: W), RHS: D, Quotient&: Q, Remainder&: R);
7626
7627 // If we are comparing with zero, then that comparison constant is okay,
7628 // else it may need to be one less than that.
7629 if (Cmp.ugt(RHS: R))
7630 Q -= 1;
7631
7632 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7633 "We are expecting that K is always less than all-ones for ShSVT");
7634
7635 // If the lane is tautological the result can be constant-folded.
7636 if (TautologicalLane) {
7637 // Set P and K amount to a bogus values so we can try to splat them.
7638 P = 0;
7639 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7640 // And ensure that comparison constant is tautological,
7641 // it will always compare true/false.
7642 Q.setAllBits();
7643 } else {
7644 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7645 }
7646
7647 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7648 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7649 return true;
7650 };
7651
7652 SDValue N = REMNode.getOperand(i: 0);
7653 SDValue D = REMNode.getOperand(i: 1);
7654
7655 // Collect the values from each element.
7656 if (!ISD::matchBinaryPredicate(LHS: D, RHS: CompTargetNode, Match: BuildUREMPattern))
7657 return SDValue();
7658
7659 // If all lanes are tautological, the result can be constant-folded.
7660 if (AllLanesAreTautological)
7661 return SDValue();
7662
7663 // If this is a urem by a powers-of-two, avoid the fold since it can be
7664 // best implemented as a bit test.
7665 if (AllDivisorsArePowerOfTwo)
7666 return SDValue();
7667
7668 SDValue PVal, KVal, QVal;
7669 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7670 if (HadTautologicalLanes) {
7671 // Try to turn PAmts into a splat, since we don't care about the values
7672 // that are currently '0'. If we can't, just keep '0'`s.
7673 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7674 // Try to turn KAmts into a splat, since we don't care about the values
7675 // that are currently '-1'. If we can't, change them to '0'`s.
7676 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7677 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7678 }
7679
7680 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7681 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7682 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7683 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7684 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
7685 "Expected matchBinaryPredicate to return one element for "
7686 "SPLAT_VECTORs");
7687 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7688 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7689 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7690 } else {
7691 PVal = PAmts[0];
7692 KVal = KAmts[0];
7693 QVal = QAmts[0];
7694 }
7695
7696 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
7697 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::SUB, VT))
7698 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
7699 assert(CompTargetNode.getValueType() == N.getValueType() &&
7700 "Expecting that the types on LHS and RHS of comparisons match.");
7701 N = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N, N2: CompTargetNode);
7702 }
7703
7704 // (mul N, P)
7705 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7706 Created.push_back(Elt: Op0.getNode());
7707
7708 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7709 // divisors as a performance improvement, since rotating by 0 is a no-op.
7710 if (HadEvenDivisor) {
7711 // We need ROTR to do this.
7712 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7713 return SDValue();
7714 // UREM: (rotr (mul N, P), K)
7715 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7716 Created.push_back(Elt: Op0.getNode());
7717 }
7718
7719 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
7720 SDValue NewCC =
7721 DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7722 Cond: ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
7723 if (!HadTautologicalInvertedLanes)
7724 return NewCC;
7725
7726 // If any lanes previously compared always-false, the NewCC will give
7727 // always-true result for them, so we need to fixup those lanes.
7728 // Or the other way around for inequality predicate.
7729 assert(VT.isVector() && "Can/should only get here for vectors.");
7730 Created.push_back(Elt: NewCC.getNode());
7731
7732 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7733 // if C2 is not less than C1, the comparison is always false.
7734 // But we have produced the comparison that will give the
7735 // opposive tautological answer. So these lanes would need to be fixed up.
7736 SDValue TautologicalInvertedChannels =
7737 DAG.getSetCC(DL, VT: SETCCVT, LHS: D, RHS: CompTargetNode, Cond: ISD::SETULE);
7738 Created.push_back(Elt: TautologicalInvertedChannels.getNode());
7739
7740 // NOTE: we avoid letting illegal types through even if we're before legalize
7741 // ops – legalization has a hard time producing good code for this.
7742 if (isOperationLegalOrCustom(Op: ISD::VSELECT, VT: SETCCVT)) {
7743 // If we have a vector select, let's replace the comparison results in the
7744 // affected lanes with the correct tautological result.
7745 SDValue Replacement = DAG.getBoolConstant(V: Cond == ISD::SETEQ ? false : true,
7746 DL, VT: SETCCVT, OpVT: SETCCVT);
7747 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: SETCCVT, N1: TautologicalInvertedChannels,
7748 N2: Replacement, N3: NewCC);
7749 }
7750
7751 // Else, we can just invert the comparison result in the appropriate lanes.
7752 //
7753 // NOTE: see the note above VSELECT above.
7754 if (isOperationLegalOrCustom(Op: ISD::XOR, VT: SETCCVT))
7755 return DAG.getNode(Opcode: ISD::XOR, DL, VT: SETCCVT, N1: NewCC,
7756 N2: TautologicalInvertedChannels);
7757
7758 return SDValue(); // Don't know how to lower.
7759}
7760
7761/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
7762/// where the divisor is constant and the comparison target is zero,
7763/// return a DAG expression that will generate the same comparison result
7764/// using only multiplications, additions and shifts/rotations.
7765/// Ref: "Hacker's Delight" 10-17.
7766SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
7767 SDValue CompTargetNode,
7768 ISD::CondCode Cond,
7769 DAGCombinerInfo &DCI,
7770 const SDLoc &DL) const {
7771 SmallVector<SDNode *, 7> Built;
7772 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7773 DCI, DL, Created&: Built)) {
7774 assert(Built.size() <= 7 && "Max size prediction failed.");
7775 for (SDNode *N : Built)
7776 DCI.AddToWorklist(N);
7777 return Folded;
7778 }
7779
7780 return SDValue();
7781}
7782
7783SDValue
7784TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
7785 SDValue CompTargetNode, ISD::CondCode Cond,
7786 DAGCombinerInfo &DCI, const SDLoc &DL,
7787 SmallVectorImpl<SDNode *> &Created) const {
7788 // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
7789 // Fold:
7790 // (seteq/ne (srem N, D), 0)
7791 // To:
7792 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
7793 //
7794 // - D must be constant, with D = D0 * 2^K where D0 is odd
7795 // - P is the multiplicative inverse of D0 modulo 2^W
7796 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
7797 // - Q = floor((2 * A) / (2^K))
7798 // where W is the width of the common type of N and D.
7799 //
7800 // When D is a power of two (and thus D0 is 1), the normal
7801 // formula for A and Q don't apply, because the derivation
7802 // depends on D not dividing 2^(W-1), and thus theorem ZRS
7803 // does not apply. This specifically fails when N = INT_MIN.
7804 //
7805 // Instead, for power-of-two D, we use:
7806 // - A = 0
7807 // | -> No offset needed. We're effectively treating it the same as urem.
7808 // - Q = 2^(W-K) - 1
7809 // |-> Test that the top K bits are zero after rotation
7810 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7811 "Only applicable for (in)equality comparisons.");
7812
7813 SelectionDAG &DAG = DCI.DAG;
7814
7815 EVT VT = REMNode.getValueType();
7816 EVT SVT = VT.getScalarType();
7817 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7818 EVT ShSVT = ShVT.getScalarType();
7819
7820 // If we are after ops legalization, and MUL is unavailable, we can not
7821 // proceed.
7822 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7823 return SDValue();
7824
7825 // TODO: Could support comparing with non-zero too.
7826 ConstantSDNode *CompTarget = isConstOrConstSplat(N: CompTargetNode);
7827 if (!CompTarget || !CompTarget->isZero())
7828 return SDValue();
7829
7830 bool HadOneDivisor = false;
7831 bool AllDivisorsAreOnes = true;
7832 bool HadEvenDivisor = false;
7833 bool AllDivisorsArePowerOfTwo = true;
7834 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
7835
7836 auto BuildSREMPattern = [&](ConstantSDNode *C) {
7837 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7838 if (C->isZero())
7839 return false;
7840
7841 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
7842
7843 // WARNING: this fold is only valid for positive divisors!
7844 // `rem %X, -C` is equivalent to `rem %X, C`
7845 APInt D = C->getAPIntValue().abs();
7846
7847 // If all divisors are ones, we will prefer to avoid the fold.
7848 HadOneDivisor |= D.isOne();
7849 AllDivisorsAreOnes &= D.isOne();
7850
7851 // Decompose D into D0 * 2^K
7852 unsigned K = D.countr_zero();
7853 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7854 APInt D0 = D.lshr(shiftAmt: K);
7855
7856 // D is even if it has trailing zeros.
7857 HadEvenDivisor |= (K != 0);
7858
7859 // D is a power-of-two if D0 is one. This includes INT_MIN.
7860 // If all divisors are power-of-two, we will prefer to avoid the fold.
7861 AllDivisorsArePowerOfTwo &= D0.isOne();
7862
7863 // P = inv(D0, 2^W)
7864 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7865 unsigned W = D.getBitWidth();
7866 APInt P = D0.multiplicativeInverse();
7867 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7868
7869 // A = floor((2^(W - 1) - 1) / D0) & -2^K
7870 APInt A = APInt::getSignedMaxValue(numBits: W).udiv(RHS: D0);
7871 A.clearLowBits(loBits: K);
7872
7873 // Q = floor((2 * A) / (2^K))
7874 APInt Q = (2 * A).udiv(RHS: APInt::getOneBitSet(numBits: W, BitNo: K));
7875
7876 assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
7877 "We are expecting that A is always less than all-ones for SVT");
7878 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7879 "We are expecting that K is always less than all-ones for ShSVT");
7880
7881 // If D was a power of two, apply the alternate constant derivation.
7882 if (D0.isOne()) {
7883 // A = 0
7884 A = APInt(W, 0);
7885 // - Q = 2^(W-K) - 1
7886 Q = APInt::getLowBitsSet(numBits: W, loBitsSet: W - K);
7887 }
7888
7889 // If the divisor is 1 the result can be constant-folded.
7890 if (D.isOne()) {
7891 // Set P, A and K to a bogus values so we can try to splat them.
7892 P = 0;
7893 A.setAllBits();
7894 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7895
7896 // x ?% 1 == 0 <--> true <--> x u<= -1
7897 Q.setAllBits();
7898 } else {
7899 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7900 }
7901
7902 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7903 AAmts.push_back(Elt: DAG.getConstant(Val: A, DL, VT: SVT));
7904 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7905 return true;
7906 };
7907
7908 SDValue N = REMNode.getOperand(i: 0);
7909 SDValue D = REMNode.getOperand(i: 1);
7910
7911 // Collect the values from each element.
7912 if (!ISD::matchUnaryPredicate(Op: D, Match: BuildSREMPattern))
7913 return SDValue();
7914
7915 // If this is a srem by a one, avoid the fold since it can be constant-folded.
7916 if (AllDivisorsAreOnes)
7917 return SDValue();
7918
7919 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
7920 // since it can be best implemented as a bit test.
7921 if (AllDivisorsArePowerOfTwo)
7922 return SDValue();
7923
7924 SDValue PVal, AVal, KVal, QVal;
7925 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7926 if (HadOneDivisor) {
7927 // Try to turn PAmts into a splat, since we don't care about the values
7928 // that are currently '0'. If we can't, just keep '0'`s.
7929 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7930 // Try to turn AAmts into a splat, since we don't care about the
7931 // values that are currently '-1'. If we can't, change them to '0'`s.
7932 turnVectorIntoSplatVector(Values: AAmts, Predicate: isAllOnesConstant,
7933 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: SVT));
7934 // Try to turn KAmts into a splat, since we don't care about the values
7935 // that are currently '-1'. If we can't, change them to '0'`s.
7936 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7937 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7938 }
7939
7940 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7941 AVal = DAG.getBuildVector(VT, DL, Ops: AAmts);
7942 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7943 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7944 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7945 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
7946 QAmts.size() == 1 &&
7947 "Expected matchUnaryPredicate to return one element for scalable "
7948 "vectors");
7949 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7950 AVal = DAG.getSplatVector(VT, DL, Op: AAmts[0]);
7951 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7952 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7953 } else {
7954 assert(isa<ConstantSDNode>(D) && "Expected a constant");
7955 PVal = PAmts[0];
7956 AVal = AAmts[0];
7957 KVal = KAmts[0];
7958 QVal = QAmts[0];
7959 }
7960
7961 // (mul N, P)
7962 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7963 Created.push_back(Elt: Op0.getNode());
7964
7965 // We need ADD to do this.
7966 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ADD, VT))
7967 return SDValue();
7968
7969 // (add (mul N, P), A)
7970 Op0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: AVal);
7971 Created.push_back(Elt: Op0.getNode());
7972
7973 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7974 // divisors as a performance improvement, since rotating by 0 is a no-op.
7975 if (HadEvenDivisor) {
7976 // We need ROTR to do this.
7977 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7978 return SDValue();
7979 // SREM: (rotr (add (mul N, P), A), K)
7980 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7981 Created.push_back(Elt: Op0.getNode());
7982 }
7983
7984 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
7985 return DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7986 Cond: (Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT);
7987}
7988
7989SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
7990 const DenormalMode &Mode,
7991 SDNodeFlags Flags) const {
7992 SDLoc DL(Op);
7993 EVT VT = Op.getValueType();
7994 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7995 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL, VT);
7996
7997 // This is specifically a check for the handling of denormal inputs, not the
7998 // result.
7999 if (Mode.Input == DenormalMode::PreserveSign ||
8000 Mode.Input == DenormalMode::PositiveZero) {
8001 // Test = X == 0.0
8002 return DAG.getSetCC(DL, VT: CCVT, LHS: Op, RHS: FPZero, Cond: ISD::SETEQ, /*Chain=*/{},
8003 /*Signaling=*/IsSignaling: false, Flags);
8004 }
8005
8006 // Testing it with denormal inputs to avoid wrong estimate.
8007 //
8008 // Test = fabs(X) < SmallestNormal
8009 const fltSemantics &FltSem = VT.getFltSemantics();
8010 APFloat SmallestNorm = APFloat::getSmallestNormalized(Sem: FltSem);
8011 SDValue NormC = DAG.getConstantFP(Val: SmallestNorm, DL, VT);
8012 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: Op, Flags);
8013 return DAG.getSetCC(DL, VT: CCVT, LHS: Fabs, RHS: NormC, Cond: ISD::SETLT, /*Chain=*/{},
8014 /*Signaling=*/IsSignaling: false, Flags);
8015}
8016
8017SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
8018 bool LegalOps, bool OptForSize,
8019 NegatibleCost &Cost,
8020 unsigned Depth) const {
8021 // fneg is removable even if it has multiple uses.
8022 if (Op.getOpcode() == ISD::FNEG) {
8023 Cost = NegatibleCost::Cheaper;
8024 return Op.getOperand(i: 0);
8025 }
8026
8027 // Don't recurse exponentially.
8028 if (Depth > SelectionDAG::MaxRecursionDepth)
8029 return SDValue();
8030
8031 // Pre-increment recursion depth for use in recursive calls.
8032 ++Depth;
8033 const SDNodeFlags Flags = Op->getFlags();
8034 EVT VT = Op.getValueType();
8035 unsigned Opcode = Op.getOpcode();
8036
8037 // Don't allow anything with multiple uses unless we know it is free.
8038 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
8039 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
8040 isFPExtFree(DestVT: VT, SrcVT: Op.getOperand(i: 0).getValueType());
8041 if (!IsFreeExtend)
8042 return SDValue();
8043 }
8044
8045 auto RemoveDeadNode = [&](SDValue N) {
8046 if (N && N.getNode()->use_empty())
8047 DAG.RemoveDeadNode(N: N.getNode());
8048 };
8049
8050 SDLoc DL(Op);
8051
8052 // Because getNegatedExpression can delete nodes we need a handle to keep
8053 // temporary nodes alive in case the recursion manages to create an identical
8054 // node.
8055 std::list<HandleSDNode> Handles;
8056
8057 switch (Opcode) {
8058 case ISD::ConstantFP: {
8059 // Don't invert constant FP values after legalization unless the target says
8060 // the negated constant is legal.
8061 bool IsOpLegal =
8062 isOperationLegal(Op: ISD::ConstantFP, VT) ||
8063 isFPImmLegal(Imm: neg(X: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF()), VT,
8064 ForCodeSize: OptForSize);
8065
8066 if (LegalOps && !IsOpLegal)
8067 break;
8068
8069 APFloat V = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
8070 V.changeSign();
8071 SDValue CFP = DAG.getConstantFP(Val: V, DL, VT);
8072
8073 // If we already have the use of the negated floating constant, it is free
8074 // to negate it even it has multiple uses.
8075 if (!Op.hasOneUse() && CFP.use_empty())
8076 break;
8077 Cost = NegatibleCost::Neutral;
8078 return CFP;
8079 }
8080 case ISD::SPLAT_VECTOR: {
8081 // fold splat_vector(fneg(X)) -> splat_vector(-X)
8082 SDValue X = Op.getOperand(i: 0);
8083 if (!isOperationLegal(Op: ISD::SPLAT_VECTOR, VT))
8084 break;
8085
8086 SDValue NegX = getCheaperNegatedExpression(Op: X, DAG, LegalOps, OptForSize);
8087 if (!NegX)
8088 break;
8089 Cost = NegatibleCost::Cheaper;
8090 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: NegX);
8091 }
8092 case ISD::BUILD_VECTOR: {
8093 // Only permit BUILD_VECTOR of constants.
8094 if (llvm::any_of(Range: Op->op_values(), P: [&](SDValue N) {
8095 return !N.isUndef() && !isa<ConstantFPSDNode>(Val: N);
8096 }))
8097 break;
8098
8099 bool IsOpLegal =
8100 (isOperationLegal(Op: ISD::ConstantFP, VT) &&
8101 isOperationLegal(Op: ISD::BUILD_VECTOR, VT)) ||
8102 llvm::all_of(Range: Op->op_values(), P: [&](SDValue N) {
8103 return N.isUndef() ||
8104 isFPImmLegal(Imm: neg(X: cast<ConstantFPSDNode>(Val&: N)->getValueAPF()), VT,
8105 ForCodeSize: OptForSize);
8106 });
8107
8108 if (LegalOps && !IsOpLegal)
8109 break;
8110
8111 SmallVector<SDValue, 4> Ops;
8112 for (SDValue C : Op->op_values()) {
8113 if (C.isUndef()) {
8114 Ops.push_back(Elt: C);
8115 continue;
8116 }
8117 APFloat V = cast<ConstantFPSDNode>(Val&: C)->getValueAPF();
8118 V.changeSign();
8119 Ops.push_back(Elt: DAG.getConstantFP(Val: V, DL, VT: C.getValueType()));
8120 }
8121 Cost = NegatibleCost::Neutral;
8122 return DAG.getBuildVector(VT, DL, Ops);
8123 }
8124 case ISD::FADD: {
8125 if (!Flags.hasNoSignedZeros())
8126 break;
8127
8128 // After operation legalization, it might not be legal to create new FSUBs.
8129 if (LegalOps && !isOperationLegalOrCustom(Op: ISD::FSUB, VT))
8130 break;
8131 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
8132
8133 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
8134 NegatibleCost CostX = NegatibleCost::Expensive;
8135 SDValue NegX =
8136 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8137 // Prevent this node from being deleted by the next call.
8138 if (NegX)
8139 Handles.emplace_back(args&: NegX);
8140
8141 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
8142 NegatibleCost CostY = NegatibleCost::Expensive;
8143 SDValue NegY =
8144 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8145
8146 // We're done with the handles.
8147 Handles.clear();
8148
8149 // Negate the X if its cost is less or equal than Y.
8150 if (NegX && (CostX <= CostY)) {
8151 Cost = CostX;
8152 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegX, N2: Y, Flags);
8153 if (NegY != N)
8154 RemoveDeadNode(NegY);
8155 return N;
8156 }
8157
8158 // Negate the Y if it is not expensive.
8159 if (NegY) {
8160 Cost = CostY;
8161 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegY, N2: X, Flags);
8162 if (NegX != N)
8163 RemoveDeadNode(NegX);
8164 return N;
8165 }
8166 break;
8167 }
8168 case ISD::FSUB: {
8169 // We can't turn -(A-B) into B-A when we honor signed zeros.
8170 if (!Flags.hasNoSignedZeros())
8171 break;
8172
8173 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
8174 // fold (fneg (fsub 0, Y)) -> Y
8175 if (ConstantFPSDNode *C = isConstOrConstSplatFP(N: X, /*AllowUndefs*/ true))
8176 if (C->isZero()) {
8177 Cost = NegatibleCost::Cheaper;
8178 return Y;
8179 }
8180
8181 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
8182 Cost = NegatibleCost::Neutral;
8183 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: X, Flags);
8184 }
8185 case ISD::FMUL:
8186 case ISD::FDIV: {
8187 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
8188
8189 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
8190 NegatibleCost CostX = NegatibleCost::Expensive;
8191 SDValue NegX =
8192 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8193 // Prevent this node from being deleted by the next call.
8194 if (NegX)
8195 Handles.emplace_back(args&: NegX);
8196
8197 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
8198 NegatibleCost CostY = NegatibleCost::Expensive;
8199 SDValue NegY =
8200 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8201
8202 // We're done with the handles.
8203 Handles.clear();
8204
8205 // Negate the X if its cost is less or equal than Y.
8206 if (NegX && (CostX <= CostY)) {
8207 Cost = CostX;
8208 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, Flags);
8209 if (NegY != N)
8210 RemoveDeadNode(NegY);
8211 return N;
8212 }
8213
8214 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
8215 if (auto *C = isConstOrConstSplatFP(N: Op.getOperand(i: 1)))
8216 if (C->isExactlyValue(V: 2.0) && Op.getOpcode() == ISD::FMUL)
8217 break;
8218
8219 // Negate the Y if it is not expensive.
8220 if (NegY) {
8221 Cost = CostY;
8222 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, Flags);
8223 if (NegX != N)
8224 RemoveDeadNode(NegX);
8225 return N;
8226 }
8227 break;
8228 }
8229 case ISD::FMA:
8230 case ISD::FMULADD:
8231 case ISD::FMAD: {
8232 if (!Flags.hasNoSignedZeros())
8233 break;
8234
8235 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), Z = Op.getOperand(i: 2);
8236 NegatibleCost CostZ = NegatibleCost::Expensive;
8237 SDValue NegZ =
8238 getNegatedExpression(Op: Z, DAG, LegalOps, OptForSize, Cost&: CostZ, Depth);
8239 // Give up if fail to negate the Z.
8240 if (!NegZ)
8241 break;
8242
8243 // Prevent this node from being deleted by the next two calls.
8244 Handles.emplace_back(args&: NegZ);
8245
8246 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
8247 NegatibleCost CostX = NegatibleCost::Expensive;
8248 SDValue NegX =
8249 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8250 // Prevent this node from being deleted by the next call.
8251 if (NegX)
8252 Handles.emplace_back(args&: NegX);
8253
8254 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
8255 NegatibleCost CostY = NegatibleCost::Expensive;
8256 SDValue NegY =
8257 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8258
8259 // We're done with the handles.
8260 Handles.clear();
8261
8262 // Negate the X if its cost is less or equal than Y.
8263 if (NegX && (CostX <= CostY)) {
8264 Cost = std::min(a: CostX, b: CostZ);
8265 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, N3: NegZ, Flags);
8266 if (NegY != N)
8267 RemoveDeadNode(NegY);
8268 return N;
8269 }
8270
8271 // Negate the Y if it is not expensive.
8272 if (NegY) {
8273 Cost = std::min(a: CostY, b: CostZ);
8274 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, N3: NegZ, Flags);
8275 if (NegX != N)
8276 RemoveDeadNode(NegX);
8277 return N;
8278 }
8279 break;
8280 }
8281
8282 case ISD::FP_EXTEND:
8283 case ISD::FSIN:
8284 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
8285 OptForSize, Cost, Depth))
8286 return DAG.getNode(Opcode, DL, VT, Operand: NegV);
8287 break;
8288 case ISD::FP_ROUND:
8289 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
8290 OptForSize, Cost, Depth))
8291 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NegV, N2: Op.getOperand(i: 1));
8292 break;
8293 case ISD::SELECT:
8294 case ISD::VSELECT: {
8295 // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
8296 // iff at least one cost is cheaper and the other is neutral/cheaper
8297 SDValue LHS = Op.getOperand(i: 1);
8298 NegatibleCost CostLHS = NegatibleCost::Expensive;
8299 SDValue NegLHS =
8300 getNegatedExpression(Op: LHS, DAG, LegalOps, OptForSize, Cost&: CostLHS, Depth);
8301 if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
8302 RemoveDeadNode(NegLHS);
8303 break;
8304 }
8305
8306 // Prevent this node from being deleted by the next call.
8307 Handles.emplace_back(args&: NegLHS);
8308
8309 SDValue RHS = Op.getOperand(i: 2);
8310 NegatibleCost CostRHS = NegatibleCost::Expensive;
8311 SDValue NegRHS =
8312 getNegatedExpression(Op: RHS, DAG, LegalOps, OptForSize, Cost&: CostRHS, Depth);
8313
8314 // We're done with the handles.
8315 Handles.clear();
8316
8317 if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
8318 (CostLHS != NegatibleCost::Cheaper &&
8319 CostRHS != NegatibleCost::Cheaper)) {
8320 RemoveDeadNode(NegLHS);
8321 RemoveDeadNode(NegRHS);
8322 break;
8323 }
8324
8325 Cost = std::min(a: CostLHS, b: CostRHS);
8326 return DAG.getSelect(DL, VT, Cond: Op.getOperand(i: 0), LHS: NegLHS, RHS: NegRHS);
8327 }
8328 }
8329
8330 return SDValue();
8331}
8332
8333//===----------------------------------------------------------------------===//
8334// Legalization Utilities
8335//===----------------------------------------------------------------------===//
8336
8337bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
8338 SDValue LHS, SDValue RHS,
8339 SmallVectorImpl<SDValue> &Result,
8340 EVT HiLoVT, SelectionDAG &DAG,
8341 MulExpansionKind Kind, SDValue LL,
8342 SDValue LH, SDValue RL, SDValue RH) const {
8343 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
8344 Opcode == ISD::SMUL_LOHI);
8345
8346 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
8347 isOperationLegalOrCustom(Op: ISD::MULHS, VT: HiLoVT);
8348 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
8349 isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT);
8350 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8351 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: HiLoVT);
8352 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8353 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT);
8354
8355 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
8356 return false;
8357
8358 unsigned OuterBitSize = VT.getScalarSizeInBits();
8359 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
8360
8361 // LL, LH, RL, and RH must be either all NULL or all set to a value.
8362 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
8363 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
8364
8365 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
8366 bool Signed) -> bool {
8367 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
8368 SDVTList VTs = DAG.getVTList(VT1: HiLoVT, VT2: HiLoVT);
8369 Lo = DAG.getNode(Opcode: Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, DL: dl, VTList: VTs, N1: L, N2: R);
8370 Hi = Lo.getValue(R: 1);
8371 return true;
8372 }
8373 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
8374 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: L, N2: R);
8375 Hi = DAG.getNode(Opcode: Signed ? ISD::MULHS : ISD::MULHU, DL: dl, VT: HiLoVT, N1: L, N2: R);
8376 return true;
8377 }
8378 return false;
8379 };
8380
8381 SDValue Lo, Hi;
8382
8383 if (!LL.getNode() && !RL.getNode() &&
8384 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8385 LL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LHS);
8386 RL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RHS);
8387 }
8388
8389 if (!LL.getNode())
8390 return false;
8391
8392 APInt HighMask = APInt::getHighBitsSet(numBits: OuterBitSize, hiBitsSet: InnerBitSize);
8393 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask) &&
8394 DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask)) {
8395 // The inputs are both zero-extended.
8396 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
8397 Result.push_back(Elt: Lo);
8398 Result.push_back(Elt: Hi);
8399 if (Opcode != ISD::MUL) {
8400 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8401 Result.push_back(Elt: Zero);
8402 Result.push_back(Elt: Zero);
8403 }
8404 return true;
8405 }
8406 }
8407
8408 if (!VT.isVector() && Opcode == ISD::MUL &&
8409 DAG.ComputeMaxSignificantBits(Op: LHS) <= InnerBitSize &&
8410 DAG.ComputeMaxSignificantBits(Op: RHS) <= InnerBitSize) {
8411 // The input values are both sign-extended.
8412 // TODO non-MUL case?
8413 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
8414 Result.push_back(Elt: Lo);
8415 Result.push_back(Elt: Hi);
8416 return true;
8417 }
8418 }
8419
8420 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
8421 SDValue Shift = DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
8422
8423 if (!LH.getNode() && !RH.getNode() &&
8424 isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
8425 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8426 LH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS, N2: Shift);
8427 LH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LH);
8428 RH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: RHS, N2: Shift);
8429 RH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RH);
8430 }
8431
8432 if (!LH.getNode())
8433 return false;
8434
8435 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
8436 return false;
8437
8438 Result.push_back(Elt: Lo);
8439
8440 if (Opcode == ISD::MUL) {
8441 RH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LL, N2: RH);
8442 LH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LH, N2: RL);
8443 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: RH);
8444 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: LH);
8445 Result.push_back(Elt: Hi);
8446 return true;
8447 }
8448
8449 // Compute the full width result.
8450 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
8451 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
8452 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8453 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
8454 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi);
8455 };
8456
8457 SDValue Next = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8458 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
8459 return false;
8460
8461 // This is effectively the add part of a multiply-add of half-sized operands,
8462 // so it cannot overflow.
8463 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8464
8465 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
8466 return false;
8467
8468 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8469 EVT BoolType = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
8470
8471 bool UseGlue = (isOperationLegalOrCustom(Op: ISD::ADDC, VT) &&
8472 isOperationLegalOrCustom(Op: ISD::ADDE, VT));
8473 if (UseGlue)
8474 Next = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Glue), N1: Next,
8475 N2: Merge(Lo, Hi));
8476 else
8477 Next = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolType), N1: Next,
8478 N2: Merge(Lo, Hi), N3: DAG.getConstant(Val: 0, DL: dl, VT: BoolType));
8479
8480 SDValue Carry = Next.getValue(R: 1);
8481 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8482 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8483
8484 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
8485 return false;
8486
8487 if (UseGlue)
8488 Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::Glue), N1: Hi, N2: Zero,
8489 N3: Carry);
8490 else
8491 Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: BoolType), N1: Hi,
8492 N2: Zero, N3: Carry);
8493
8494 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8495
8496 if (Opcode == ISD::SMUL_LOHI) {
8497 SDValue NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8498 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RL));
8499 Next = DAG.getSelectCC(DL: dl, LHS: LH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8500
8501 NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8502 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LL));
8503 Next = DAG.getSelectCC(DL: dl, LHS: RH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8504 }
8505
8506 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8507 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8508 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8509 return true;
8510}
8511
8512bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
8513 SelectionDAG &DAG, MulExpansionKind Kind,
8514 SDValue LL, SDValue LH, SDValue RL,
8515 SDValue RH) const {
8516 SmallVector<SDValue, 2> Result;
8517 bool Ok = expandMUL_LOHI(Opcode: N->getOpcode(), VT: N->getValueType(ResNo: 0), dl: SDLoc(N),
8518 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Result, HiLoVT,
8519 DAG, Kind, LL, LH, RL, RH);
8520 if (Ok) {
8521 assert(Result.size() == 2);
8522 Lo = Result[0];
8523 Hi = Result[1];
8524 }
8525 return Ok;
8526}
8527
8528// Optimize unsigned division or remainder by constants for types twice as large
8529// as a legal VT.
8530//
8531// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
8532// can be computed
8533// as:
8534// Sum = __builtin_uadd_overflow(Lo, High, &Sum);
8535// Remainder = Sum % Constant;
8536//
8537// If (1 << (BitWidth / 2)) % Constant != 1, we can search for a smaller value
8538// W such that W != (BitWidth / 2) and (1 << W) % Constant == 1. We can break
8539// High:Low into 3 chunks of W bits and compute remainder as
8540// Sum = Chunk0 + Chunk1 + Chunk2;
8541// Remainder = Sum % Constant;
8542//
8543// This is based on "Remainder by Summing Digits" from Hacker's Delight.
8544//
8545// For division, we can compute the remainder using the algorithm described
8546// above, subtract it from the dividend to get an exact multiple of Constant.
8547// Then multiply that exact multiply by the multiplicative inverse modulo
8548// (1 << (BitWidth / 2)) to get the quotient.
8549
8550// If Constant is even, we can shift right the dividend and the divisor by the
8551// number of trailing zeros in Constant before applying the remainder algorithm.
8552// If we're after the quotient, we can subtract this value from the shifted
8553// dividend and multiply by the multiplicative inverse of the shifted divisor.
8554// If we want the remainder, we shift the value left by the number of trailing
8555// zeros and add the bits that were shifted out of the dividend.
8556bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
8557 SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
8558 SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8559 unsigned Opcode = N->getOpcode();
8560 EVT VT = N->getValueType(ResNo: 0);
8561
8562 unsigned BitWidth = Divisor.getBitWidth();
8563 unsigned HBitWidth = BitWidth / 2;
8564 assert(VT.getScalarSizeInBits() == BitWidth &&
8565 HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");
8566
8567 // If the divisor is even, shift it until it becomes odd.
8568 unsigned TrailingZeros = 0;
8569 if (!Divisor[0]) {
8570 TrailingZeros = Divisor.countr_zero();
8571 Divisor.lshrInPlace(ShiftAmt: TrailingZeros);
8572 }
8573
8574 // After removing trailing zeros, the divisor needs to be less than
8575 // (1 << HBitWidth).
8576 APInt HalfMaxPlus1 = APInt::getOneBitSet(numBits: BitWidth, BitNo: HBitWidth);
8577 if (Divisor.uge(RHS: HalfMaxPlus1))
8578 return false;
8579
8580 // Look for the largest chunk width W such that (1 << W) % Divisor == 1 or
8581 // (1 << W) % Divisor == -1.
8582 unsigned BestChunkWidth = 0, AltChunkWidth = 0;
8583 for (unsigned I = HBitWidth, E = HBitWidth / 2; I > E; --I) {
8584 // Skip HBitWidth-1, it doesn't have enough bits for carries.
8585 if (I == HBitWidth - 1)
8586 continue;
8587
8588 APInt Mod = APInt::getOneBitSet(numBits: Divisor.getBitWidth(), BitNo: I).urem(RHS: Divisor);
8589
8590 if (Mod.isOne()) {
8591 BestChunkWidth = I;
8592 break;
8593 }
8594
8595 // We have an alternate strategy for Remainder == Divisor - 1.
8596 // FIXME: Support HBitWidth.
8597 if (I != HBitWidth && Mod == Divisor - 1)
8598 AltChunkWidth = I;
8599 }
8600
8601 bool Alternate = false;
8602 if (!BestChunkWidth) {
8603 if (!AltChunkWidth)
8604 return false;
8605 Alternate = true;
8606 BestChunkWidth = AltChunkWidth;
8607 }
8608
8609 SDLoc dl(N);
8610
8611 assert(!LL == !LH && "Expected both input halves or no input halves!");
8612 if (!LL)
8613 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8614
8615 bool HasFSHR = isOperationLegal(Op: ISD::FSHR, VT: HiLoVT);
8616
8617 auto GetFSHR = [&](SDValue Lo, SDValue Hi, unsigned ShiftAmt) {
8618 assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
8619 if (HasFSHR)
8620 return DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: HiLoVT, N1: Hi, N2: Lo,
8621 N3: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8622 return DAG.getNode(
8623 Opcode: ISD::OR, DL: dl, VT: HiLoVT,
8624 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Lo,
8625 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl)),
8626 N2: DAG.getNode(
8627 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: Hi,
8628 N2: DAG.getShiftAmountConstant(Val: HBitWidth - ShiftAmt, VT: HiLoVT, DL: dl)));
8629 };
8630
8631 // Helper to perform a right shift on a 128-bit value split into two halves.
8632 // Handles shifts >= HBitWidth by moving Hi to Lo and shifting Hi.
8633 auto ShiftRight = [&](SDValue &Lo, SDValue &Hi, unsigned ShiftAmt) {
8634 if (ShiftAmt == 0)
8635 return;
8636 if (ShiftAmt < HBitWidth) {
8637 Lo = GetFSHR(Lo, Hi, ShiftAmt);
8638 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8639 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8640 } else if (ShiftAmt == HBitWidth) {
8641 Lo = Hi;
8642 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8643 } else {
8644 Lo = DAG.getNode(
8645 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8646 N2: DAG.getShiftAmountConstant(Val: ShiftAmt - HBitWidth, VT: HiLoVT, DL: dl));
8647 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8648 }
8649 };
8650
8651 // Shift the input by the number of TrailingZeros in the divisor. The
8652 // shifted out bits will be added to the remainder later.
8653 SDValue PartialRemL, PartialRemH;
8654 if (TrailingZeros && Opcode != ISD::UDIV) {
8655 // Save the shifted off bits if we need the remainder.
8656 if (TrailingZeros < HBitWidth) {
8657 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros);
8658 PartialRemL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LL,
8659 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8660 } else if (TrailingZeros == HBitWidth) {
8661 // All of LL is part of the remainder.
8662 PartialRemL = LL;
8663 } else {
8664 // TrailingZeros > HBitWidth: LL and part of LH are the remainder.
8665 PartialRemL = LL;
8666 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros - HBitWidth);
8667 PartialRemH = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LH,
8668 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8669 }
8670 }
8671
8672 SDValue Sum;
8673 // If BestChunkWidth is HBitWidth add low and high half. If there is a carry
8674 // out, add that to the final sum.
8675 if (BestChunkWidth == HBitWidth) {
8676 assert(!Alternate);
8677 // Shift LH:LL right if there were trailing zeros in the divisor.
8678 ShiftRight(LL, LH, TrailingZeros);
8679
8680 // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
8681 EVT SetCCType =
8682 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: HiLoVT);
8683 if (isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT: HiLoVT)) {
8684 SDVTList VTList = DAG.getVTList(VT1: HiLoVT, VT2: SetCCType);
8685 Sum = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, N1: LL, N2: LH);
8686 Sum = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, N1: Sum,
8687 N2: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT), N3: Sum.getValue(R: 1));
8688 } else {
8689 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: LL, N2: LH);
8690 SDValue Carry = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LL, Cond: ISD::SETULT);
8691 // If the boolean for the target is 0 or 1, we can add the setcc result
8692 // directly.
8693 if (getBooleanContents(Type: HiLoVT) ==
8694 TargetLoweringBase::ZeroOrOneBooleanContent)
8695 Carry = DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: HiLoVT);
8696 else
8697 Carry = DAG.getSelect(DL: dl, VT: HiLoVT, Cond: Carry, LHS: DAG.getConstant(Val: 1, DL: dl, VT: HiLoVT),
8698 RHS: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT));
8699 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum, N2: Carry);
8700 }
8701 } else {
8702 // Otherwise split into multple chunks and add them together. We chose
8703 // BestChunkWidth so that the sum will not overflow.
8704 SDValue Mask = DAG.getConstant(
8705 Val: APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: BestChunkWidth), DL: dl, VT: HiLoVT);
8706
8707 for (unsigned I = 0; I < BitWidth - TrailingZeros; I += BestChunkWidth) {
8708 // If there were trailing zeros in the divisor, increase the shift amount.
8709 unsigned Shift = I + TrailingZeros;
8710 SDValue Chunk;
8711 if (Shift == 0)
8712 Chunk = LL;
8713 else if (Shift >= HBitWidth)
8714 Chunk = DAG.getNode(
8715 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: LH,
8716 N2: DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL: dl));
8717 else
8718 Chunk = GetFSHR(LL, LH, Shift);
8719 // If we're on the last chunk, we don't need an AND.
8720 if (I + BestChunkWidth < BitWidth - TrailingZeros)
8721 Chunk = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: Chunk, N2: Mask);
8722 if (!Sum) {
8723 Sum = Chunk;
8724 } else {
8725 // For Alternate, we need to subtract odd chunks.
8726 unsigned ChunkNum = I / BestChunkWidth;
8727 unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ? ISD::SUB : ISD::ADD;
8728 Sum = DAG.getNode(Opcode: Opc, DL: dl, VT: HiLoVT, N1: Sum, N2: Chunk);
8729 }
8730 }
8731
8732 // For Alternate, the sum may be negative, but we need a positive sum. We
8733 // can increase it by a multiple of the divisor to make it positive. For 3
8734 // chunks the largest negative value is -(2^BestChunkWidth - 1). For 4
8735 // chunks, it's 2*-(2^BestChunkWidth - 1). We know that 2^BestChunkWidth + 1
8736 // is a multiple of the divisor. Add that 1 or 2 times to make the sum
8737 // positive.
8738 if (Alternate) {
8739 unsigned NumChunks = divideCeil(Numerator: BitWidth - TrailingZeros, Denominator: BestChunkWidth);
8740 assert(NumChunks <= 4);
8741
8742 APInt Adjust = APInt::getOneBitSet(numBits: HBitWidth, BitNo: BestChunkWidth);
8743 Adjust.setBit(0);
8744 // If there are 4 chunks, we need to adjust twice.
8745 if (NumChunks == 4)
8746 Adjust <<= 1;
8747 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum,
8748 N2: DAG.getConstant(Val: Adjust, DL: dl, VT: HiLoVT));
8749 }
8750 }
8751
8752 // Perform a HiLoVT urem on the Sum using truncated divisor.
8753 SDValue RemL =
8754 DAG.getNode(Opcode: ISD::UREM, DL: dl, VT: HiLoVT, N1: Sum,
8755 N2: DAG.getConstant(Val: Divisor.trunc(width: HBitWidth), DL: dl, VT: HiLoVT));
8756 SDValue RemH = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8757
8758 if (Opcode != ISD::UREM) {
8759 // If we didn't shift LH/LR earlier, do it now.
8760 if (BestChunkWidth != HBitWidth)
8761 ShiftRight(LL, LH, TrailingZeros);
8762
8763 // Subtract the remainder from the shifted dividend.
8764 SDValue Dividend = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: LL, N2: LH);
8765 SDValue Rem = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: RemL, N2: RemH);
8766
8767 Dividend = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Rem);
8768
8769 // Multiply by the multiplicative inverse of the divisor modulo
8770 // (1 << BitWidth).
8771 APInt MulFactor = Divisor.multiplicativeInverse();
8772
8773 SDValue Quotient = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Dividend,
8774 N2: DAG.getConstant(Val: MulFactor, DL: dl, VT));
8775
8776 // Split the quotient into low and high parts.
8777 SDValue QuotL, QuotH;
8778 std::tie(args&: QuotL, args&: QuotH) = DAG.SplitScalar(N: Quotient, DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8779 Result.push_back(Elt: QuotL);
8780 Result.push_back(Elt: QuotH);
8781 }
8782
8783 if (Opcode != ISD::UDIV) {
8784 // If we shifted the input, shift the remainder left and add the bits we
8785 // shifted off the input.
8786 if (TrailingZeros) {
8787 if (TrailingZeros < HBitWidth) {
8788 // Shift RemH:RemL left by TrailingZeros.
8789 // RemH gets the high bits shifted out of RemL.
8790 RemH = DAG.getNode(
8791 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: RemL,
8792 N2: DAG.getShiftAmountConstant(Val: HBitWidth - TrailingZeros, VT: HiLoVT, DL: dl));
8793 RemL =
8794 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8795 N2: DAG.getShiftAmountConstant(Val: TrailingZeros, VT: HiLoVT, DL: dl));
8796 // OR in the partial remainder.
8797 RemL = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemL, N2: PartialRemL,
8798 Flags: SDNodeFlags::Disjoint);
8799 } else if (TrailingZeros == HBitWidth) {
8800 // Shift left by exactly HBitWidth: RemH becomes RemL, RemL becomes
8801 // PartialRemL.
8802 RemH = RemL;
8803 RemL = PartialRemL;
8804 } else {
8805 // Shift left by more than HBitWidth.
8806 RemH = DAG.getNode(
8807 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8808 N2: DAG.getShiftAmountConstant(Val: TrailingZeros - HBitWidth, VT: HiLoVT, DL: dl));
8809 RemH = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemH, N2: PartialRemH,
8810 Flags: SDNodeFlags::Disjoint);
8811 RemL = PartialRemL;
8812 }
8813 }
8814 Result.push_back(Elt: RemL);
8815 Result.push_back(Elt: RemH);
8816 }
8817
8818 return true;
8819}
8820
8821bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
8822 SDNode *N, const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
8823 EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8824
8825 SDValue N0 = N->getOperand(Num: 0);
8826 EVT VT = N0->getValueType(ResNo: 0);
8827 SDLoc DL{N};
8828
8829 assert(!Divisor.isOne() && "Magic algorithm does not work for division by 1");
8830
8831 // This helper creates a MUL_LOHI of the pair (LL, LH) by a constant.
8832 auto MakeMUL_LOHIByConst = [&](unsigned Opc, SDValue LL, SDValue LH,
8833 const APInt &Const,
8834 SmallVectorImpl<SDValue> &Result) {
8835 SDValue LHS = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT, N1: LL, N2: LH);
8836 SDValue RHS = DAG.getConstant(Val: Const, DL, VT);
8837 auto [RL, RH] = DAG.SplitScalar(N: RHS, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8838 return expandMUL_LOHI(Opcode: Opc, VT, dl: DL, LHS, RHS, Result, HiLoVT, DAG,
8839 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
8840 LL, LH, RL, RH);
8841 };
8842
8843 // This helper creates an ADD/SUB of the pairs (LL, LH) and (RL, RH).
8844 auto MakeAddSubLong = [&](unsigned Opc, SDValue LL, SDValue LH, SDValue RL,
8845 SDValue RH) {
8846 SDValue AddSubNode =
8847 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO : ISD::USUBO, DL,
8848 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LL, N2: RL);
8849 SDValue OutL = AddSubNode.getValue(R: 0);
8850 SDValue Overflow = AddSubNode.getValue(R: 1);
8851 SDValue AddSubWithOverflow =
8852 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, DL,
8853 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LH, N2: RH, N3: Overflow);
8854 SDValue OutH = AddSubWithOverflow.getValue(R: 0);
8855 return std::make_pair(x&: OutL, y&: OutH);
8856 };
8857
8858 // This helper creates a SRL of the pair (LL, LH) by Shift.
8859 auto MakeSRLLong = [&](SDValue LL, SDValue LH, unsigned Shift) {
8860 unsigned HBitWidth = HiLoVT.getScalarSizeInBits();
8861 if (Shift < HBitWidth) {
8862 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift, VT: HiLoVT, DL);
8863 SDValue ResL = DAG.getNode(Opcode: ISD::FSHR, DL, VT: HiLoVT, N1: LH, N2: LL, N3: ShAmt);
8864 SDValue ResH = DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt);
8865 return std::make_pair(x&: ResL, y&: ResH);
8866 }
8867 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HiLoVT);
8868 if (Shift == HBitWidth)
8869 return std::make_pair(x&: LH, y&: Zero);
8870 assert(Shift - HBitWidth < HBitWidth &&
8871 "We shouldn't generate an undefined shift");
8872 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL);
8873 return std::make_pair(x: DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt), y&: Zero);
8874 };
8875
8876 // Knowledge of leading zeros may help to reduce the multiplier.
8877 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
8878
8879 UnsignedDivisionByConstantInfo Magics = UnsignedDivisionByConstantInfo::get(
8880 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
8881
8882 assert(!LL == !LH && "Expected both input halves or no input halves!");
8883 if (!LL)
8884 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N0, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8885 SDValue QL = LL;
8886 SDValue QH = LH;
8887 if (Magics.PreShift != 0)
8888 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PreShift);
8889
8890 SmallVector<SDValue, 4> UMulResult;
8891 if (!MakeMUL_LOHIByConst(ISD::UMUL_LOHI, QL, QH, Magics.Magic, UMulResult))
8892 return false;
8893
8894 QL = UMulResult[2];
8895 QH = UMulResult[3];
8896
8897 if (Magics.IsAdd) {
8898 auto [NPQL, NPQH] = MakeAddSubLong(ISD::SUB, LL, LH, QL, QH);
8899 std::tie(args&: NPQL, args&: NPQH) = MakeSRLLong(NPQL, NPQH, 1);
8900 std::tie(args&: QL, args&: QH) = MakeAddSubLong(ISD::ADD, NPQL, NPQH, QL, QH);
8901 }
8902
8903 if (Magics.PostShift != 0)
8904 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PostShift);
8905
8906 unsigned Opcode = N->getOpcode();
8907 if (Opcode != ISD::UREM) {
8908 Result.push_back(Elt: QL);
8909 Result.push_back(Elt: QH);
8910 }
8911
8912 if (Opcode != ISD::UDIV) {
8913 SmallVector<SDValue, 2> MulResult;
8914 if (!MakeMUL_LOHIByConst(ISD::MUL, QL, QH, Divisor, MulResult))
8915 return false;
8916
8917 assert(MulResult.size() == 2);
8918
8919 auto [RemL, RemH] =
8920 MakeAddSubLong(ISD::SUB, LL, LH, MulResult[0], MulResult[1]);
8921
8922 Result.push_back(Elt: RemL);
8923 Result.push_back(Elt: RemH);
8924 }
8925
8926 return true;
8927}
8928
8929bool TargetLowering::expandDIVREMByConstant(SDNode *N,
8930 SmallVectorImpl<SDValue> &Result,
8931 EVT HiLoVT, SelectionDAG &DAG,
8932 SDValue LL, SDValue LH) const {
8933 unsigned Opcode = N->getOpcode();
8934
8935 // TODO: Support signed division/remainder.
8936 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
8937 return false;
8938 assert(
8939 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
8940 "Unexpected opcode");
8941
8942 auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8943 if (!CN)
8944 return false;
8945
8946 APInt Divisor = CN->getAPIntValue();
8947
8948 // The generated half-width UREM is normally optimized using high multiply.
8949 // If the wide UREM libcall is unavailable, a legal or custom half-width
8950 // UDIVREM can lower it instead.
8951 bool CanDecomposeUREMWithoutMulHi =
8952 Opcode == ISD::UREM &&
8953 DAG.getLibcalls().getLibcallImpl(Call: RTLIB::getUREM(VT: N->getValueType(ResNo: 0))) ==
8954 RTLIB::Unsupported &&
8955 isOperationLegalOrCustom(Op: ISD::UDIVREM, VT: HiLoVT);
8956 if (!CanDecomposeUREMWithoutMulHi &&
8957 !isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT) &&
8958 !isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT))
8959 return false;
8960
8961 // Prefer the smaller libcall when one is available.
8962 if (DAG.shouldOptForSize() && !CanDecomposeUREMWithoutMulHi)
8963 return false;
8964
8965 // Early out for 0 or 1 divisors.
8966 if (Divisor.ule(RHS: 1))
8967 return false;
8968
8969 if (expandUDIVREMByConstantViaUREMDecomposition(N, Divisor, Result, HiLoVT,
8970 DAG, LL, LH))
8971 return true;
8972
8973 if (expandUDIVREMByConstantViaUMulHiMagic(N, Divisor, Result, HiLoVT, DAG, LL,
8974 LH))
8975 return true;
8976
8977 return false;
8978}
8979
8980// Check that (every element of) Z is undef or not an exact multiple of BW.
8981static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
8982 return ISD::matchUnaryPredicate(
8983 Op: Z,
8984 Match: [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(RHS: BW) != 0; },
8985 /*AllowUndefs=*/true, /*AllowTruncation=*/true);
8986}
8987
8988SDValue TargetLowering::expandFunnelShift(SDNode *Node,
8989 SelectionDAG &DAG) const {
8990 EVT VT = Node->getValueType(ResNo: 0);
8991
8992 if (VT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8993 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8994 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8995 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
8996 return SDValue();
8997
8998 SDValue X = Node->getOperand(Num: 0);
8999 SDValue Y = Node->getOperand(Num: 1);
9000 SDValue Z = Node->getOperand(Num: 2);
9001
9002 unsigned BW = VT.getScalarSizeInBits();
9003 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
9004 SDLoc DL(SDValue(Node, 0));
9005
9006 EVT ShVT = Z.getValueType();
9007
9008 // If a funnel shift in the other direction is more supported, use it.
9009 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
9010 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
9011 isOperationLegalOrCustom(Op: RevOpcode, VT) && isPowerOf2_32(Value: BW)) {
9012 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
9013 // fshl X, Y, Z -> fshr X, Y, -Z
9014 // fshr X, Y, Z -> fshl X, Y, -Z
9015 Z = DAG.getNegative(Val: Z, DL, VT: ShVT);
9016 } else {
9017 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
9018 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
9019 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
9020 if (IsFSHL) {
9021 Y = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
9022 X = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: One);
9023 } else {
9024 X = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
9025 Y = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Y, N2: One);
9026 }
9027 Z = DAG.getNOT(DL, Val: Z, VT: ShVT);
9028 }
9029 return DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: Z);
9030 }
9031
9032 SDValue ShX, ShY;
9033 SDValue ShAmt, InvShAmt;
9034 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
9035 // fshl: X << C | Y >> (BW - C)
9036 // fshr: X << (BW - C) | Y >> C
9037 // where C = Z % BW is not zero
9038 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
9039 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
9040 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthC, N2: ShAmt);
9041 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: IsFSHL ? ShAmt : InvShAmt);
9042 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: IsFSHL ? InvShAmt : ShAmt);
9043 } else {
9044 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
9045 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
9046 SDValue Mask = DAG.getConstant(Val: BW - 1, DL, VT: ShVT);
9047 if (isPowerOf2_32(Value: BW)) {
9048 // Z % BW -> Z & (BW - 1)
9049 ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Z, N2: Mask);
9050 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
9051 InvShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: DAG.getNOT(DL, Val: Z, VT: ShVT), N2: Mask);
9052 } else {
9053 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
9054 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
9055 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Mask, N2: ShAmt);
9056 }
9057
9058 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
9059 if (IsFSHL) {
9060 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShAmt);
9061 SDValue ShY1 = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: One);
9062 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShY1, N2: InvShAmt);
9063 } else {
9064 SDValue ShX1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: One);
9065 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShX1, N2: InvShAmt);
9066 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt);
9067 }
9068 }
9069 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShX, N2: ShY);
9070}
9071
9072// TODO: Merge with expandFunnelShift.
9073SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
9074 SelectionDAG &DAG) const {
9075 EVT VT = Node->getValueType(ResNo: 0);
9076 unsigned EltSizeInBits = VT.getScalarSizeInBits();
9077 bool IsLeft = Node->getOpcode() == ISD::ROTL;
9078 SDValue Op0 = Node->getOperand(Num: 0);
9079 SDValue Op1 = Node->getOperand(Num: 1);
9080 SDLoc DL(SDValue(Node, 0));
9081
9082 EVT ShVT = Op1.getValueType();
9083 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: ShVT);
9084
9085 // If a rotate in the other direction is more supported, use it.
9086 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
9087 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
9088 isOperationLegalOrCustom(Op: RevRot, VT) && isPowerOf2_32(Value: EltSizeInBits)) {
9089 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
9090 return DAG.getNode(Opcode: RevRot, DL, VT, N1: Op0, N2: Sub);
9091 }
9092
9093 if (!AllowVectorOps && VT.isVector() &&
9094 (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
9095 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
9096 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
9097 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT) ||
9098 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT)))
9099 return SDValue();
9100
9101 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
9102 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
9103 SDValue BitWidthMinusOneC = DAG.getConstant(Val: EltSizeInBits - 1, DL, VT: ShVT);
9104 SDValue ShVal;
9105 SDValue HsVal;
9106 if (isPowerOf2_32(Value: EltSizeInBits)) {
9107 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
9108 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
9109 SDValue NegOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
9110 SDValue ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Op1, N2: BitWidthMinusOneC);
9111 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
9112 SDValue HsAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: NegOp1, N2: BitWidthMinusOneC);
9113 HsVal = DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: HsAmt);
9114 } else {
9115 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
9116 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
9117 SDValue BitWidthC = DAG.getConstant(Val: EltSizeInBits, DL, VT: ShVT);
9118 SDValue ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Op1, N2: BitWidthC);
9119 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
9120 SDValue HsAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthMinusOneC, N2: ShAmt);
9121 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
9122 HsVal =
9123 DAG.getNode(Opcode: HsOpc, DL, VT, N1: DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: One), N2: HsAmt);
9124 }
9125 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShVal, N2: HsVal);
9126}
9127
9128/// Check if CLMUL on VT can eventually reach a type with legal CLMUL through
9129/// a chain of halving decompositions (halving element width) and/or vector
9130/// widening (doubling element count). This guides expansion strategy selection:
9131/// if true, the halving/widening path produces better code than bit-by-bit.
9132///
9133/// HalveDepth tracks halving steps only (each creates ~4x more operations).
9134/// Widening steps are cheap (O(1) pad/extract) and don't count.
9135/// Limiting halvings to 2 prevents exponential blowup:
9136/// 1 halving: ~4 sub-CLMULs (good, e.g. v8i16 -> v8i8)
9137/// 2 halvings: ~16 sub-CLMULs (acceptable, e.g. v4i32 -> v4i16 -> v8i8)
9138/// 3 halvings: ~64 sub-CLMULs (worse than bit-by-bit expansion)
9139static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx,
9140 EVT VT, unsigned HalveDepth = 0,
9141 unsigned TotalDepth = 0) {
9142 if (HalveDepth > 2 || TotalDepth > 8 || !VT.isFixedLengthVector())
9143 return false;
9144 if (TLI.isOperationLegalOrCustom(Op: ISD::CLMUL, VT))
9145 return true;
9146 if (!TLI.isTypeLegal(VT))
9147 return false;
9148
9149 unsigned BW = VT.getScalarSizeInBits();
9150
9151 // Halve: halve element width, same element count.
9152 // This is the expensive step -- each halving creates ~4x more operations.
9153 if (BW % 2 == 0) {
9154 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: BW / 2);
9155 EVT HalfVT = VT.changeVectorElementType(Context&: Ctx, EltVT: HalfEltVT);
9156 if (TLI.isTypeLegal(VT: HalfVT) &&
9157 canNarrowCLMULToLegal(TLI, Ctx, VT: HalfVT, HalveDepth: HalveDepth + 1, TotalDepth: TotalDepth + 1))
9158 return true;
9159 }
9160
9161 // Widen: double element count (fixed-width vectors only).
9162 // This is cheap -- just INSERT_SUBVECTOR + EXTRACT_SUBVECTOR.
9163 EVT WideVT = VT.getDoubleNumVectorElementsVT(Context&: Ctx);
9164 if (TLI.isTypeLegal(VT: WideVT) &&
9165 canNarrowCLMULToLegal(TLI, Ctx, VT: WideVT, HalveDepth, TotalDepth: TotalDepth + 1))
9166 return true;
9167
9168 return false;
9169}
9170
9171SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
9172 SDLoc DL(Node);
9173 EVT VT = Node->getValueType(ResNo: 0);
9174 SDValue X = Node->getOperand(Num: 0);
9175 SDValue Y = Node->getOperand(Num: 1);
9176 unsigned BW = VT.getScalarSizeInBits();
9177 unsigned Opcode = Node->getOpcode();
9178 LLVMContext &Ctx = *DAG.getContext();
9179
9180 switch (Opcode) {
9181 case ISD::CLMUL: {
9182 // For vector types, try decomposition strategies that leverage legal
9183 // CLMUL on narrower or wider element types, avoiding the expensive
9184 // bit-by-bit expansion.
9185 if (VT.isVector()) {
9186 // Strategy 1: Halving decomposition to half-element-width CLMUL.
9187 // Applies ExpandIntRes_CLMUL's identity element-wise:
9188 // CLMUL(X, Y) = (Hi << HalfBW) | Lo
9189 // where:
9190 // Lo = CLMUL(XLo, YLo)
9191 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9192 unsigned HalfBW = BW / 2;
9193 if (BW % 2 == 0) {
9194 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: HalfBW);
9195 EVT HalfVT =
9196 EVT::getVectorVT(Context&: Ctx, VT: HalfEltVT, EC: VT.getVectorElementCount());
9197 if (isTypeLegal(VT: HalfVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: HalfVT,
9198 /*HalveDepth=*/1)) {
9199 SDValue ShAmt = DAG.getShiftAmountConstant(Val: HalfBW, VT, DL);
9200
9201 // Extract low and high halves of each element.
9202 SDValue XLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: X);
9203 SDValue XHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
9204 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: ShAmt));
9205 SDValue YLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: Y);
9206 SDValue YHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
9207 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt));
9208
9209 // Lo = CLMUL(XLo, YLo)
9210 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YLo);
9211
9212 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9213 SDValue LoH = DAG.getNode(Opcode: ISD::CLMULH, DL, VT: HalfVT, N1: XLo, N2: YLo);
9214 SDValue Cross1 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YHi);
9215 SDValue Cross2 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XHi, N2: YLo);
9216 SDValue Cross = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: Cross1, N2: Cross2);
9217 SDValue Hi = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: LoH, N2: Cross);
9218
9219 // Reassemble: Result = ZExt(Lo) | (AnyExt(Hi) << HalfBW)
9220 SDValue LoExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Lo);
9221 SDValue HiExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Hi);
9222 SDValue HiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: HiExt, N2: ShAmt);
9223 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoExt, N2: HiShifted);
9224 }
9225 }
9226
9227 // Strategy 2: Promote to double-element-width CLMUL.
9228 // CLMUL(X, Y) = Trunc(CLMUL(AnyExt(X), AnyExt(Y)))
9229 {
9230 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9231 if (isTypeLegal(VT: ExtVT) && isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT)) {
9232 // If CLMUL on ExtVT is Custom (not Legal), the target may
9233 // scalarize it, costing O(NumElements) scalar ops. The bit-by-bit
9234 // fallback costs O(BW) vectorized iterations. Only widen when
9235 // element count is small enough that scalarization is cheaper.
9236 unsigned NumElts = VT.getVectorMinNumElements();
9237 if (isOperationLegal(Op: ISD::CLMUL, VT: ExtVT) || NumElts < BW) {
9238 SDValue XExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: X);
9239 SDValue YExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: Y);
9240 SDValue Mul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9241 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Mul);
9242 }
9243 }
9244 }
9245
9246 // Strategy 3: Widen element count (pad with undef, do CLMUL on wider
9247 // vector, extract lower result). CLMUL is element-wise, so upper
9248 // (undef) lanes don't affect the lower results.
9249 // e.g. v4i16 => pad to v8i16 => halve to v8i8 PMUL => extract v4i16.
9250 if (auto EC = VT.getVectorElementCount(); EC.isFixed()) {
9251 EVT WideVT = EVT::getVectorVT(Context&: Ctx, VT: VT.getVectorElementType(), EC: EC * 2);
9252 if (isTypeLegal(VT: WideVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: WideVT)) {
9253 SDValue Undef = DAG.getUNDEF(VT: WideVT);
9254 SDValue XWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
9255 N2: X, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9256 SDValue YWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
9257 N2: Y, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9258 SDValue WideRes = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: WideVT, N1: XWide, N2: YWide);
9259 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WideRes,
9260 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9261 }
9262 }
9263 }
9264
9265 // Special case: clmul(X, Y) where Y is a known constant (splat) that forms
9266 // a contiguous block of trailing ones whose length N is a power of two
9267 // (e.g. i8 0xFF, i8 0x0F, ...) or equal to the operand width. In this
9268 // special case, clmul(X, Y) is equivalent to a "parallel prefix XOR" or
9269 // "bitwise parity" operation on X.
9270 //
9271 // Note: This special currently dose NOT apply when the mask is neither a
9272 // power of two nor equal to the operand width because the loop inside
9273 // behaves as if the mask was bit-ceiled, and "undoing" the XOR with parts
9274 // of that CLMUL is a recursive problem (e.g. CLMUL with a 20-bit mask
9275 // requires correction XOR with CLMUL with 12-bit mask).
9276 if (auto *C = isConstOrConstSplat(N: Y, /*AllowUndefs=*/true)) {
9277 const APInt &YVal = C->getAPIntValue();
9278 unsigned N = YVal.countr_one();
9279 if (YVal.isAllOnes() || (YVal.isMask() && isPowerOf2_32(Value: N))) {
9280 SDValue R = X;
9281 for (unsigned I = 1; I < N; I <<= 1) {
9282 SDValue ShAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9283 SDValue Shifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: R, N2: ShAmt);
9284 R = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: R, N2: Shifted);
9285 }
9286 return R;
9287 }
9288 }
9289
9290 // NOTE: If you change this expansion, please update the cost model
9291 // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
9292 // Intrinsic::clmul.
9293
9294 // Strategy 4: multiplication with holes.
9295 //
9296 // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
9297 // do occur, they wind up in a "hole" and are subsequently masked out of the
9298 // result.
9299 //
9300 // https://www.bearssl.org/constanttime.html#ghash-for-gcm describes this
9301 // approach.
9302
9303 // Stride S handles operands up to S·2^S bits using S² multiplies.
9304 //
9305 // * BW <= 8 uses S = 2 (holes of 1 bit)
9306 // * BW <= 24 uses S = 3 (holes of 2 bits)
9307 // * BW <= 64 uses S = 4 (holes of 3 bits)
9308 // * BW <= 160 uses S = 5 (holes of 4 bits)
9309 // * BW <= 384 uses S = 6 (holes of 5 bits)
9310 //
9311 // We distribute the BW bits over S phases:
9312 //
9313 // phase 0 keeps bits: 0, S, 2S, ...
9314 // phase 1 keeps bits: 1, S + 1, 2S + 1, ...
9315 // ...
9316 //
9317 // Each phase has up to n = ceil(BW / S) bits set, and the holes are S-1
9318 // bits wide.
9319 //
9320 // Take BW = 4, S = 2, n = 2. The worst case is a fully populated phase (all
9321 // non-hole bits are set to 1) multiplied by itself, 0b0101 * 0b0101. Each
9322 // set bit of one operand shifts a copy of the other, and we add the copies:
9323 //
9324 // col: 4 3 2 1 0
9325 // 0b0101 << 0: 0 0 1 0 1
9326 // 0b0101 << 2: 1 0 1 0 0
9327 // ----------------- +
9328 // count: 1 0 2 0 1
9329 //
9330 // Counting the number of one-bits in each column gives a triangle: the
9331 // counts climb 1, 2, ..., n and back down (here 1, 2, 1 across the data
9332 // columns). So a column holds at most n one-bits, and that maximum n is
9333 // reached in only one column: the peak. Every other column holds at most n
9334 // - 1 one-bits.
9335 //
9336 // A stack of one-bits in a column turns into carries: column 2 above really
9337 // stores the value 1 + 1 = 2 = n. A column spans S bits, its kept bit
9338 // plus S-1 hole bits, and the count is written from the kept bit upward,
9339 // so any count <= 2^S - 1 stays within the column and never interferes with
9340 // the next data bit S positions up. Every non-peak column holds at most n -
9341 // 1, so they all fit as soon as n - 1 <= 2^S - 1.
9342 //
9343 // That leaves only the peak column. Because both operands set all data
9344 // bits, the triangle peaks at the top of the word at the highest data bit
9345 // still inside BW. Here the count reaches exactly n = 2^S and overflows.
9346 // But its carry lands at bit n*S >= BW, off the top, where it (and the
9347 // whole descending half of the triangle) is truncated.
9348 //
9349 // Hence the holes suffice exactly when n = ceil(BW / S) <= 2^S, i.e. BW <=
9350 // S*2^S.
9351 //
9352 // Here we find the smallest S that satisfies this inequality.
9353 unsigned S = 1;
9354 while (S < 32 && divideCeil(Numerator: BW, Denominator: S) > (1u << S))
9355 ++S;
9356
9357 // The "multiplication with holes" expansion emits S*S MULs, 3*S ANDs,
9358 // S*(S-1) XORs and S-1 ORs.
9359 unsigned HolesCost = S * S + 3 * S + S * (S - 1) + (S - 1);
9360
9361 // Estimate the cost of the naive algorithm.
9362 KnownBits KnownY = DAG.computeKnownBits(Op: Y);
9363 unsigned NaiveCost = 0;
9364 for (unsigned I = 0; I < BW; ++I) {
9365 // The iteration folds away entirely and is free.
9366 if (KnownY.Zero[I])
9367 continue;
9368
9369 // On targets with a fast bit test instruction more instructions are used
9370 // to not need a (potentially expensive) multiplication. See also below.
9371 if (hasBitTest(X: Y, Y: DAG.getShiftAmountConstant(Val: I, VT, DL))) {
9372 // AND + SETCC + SHL + SELECT + XOR.
9373 NaiveCost += 5;
9374 } else {
9375 // AND + MUL + XOR.
9376 NaiveCost += 3;
9377 }
9378 }
9379
9380 // Only use multiplication with holes when it is cheaper, else use the naive
9381 // fallback below.
9382 if (HolesCost < NaiveCost &&
9383 isOperationLegalOrCustom(Op: ISD::MUL, VT: getTypeToTransformTo(Context&: Ctx, VT))) {
9384
9385 // Set a bit every S positions, e.g. for S = 4 this is equivalent to
9386 // 0b...00010001...0001.
9387 APInt MaskVal = APInt::getSplat(NewLen: BW, V: APInt(S, 1));
9388
9389 // Create versions of X and Y that keep only the I-th bit of each S-bit
9390 // slice.
9391 SmallVector<SDValue, 4> M(S), Xp(S), Yp(S);
9392 for (unsigned I = 0; I < S; ++I) {
9393 M[I] = DAG.getConstant(Val: MaskVal.shl(shiftAmt: I), DL, VT);
9394 Xp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: M[I]);
9395 Yp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: M[I]);
9396 }
9397
9398 // Codegens these expressions (S*S multiplications), e.g. for S=4:
9399 //
9400 // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
9401 // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
9402 // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
9403 // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
9404 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9405 for (unsigned I = 0; I < S; ++I) {
9406 SDValue Zi = DAG.getConstant(Val: 0, DL, VT);
9407 for (unsigned J = 0; J < S; ++J) {
9408 unsigned K = (I + S - J) % S;
9409 SDValue P = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Xp[J], N2: Yp[K]);
9410 Zi = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Zi, N2: P);
9411 }
9412
9413 // Keep only the bits belonging to this iteration, and bitwise or it all
9414 // together.
9415 Zi = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Zi, N2: M[I]);
9416 Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Res, N2: Zi, Flags: SDNodeFlags::Disjoint);
9417 }
9418 return Res;
9419 }
9420
9421 // Strategy 5: the naive fallback.
9422 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
9423
9424 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9425 for (unsigned I = 0; I < BW; ++I) {
9426 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9427 SDValue Mask = DAG.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: I), DL, VT);
9428 SDValue YMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: Mask);
9429
9430 // For targets with a fast bit test instruction (e.g., x86 BT) or without
9431 // multiply, use a shift-based expansion to avoid expensive MUL
9432 // instructions.
9433 SDValue Part;
9434 if (!hasBitTest(X: Y, Y: ShiftAmt) &&
9435 isOperationLegalOrCustom(
9436 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
9437 Part = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: YMasked);
9438 } else {
9439 // Canonical bit test: (Y & (1 << I)) != 0
9440 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
9441 SDValue Cond = DAG.getSetCC(DL, VT: SetCCVT, LHS: YMasked, RHS: Zero, Cond: ISD::SETEQ);
9442 SDValue XShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftAmt);
9443 Part = DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: XShifted);
9444 }
9445 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Res, N2: Part);
9446 }
9447 return Res;
9448 }
9449 case ISD::CLMULR:
9450 // If we have CLMUL/CLMULH, merge the shifted results to form CLMULR.
9451 if (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) &&
9452 isOperationLegalOrCustom(Op: ISD::CLMULH, VT)) {
9453 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
9454 SDValue Hi = DAG.getNode(Opcode: ISD::CLMULH, DL, VT, N1: X, N2: Y);
9455 Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo,
9456 N2: DAG.getShiftAmountConstant(Val: BW - 1, VT, DL));
9457 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi,
9458 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9459 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Lo, N2: Hi);
9460 }
9461 [[fallthrough]];
9462 case ISD::CLMULH: {
9463 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9464 // Use bitreverse-based lowering (CLMULR/H = rev(CLMUL(rev,rev)) >> S)
9465 // when any of these hold:
9466 // (a) ZERO_EXTEND to ExtVT or SRL on ExtVT isn't legal.
9467 // (b) CLMUL is legal on VT but not on ExtVT (e.g. v8i8 on AArch64).
9468 // (c) CLMUL on ExtVT isn't legal, but CLMUL on VT can be efficiently
9469 // expanded via halving/widening to reach legal CLMUL. The bitreverse
9470 // path creates CLMUL(VT) which will be expanded efficiently. The
9471 // promote path would create CLMUL(ExtVT) => halving => CLMULH(VT),
9472 // causing a cycle.
9473 // Note: when CLMUL is legal on ExtVT, the zext => CLMUL(ExtVT) => shift
9474 // => trunc path is preferred over the bitreverse path, as it avoids the
9475 // cost of 3 bitreverse operations.
9476 if (!isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT: ExtVT) ||
9477 !isOperationLegalOrCustom(Op: ISD::SRL, VT: ExtVT) ||
9478 (!isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT) &&
9479 (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) ||
9480 canNarrowCLMULToLegal(TLI: *this, Ctx, VT)))) {
9481 SDValue XRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: X);
9482 SDValue YRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: Y);
9483 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: XRev, N2: YRev);
9484 SDValue Res = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: ClMul);
9485 if (Opcode == ISD::CLMULH)
9486 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res,
9487 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9488 return Res;
9489 }
9490 SDValue XExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: X);
9491 SDValue YExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: Y);
9492 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9493 unsigned ShAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
9494 SDValue HiBits = DAG.getNode(Opcode: ISD::SRL, DL, VT: ExtVT, N1: ClMul,
9495 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: ExtVT, DL));
9496 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: HiBits);
9497 }
9498 }
9499 llvm_unreachable("Expected CLMUL, CLMULR, or CLMULH");
9500}
9501
9502bool TargetLowering::expandMODF(SDNode *Node, SelectionDAG &DAG,
9503 SmallVectorImpl<SDValue> &Results) const {
9504 SDLoc dl(Node);
9505 SDValue Val = Node->getOperand(Num: 0);
9506 EVT VT = Val.getValueType();
9507 SDNodeFlags Flags = Node->getFlags();
9508
9509 SDValue IntPart = DAG.getNode(Opcode: ISD::FTRUNC, DL: dl, VT, Operand: Val, Flags);
9510 SDValue FracPart = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT, N1: Val, N2: IntPart, Flags);
9511
9512 SDValue FracToUse = FracPart;
9513 if (!Flags.hasNoInfs()) {
9514 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Val, Flags);
9515 SDValue Inf =
9516 DAG.getConstantFP(Val: APFloat::getInf(Sem: VT.getFltSemantics()), DL: dl, VT);
9517 EVT SetCCVT =
9518 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
9519 SDValue IsInf = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Abs, RHS: Inf, Cond: ISD::SETOEQ);
9520 SDValue Zero = DAG.getConstantFP(Val: 0.0, DL: dl, VT);
9521 FracToUse = DAG.getSelect(DL: dl, VT, Cond: IsInf, LHS: Zero, RHS: FracPart);
9522 }
9523
9524 SDValue ResultFrac =
9525 DAG.getNode(Opcode: ISD::FCOPYSIGN, DL: dl, VT, N1: FracToUse, N2: Val, Flags);
9526 Results.push_back(Elt: ResultFrac);
9527 Results.push_back(Elt: IntPart);
9528 return true;
9529}
9530
9531SDValue TargetLowering::expandPEXT(SDNode *Node, SelectionDAG &DAG) const {
9532 SDLoc DL(Node);
9533 EVT VT = Node->getValueType(ResNo: 0);
9534 SDValue Val = Node->getOperand(Num: 0);
9535 SDValue Msk = Node->getOperand(Num: 1);
9536 unsigned BW = VT.getScalarSizeInBits();
9537
9538 // Just scalarize if scalar PEXT is legal
9539 if (VT.isVector() && isOperationLegal(Op: ISD::PEXT, VT: VT.getVectorElementType()))
9540 return DAG.UnrollVectorOp(N: Node);
9541
9542 // Hacker's Delight §7-4: Compress, or Generalized Extract
9543 SDValue X = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Val, N2: Msk);
9544 SDValue M = Msk;
9545 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9546 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: M, VT), N2: One);
9547
9548 // Repeatedly compute which bits would shift to the right by an odd amount,
9549 // shift all such bits in parallel using a mask, and double the shift amount.
9550 for (unsigned I = 1; I < BW; I *= 2) {
9551 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9552 SDValue Mp =
9553 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9554 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: M);
9555 SDValue ShiftI = DAG.getShiftAmountConstant(Val: I, VT, DL);
9556 SDValue MvS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mv, N2: ShiftI);
9557 M = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: M, N2: Mv), N2: MvS,
9558 Flags: SDNodeFlags::Disjoint);
9559 SDValue T = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Mv);
9560 SDValue TS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: T, N2: ShiftI);
9561 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: X, N2: T), N2: TS,
9562 Flags: SDNodeFlags::Disjoint);
9563 if (I * 2 < BW)
9564 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9565 }
9566
9567 return X;
9568}
9569
9570SDValue TargetLowering::expandPDEP(SDNode *Node, SelectionDAG &DAG) const {
9571 SDLoc DL(Node);
9572 EVT VT = Node->getValueType(ResNo: 0);
9573 SDValue Val = Node->getOperand(Num: 0);
9574 SDValue Msk = Node->getOperand(Num: 1);
9575 unsigned BW = VT.getScalarSizeInBits();
9576
9577 // Just scalarize if scalar PDEP is legal
9578 if (VT.isVector() && isOperationLegal(Op: ISD::PDEP, VT: VT.getVectorElementType()))
9579 return DAG.UnrollVectorOp(N: Node);
9580
9581 // Hacker's Delight §7-5: Expand, or Generalized Insert.
9582 unsigned LogBW = Log2_32_Ceil(Value: BW);
9583 SmallVector<SDValue, 8> MvArray(LogBW);
9584 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9585 SDValue Mc = Msk;
9586 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: Msk, VT), N2: One);
9587
9588 // First pass: compute move masks for each power of two that a bit moves by.
9589 for (unsigned S = 0; S < LogBW; ++S) {
9590 unsigned ShiftS = 1u << S;
9591 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9592 SDValue Mp =
9593 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9594 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: Mc);
9595 MvArray[S] = Mv;
9596 if (S + 1 < LogBW) {
9597 SDValue McXorMv = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Mc, N2: Mv);
9598 SDValue MvShifted = DAG.getNode(
9599 Opcode: ISD::SRL, DL, VT, N1: Mv, N2: DAG.getShiftAmountConstant(Val: ShiftS, VT, DL));
9600 Mc = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: McXorMv, N2: MvShifted,
9601 Flags: SDNodeFlags::Disjoint);
9602 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9603 }
9604 }
9605
9606 // Second pass: move bits by 32, 16, 8, 4, 2, 1, using masks, in parallel.
9607 // Each pass handles half the shift amount of the previous pass.
9608 SDValue X = Val;
9609 for (int S = (int)LogBW - 1; S >= 0; --S) {
9610 SDValue ShiftSv = DAG.getShiftAmountConstant(Val: 1ull << S, VT, DL);
9611 SDValue T = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftSv);
9612 SDValue UnshiftedBits =
9613 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: DAG.getNOT(DL, Val: MvArray[S], VT));
9614 SDValue ShiftedBits = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T, N2: MvArray[S]);
9615 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: UnshiftedBits, N2: ShiftedBits,
9616 Flags: SDNodeFlags::Disjoint);
9617 }
9618
9619 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Msk);
9620}
9621
9622void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
9623 SelectionDAG &DAG) const {
9624 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
9625 EVT VT = Node->getValueType(ResNo: 0);
9626 unsigned VTBits = VT.getScalarSizeInBits();
9627 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
9628
9629 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
9630 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
9631 SDValue ShOpLo = Node->getOperand(Num: 0);
9632 SDValue ShOpHi = Node->getOperand(Num: 1);
9633 SDValue ShAmt = Node->getOperand(Num: 2);
9634 EVT ShAmtVT = ShAmt.getValueType();
9635 EVT ShAmtCCVT =
9636 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ShAmtVT);
9637 SDLoc dl(Node);
9638
9639 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
9640 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
9641 // away during isel.
9642 SDValue SafeShAmt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9643 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT));
9644 SDValue Tmp1 = IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: ShOpHi,
9645 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT))
9646 : DAG.getConstant(Val: 0, DL: dl, VT);
9647
9648 SDValue Tmp2, Tmp3;
9649 if (IsSHL) {
9650 Tmp2 = DAG.getNode(Opcode: ISD::FSHL, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9651 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: SafeShAmt);
9652 } else {
9653 Tmp2 = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9654 Tmp3 = DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: ShOpHi, N2: SafeShAmt);
9655 }
9656
9657 // If the shift amount is larger or equal than the width of a part we don't
9658 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
9659 // values for large shift amounts.
9660 SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9661 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: ShAmtVT));
9662 SDValue Cond = DAG.getSetCC(DL: dl, VT: ShAmtCCVT, LHS: AndNode,
9663 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShAmtVT), Cond: ISD::SETNE);
9664
9665 if (IsSHL) {
9666 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9667 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9668 } else {
9669 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9670 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9671 }
9672}
9673
9674SDValue TargetLowering::expandFCANONICALIZE(SDNode *Node,
9675 SelectionDAG &DAG) const {
9676 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
9677 // suggested in
9678 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
9679 // It uses strict_fp operations even outside a strict_fp context in order
9680 // to guarantee that the canonicalization is not optimized away by later
9681 // passes. The result chain introduced by that is intentionally ignored
9682 // since no ordering requirement is intended here.
9683 EVT VT = Node->getValueType(ResNo: 0);
9684 SDLoc DL(Node);
9685 SDNodeFlags Flags = Node->getFlags();
9686 Flags.setNoFPExcept(true);
9687 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT);
9688 SDValue Mul =
9689 DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {VT, MVT::Other},
9690 Ops: {DAG.getEntryNode(), Node->getOperand(Num: 0), One}, Flags);
9691 return Mul;
9692}
9693
9694SDValue TargetLowering::expandCONVERT_TO_ARBITRARY_FP(SDNode *Node,
9695 SelectionDAG &DAG) const {
9696 // Expand conversion from a native IEEE float type to an arbitrary FP format
9697 // returning the result as an integer using bit manipulation.
9698 EVT ResVT = Node->getValueType(ResNo: 0);
9699 SDLoc dl(Node);
9700
9701 SDValue FloatVal = Node->getOperand(Num: 0);
9702 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9703 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9704 const auto RoundMode =
9705 static_cast<RoundingMode>(Node->getConstantOperandVal(Num: 2));
9706 const bool Saturate = Node->getConstantOperandVal(Num: 3) != 0;
9707
9708 // Supported destination formats.
9709 switch (Sem) {
9710 case APFloatBase::S_Float8E5M2:
9711 case APFloatBase::S_Float8E4M3FN:
9712 case APFloatBase::S_Float8E5M3FNU:
9713 case APFloatBase::S_Float6E3M2FN:
9714 case APFloatBase::S_Float6E2M3FN:
9715 case APFloatBase::S_Float4E2M1FN:
9716 break;
9717 default:
9718 DAG.getContext()->emitError(ErrorStr: "CONVERT_TO_ARBITRARY_FP: not implemented "
9719 "destination format (semantics enum " +
9720 Twine(SemEnum) + ")");
9721 return SDValue();
9722 }
9723
9724 // Supported rounding modes.
9725 switch (RoundMode) {
9726 case RoundingMode::NearestTiesToEven:
9727 case RoundingMode::TowardZero:
9728 case RoundingMode::TowardPositive:
9729 case RoundingMode::TowardNegative:
9730 case RoundingMode::NearestTiesToAway:
9731 break;
9732 default:
9733 DAG.getContext()->emitError(
9734 ErrorStr: "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
9735 Twine(static_cast<int>(RoundMode)) + ")");
9736 return SDValue();
9737 }
9738
9739 // Destination format parameters.
9740 const fltSemantics &DstSem = APFloatBase::EnumToSemantics(S: Sem);
9741 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9742 const unsigned DstPrecision = APFloat::semanticsPrecision(DstSem);
9743 const unsigned DstMant = DstPrecision - 1;
9744 // Unsigned formats spend no bit on the sign.
9745 const bool DstHasSign = APFloat::semanticsHasSignedRepr(DstSem);
9746 const unsigned DstExpBits = DstBits - (DstHasSign ? 1 : 0) - DstMant;
9747 const int DstBias = 1 - APFloat::semanticsMinExponent(DstSem);
9748 const unsigned DstExpMax = (1U << DstExpBits) - 1;
9749 const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
9750 const fltNonfiniteBehavior DstNFBehavior = DstSem.nonFiniteBehavior;
9751 const fltNanEncoding DstNanEnc = DstSem.nanEncoding;
9752
9753 // Compute the maximum normal exponent for the destination format.
9754 const unsigned DstExpMaxNormal =
9755 DstNFBehavior == fltNonfiniteBehavior::IEEE754 ? DstExpMax - 1
9756 : DstExpMax;
9757
9758 // For NanOnly formats the max exponent field for finite values
9759 // is DstExpMax, but the encoding with exp = DstExpMax and
9760 // mant = all-ones is NaN. So DstExpMaxNormal = DstExpMax, but max
9761 // mantissa at that exponent is DstMantMask - 1 (if NanEnc == AllOnes) to
9762 // avoid the NaN encoding.
9763 uint64_t DstMaxMantAtMaxExp = DstMantMask;
9764 if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9765 DstNanEnc == fltNanEncoding::AllOnes)
9766 DstMaxMantAtMaxExp = DstMantMask - 1;
9767
9768 // Source format parameters.
9769 EVT SrcVT = FloatVal.getValueType();
9770 const fltSemantics &SrcSem = SrcVT.getScalarType().getFltSemantics();
9771 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9772 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9773 const unsigned SrcMant = SrcPrecision - 1;
9774 const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;
9775
9776 // Work in the source integer type. Match the destination shape so the
9777 // expansion stays vector when ResVT is a vector.
9778 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcBits);
9779 EVT IntVT = ResVT.changeElementType(Context&: *DAG.getContext(), EltVT: IntScalarVT);
9780 EVT SetCCVT =
9781 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9782 EVT FPSetCCVT =
9783 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
9784
9785 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9786 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9787
9788 // Bitcast source float to integer to extract the sign bit.
9789 SDValue Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FloatVal);
9790 SDValue SignBit =
9791 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9792 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
9793
9794 // Classify the input.
9795 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT);
9796 SDValue FPInf = DAG.getConstantFP(Val: APFloat::getInf(Sem: SrcSem), DL: dl, VT: SrcVT);
9797 SDValue AbsVal = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: SrcVT, Operand: FloatVal);
9798 SDValue IsNaN = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETUO);
9799 SDValue IsInf = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: AbsVal, RHS: FPInf, Cond: ISD::SETOEQ);
9800 SDValue IsZero = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOEQ);
9801
9802 // Split into a normalized fraction and unbiased exponent. FFREXP normalizes
9803 // source denormals automatically. The result is unspecified for Inf/NaN, but
9804 // those inputs are detected above and override the final result.
9805 EVT FrexpExpScalarVT =
9806 getValueType(DL: DAG.getDataLayout(), Ty: Type::getInt32Ty(C&: *DAG.getContext()));
9807 EVT FrexpExpVT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: FrexpExpScalarVT);
9808 SDValue Frexp =
9809 DAG.getNode(Opcode: ISD::FFREXP, DL: dl, VTList: DAG.getVTList(VT1: SrcVT, VT2: FrexpExpVT), N: FloatVal);
9810 SDValue FrexpFrac = Frexp.getValue(R: 0);
9811 SDValue FrexpExp = Frexp.getValue(R: 1);
9812
9813 SDValue FrexpFracInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FrexpFrac);
9814 SDValue EffSrcMant = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FrexpFracInt,
9815 N2: DAG.getConstant(Val: SrcMantMask, DL: dl, VT: IntVT));
9816
9817 SDValue FrexpExpExt = DAG.getSExtOrTrunc(Op: FrexpExp, DL: dl, VT: IntVT);
9818 SDValue NewExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: FrexpExpExt,
9819 N2: DAG.getConstant(Val: DstBias - 1, DL: dl, VT: IntVT));
9820
9821 // Compute rounding increment given the round bit, sticky bits, and LSB
9822 // of the truncated mantissa.
9823 auto ComputeRoundUp = [&](SDValue RoundBit, SDValue StickyBits,
9824 SDValue LSB) -> SDValue {
9825 switch (RoundMode) {
9826 case RoundingMode::NearestTiesToEven: {
9827 // Round up if round_bit && (sticky || lsb)
9828 SDValue StickyOrLSB = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: StickyBits, N2: LSB);
9829 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyOrLSB);
9830 }
9831 case RoundingMode::TowardZero:
9832 return Zero;
9833 case RoundingMode::TowardPositive: {
9834 // Round up if positive and any truncated bits are set.
9835 SDValue AnyTruncBits =
9836 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9837 SDValue HasTruncBits =
9838 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9839 SDValue IsPositive = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETEQ);
9840 SDValue DoRound =
9841 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsPositive);
9842 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9843 }
9844 case RoundingMode::TowardNegative: {
9845 // Round up if negative and any truncated bits are set (to -Inf).
9846 SDValue AnyTruncBits =
9847 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9848 SDValue HasTruncBits =
9849 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9850 SDValue IsNegative = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETNE);
9851 SDValue DoRound =
9852 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsNegative);
9853 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9854 }
9855 case RoundingMode::NearestTiesToAway:
9856 return RoundBit;
9857 default:
9858 llvm_unreachable("unsupported rounding mode");
9859 }
9860 };
9861
9862 // Round mantissa from SrcMant bits to DstMant bits.
9863 SDValue TruncMant;
9864 SDValue RoundUp;
9865 if (SrcMant > DstMant) {
9866 const unsigned Shift = SrcMant - DstMant;
9867 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: Shift, VT: IntVT, DL: dl);
9868 TruncMant = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ShiftConst);
9869
9870 // Check bit at position Shift - 1 aka the round bit.
9871 SDValue RoundBit;
9872 if (Shift >= 1) {
9873 SDValue RoundBitShift = DAG.getShiftAmountConstant(Val: Shift - 1, VT: IntVT, DL: dl);
9874 SDValue ShiftedMant =
9875 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: RoundBitShift);
9876 RoundBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: ShiftedMant, N2: One);
9877 } else {
9878 RoundBit = Zero;
9879 }
9880
9881 // OR of all bits below the round bit to get sticky bits.
9882 SDValue StickyBits;
9883 if (Shift >= 2) {
9884 uint64_t StickyMask = maskTrailingOnes<uint64_t>(N: Shift - 1);
9885 StickyBits = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: EffSrcMant,
9886 N2: DAG.getConstant(Val: StickyMask, DL: dl, VT: IntVT));
9887 StickyBits = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: StickyBits, RHS: Zero, Cond: ISD::SETNE);
9888 StickyBits = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: StickyBits);
9889 } else {
9890 StickyBits = Zero;
9891 }
9892
9893 // LSB of truncated mantissa.
9894 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: TruncMant, N2: One);
9895
9896 RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
9897 } else {
9898 // If DstMant >= SrcMant, then no rounding needed, just shift left.
9899 SDValue MantShift =
9900 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
9901 TruncMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: MantShift);
9902 RoundUp = Zero;
9903 }
9904
9905 // Apply rounding.
9906 SDValue RoundedMant = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: TruncMant, N2: RoundUp);
9907
9908 // Handle mantissa overflow from rounding.
9909 // If rounded_mant > DstMantMask, carry into exponent.
9910 SDValue MantOverflow =
9911 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: RoundedMant,
9912 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9913 // On overflow: mant = 0, exp += 1.
9914 SDValue AdjMant = DAG.getSelect(DL: dl, VT: IntVT, Cond: MantOverflow, LHS: Zero, RHS: RoundedMant);
9915 SDValue AdjExp =
9916 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: NewExp,
9917 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: MantOverflow));
9918
9919 // Precompute sign shifted to MSB of destination. Unsigned formats have no
9920 // sign bit to merge in.
9921 SDValue SignShifted =
9922 DstHasSign
9923 ? DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
9924 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
9925 : Zero;
9926
9927 // Destination denormal conversion (when new_exp <= 0).
9928 // Shift the mantissa right by 1 - new_exp additional bits and set the
9929 // exponent field to 0.
9930 SDValue ExpIsNeg = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9931 RHS: DAG.getConstant(Val: 1, DL: dl, VT: IntVT), Cond: ISD::SETLT);
9932
9933 SDValue DenormResult;
9934 {
9935 // denorm_shift = 1 - NewExp.
9936 SDValue DenormShift = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: One, N2: NewExp);
9937
9938 // full_src_mant = (1 << SrcMant) | EffSrcMant.
9939 SDValue ImplicitOne =
9940 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One,
9941 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl));
9942 SDValue FullSrcMant =
9943 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ImplicitOne);
9944
9945 // Total right shift = DenormShift + (SrcMant - DstMant).
9946 int64_t MantDelta = static_cast<int64_t>(SrcMant) - DstMant;
9947 SDValue TotalShift =
9948 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormShift,
9949 N2: DAG.getSignedConstant(Val: MantDelta, DL: dl, VT: IntVT));
9950
9951 // Clamp total shift to avoid UB, then truncate denorm mantissa.
9952 EVT ShiftVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
9953 SDValue MaxShift = DAG.getConstant(Val: SrcBits - 1, DL: dl, VT: IntVT);
9954 SDValue ClampedShift =
9955 DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IntVT, N1: TotalShift, N2: MaxShift);
9956 SDValue DenormTruncMant =
9957 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant,
9958 N2: DAG.getZExtOrTrunc(Op: ClampedShift, DL: dl, VT: ShiftVT));
9959
9960 // Rounding for denorm path.
9961 SDValue DenormRoundUp;
9962 {
9963 // Round bit is at position TotalShift - 1 of FullSrcMant.
9964 // Clamp to at least 1 so the subtraction doesn't underflow and create
9965 // shift nodes with invalid shift amounts.
9966 SDValue SafeShift = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT: IntVT, N1: ClampedShift, N2: One);
9967 SDValue RoundBitPos = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: SafeShift, N2: One);
9968 SDValue RoundBitPosAmt = DAG.getZExtOrTrunc(Op: RoundBitPos, DL: dl, VT: ShiftVT);
9969 SDValue DenormRoundBit = DAG.getNode(
9970 Opcode: ISD::AND, DL: dl, VT: IntVT,
9971 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant, N2: RoundBitPosAmt), N2: One);
9972
9973 // Sticky: all bits below round bit.
9974 // sticky_mask = (1 << RoundBitPos) - 1
9975 SDValue StickyMask = DAG.getNode(
9976 Opcode: ISD::SUB, DL: dl, VT: IntVT,
9977 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: RoundBitPosAmt), N2: One);
9978 SDValue DenormStickyBits =
9979 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FullSrcMant, N2: StickyMask);
9980 SDValue HasSticky = DAG.getNode(
9981 Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT,
9982 Operand: DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormStickyBits, RHS: Zero, Cond: ISD::SETNE));
9983
9984 SDValue DenormLSB =
9985 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: One);
9986
9987 DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);
9988
9989 // Only apply rounding if TotalShift >= 1 (i.e., there are bits to round).
9990 SDValue ShiftGEOne =
9991 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ClampedShift, RHS: One, Cond: ISD::SETUGE);
9992 DenormRoundUp = DAG.getSelect(DL: dl, VT: IntVT, Cond: ShiftGEOne, LHS: DenormRoundUp, RHS: Zero);
9993 }
9994
9995 SDValue DenormRoundedMant =
9996 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: DenormRoundUp);
9997
9998 // If rounding caused overflow into the normal range, then we get the
9999 // smallest normal number.
10000 SDValue DenormMantOF =
10001 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormRoundedMant,
10002 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
10003 SDValue DenormFinalMant =
10004 DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: Zero, RHS: DenormRoundedMant);
10005 SDValue DenormFinalExp = DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: One, RHS: Zero);
10006
10007 // Assemble: sign | (exp << DstMant) | mant
10008 SDValue DenormExpShifted =
10009 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormFinalExp,
10010 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
10011 DenormResult = DAG.getNode(
10012 Opcode: ISD::OR, DL: dl, VT: IntVT,
10013 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
10014 N2: DenormFinalMant);
10015 }
10016
10017 // Exponent overflow detection.
10018 SDValue ExpOF =
10019 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
10020 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETGT);
10021
10022 // Also check if AdjExp == DstExpMaxNormal and mantissa overflow into
10023 // a value that exceeds the max allowed mantissa at that exponent.
10024 SDValue ExpAtMax =
10025 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
10026 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETEQ);
10027 SDValue MantExceedsMax =
10028 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjMant,
10029 RHS: DAG.getConstant(Val: DstMaxMantAtMaxExp, DL: dl, VT: IntVT), Cond: ISD::SETGT);
10030 SDValue ExpMantOF =
10031 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: ExpAtMax, N2: MantExceedsMax);
10032 SDValue IsOverflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCVT, N1: ExpOF, N2: ExpMantOF);
10033
10034 // Build overflow result.
10035 SDValue OverflowResult;
10036
10037 if (Saturate) {
10038 // Clamp to max finite value:
10039 // sign | (DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp
10040 uint64_t MaxFinite =
10041 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
10042 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
10043 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
10044 } else if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
10045 // Produce infinity.
10046 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
10047 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
10048 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
10049 } else {
10050 // Emit poison if no Inf in format and not saturating.
10051 OverflowResult = DAG.getPOISON(VT: IntVT);
10052 }
10053
10054 // Assemble normal result: sign | (AdjExp << DstMant) | AdjMant
10055 SDValue NormExpShifted =
10056 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: AdjExp,
10057 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
10058 SDValue NormResult = DAG.getNode(
10059 Opcode: ISD::OR, DL: dl, VT: IntVT,
10060 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted), N2: AdjMant);
10061
10062 // Build special-value results.
10063 SDValue NaNResult;
10064 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
10065 // Produce canonical NaN.
10066 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10067 NaNResult =
10068 DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | QNaNBit, DL: dl, VT: IntVT);
10069 } else if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
10070 DstNanEnc == fltNanEncoding::AllOnes) {
10071 // E4M3FN-style: NaN is exp=all-ones, mant=all-ones.
10072 NaNResult = DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | DstMantMask,
10073 DL: dl, VT: IntVT);
10074 } else {
10075 // NaN -> poison for finite only values.
10076 NaNResult = DAG.getPOISON(VT: IntVT);
10077 }
10078
10079 // Inf handling.
10080 SDValue InfResult;
10081 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
10082 // Produce signed infinity.
10083 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
10084 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
10085 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
10086 } else if (Saturate) {
10087 // Inf saturates to max finite.
10088 uint64_t MaxFinite =
10089 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
10090 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
10091 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
10092 } else {
10093 // No Inf and not saturating -> poison.
10094 InfResult = DAG.getPOISON(VT: IntVT);
10095 }
10096
10097 SDValue ZeroResult = SignShifted;
10098
10099 // Final selection in an order: NaN takes priority, then Inf, then Zero.
10100 SDValue FiniteResult =
10101 DAG.getSelect(DL: dl, VT: IntVT, Cond: ExpIsNeg, LHS: DenormResult, RHS: NormResult);
10102 FiniteResult =
10103 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: OverflowResult, RHS: FiniteResult);
10104
10105 SDValue Result = FiniteResult;
10106 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
10107 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
10108
10109 // Negative values are unrepresentable in an unsigned format: clamp to zero
10110 // when saturating, poison otherwise so no select is needed. -0.0 is handled
10111 // by IsZero above. Run before the NaN case so a negative NaN still yields
10112 // NaN.
10113 if (!DstHasSign && Saturate) {
10114 SDValue IsNegative =
10115 DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOLT);
10116 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNegative, LHS: Zero, RHS: Result);
10117 }
10118
10119 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
10120
10121 // Truncate to destination integer type.
10122 return DAG.getZExtOrTrunc(Op: Result, DL: dl, VT: ResVT);
10123}
10124
10125SDValue
10126TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
10127 SelectionDAG &DAG) const {
10128 SDLoc dl(Node);
10129 EVT DstVT = Node->getValueType(ResNo: 0);
10130 EVT DstScalarVT = DstVT.getScalarType();
10131
10132 SDValue IntVal = Node->getOperand(Num: 0);
10133 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
10134 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
10135
10136 // Supported source formats.
10137 switch (Sem) {
10138 case APFloatBase::S_Float8E5M2:
10139 case APFloatBase::S_Float8E4M3FN:
10140 case APFloatBase::S_Float8E5M3FNU:
10141 case APFloatBase::S_Float6E3M2FN:
10142 case APFloatBase::S_Float6E2M3FN:
10143 case APFloatBase::S_Float4E2M1FN:
10144 break;
10145 default:
10146 DAG.getContext()->emitError(ErrorStr: "CONVERT_FROM_ARBITRARY_FP: not implemented "
10147 "source format (semantics enum " +
10148 Twine(SemEnum) + ")");
10149 return SDValue();
10150 }
10151
10152 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(S: Sem);
10153 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
10154 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
10155 const unsigned SrcMant = SrcPrecision - 1;
10156 // Unsigned formats spend no bit on the sign.
10157 const bool SrcHasSign = APFloat::semanticsHasSignedRepr(SrcSem);
10158 const unsigned SrcExp = SrcBits - (SrcHasSign ? 1 : 0) - SrcMant;
10159 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
10160 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
10161
10162 // Destination format parameters.
10163 const fltSemantics &DstSem = DstScalarVT.getFltSemantics();
10164 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
10165 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
10166 const unsigned DstExpBits = DstBits - DstMant - 1;
10167 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
10168 const int DstBias = 1 - DstMinExp;
10169 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
10170
10171 // Work in an integer type matching the destination float width.
10172 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstBits);
10173 EVT IntVT = IntScalarVT;
10174 if (DstVT.isVector()) {
10175 IntVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IntScalarVT,
10176 EC: DstVT.getVectorElementCount());
10177 } else if (!isTypeLegal(VT: IntScalarVT)) {
10178 // Avoid generating illegal type as there is no other places that'll
10179 // legalize it. Vector types don't have this problem because they
10180 // are subject to LegalizeVectorOps and another type legalization phase
10181 // will follow.
10182 if (getTypeAction(Context&: *DAG.getContext(), VT: IntScalarVT) != TypePromoteInteger) {
10183 // We only know how to handle situations where the legal type is wider.
10184 DAG.getContext()->emitError(
10185 ErrorStr: "CONVERT_FROM_ARBITRARY_FP: the requested integer value type for its "
10186 "legalization is not supported");
10187 return SDValue();
10188 }
10189 IntVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: IntScalarVT);
10190 }
10191
10192 SDValue Src = DAG.getZExtOrTrunc(Op: IntVal, DL: dl, VT: IntVT);
10193
10194 EVT SetCCVT =
10195 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
10196
10197 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
10198 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
10199
10200 // Extract bit fields.
10201 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
10202 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
10203
10204 SDValue MantField = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Src,
10205 N2: DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT));
10206
10207 SDValue ExpField =
10208 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT,
10209 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
10210 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl)),
10211 N2: DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT));
10212
10213 // An unsigned source has no sign bit; bit SrcBits - 1 is part of the
10214 // exponent.
10215 SDValue SignShifted =
10216 SrcHasSign
10217 ? DAG.getNode(
10218 Opcode: ISD::SHL, DL: dl, VT: IntVT,
10219 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
10220 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl)),
10221 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
10222 : Zero;
10223
10224 // Classify the input.
10225 SDValue ExpAllOnes = DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT);
10226 SDValue IsExpAllOnes =
10227 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: ExpAllOnes, Cond: ISD::SETEQ);
10228 SDValue IsExpZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: Zero, Cond: ISD::SETEQ);
10229 SDValue IsMantZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETEQ);
10230 SDValue IsMantNonZero =
10231 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETNE);
10232
10233 SDValue IsNaN;
10234 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
10235 IsNaN = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
10236 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
10237 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantNonZero);
10238 } else {
10239 assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
10240 SDValue MantAllOnes = DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT);
10241 SDValue IsMantAllOnes =
10242 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: MantAllOnes, Cond: ISD::SETEQ);
10243 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantAllOnes);
10244 }
10245
10246 SDValue IsInf;
10247 if (NFBehavior == fltNonfiniteBehavior::IEEE754)
10248 IsInf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantZero);
10249 else
10250 IsInf = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
10251
10252 SDValue IsZero = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantZero);
10253 SDValue IsDenorm =
10254 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantNonZero);
10255
10256 // Normal value conversion.
10257 const int BiasAdjust = DstBias - SrcBias;
10258 SDValue NormDstExp = DAG.getNode(
10259 Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: ExpField,
10260 N2: DAG.getConstant(Val: APInt(IntVT.getScalarSizeInBits(), BiasAdjust, true), DL: dl,
10261 VT: IntVT));
10262
10263 SDValue NormDstMant;
10264 if (DstMant > SrcMant) {
10265 SDValue NormDstMantShift =
10266 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
10267 NormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: MantField, N2: NormDstMantShift);
10268 } else {
10269 NormDstMant = MantField;
10270 }
10271
10272 SDValue DstMantShift = DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl);
10273 SDValue NormExpShifted =
10274 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: NormDstExp, N2: DstMantShift);
10275 SDValue NormResult =
10276 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
10277 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted),
10278 N2: NormDstMant);
10279
10280 // With identical exponent biases, denormal values remain denormal and the
10281 // normal conversion's mantissa shift is sufficient.
10282 SDValue DenormResult = NormResult;
10283 if (BiasAdjust != 0) {
10284 const unsigned IntVTBits = IntVT.getScalarSizeInBits();
10285 SDValue LeadingZeros =
10286 DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT: IntVT, Operand: MantField);
10287
10288 const int DenormExpConst =
10289 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
10290 SDValue DenormDstExp = DAG.getNode(
10291 Opcode: ISD::SUB, DL: dl, VT: IntVT,
10292 N1: DAG.getConstant(Val: APInt(IntVTBits, DenormExpConst, true), DL: dl, VT: IntVT),
10293 N2: LeadingZeros);
10294
10295 SDValue MantMSB =
10296 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT,
10297 N1: DAG.getConstant(Val: IntVTBits - 1, DL: dl, VT: IntVT), N2: LeadingZeros);
10298
10299 SDValue LeadingOne = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: MantMSB);
10300 SDValue Frac = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: IntVT, N1: MantField, N2: LeadingOne);
10301
10302 const unsigned ShiftSub = IntVTBits - 1 - DstMant;
10303 SDValue ShiftAmount = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: LeadingZeros,
10304 N2: DAG.getConstant(Val: ShiftSub, DL: dl, VT: IntVT));
10305
10306 SDValue DenormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: Frac, N2: ShiftAmount);
10307
10308 SDValue DenormExpShifted =
10309 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormDstExp, N2: DstMantShift);
10310 DenormResult = DAG.getNode(
10311 Opcode: ISD::OR, DL: dl, VT: IntVT,
10312 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
10313 N2: DenormDstMant);
10314 }
10315
10316 SDValue FiniteResult =
10317 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsDenorm, LHS: DenormResult, RHS: NormResult);
10318
10319 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10320 SDValue NaNResult =
10321 DAG.getConstant(Val: (DstExpAllOnes << DstMant) | QNaNBit, DL: dl, VT: IntVT);
10322
10323 SDValue InfResult =
10324 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
10325 N2: DAG.getConstant(Val: DstExpAllOnes << DstMant, DL: dl, VT: IntVT));
10326
10327 // A source format may have a larger finite exponent range despite having
10328 // fewer bits, as with Float8E5M3FNU converted to half. Its overflowing finite
10329 // values become infinity. The NaN selection below still takes precedence.
10330 if (APFloat::semanticsMaxExponent(SrcSem) >
10331 APFloat::semanticsMaxExponent(DstSem)) {
10332 SDValue IsOverflow =
10333 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NormDstExp,
10334 RHS: DAG.getConstant(Val: DstExpAllOnes, DL: dl, VT: IntVT), Cond: ISD::SETUGE);
10335 FiniteResult =
10336 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: InfResult, RHS: FiniteResult);
10337 }
10338
10339 SDValue ZeroResult = SignShifted;
10340
10341 SDValue Result = FiniteResult;
10342 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
10343 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
10344 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
10345
10346 if (!DstVT.bitsEq(VT: IntVT)) {
10347 // Store to stack before loading it back.
10348 assert(!IntVT.isVector() && IntVT.bitsGT(DstVT));
10349 // IntScalarVT is the original type that has the same width as DstVT.
10350 Align Alignment = DAG.getReducedAlign(VT: IntScalarVT, /*UseABI=*/false);
10351 SDValue StackPtr =
10352 DAG.CreateStackTemporary(Bytes: IntScalarVT.getStoreSize(), Alignment);
10353 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
10354 MachineFunction &MF = DAG.getMachineFunction();
10355 MachinePointerInfo PtrInfo =
10356 MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
10357 SDValue Store = DAG.getTruncStore(Chain: DAG.getEntryNode(), dl, Val: Result, Ptr: StackPtr,
10358 PtrInfo, SVT: IntScalarVT, Alignment);
10359
10360 SDValue Load = DAG.getLoad(VT: DstVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment);
10361 return DAG.getMergeValues(Ops: {Load, Load.getValue(R: 1)}, dl);
10362 }
10363
10364 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT, Operand: Result);
10365}
10366
10367bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
10368 SelectionDAG &DAG) const {
10369 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10370 SDValue Src = Node->getOperand(Num: OpNo);
10371 EVT SrcVT = Src.getValueType();
10372 EVT DstVT = Node->getValueType(ResNo: 0);
10373 SDLoc dl(SDValue(Node, 0));
10374
10375 // FIXME: Only f32 to i64 conversions are supported.
10376 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
10377 return false;
10378
10379 if (Node->isStrictFPOpcode())
10380 // When a NaN is converted to an integer a trap is allowed. We can't
10381 // use this expansion here because it would eliminate that trap. Other
10382 // traps are also allowed and cannot be eliminated. See
10383 // IEEE 754-2008 sec 5.8.
10384 return false;
10385
10386 // Expand f32 -> i64 conversion
10387 // This algorithm comes from compiler-rt's implementation of fixsfdi:
10388 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
10389 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
10390 EVT IntVT = SrcVT.changeTypeToInteger();
10391 EVT IntShVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
10392
10393 SDValue ExponentMask = DAG.getConstant(Val: 0x7F800000, DL: dl, VT: IntVT);
10394 SDValue ExponentLoBit = DAG.getConstant(Val: 23, DL: dl, VT: IntVT);
10395 SDValue Bias = DAG.getConstant(Val: 127, DL: dl, VT: IntVT);
10396 SDValue SignMask = DAG.getConstant(Val: APInt::getSignMask(BitWidth: SrcEltBits), DL: dl, VT: IntVT);
10397 SDValue SignLowBit = DAG.getConstant(Val: SrcEltBits - 1, DL: dl, VT: IntVT);
10398 SDValue MantissaMask = DAG.getConstant(Val: 0x007FFFFF, DL: dl, VT: IntVT);
10399
10400 SDValue Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: Src);
10401
10402 SDValue ExponentBits = DAG.getNode(
10403 Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: ExponentMask),
10404 N2: DAG.getZExtOrTrunc(Op: ExponentLoBit, DL: dl, VT: IntShVT));
10405 SDValue Exponent = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentBits, N2: Bias);
10406
10407 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: IntVT,
10408 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: SignMask),
10409 N2: DAG.getZExtOrTrunc(Op: SignLowBit, DL: dl, VT: IntShVT));
10410 Sign = DAG.getSExtOrTrunc(Op: Sign, DL: dl, VT: DstVT);
10411
10412 SDValue R = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
10413 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: MantissaMask),
10414 N2: DAG.getConstant(Val: 0x00800000, DL: dl, VT: IntVT));
10415
10416 R = DAG.getZExtOrTrunc(Op: R, DL: dl, VT: DstVT);
10417
10418 R = DAG.getSelectCC(
10419 DL: dl, LHS: Exponent, RHS: ExponentLoBit,
10420 True: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: DstVT, N1: R,
10421 N2: DAG.getZExtOrTrunc(
10422 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: Exponent, N2: ExponentLoBit),
10423 DL: dl, VT: IntShVT)),
10424 False: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: DstVT, N1: R,
10425 N2: DAG.getZExtOrTrunc(
10426 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentLoBit, N2: Exponent),
10427 DL: dl, VT: IntShVT)),
10428 Cond: ISD::SETGT);
10429
10430 SDValue Ret = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: DstVT,
10431 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: R, N2: Sign), N2: Sign);
10432
10433 Result = DAG.getSelectCC(DL: dl, LHS: Exponent, RHS: DAG.getConstant(Val: 0, DL: dl, VT: IntVT),
10434 True: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), False: Ret, Cond: ISD::SETLT);
10435 return true;
10436}
10437
10438bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
10439 SDValue &Chain,
10440 SelectionDAG &DAG) const {
10441 SDLoc dl(SDValue(Node, 0));
10442 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10443 SDValue Src = Node->getOperand(Num: OpNo);
10444
10445 EVT SrcVT = Src.getValueType();
10446 EVT DstVT = Node->getValueType(ResNo: 0);
10447 EVT SetCCVT =
10448 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
10449 EVT DstSetCCVT =
10450 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: DstVT);
10451
10452 // Only expand vector types if we have the appropriate vector bit operations.
10453 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
10454 ISD::FP_TO_SINT;
10455 if (DstVT.isVector() && (!isOperationLegalOrCustom(Op: SIntOpcode, VT: DstVT) ||
10456 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT: SrcVT)))
10457 return false;
10458
10459 // If the maximum float value is smaller then the signed integer range,
10460 // the destination signmask can't be represented by the float, so we can
10461 // just use FP_TO_SINT directly.
10462 const fltSemantics &APFSem = SrcVT.getFltSemantics();
10463 APFloat APF(APFSem, APInt::getZero(numBits: SrcVT.getScalarSizeInBits()));
10464 APInt SignMask = APInt::getSignMask(BitWidth: DstVT.getScalarSizeInBits());
10465 if (APFloat::opOverflow &
10466 APF.convertFromAPInt(Input: SignMask, IsSigned: false, RM: APFloat::rmNearestTiesToEven)) {
10467 if (Node->isStrictFPOpcode()) {
10468 Result = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10469 Ops: { Node->getOperand(Num: 0), Src });
10470 Chain = Result.getValue(R: 1);
10471 } else
10472 Result = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10473 return true;
10474 }
10475
10476 // Don't expand it if there isn't cheap fsub instruction.
10477 if (!isOperationLegalOrCustom(
10478 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, VT: SrcVT))
10479 return false;
10480
10481 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
10482 SDValue Sel;
10483
10484 if (Node->isStrictFPOpcode()) {
10485 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
10486 Chain: Node->getOperand(Num: 0), /*IsSignaling*/ true);
10487 Chain = Sel.getValue(R: 1);
10488 } else {
10489 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT);
10490 }
10491
10492 bool Strict = Node->isStrictFPOpcode() ||
10493 shouldUseStrictFP_TO_INT(FpVT: SrcVT, IntVT: DstVT, /*IsSigned*/ false);
10494
10495 if (Strict) {
10496 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
10497 // signmask then offset (the result of which should be fully representable).
10498 // Sel = Src < 0x8000000000000000
10499 // FltOfs = select Sel, 0, 0x8000000000000000
10500 // IntOfs = select Sel, 0, 0x8000000000000000
10501 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
10502
10503 // TODO: Should any fast-math-flags be set for the FSUB?
10504 SDValue FltOfs = DAG.getSelect(DL: dl, VT: SrcVT, Cond: Sel,
10505 LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
10506 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10507 SDValue IntOfs = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel,
10508 LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT),
10509 RHS: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10510 SDValue SInt;
10511 if (Node->isStrictFPOpcode()) {
10512 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: { SrcVT, MVT::Other },
10513 Ops: { Chain, Src, FltOfs });
10514 SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10515 Ops: { Val.getValue(R: 1), Val });
10516 Chain = SInt.getValue(R: 1);
10517 } else {
10518 SDValue Val = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: FltOfs);
10519 SInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Val);
10520 }
10521 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
10522 } else {
10523 // Expand based on maximum range of FP_TO_SINT:
10524 // True = fp_to_sint(Src)
10525 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
10526 // Result = select (Src < 0x8000000000000000), True, False
10527
10528 SDValue True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10529 // TODO: Should any fast-math-flags be set for the FSUB?
10530 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT,
10531 Operand: DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: Cst));
10532 False = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: False,
10533 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10534 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10535 Result = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel, LHS: True, RHS: False);
10536 }
10537 return true;
10538}
10539
10540bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
10541 SDValue &Chain, SelectionDAG &DAG) const {
10542 // This transform is not correct for converting 0 when rounding mode is set
10543 // to round toward negative infinity which will produce -0.0. So disable
10544 // under strictfp.
10545 if (Node->isStrictFPOpcode())
10546 return false;
10547
10548 SDValue Src = Node->getOperand(Num: 0);
10549 EVT SrcVT = Src.getValueType();
10550 EVT DstVT = Node->getValueType(ResNo: 0);
10551
10552 // If the input is known to be non-negative and SINT_TO_FP is legal then use
10553 // it.
10554 if (Node->getFlags().hasNonNeg() &&
10555 isOperationLegalOrCustom(Op: ISD::SINT_TO_FP, VT: SrcVT)) {
10556 Result =
10557 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Node->getOperand(Num: 0));
10558 return true;
10559 }
10560
10561 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
10562 return false;
10563
10564 // Only expand vector types if we have the appropriate vector bit
10565 // operations.
10566 if (SrcVT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SRL, VT: SrcVT) ||
10567 !isOperationLegalOrCustom(Op: ISD::FADD, VT: DstVT) ||
10568 !isOperationLegalOrCustom(Op: ISD::FSUB, VT: DstVT) ||
10569 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: SrcVT) ||
10570 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: SrcVT)))
10571 return false;
10572
10573 SDLoc dl(SDValue(Node, 0));
10574
10575 // Implementation of unsigned i64 to f64 following the algorithm in
10576 // __floatundidf in compiler_rt. This implementation performs rounding
10577 // correctly in all rounding modes with the exception of converting 0
10578 // when rounding toward negative infinity. In that case the fsub will
10579 // produce -0.0. This will be added to +0.0 and produce -0.0 which is
10580 // incorrect.
10581 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), DL: dl, VT: SrcVT);
10582 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
10583 Val: llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), DL: dl, VT: DstVT);
10584 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), DL: dl, VT: SrcVT);
10585 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), DL: dl, VT: SrcVT);
10586 SDValue HiShift = DAG.getShiftAmountConstant(Val: 32, VT: SrcVT, DL: dl);
10587
10588 SDValue Lo = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Src, N2: LoMask);
10589 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Src, N2: HiShift);
10590 SDValue LoOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Lo, N2: TwoP52);
10591 SDValue HiOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Hi, N2: TwoP84);
10592 SDValue LoFlt = DAG.getBitcast(VT: DstVT, V: LoOr);
10593 SDValue HiFlt = DAG.getBitcast(VT: DstVT, V: HiOr);
10594 SDValue HiSub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: DstVT, N1: HiFlt, N2: TwoP84PlusTwoP52);
10595 Result = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DstVT, N1: LoFlt, N2: HiSub);
10596 return true;
10597}
10598
10599SDValue
10600TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node,
10601 SelectionDAG &DAG) const {
10602 unsigned Opcode = Node->getOpcode();
10603 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
10604 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
10605 "Wrong opcode");
10606
10607 if (Node->getFlags().hasNoNaNs()) {
10608 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
10609 EVT VT = Node->getValueType(ResNo: 0);
10610 if ((!isCondCodeLegal(CC: Pred, VT: VT.getSimpleVT()) ||
10611 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)) &&
10612 VT.isVector())
10613 return SDValue();
10614 SDValue Op1 = Node->getOperand(Num: 0);
10615 SDValue Op2 = Node->getOperand(Num: 1);
10616 return DAG.getSelectCC(DL: SDLoc(Node), LHS: Op1, RHS: Op2, True: Op1, False: Op2, Cond: Pred,
10617 Flags: Node->getFlags());
10618 }
10619
10620 return SDValue();
10621}
10622
10623SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
10624 SelectionDAG &DAG) const {
10625 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node, DAG))
10626 return Expanded;
10627
10628 EVT VT = Node->getValueType(ResNo: 0);
10629 if (VT.isScalableVector())
10630 report_fatal_error(
10631 reason: "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
10632
10633 SDLoc dl(Node);
10634 unsigned NewOp =
10635 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10636
10637 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10638 SDValue Quiet0 = Node->getOperand(Num: 0);
10639 SDValue Quiet1 = Node->getOperand(Num: 1);
10640
10641 if (!Node->getFlags().hasNoNaNs()) {
10642 // Insert canonicalizes if it's possible we need to quiet to get correct
10643 // sNaN behavior.
10644 if (!DAG.isKnownNeverSNaN(Op: Quiet0)) {
10645 Quiet0 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet0,
10646 Flags: Node->getFlags());
10647 }
10648 if (!DAG.isKnownNeverSNaN(Op: Quiet1)) {
10649 Quiet1 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet1,
10650 Flags: Node->getFlags());
10651 }
10652 }
10653
10654 return DAG.getNode(Opcode: NewOp, DL: dl, VT, N1: Quiet0, N2: Quiet1, Flags: Node->getFlags());
10655 }
10656
10657 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
10658 // instead if there are no NaNs.
10659 if (Node->getFlags().hasNoNaNs() ||
10660 (DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 0)) &&
10661 DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 1)))) {
10662 unsigned IEEE2018Op =
10663 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10664 if (isOperationLegalOrCustom(Op: IEEE2018Op, VT))
10665 return DAG.getNode(Opcode: IEEE2018Op, DL: dl, VT, N1: Node->getOperand(Num: 0),
10666 N2: Node->getOperand(Num: 1), Flags: Node->getFlags());
10667 }
10668
10669 if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
10670 return SelCC;
10671
10672 return SDValue();
10673}
10674
10675static SDValue isSpecificZeroAfterMaybeRounding(SelectionDAG &DAG,
10676 const TargetLowering &TLI,
10677 const SDLoc &DL, SDValue Val,
10678 FPClassTest FPClass) {
10679 EVT VT = Val.getValueType();
10680 EVT CCVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10681 EVT IntVT = VT.changeTypeToInteger();
10682 EVT FloatVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
10683 SDValue TestZero = DAG.getTargetConstant(Val: FPClass, DL, VT: MVT::i32);
10684 if (!TLI.isTypeLegal(VT: IntVT) &&
10685 !TLI.isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT))
10686 Val = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: FloatVT, N1: Val,
10687 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
10688 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: CCVT, N1: Val, N2: TestZero);
10689}
10690
10691SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
10692 SelectionDAG &DAG) const {
10693 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node: N, DAG))
10694 return Expanded;
10695
10696 SDLoc DL(N);
10697 SDValue LHS = N->getOperand(Num: 0);
10698 SDValue RHS = N->getOperand(Num: 1);
10699 unsigned Opc = N->getOpcode();
10700 EVT VT = N->getValueType(ResNo: 0);
10701 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10702 bool IsMax = Opc == ISD::FMAXIMUM;
10703 SDNodeFlags Flags = N->getFlags();
10704
10705 // First, implement comparison not propagating NaN. If no native fmin or fmax
10706 // available, use plain select with setcc instead.
10707 SDValue MinMax;
10708 unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
10709 unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;
10710
10711 // FIXME: We should probably define fminnum/fmaxnum variants with correct
10712 // signed zero behavior.
10713 bool MinMaxMustRespectOrderedZero = false;
10714
10715 if (isOperationLegalOrCustom(Op: CompOpcIeee, VT)) {
10716 MinMax = DAG.getNode(Opcode: CompOpcIeee, DL, VT, N1: LHS, N2: RHS, Flags);
10717 MinMaxMustRespectOrderedZero = true;
10718 } else if (isOperationLegalOrCustom(Op: CompOpc, VT)) {
10719 MinMax = DAG.getNode(Opcode: CompOpc, DL, VT, N1: LHS, N2: RHS, Flags);
10720 } else {
10721 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
10722 return DAG.UnrollVectorOp(N);
10723
10724 // NaN (if exists) will be propagated later, so orderness doesn't matter.
10725 SDValue Compare =
10726 DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: IsMax ? ISD::SETOGT : ISD::SETOLT);
10727 MinMax = DAG.getSelect(DL, VT, Cond: Compare, LHS, RHS, Flags);
10728 }
10729
10730 // Propagate any NaN of both operands
10731 if (!N->getFlags().hasNoNaNs() &&
10732 (!DAG.isKnownNeverNaN(Op: RHS) || !DAG.isKnownNeverNaN(Op: LHS))) {
10733 ConstantFP *FPNaN = ConstantFP::get(Context&: *DAG.getContext(),
10734 V: APFloat::getNaN(Sem: VT.getFltSemantics()));
10735 MinMax = DAG.getSelect(DL, VT, Cond: DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETUO),
10736 LHS: DAG.getConstantFP(V: *FPNaN, DL, VT), RHS: MinMax, Flags);
10737 }
10738
10739 // fminimum/fmaximum requires -0.0 less than +0.0
10740 if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
10741 !DAG.isKnownNeverLogicalZero(Op: RHS) && !DAG.isKnownNeverLogicalZero(Op: LHS)) {
10742 SDValue IsEqual = DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETOEQ);
10743 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10744 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10745 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS, Flags);
10746 MinMax = DAG.getSelect(DL, VT, Cond: IsEqual, LHS: RetZero, RHS: MinMax, Flags);
10747 }
10748
10749 return MinMax;
10750}
10751
10752SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
10753 SelectionDAG &DAG) const {
10754 SDLoc DL(Node);
10755 SDValue LHS = Node->getOperand(Num: 0);
10756 SDValue RHS = Node->getOperand(Num: 1);
10757 unsigned Opc = Node->getOpcode();
10758 EVT VT = Node->getValueType(ResNo: 0);
10759 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10760 bool IsMax = Opc == ISD::FMAXIMUMNUM;
10761 SDNodeFlags Flags = Node->getFlags();
10762
10763 unsigned NewOp =
10764 Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10765
10766 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10767 if (!Flags.hasNoNaNs()) {
10768 // Insert canonicalizes if it's possible we need to quiet to get correct
10769 // sNaN behavior.
10770 if (!DAG.isKnownNeverSNaN(Op: LHS)) {
10771 LHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: LHS, Flags);
10772 }
10773 if (!DAG.isKnownNeverSNaN(Op: RHS)) {
10774 RHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: RHS, Flags);
10775 }
10776 }
10777
10778 return DAG.getNode(Opcode: NewOp, DL, VT, N1: LHS, N2: RHS, Flags);
10779 }
10780
10781 // We can use FMINIMUM/FMAXIMUM if there is no NaN, since it has
10782 // same behaviors for all of other cases: +0.0 vs -0.0 included.
10783 if (Flags.hasNoNaNs() ||
10784 (DAG.isKnownNeverNaN(Op: LHS) && DAG.isKnownNeverNaN(Op: RHS))) {
10785 unsigned IEEE2019Op =
10786 Opc == ISD::FMINIMUMNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10787 if (isOperationLegalOrCustom(Op: IEEE2019Op, VT))
10788 return DAG.getNode(Opcode: IEEE2019Op, DL, VT, N1: LHS, N2: RHS, Flags);
10789 }
10790
10791 // FMINNUM/FMAXMUM returns qNaN if either operand is sNaN, and it may return
10792 // either one for +0.0 vs -0.0.
10793 if ((Flags.hasNoNaNs() ||
10794 (DAG.isKnownNeverSNaN(Op: LHS) && DAG.isKnownNeverSNaN(Op: RHS))) &&
10795 (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10796 DAG.isKnownNeverLogicalZero(Op: RHS))) {
10797 unsigned IEEE2008Op = Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM : ISD::FMAXNUM;
10798 if (isOperationLegalOrCustom(Op: IEEE2008Op, VT))
10799 return DAG.getNode(Opcode: IEEE2008Op, DL, VT, N1: LHS, N2: RHS, Flags);
10800 }
10801
10802 if (VT.isVector() &&
10803 (isOperationLegalOrCustomOrPromote(Op: Opc, VT: VT.getVectorElementType()) ||
10804 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)))
10805 return DAG.UnrollVectorOp(N: Node);
10806
10807 // If only one operand is NaN, override it with another operand.
10808 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: LHS)) {
10809 LHS = DAG.getSelectCC(DL, LHS, RHS: LHS, True: RHS, False: LHS, Cond: ISD::SETUO);
10810 }
10811 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: RHS)) {
10812 RHS = DAG.getSelectCC(DL, LHS: RHS, RHS, True: LHS, False: RHS, Cond: ISD::SETUO);
10813 }
10814
10815 // Always prefer RHS if equal.
10816 SDValue MinMax =
10817 DAG.getSelectCC(DL, LHS, RHS, True: LHS, False: RHS, Cond: IsMax ? ISD::SETGT : ISD::SETLT);
10818
10819 // TODO: We need quiet sNaN if strictfp.
10820
10821 // Fixup signed zero behavior.
10822 if (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10823 DAG.isKnownNeverLogicalZero(Op: RHS)) {
10824 return MinMax;
10825 }
10826 SDValue IsZero = DAG.getSetCC(DL, VT: CCVT, LHS: MinMax,
10827 RHS: DAG.getConstantFP(Val: 0.0, DL, VT), Cond: ISD::SETEQ);
10828 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10829 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10830 // It's OK to select from LHS and MinMax, with only one ISD::IS_FPCLASS, as
10831 // we preferred RHS when generate MinMax, if the operands are equal.
10832 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS: MinMax, Flags);
10833 return DAG.getSelect(DL, VT, Cond: IsZero, LHS: RetZero, RHS: MinMax, Flags);
10834}
10835
10836/// Returns a true value if if this FPClassTest can be performed with an ordered
10837/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
10838/// std::nullopt if it cannot be performed as a compare with 0.
10839static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
10840 const fltSemantics &Semantics,
10841 const MachineFunction &MF) {
10842 FPClassTest OrderedMask = Test & ~fcNan;
10843 FPClassTest NanTest = Test & fcNan;
10844 bool IsOrdered = NanTest == fcNone;
10845 bool IsUnordered = NanTest == fcNan;
10846
10847 // Skip cases that are testing for only a qnan or snan.
10848 if (!IsOrdered && !IsUnordered)
10849 return std::nullopt;
10850
10851 if (OrderedMask == fcZero &&
10852 MF.getDenormalMode(FPType: Semantics).Input == DenormalMode::IEEE)
10853 return IsOrdered;
10854 if (OrderedMask == (fcZero | fcSubnormal) &&
10855 MF.getDenormalMode(FPType: Semantics).inputsAreZero())
10856 return IsOrdered;
10857 return std::nullopt;
10858}
10859
10860SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
10861 const FPClassTest OrigTestMask,
10862 SDNodeFlags Flags, const SDLoc &DL,
10863 SelectionDAG &DAG) const {
10864 EVT OperandVT = Op.getValueType();
10865 assert(OperandVT.isFloatingPoint());
10866 FPClassTest Test = OrigTestMask;
10867
10868 // Degenerated cases.
10869 if (Test == fcNone)
10870 return DAG.getBoolConstant(V: false, DL, VT: ResultVT, OpVT: OperandVT);
10871 if (Test == fcAllFlags)
10872 return DAG.getBoolConstant(V: true, DL, VT: ResultVT, OpVT: OperandVT);
10873
10874 // PPC double double is a pair of doubles, of which the higher part determines
10875 // the value class.
10876 if (OperandVT == MVT::ppcf128) {
10877 Op = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::f64, N1: Op,
10878 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
10879 OperandVT = MVT::f64;
10880 }
10881
10882 // Floating-point type properties.
10883 EVT ScalarFloatVT = OperandVT.getScalarType();
10884 const Type *FloatTy = ScalarFloatVT.getTypeForEVT(Context&: *DAG.getContext());
10885 const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
10886 bool IsF80 = (ScalarFloatVT == MVT::f80);
10887
10888 // Some checks can be implemented using float comparisons, if floating point
10889 // exceptions are ignored.
10890 if (Flags.hasNoFPExcept() &&
10891 isOperationLegalOrCustom(Op: ISD::SETCC, VT: OperandVT.getScalarType())) {
10892 FPClassTest FPTestMask = Test;
10893 bool IsInvertedFP = false;
10894
10895 if (FPClassTest InvertedFPCheck =
10896 invertFPClassTestIfSimpler(Test: FPTestMask, UseFCmp: true)) {
10897 FPTestMask = InvertedFPCheck;
10898 IsInvertedFP = true;
10899 }
10900
10901 ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ;
10902 ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ;
10903
10904 // See if we can fold an | fcNan into an unordered compare.
10905 FPClassTest OrderedFPTestMask = FPTestMask & ~fcNan;
10906
10907 // Can't fold the ordered check if we're only testing for snan or qnan
10908 // individually.
10909 if ((FPTestMask & fcNan) != fcNan)
10910 OrderedFPTestMask = FPTestMask;
10911
10912 const bool IsOrdered = FPTestMask == OrderedFPTestMask;
10913
10914 if (std::optional<bool> IsCmp0 =
10915 isFCmpEqualZero(Test: FPTestMask, Semantics, MF: DAG.getMachineFunction());
10916 IsCmp0 && (isCondCodeLegalOrCustom(
10917 CC: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
10918 VT: OperandVT.getScalarType().getSimpleVT()))) {
10919
10920 // If denormals could be implicitly treated as 0, this is not equivalent
10921 // to a compare with 0 since it will also be true for denormals.
10922 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op,
10923 RHS: DAG.getConstantFP(Val: 0.0, DL, VT: OperandVT),
10924 Cond: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
10925 }
10926
10927 if (FPTestMask == fcNan &&
10928 isCondCodeLegalOrCustom(CC: IsInvertedFP ? ISD::SETO : ISD::SETUO,
10929 VT: OperandVT.getScalarType().getSimpleVT()))
10930 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Op,
10931 Cond: IsInvertedFP ? ISD::SETO : ISD::SETUO);
10932
10933 bool IsOrderedInf = FPTestMask == fcInf;
10934 if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) &&
10935 isCondCodeLegalOrCustom(CC: IsOrderedInf ? OrderedCmpOpcode
10936 : UnorderedCmpOpcode,
10937 VT: OperandVT.getScalarType().getSimpleVT()) &&
10938 isOperationLegalOrCustom(Op: ISD::FABS, VT: OperandVT.getScalarType()) &&
10939 (isOperationLegal(Op: ISD::ConstantFP, VT: OperandVT.getScalarType()) ||
10940 (OperandVT.isVector() &&
10941 isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: OperandVT)))) {
10942 // isinf(x) --> fabs(x) == inf
10943 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10944 SDValue Inf =
10945 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10946 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf,
10947 Cond: IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
10948 }
10949
10950 if ((OrderedFPTestMask == fcPosInf || OrderedFPTestMask == fcNegInf) &&
10951 isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedCmpOpcode
10952 : UnorderedCmpOpcode,
10953 VT: OperandVT.getSimpleVT())) {
10954 // isposinf(x) --> x == inf
10955 // isneginf(x) --> x == -inf
10956 // isposinf(x) || nan --> x u== inf
10957 // isneginf(x) || nan --> x u== -inf
10958
10959 SDValue Inf = DAG.getConstantFP(
10960 Val: APFloat::getInf(Sem: Semantics, Negative: OrderedFPTestMask == fcNegInf), DL,
10961 VT: OperandVT);
10962 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Inf,
10963 Cond: IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
10964 }
10965
10966 if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
10967 // TODO: Could handle ordered case, but it produces worse code for
10968 // x86. Maybe handle ordered if fabs is free?
10969
10970 ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10971 ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;
10972
10973 if (isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedOp : UnorderedOp,
10974 VT: OperandVT.getScalarType().getSimpleVT())) {
10975 // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal
10976
10977 // TODO: Maybe only makes sense if fabs is free. Integer test of
10978 // exponent bits seems better for x86.
10979 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10980 SDValue SmallestNormal = DAG.getConstantFP(
10981 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10982 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal,
10983 Cond: IsOrdered ? OrderedOp : UnorderedOp);
10984 }
10985 }
10986
10987 if (FPTestMask == fcNormal) {
10988 // TODO: Handle unordered
10989 ISD::CondCode IsFiniteOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10990 ISD::CondCode IsNormalOp = IsInvertedFP ? ISD::SETOLT : ISD::SETUGE;
10991
10992 if (isCondCodeLegalOrCustom(CC: IsFiniteOp,
10993 VT: OperandVT.getScalarType().getSimpleVT()) &&
10994 isCondCodeLegalOrCustom(CC: IsNormalOp,
10995 VT: OperandVT.getScalarType().getSimpleVT()) &&
10996 isFAbsFree(VT: OperandVT)) {
10997 // isnormal(x) --> fabs(x) < infinity && !(fabs(x) < smallest_normal)
10998 SDValue Inf =
10999 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
11000 SDValue SmallestNormal = DAG.getConstantFP(
11001 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
11002
11003 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
11004 SDValue IsFinite = DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf, Cond: IsFiniteOp);
11005 SDValue IsNormal =
11006 DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal, Cond: IsNormalOp);
11007 unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
11008 return DAG.getNode(Opcode: LogicOp, DL, VT: ResultVT, N1: IsFinite, N2: IsNormal);
11009 }
11010 }
11011 }
11012
11013 // Some checks may be represented as inversion of simpler check, for example
11014 // "inf|normal|subnormal|zero" => !"nan".
11015 bool IsInverted = false;
11016
11017 if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, UseFCmp: false)) {
11018 Test = InvertedCheck;
11019 IsInverted = true;
11020 }
11021
11022 // In the general case use integer operations.
11023 unsigned BitSize = OperandVT.getScalarSizeInBits();
11024 EVT IntVT = OperandVT.changeElementType(
11025 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BitSize));
11026 SDValue OpAsInt = DAG.getBitcast(VT: IntVT, V: Op);
11027
11028 // Various masks.
11029 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
11030 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
11031 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
11032 const unsigned ExplicitIntBitInF80 = 63;
11033 APInt ExpMask = Inf;
11034 if (IsF80)
11035 ExpMask.clearBit(BitPosition: ExplicitIntBitInF80);
11036 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
11037 APInt QNaNBitMask =
11038 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
11039 APInt InversionMask = APInt::getAllOnes(numBits: ResultVT.getScalarSizeInBits());
11040
11041 SDValue ValueMaskV = DAG.getConstant(Val: ValueMask, DL, VT: IntVT);
11042 SDValue SignBitV = DAG.getConstant(Val: SignBit, DL, VT: IntVT);
11043 SDValue ExpMaskV = DAG.getConstant(Val: ExpMask, DL, VT: IntVT);
11044 SDValue ZeroV = DAG.getConstant(Val: 0, DL, VT: IntVT);
11045 SDValue InfV = DAG.getConstant(Val: Inf, DL, VT: IntVT);
11046 SDValue ResultInversionMask = DAG.getConstant(Val: InversionMask, DL, VT: ResultVT);
11047
11048 SDValue Res;
11049 const auto appendResult = [&](SDValue PartialRes) {
11050 if (PartialRes) {
11051 if (Res)
11052 Res = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: Res, N2: PartialRes);
11053 else
11054 Res = PartialRes;
11055 }
11056 };
11057
11058 SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
11059 const auto getIntBitIsSet = [&]() -> SDValue {
11060 if (!IntBitIsSetV) {
11061 APInt IntBitMask(BitSize, 0);
11062 IntBitMask.setBit(ExplicitIntBitInF80);
11063 SDValue IntBitMaskV = DAG.getConstant(Val: IntBitMask, DL, VT: IntVT);
11064 SDValue IntBitV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: IntBitMaskV);
11065 IntBitIsSetV = DAG.getSetCC(DL, VT: ResultVT, LHS: IntBitV, RHS: ZeroV, Cond: ISD::SETNE);
11066 }
11067 return IntBitIsSetV;
11068 };
11069
11070 // Split the value into sign bit and absolute value.
11071 SDValue AbsV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ValueMaskV);
11072 SDValue SignV = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt,
11073 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETLT);
11074
11075 // Tests that involve more than one class should be processed first.
11076 SDValue PartialRes;
11077
11078 if (IsF80)
11079 ; // Detect finite numbers of f80 by checking individual classes because
11080 // they have different settings of the explicit integer bit.
11081 else if ((Test & fcFinite) == fcFinite) {
11082 // finite(V) ==> (a << 1) < (inf << 1)
11083 //
11084 // See https://github.com/llvm/llvm-project/issues/169270, this is slightly
11085 // shorter than the `finite(V) ==> abs(V) < exp_mask` formula used before.
11086
11087 assert(APFloat::isIEEELikeFP(OperandVT.getFltSemantics()) &&
11088 "finite check requires IEEE-like FP");
11089
11090 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: IntVT, DL);
11091 SDValue TwiceOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: OpAsInt, N2: One);
11092 SDValue TwiceInf = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: ExpMaskV, N2: One);
11093
11094 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: TwiceOp, RHS: TwiceInf, Cond: ISD::SETULT);
11095 Test &= ~fcFinite;
11096 } else if ((Test & fcFinite) == fcPosFinite) {
11097 // finite(V) && V > 0 ==> V < exp_mask
11098 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ExpMaskV, Cond: ISD::SETULT);
11099 Test &= ~fcPosFinite;
11100 } else if ((Test & fcFinite) == fcNegFinite) {
11101 // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
11102 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ExpMaskV, Cond: ISD::SETLT);
11103 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
11104 Test &= ~fcNegFinite;
11105 }
11106 appendResult(PartialRes);
11107
11108 if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
11109 // fcZero | fcSubnormal => test all exponent bits are 0
11110 // TODO: Handle sign bit specific cases
11111 if (PartialCheck == (fcZero | fcSubnormal)) {
11112 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ExpMaskV);
11113 SDValue ExpIsZero =
11114 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
11115 appendResult(ExpIsZero);
11116 Test &= ~PartialCheck & fcAllFlags;
11117 }
11118 }
11119
11120 // Check for individual classes.
11121
11122 if (unsigned PartialCheck = Test & fcZero) {
11123 if (PartialCheck == fcPosZero)
11124 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ZeroV, Cond: ISD::SETEQ);
11125 else if (PartialCheck == fcZero)
11126 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ZeroV, Cond: ISD::SETEQ);
11127 else // ISD::fcNegZero
11128 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: SignBitV, Cond: ISD::SETEQ);
11129 appendResult(PartialRes);
11130 }
11131
11132 if (unsigned PartialCheck = Test & fcSubnormal) {
11133 // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
11134 // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
11135 SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
11136 SDValue MantissaV = DAG.getConstant(Val: AllOneMantissa, DL, VT: IntVT);
11137 SDValue VMinusOneV =
11138 DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: V, N2: DAG.getConstant(Val: 1, DL, VT: IntVT));
11139 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: VMinusOneV, RHS: MantissaV, Cond: ISD::SETULT);
11140 if (PartialCheck == fcNegSubnormal)
11141 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
11142 appendResult(PartialRes);
11143 }
11144
11145 if (unsigned PartialCheck = Test & fcInf) {
11146 if (PartialCheck == fcPosInf)
11147 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: InfV, Cond: ISD::SETEQ);
11148 else if (PartialCheck == fcInf)
11149 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETEQ);
11150 else { // ISD::fcNegInf
11151 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
11152 SDValue NegInfV = DAG.getConstant(Val: NegInf, DL, VT: IntVT);
11153 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: NegInfV, Cond: ISD::SETEQ);
11154 }
11155 appendResult(PartialRes);
11156 }
11157
11158 if (unsigned PartialCheck = Test & fcNan) {
11159 APInt InfWithQnanBit = Inf | QNaNBitMask;
11160 SDValue InfWithQnanBitV = DAG.getConstant(Val: InfWithQnanBit, DL, VT: IntVT);
11161 if (PartialCheck == fcNan) {
11162 // isnan(V) ==> abs(V) > int(inf)
11163 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
11164 if (IsF80) {
11165 // Recognize unsupported values as NaNs for compatibility with glibc.
11166 // In them (exp(V)==0) == int_bit.
11167 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: AbsV, N2: ExpMaskV);
11168 SDValue ExpIsZero =
11169 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
11170 SDValue IsPseudo =
11171 DAG.getSetCC(DL, VT: ResultVT, LHS: getIntBitIsSet(), RHS: ExpIsZero, Cond: ISD::SETEQ);
11172 PartialRes = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: PartialRes, N2: IsPseudo);
11173 }
11174 } else if (PartialCheck == fcQNan) {
11175 // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
11176 PartialRes =
11177 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETGE);
11178 } else { // ISD::fcSNan
11179 // issignaling(V) ==> abs(V) > unsigned(Inf) &&
11180 // abs(V) < (unsigned(Inf) | quiet_bit)
11181 SDValue IsNan = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
11182 SDValue IsNotQnan =
11183 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETLT);
11184 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: IsNan, N2: IsNotQnan);
11185 }
11186 appendResult(PartialRes);
11187 }
11188
11189 if (unsigned PartialCheck = Test & fcNormal) {
11190 // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
11191 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
11192 SDValue ExpLSBV = DAG.getConstant(Val: ExpLSB, DL, VT: IntVT);
11193 SDValue ExpMinus1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: AbsV, N2: ExpLSBV);
11194 APInt ExpLimit = ExpMask - ExpLSB;
11195 SDValue ExpLimitV = DAG.getConstant(Val: ExpLimit, DL, VT: IntVT);
11196 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: ExpMinus1, RHS: ExpLimitV, Cond: ISD::SETULT);
11197 if (PartialCheck == fcNegNormal)
11198 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
11199 else if (PartialCheck == fcPosNormal) {
11200 SDValue PosSignV =
11201 DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: SignV, N2: ResultInversionMask);
11202 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: PosSignV);
11203 }
11204 if (IsF80)
11205 PartialRes =
11206 DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: getIntBitIsSet());
11207 appendResult(PartialRes);
11208 }
11209
11210 if (!Res)
11211 return DAG.getConstant(Val: IsInverted, DL, VT: ResultVT);
11212 if (IsInverted)
11213 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: Res, N2: ResultInversionMask);
11214 return Res;
11215}
11216
11217// Only expand vector types if we have the appropriate vector bit operations.
11218static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
11219 assert(VT.isVector() && "Expected vector type");
11220 unsigned Len = VT.getScalarSizeInBits();
11221 return TLI.isOperationLegalOrCustom(Op: ISD::ADD, VT) &&
11222 TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT) &&
11223 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
11224 (Len == 8 || TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT)) &&
11225 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT);
11226}
11227
11228SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
11229 SDLoc dl(Node);
11230 EVT VT = Node->getValueType(ResNo: 0);
11231 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11232 SDValue Op = Node->getOperand(Num: 0);
11233 unsigned Len = VT.getScalarSizeInBits();
11234
11235 // Compute effective bit width from known bits, allowing us to shift the
11236 // active bits down if necessary to fit into smaller specialized expansions.
11237 KnownBits Known = DAG.computeKnownBits(Op);
11238 unsigned LZ = Known.countMinLeadingZeros();
11239 unsigned TZ = Known.countMinTrailingZeros();
11240 unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ);
11241
11242 // Round up to 8-bit boundary for byte-oriented SWAR algorithm
11243 unsigned EffectiveLen = Len;
11244 if (ShiftedActiveBits > 0 && ShiftedActiveBits < Len)
11245 EffectiveLen = std::min(a: alignTo(Value: ShiftedActiveBits, Align: 8), b: Len);
11246
11247 assert(VT.isInteger() && "CTPOP not implemented for this type.");
11248
11249 // TODO: Add support for irregular type lengths.
11250 if (!(Len <= 128 && Len % 8 == 0))
11251 return SDValue();
11252
11253 // Only expand vector types if we have the appropriate vector bit operations.
11254 if (VT.isVector() && !canExpandVectorCTPOP(TLI: *this, VT))
11255 return SDValue();
11256
11257 // If the active bits are not at the low end, shift them down
11258 if (EffectiveLen < Len && TZ > 0) {
11259 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11260 N2: DAG.getShiftAmountConstant(Val: TZ, VT, DL: dl));
11261 }
11262
11263 // This is the "best" algorithm from
11264 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
11265 SDValue Mask55 =
11266 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x55)), DL: dl, VT);
11267 SDValue Mask33 =
11268 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x33)), DL: dl, VT);
11269 SDValue Mask0F =
11270 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x0F)), DL: dl, VT);
11271
11272 // v = v - ((v >> 1) & 0x55555555...)
11273 Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op,
11274 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11275 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11276 N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT)),
11277 N2: Mask55));
11278 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
11279 Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: Mask33),
11280 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11281 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11282 N2: DAG.getConstant(Val: 2, DL: dl, VT: ShVT)),
11283 N2: Mask33));
11284 // v = (v + (v >> 4)) & 0x0F0F0F0F...
11285 Op = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11286 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
11287 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11288 N2: DAG.getConstant(Val: 4, DL: dl, VT: ShVT))),
11289 N2: Mask0F);
11290
11291 if (EffectiveLen <= 8)
11292 return Op;
11293
11294 // Avoid the multiply if we only have 2 bytes to add.
11295 // TODO: Only doing this for scalars because vectors weren't as obviously
11296 // improved.
11297 if (EffectiveLen == 16 && !VT.isVector()) {
11298 // v = (v + (v >> 8)) & 0x00FF;
11299 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11300 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
11301 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11302 N2: DAG.getConstant(Val: 8, DL: dl, VT: ShVT))),
11303 N2: DAG.getConstant(Val: 0xFF, DL: dl, VT));
11304 }
11305
11306 // v = (v * 0x01010101...) >> (Len - 8)
11307 SDValue V;
11308 if (isOperationLegalOrCustomOrPromote(
11309 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11310 SDValue Mask01 =
11311 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x01)), DL: dl, VT);
11312 V = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Op, N2: Mask01);
11313 } else {
11314 V = Op;
11315 for (unsigned Shift = 8; Shift < EffectiveLen; Shift *= 2) {
11316 SDValue ShiftC = DAG.getShiftAmountConstant(Val: Shift, VT, DL: dl);
11317 V = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: V,
11318 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: ShiftC));
11319 }
11320 }
11321 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: V, N2: DAG.getConstant(Val: Len - 8, DL: dl, VT: ShVT));
11322}
11323
11324SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
11325 SDLoc dl(Node);
11326 EVT VT = Node->getValueType(ResNo: 0);
11327 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11328 SDValue Op = Node->getOperand(Num: 0);
11329 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11330
11331 // If the non-ZERO_POISON version is supported we can use that instead.
11332 if (Node->getOpcode() == ISD::CTLZ_ZERO_POISON &&
11333 isOperationLegalOrCustom(Op: ISD::CTLZ, VT))
11334 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Op);
11335
11336 // If the ZERO_POISON version is supported use that and handle the zero case.
11337 if (isOperationLegalOrCustom(Op: ISD::CTLZ_ZERO_POISON, VT)) {
11338 EVT SetCCVT =
11339 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11340 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11341 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11342 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11343 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11344 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTLZ);
11345 }
11346
11347 // Only expand vector types if we have the appropriate vector bit operations.
11348 // This includes the operations needed to expand CTPOP if it isn't supported.
11349 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11350 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11351 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11352 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
11353 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
11354 return SDValue();
11355
11356 // for now, we do this:
11357 // x = x | (x >> 1);
11358 // x = x | (x >> 2);
11359 // ...
11360 // x = x | (x >>16);
11361 // x = x | (x >>32); // for 64-bit input
11362 // return popcount(~x);
11363 //
11364 // Ref: "Hacker's Delight" by Henry Warren
11365 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
11366 SDValue Tmp = DAG.getConstant(Val: 1ULL << i, DL: dl, VT: ShVT);
11367 Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op,
11368 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: Tmp));
11369 }
11370 Op = DAG.getNOT(DL: dl, Val: Op, VT);
11371 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
11372}
11373
11374SDValue TargetLowering::expandCTLS(SDNode *Node, SelectionDAG &DAG) const {
11375 SDLoc dl(Node);
11376 EVT VT = Node->getValueType(ResNo: 0);
11377 SDValue Op = DAG.getFreeze(V: Node->getOperand(Num: 0));
11378 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11379
11380 // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
11381 // This transforms the sign bits into leading zeros that can be counted.
11382 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: NumBitsPerElt - 1, VT, DL: dl);
11383 SDValue SignBit = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Op, N2: ShiftAmt);
11384 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: SignBit);
11385 SDValue Shl =
11386 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11387 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Shl, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11388 return DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Or);
11389}
11390
11391SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG,
11392 const SDLoc &DL, EVT VT, SDValue Op,
11393 unsigned BitWidth) const {
11394 if (BitWidth != 32 && BitWidth != 64)
11395 return SDValue();
11396
11397 const DataLayout &TD = DAG.getDataLayout();
11398 if (!isOperationCustom(Op: ISD::ConstantPool, VT: getPointerTy(DL: TD)))
11399 return SDValue();
11400
11401 APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
11402 : APInt(64, 0x0218A392CD3D5DBFULL);
11403 MachinePointerInfo PtrInfo =
11404 MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction());
11405 unsigned ShiftAmt = BitWidth - Log2_32(Value: BitWidth);
11406 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
11407 SDValue Lookup = DAG.getNode(
11408 Opcode: ISD::SRL, DL, VT,
11409 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op, N2: Neg),
11410 N2: DAG.getConstant(Val: DeBruijn, DL, VT)),
11411 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
11412 Lookup = DAG.getSExtOrTrunc(Op: Lookup, DL, VT: getPointerTy(DL: TD));
11413
11414 SmallVector<uint8_t> Table(BitWidth, 0);
11415 for (unsigned i = 0; i < BitWidth; i++) {
11416 APInt Shl = DeBruijn.shl(shiftAmt: i);
11417 APInt Lshr = Shl.lshr(shiftAmt: ShiftAmt);
11418 Table[Lshr.getZExtValue()] = i;
11419 }
11420
11421 // Create a ConstantArray in Constant Pool
11422 auto *CA = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: Table);
11423 SDValue CPIdx = DAG.getConstantPool(C: CA, VT: getPointerTy(DL: TD),
11424 Align: TD.getPrefTypeAlign(Ty: CA->getType()));
11425 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: DL, VT, Chain: DAG.getEntryNode(),
11426 Ptr: DAG.getMemBasePlusOffset(Base: CPIdx, Offset: Lookup, DL),
11427 PtrInfo, MemVT: MVT::i8);
11428 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
11429 return ExtLoad;
11430
11431 EVT SetCCVT =
11432 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11433 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
11434 SDValue SrcIsZero = DAG.getSetCC(DL, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11435 return DAG.getSelect(DL, VT, Cond: SrcIsZero,
11436 LHS: DAG.getConstant(Val: BitWidth, DL, VT), RHS: ExtLoad);
11437}
11438
11439SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
11440 SDLoc dl(Node);
11441 EVT VT = Node->getValueType(ResNo: 0);
11442 SDValue Op = Node->getOperand(Num: 0);
11443 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11444
11445 // If the non-ZERO_POISON version is supported we can use that instead.
11446 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON &&
11447 isOperationLegalOrCustom(Op: ISD::CTTZ, VT))
11448 return DAG.getNode(Opcode: ISD::CTTZ, DL: dl, VT, Operand: Op);
11449
11450 // If the ZERO_POISON version is supported use that and handle the zero case.
11451 if (isOperationLegalOrCustom(Op: ISD::CTTZ_ZERO_POISON, VT)) {
11452 EVT SetCCVT =
11453 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11454 SDValue CTTZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11455 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11456 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11457 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11458 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTTZ);
11459 }
11460
11461 // Only expand vector types if we have the appropriate vector bit operations.
11462 // This includes the operations needed to expand CTPOP if it isn't supported.
11463 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11464 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11465 !isOperationLegalOrCustom(Op: ISD::CTLZ, VT) &&
11466 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11467 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
11468 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) ||
11469 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11470 return SDValue();
11471
11472 // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
11473 // to be expanded or converted to a libcall.
11474 if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT) &&
11475 !isOperationLegal(Op: ISD::CTLZ, VT))
11476 if (SDValue V = CTTZTableLookup(Node, DAG, DL: dl, VT, Op, BitWidth: NumBitsPerElt))
11477 return V;
11478
11479 bool UseCTLZ =
11480 isOperationLegal(Op: ISD::CTLZ, VT) && !isOperationLegal(Op: ISD::CTPOP, VT);
11481
11482 // When only ctlz is available and the operand is nonzero we can use:
11483 // { return nlz(x & -x) ^ 31; }
11484 // which is more efficient than:
11485 // { return 32 - nlz(~x & (x - 1)); }.
11486 if (UseCTLZ && Node->getOpcode() == ISD::CTTZ_ZERO_POISON) {
11487 SDValue LowestBit =
11488 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: DAG.getNegative(Val: Op, DL: dl, VT));
11489 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT,
11490 N1: DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: LowestBit),
11491 N2: DAG.getConstant(Val: NumBitsPerElt - 1, DL: dl, VT));
11492 }
11493
11494 // If ctpop is available, we use:
11495 // { return popcount(~x & (x-1)); }
11496 // If the target has ctlz but not ctpop, we use:
11497 // { return 32 - nlz(~x & (x-1)); }
11498 // Ref: "Hacker's Delight" by Henry Warren
11499 SDValue Tmp = DAG.getNode(
11500 Opcode: ISD::AND, DL: dl, VT, N1: DAG.getNOT(DL: dl, Val: Op, VT),
11501 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 1, DL: dl, VT)));
11502
11503 if (UseCTLZ)
11504 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT),
11505 N2: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Tmp));
11506
11507 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Tmp);
11508}
11509
11510SDValue TargetLowering::expandVPCTTZElements(SDNode *N,
11511 SelectionDAG &DAG) const {
11512 // %cond = to_bool_vec %source
11513 // %splat = splat /*val=*/VL
11514 // %tz = step_vector
11515 // %v = select %cond, /*true=*/tz, /*false=*/%splat
11516 // %r = vp.reduce.umin %v
11517 SDLoc DL(N);
11518 SDValue Source = N->getOperand(Num: 0);
11519 SDValue Mask = N->getOperand(Num: 1);
11520 SDValue EVL = N->getOperand(Num: 2);
11521 EVT SrcVT = Source.getValueType();
11522 EVT ResVT = N->getValueType(ResNo: 0);
11523 EVT ResVecVT =
11524 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT, EC: SrcVT.getVectorElementCount());
11525
11526 // Convert to boolean vector.
11527 if (SrcVT.getScalarType() != MVT::i1) {
11528 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
11529 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
11530 EC: SrcVT.getVectorElementCount());
11531 Source = DAG.getSetCC(DL, VT: SrcVT, LHS: Source, RHS: AllZero, Cond: ISD::SETNE);
11532 }
11533
11534 SDValue ExtEVL = DAG.getZExtOrTrunc(Op: EVL, DL, VT: ResVT);
11535 SDValue Splat = DAG.getSplat(VT: ResVecVT, DL, Op: ExtEVL);
11536 SDValue StepVec = DAG.getStepVector(DL, ResVT: ResVecVT);
11537 SDValue Select = DAG.getSelect(DL, VT: ResVecVT, Cond: Source, LHS: StepVec, RHS: Splat);
11538 return DAG.getNode(Opcode: ISD::VP_REDUCE_UMIN, DL, VT: ResVT, N1: ExtEVL, N2: Select, N3: Mask, N4: EVL);
11539}
11540
11541/// Returns a type-legalized version of \p Mask as the first item in the
11542/// pair. The second item contains a type-legalized step vector that's
11543/// guaranteed to fit the number of elements in \p Mask.
11544/// If the stepvector would require splitting, returns an empty SDValue
11545/// as the second item to signal that the operation should be split instead.
11546static std::pair<SDValue, SDValue>
11547getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL,
11548 SelectionDAG &DAG) {
11549 EVT MaskVT = Mask.getValueType();
11550 EVT BoolVT = MaskVT.getScalarType();
11551
11552 // Find a suitable type for a stepvector.
11553 // If zero is poison, we can assume the upper limit of the result is VF-1.
11554 ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
11555 if (MaskVT.isScalableVector())
11556 VScaleRange = getVScaleRange(F: &DAG.getMachineFunction().getFunction(), BitWidth: 64);
11557 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11558 uint64_t EltWidth = TLI.getBitWidthForCttzElements(
11559 RetVT: EVT(TLI.getVectorIdxTy(DL: DAG.getDataLayout())),
11560 EC: MaskVT.getVectorElementCount(), ZeroIsPoison, VScaleRange: &VScaleRange);
11561 // If the step vector element type is smaller than the mask element type,
11562 // use the mask type directly to avoid widening issues.
11563 EltWidth = std::max(a: EltWidth, b: BoolVT.getFixedSizeInBits());
11564 EVT StepVT = MVT::getIntegerVT(BitWidth: EltWidth);
11565 EVT StepVecVT = MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: StepVT);
11566
11567 // If promotion or widening is required to make the type legal, do it here.
11568 // Promotion of integers within LegalizeVectorOps is looking for types of
11569 // the same size but with a smaller number of larger elements, not the usual
11570 // larger size with the same number of larger elements.
11571 TargetLowering::LegalizeTypeAction TypeAction =
11572 TLI.getTypeAction(Context&: *DAG.getContext(), VT: StepVecVT);
11573 SDValue StepVec;
11574 if (TypeAction == TargetLowering::TypePromoteInteger) {
11575 StepVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11576 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11577 } else if (TypeAction == TargetLowering::TypeWidenVector) {
11578 // For widening, the element count changes. Create a step vector with only
11579 // the original elements valid and zeros for padding. Also widen the mask.
11580 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11581 unsigned WideNumElts = WideVecVT.getVectorNumElements();
11582
11583 // Build widened step vector: <0, 1, ..., OrigNumElts-1, poison, poison, ..>
11584 SDValue OrigStepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11585 SDValue UndefStep = DAG.getPOISON(VT: WideVecVT);
11586 StepVec = DAG.getInsertSubvector(DL, Vec: UndefStep, SubVec: OrigStepVec, Idx: 0);
11587
11588 // Widen mask: pad with zeros.
11589 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: BoolVT, NumElements: WideNumElts);
11590 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
11591 Mask = DAG.getInsertSubvector(DL, Vec: ZeroMask, SubVec: Mask, Idx: 0);
11592 } else if (TypeAction == TargetLowering::TypeSplitVector) {
11593 // The stepvector type would require splitting. Signal to the caller
11594 // that the operation should be split instead of expanded.
11595 return {Mask, SDValue()};
11596 } else {
11597 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11598 }
11599
11600 return {Mask, StepVec};
11601}
11602
11603SDValue TargetLowering::expandVectorFindLastActive(SDNode *N,
11604 SelectionDAG &DAG) const {
11605 SDLoc DL(N);
11606 auto [Mask, StepVec] = getLegalMaskAndStepVector(
11607 Mask: N->getOperand(Num: 0), /*ZeroIsPoison=*/true, DL, DAG);
11608
11609 // If StepVec is empty, the stepvector would require splitting.
11610 // Split the operation instead and let it be recursively legalized.
11611 if (!StepVec) {
11612 EVT MaskVT = N->getOperand(Num: 0).getValueType();
11613 EVT ResVT = N->getValueType(ResNo: 0);
11614
11615 // Split the mask
11616 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: MaskVT);
11617 auto [MaskLo, MaskHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
11618
11619 // Create split VECTOR_FIND_LAST_ACTIVE operations
11620 SDValue LoResult =
11621 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskLo);
11622 SDValue HiResult =
11623 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskHi);
11624
11625 // Check if any lane is active in the high mask.
11626 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: MaskHi);
11627 SDValue Cond = DAG.getBoolExtOrTrunc(
11628 Op: AnyHiActive, SL: DL,
11629 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i1),
11630 OpVT: MVT::i1);
11631
11632 // Adjust HiResult by adding the number of elements in Lo
11633 SDValue LoNumElts =
11634 DAG.getElementCount(DL, VT: ResVT, EC: LoVT.getVectorElementCount());
11635 SDValue AdjustedHiResult =
11636 DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: HiResult, N2: LoNumElts);
11637
11638 // Return: AnyHiActive ? AdjustedHiResult : LoResult;
11639 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: ResVT, N1: Cond, N2: AdjustedHiResult,
11640 N3: LoResult);
11641 }
11642
11643 EVT StepVecVT = StepVec.getValueType();
11644 EVT StepVT = StepVec.getValueType().getVectorElementType();
11645
11646 // Zero out lanes with inactive elements, then find the highest remaining
11647 // value from the stepvector.
11648 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
11649 SDValue ActiveElts = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
11650 SDValue HighestIdx = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL, VT: StepVT, Operand: ActiveElts);
11651 return DAG.getZExtOrTrunc(Op: HighestIdx, DL, VT: N->getValueType(ResNo: 0));
11652}
11653
11654SDValue TargetLowering::expandGetActiveLaneMask(SDNode *N,
11655 SelectionDAG &DAG) const {
11656 SDLoc DL(N);
11657
11658 SDValue Start = N->getOperand(Num: 0);
11659 SDValue End = N->getOperand(Num: 1);
11660 EVT VT = N->getValueType(ResNo: 0);
11661 EVT OpVT = Start.getValueType();
11662
11663 auto [Mask, StepVector] = getLegalMaskAndStepVector(
11664 Mask: DAG.getPOISON(VT), /*ZeroIsPoison=*/false, DL, DAG);
11665
11666 // No legal step vector: split the mask in half and recombine the results.
11667 // Lo = get_active_lane_mask Start, End
11668 // Hi = get_active_lane_mask (uaddsat Start, LoNumElts), End
11669 if (!StepVector) {
11670 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
11671 SDValue Lo = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: LoVT, N1: Start, N2: End);
11672 SDValue LoNumElts =
11673 DAG.getElementCount(DL, VT: OpVT, EC: LoVT.getVectorElementCount());
11674 SDValue HiStart = DAG.getNode(Opcode: ISD::UADDSAT, DL, VT: OpVT, N1: Start, N2: LoNumElts);
11675 SDValue Hi = DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: HiVT, N1: HiStart, N2: End);
11676 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
11677 }
11678
11679 // Rebase and saturate the termination value.
11680 SDValue Max =
11681 DAG.getConstant(Val: maxUIntN(N: std::min(a: StepVector.getScalarValueSizeInBits(),
11682 b: OpVT.getScalarSizeInBits())),
11683 DL, VT: OpVT);
11684 End = DAG.getNode(Opcode: ISD::USUBSAT, DL, VT: OpVT, N1: End, N2: Start);
11685 End = DAG.getNode(Opcode: ISD::UMIN, DL, VT: OpVT, N1: End, N2: Max);
11686 End = DAG.getZExtOrTrunc(Op: End, DL, VT: StepVector.getValueType().getScalarType());
11687
11688 // cmp <0, 1, 2, 3...>, End
11689 SDValue EndV = DAG.getSplat(VT: StepVector.getValueType(), DL, Op: End);
11690 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
11691 VT: StepVector.getValueType());
11692 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: StepVector, RHS: EndV, Cond: ISD::SETULT);
11693 SDValue Res = DAG.getBoolExtOrTrunc(Op: Cmp, SL: DL, VT: Mask.getValueType(),
11694 OpVT: StepVector.getValueType());
11695 if (Mask.getValueType() != VT)
11696 Res = DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0);
11697 return Res;
11698}
11699
11700SDValue TargetLowering::expandLoopDependenceMask(SDNode *N,
11701 SelectionDAG &DAG) const {
11702 SDLoc DL(N);
11703 EVT VT = N->getValueType(ResNo: 0);
11704 SDValue SourceValue = N->getOperand(Num: 0);
11705 SDValue SinkValue = N->getOperand(Num: 1);
11706 SDValue EltSizeInBytes = N->getOperand(Num: 2);
11707
11708 // Note: The lane offset is scalable if the mask is scalable.
11709 ElementCount LaneOffsetEC =
11710 ElementCount::get(MinVal: N->getConstantOperandVal(Num: 3), Scalable: VT.isScalableVT());
11711
11712 EVT AddrVT = SourceValue->getValueType(ResNo: 0);
11713 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
11714
11715 EVT CmpVT =
11716 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: AddrVT);
11717
11718 // Unsigned compare: Source >= Sink.
11719 SDValue SourceAheadOfOrEqualToSink =
11720 DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETUGE);
11721
11722 // Take the difference between the pointers and divided by the element size,
11723 // to see how many lanes separate them.
11724 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: AddrVT, N1: SinkValue, N2: SourceValue);
11725
11726 // RAW_MASK: Diff = Source >= Sink ? (Source - Sink) : (Sink - Source)
11727 if (IsReadAfterWrite)
11728 Diff = DAG.getSelect(DL, VT: AddrVT, Cond: SourceAheadOfOrEqualToSink,
11729 LHS: DAG.getNegative(Val: Diff, DL, VT: AddrVT), RHS: Diff);
11730
11731 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: AddrVT, N1: Diff, N2: EltSizeInBytes);
11732
11733 // The pointers do not alias if:
11734 // - Source >= Sink (WAR_MASK)
11735 // - Source == Sink (RAW_MASK)
11736 SDValue NoAlias = SourceAheadOfOrEqualToSink;
11737 if (IsReadAfterWrite)
11738 NoAlias = DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETEQ);
11739
11740 // The pointers do not alias if:
11741 // Lane + LaneOffset < Diff (WAR/RAW_MASK)
11742 SDValue LaneOffset = DAG.getElementCount(DL, VT: AddrVT, EC: LaneOffsetEC);
11743 SDValue MaskN = DAG.getSelect(
11744 DL, VT: AddrVT, Cond: NoAlias,
11745 LHS: DAG.getConstant(Val: APInt::getMaxValue(numBits: AddrVT.getScalarSizeInBits()), DL,
11746 VT: AddrVT),
11747 RHS: Diff);
11748
11749 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT, N1: LaneOffset, N2: MaskN);
11750}
11751
11752SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
11753 bool IsNegative) const {
11754 SDLoc dl(N);
11755 EVT VT = N->getValueType(ResNo: 0);
11756 SDValue Op = N->getOperand(Num: 0);
11757
11758 // If expanding ABS_MIN_POISON, fall back to ABS if the target supports it.
11759 if (N->getOpcode() == ISD::ABS_MIN_POISON &&
11760 isOperationLegalOrCustom(Op: ISD::ABS, VT)) {
11761 SDValue AbsVal = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT, Operand: Op);
11762 if (IsNegative)
11763 return DAG.getNegative(Val: AbsVal, DL: dl, VT);
11764 return AbsVal;
11765 }
11766
11767 // abs(x) -> smax(x,sub(0,x))
11768 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11769 isOperationLegal(Op: ISD::SMAX, VT)) {
11770 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11771 Op = DAG.getFreeze(V: Op);
11772 return DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: Op,
11773 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11774 }
11775
11776 // abs(x) -> umin(x,sub(0,x))
11777 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11778 isOperationLegal(Op: ISD::UMIN, VT)) {
11779 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11780 Op = DAG.getFreeze(V: Op);
11781 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: Op,
11782 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11783 }
11784
11785 // 0 - abs(x) -> smin(x, sub(0,x))
11786 if (IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11787 isOperationLegal(Op: ISD::SMIN, VT)) {
11788 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11789 Op = DAG.getFreeze(V: Op);
11790 return DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: Op,
11791 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11792 }
11793
11794 // Only expand vector types if we have the appropriate vector operations.
11795 if (VT.isVector() &&
11796 (!isOperationLegalOrCustom(Op: ISD::SRA, VT) ||
11797 (!IsNegative && !isOperationLegalOrCustom(Op: ISD::ADD, VT)) ||
11798 (IsNegative && !isOperationLegalOrCustom(Op: ISD::SUB, VT)) ||
11799 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11800 return SDValue();
11801
11802 Op = DAG.getFreeze(V: Op);
11803 SDValue Shift = DAG.getNode(
11804 Opcode: ISD::SRA, DL: dl, VT, N1: Op,
11805 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11806 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Shift);
11807
11808 // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
11809 if (!IsNegative)
11810 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Shift);
11811
11812 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
11813 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Shift, N2: Xor);
11814}
11815
11816SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
11817 SDLoc dl(N);
11818 EVT VT = N->getValueType(ResNo: 0);
11819 SDValue LHS = N->getOperand(Num: 0);
11820 SDValue RHS = N->getOperand(Num: 1);
11821 bool IsSigned = N->getOpcode() == ISD::ABDS;
11822
11823 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
11824 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
11825 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
11826 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
11827 if (isOperationLegal(Op: MaxOpc, VT) && isOperationLegal(Op: MinOpc, VT)) {
11828 LHS = DAG.getFreeze(V: LHS);
11829 RHS = DAG.getFreeze(V: RHS);
11830 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
11831 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
11832 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
11833 }
11834
11835 // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
11836 if (!IsSigned && isOperationLegal(Op: ISD::USUBSAT, VT)) {
11837 LHS = DAG.getFreeze(V: LHS);
11838 RHS = DAG.getFreeze(V: RHS);
11839 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT,
11840 N1: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: LHS, N2: RHS),
11841 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: RHS, N2: LHS));
11842 }
11843
11844 // abds: if the sub is nsw -> abs(sub)
11845 // abdu with non-negative operands -> abs(sub)
11846 // The abdu case where the sub is nuw is handled in DAGCombine.
11847 if (IsSigned) {
11848 if (DAG.willNotOverflowSub(/*IsSigned=*/true, N0: LHS, N1: RHS))
11849 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11850 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS));
11851
11852 if (DAG.willNotOverflowSub(/*IsSigned=*/true, N0: RHS, N1: LHS))
11853 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11854 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11855 } else if (DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS)) {
11856 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11857 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS));
11858 }
11859
11860 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11861 ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
11862 LHS = DAG.getFreeze(V: LHS);
11863 RHS = DAG.getFreeze(V: RHS);
11864 SDValue Cmp = DAG.getSetCC(DL: dl, VT: CCVT, LHS, RHS, Cond: CC);
11865
11866 // Branchless expansion iff cmp result is allbits:
11867 // abds(lhs, rhs) -> sub(sgt(lhs, rhs), xor(sgt(lhs, rhs), sub(lhs, rhs)))
11868 // abdu(lhs, rhs) -> sub(ugt(lhs, rhs), xor(ugt(lhs, rhs), sub(lhs, rhs)))
11869 if (CCVT == VT && getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
11870 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
11871 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Diff, N2: Cmp);
11872 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Cmp, N2: Xor);
11873 }
11874
11875 // Similar to the branchless expansion, if we don't prefer selects, use the
11876 // (sign-extended) usubo overflow flag if the (scalar) type is illegal as this
11877 // is more likely to legalize cleanly: abdu(lhs, rhs) -> sub(xor(sub(lhs,
11878 // rhs), uof(lhs, rhs)), uof(lhs, rhs))
11879 if (!IsSigned && VT.isScalarInteger() && !isTypeLegal(VT) &&
11880 !preferSelectsOverBooleanArithmetic(VT)) {
11881 SDValue USubO =
11882 DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {LHS, RHS});
11883 SDValue Cmp = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT, Operand: USubO.getValue(R: 1));
11884 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: USubO.getValue(R: 0), N2: Cmp);
11885 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Cmp);
11886 }
11887
11888 // FIXME: Should really try to split the vector in case it's legal on a
11889 // subvector.
11890 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
11891 return DAG.UnrollVectorOp(N);
11892
11893 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11894 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11895 return DAG.getSelect(DL: dl, VT, Cond: Cmp, LHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS),
11896 RHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11897}
11898
11899SDValue TargetLowering::expandAVG(SDNode *N, SelectionDAG &DAG) const {
11900 SDLoc dl(N);
11901 EVT VT = N->getValueType(ResNo: 0);
11902 SDValue LHS = N->getOperand(Num: 0);
11903 SDValue RHS = N->getOperand(Num: 1);
11904
11905 unsigned Opc = N->getOpcode();
11906 bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
11907 bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
11908 unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
11909 unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
11910 unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
11911 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
11912 assert((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS ||
11913 Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
11914 "Unknown AVG node");
11915
11916 // If the operands are already extended, we can add+shift.
11917 bool IsExt =
11918 (IsSigned && DAG.ComputeNumSignBits(Op: LHS) >= 2 &&
11919 DAG.ComputeNumSignBits(Op: RHS) >= 2) ||
11920 (!IsSigned && DAG.computeKnownBits(Op: LHS).countMinLeadingZeros() >= 1 &&
11921 DAG.computeKnownBits(Op: RHS).countMinLeadingZeros() >= 1);
11922 if (IsExt) {
11923 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
11924 if (!IsFloor)
11925 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Sum, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11926 return DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Sum,
11927 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11928 }
11929
11930 // For scalars, see if we can efficiently extend/truncate to use add+shift.
11931 if (VT.isScalarInteger()) {
11932 EVT ExtVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
11933 if (isTypeLegal(VT: ExtVT) && isTruncateFree(FromVT: ExtVT, ToVT: VT)) {
11934 LHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: LHS);
11935 RHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: RHS);
11936 SDValue Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: LHS, N2: RHS);
11937 if (!IsFloor)
11938 Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: Avg,
11939 N2: DAG.getConstant(Val: 1, DL: dl, VT: ExtVT));
11940 // Just use SRL as we will be truncating away the extended sign bits.
11941 Avg = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ExtVT, N1: Avg,
11942 N2: DAG.getShiftAmountConstant(Val: 1, VT: ExtVT, DL: dl));
11943 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Avg);
11944 }
11945 }
11946
11947 // avgflooru(lhs, rhs) -> or(lshr(add(lhs, rhs),1),shl(overflow, typesize-1))
11948 if (Opc == ISD::AVGFLOORU && VT.isScalarInteger() && !isTypeLegal(VT) &&
11949 isOperationLegalOrCustom(
11950 Op: ISD::UADDO, VT: getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11951 SDValue UAddWithOverflow =
11952 DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {RHS, LHS});
11953
11954 SDValue Sum = UAddWithOverflow.getValue(R: 0);
11955 SDValue Overflow = UAddWithOverflow.getValue(R: 1);
11956
11957 // Right shift the sum by 1
11958 SDValue LShrVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Sum,
11959 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11960
11961 SDValue ZeroExtOverflow = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Overflow);
11962 SDValue OverflowShl = DAG.getNode(
11963 Opcode: ISD::SHL, DL: dl, VT, N1: ZeroExtOverflow,
11964 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11965
11966 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: LShrVal, N2: OverflowShl);
11967 }
11968
11969 // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
11970 // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
11971 // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
11972 // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
11973 LHS = DAG.getFreeze(V: LHS);
11974 RHS = DAG.getFreeze(V: RHS);
11975 SDValue Sign = DAG.getNode(Opcode: SignOpc, DL: dl, VT, N1: LHS, N2: RHS);
11976 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
11977 SDValue Shift =
11978 DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11979 return DAG.getNode(Opcode: SumOpc, DL: dl, VT, N1: Sign, N2: Shift);
11980}
11981
11982SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
11983 SDLoc dl(N);
11984 EVT VT = N->getValueType(ResNo: 0);
11985 SDValue Op = N->getOperand(Num: 0);
11986
11987 if (!VT.isSimple())
11988 return SDValue();
11989
11990 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11991 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
11992 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
11993 default:
11994 return SDValue();
11995 case MVT::i16:
11996 // Use a rotate by 8. This can be further expanded if necessary.
11997 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11998 case MVT::i32:
11999 // This is meant for ARM specifically, which has ROTR but no ROTL.
12000 // t = x ^ rotr(x, 16)
12001 // t = bic(t, 0x00ff0000)
12002 // t = lshr(t, 8)
12003 // x = t ^ rotr(x, 8)
12004 if (isOperationLegalOrCustom(Op: ISD::ROTR, VT)) {
12005 SDValue Rotr16 =
12006 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 16, DL: dl, VT: SHVT));
12007 SDValue Tmp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Rotr16);
12008 Tmp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp,
12009 N2: DAG.getConstant(Val: 0xFF00FFFF, DL: dl, VT));
12010 Tmp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
12011 SDValue Rotr8 =
12012 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
12013 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Tmp, N2: Rotr8);
12014 }
12015 Tmp4 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
12016 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
12017 N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
12018 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
12019 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
12020 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
12021 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
12022 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
12023 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
12024 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
12025 case MVT::i64:
12026 Tmp8 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
12027 Tmp7 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
12028 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
12029 Tmp7 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp7, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
12030 Tmp6 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
12031 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
12032 Tmp6 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp6, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
12033 Tmp5 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
12034 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
12035 Tmp5 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp5, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
12036 Tmp4 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
12037 Tmp4 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp4,
12038 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
12039 Tmp3 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
12040 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp3,
12041 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
12042 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
12043 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2,
12044 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
12045 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
12046 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp7);
12047 Tmp6 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp6, N2: Tmp5);
12048 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
12049 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
12050 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp6);
12051 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
12052 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp4);
12053 }
12054}
12055
12056SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
12057 SDLoc dl(N);
12058 EVT VT = N->getValueType(ResNo: 0);
12059 SDValue Op = N->getOperand(Num: 0);
12060 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
12061 unsigned Sz = VT.getScalarSizeInBits();
12062
12063 SDValue Tmp, Tmp2, Tmp3;
12064
12065 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
12066 // and finally the i1 pairs.
12067 // TODO: We can easily support i4/i2 legal types if any target ever does.
12068 if (Sz >= 8 && isPowerOf2_32(Value: Sz)) {
12069 // Create the masks - repeating the pattern every byte.
12070 APInt Mask4 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x0F));
12071 APInt Mask2 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x33));
12072 APInt Mask1 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x55));
12073
12074 // BSWAP if the type is wider than a single byte.
12075 Tmp = (Sz > 8 ? DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT, Operand: Op) : Op);
12076
12077 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
12078 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
12079 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
12080 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
12081 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
12082 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
12083
12084 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
12085 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
12086 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
12087 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
12088 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
12089 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
12090
12091 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
12092 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
12093 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
12094 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
12095 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
12096 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
12097 return Tmp;
12098 }
12099
12100 Tmp = DAG.getConstant(Val: 0, DL: dl, VT);
12101 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
12102 if (I < J)
12103 Tmp2 =
12104 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: J - I, DL: dl, VT: SHVT));
12105 else
12106 Tmp2 =
12107 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: I - J, DL: dl, VT: SHVT));
12108
12109 APInt Shift = APInt::getOneBitSet(numBits: Sz, BitNo: J);
12110 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Shift, DL: dl, VT));
12111 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp, N2: Tmp2);
12112 }
12113
12114 return Tmp;
12115}
12116
12117std::pair<SDValue, SDValue>
12118TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
12119 SelectionDAG &DAG) const {
12120 SDLoc SL(LD);
12121 SDValue Chain = LD->getChain();
12122 SDValue BasePTR = LD->getBasePtr();
12123 EVT SrcVT = LD->getMemoryVT();
12124 EVT DstVT = LD->getValueType(ResNo: 0);
12125 ISD::LoadExtType ExtType = LD->getExtensionType();
12126
12127 if (SrcVT.isScalableVector())
12128 report_fatal_error(reason: "Cannot scalarize scalable vector loads");
12129
12130 unsigned NumElem = SrcVT.getVectorNumElements();
12131
12132 EVT SrcEltVT = SrcVT.getScalarType();
12133 EVT DstEltVT = DstVT.getScalarType();
12134
12135 // A vector must always be stored in memory as-is, i.e. without any padding
12136 // between the elements, since various code depend on it, e.g. in the
12137 // handling of a bitcast of a vector type to int, which may be done with a
12138 // vector store followed by an integer load. A vector that does not have
12139 // elements that are byte-sized must therefore be stored as an integer
12140 // built out of the extracted vector elements.
12141 if (!SrcEltVT.isByteSized()) {
12142 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
12143 EVT LoadVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumLoadBits);
12144
12145 unsigned NumSrcBits = SrcVT.getSizeInBits();
12146 EVT SrcIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumSrcBits);
12147
12148 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
12149 SDValue SrcEltBitMask = DAG.getConstant(
12150 Val: APInt::getLowBitsSet(numBits: NumLoadBits, loBitsSet: SrcEltBits), DL: SL, VT: LoadVT);
12151
12152 // Load the whole vector and avoid masking off the top bits as it makes
12153 // the codegen worse.
12154 SDValue Load = DAG.getExtLoad(
12155 ExtType: ISD::EXTLOAD, dl: SL, VT: LoadVT, Chain, Ptr: BasePTR, PtrInfo: LD->getPointerInfo(),
12156 MemVT: SrcIntVT, Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
12157 Metadata: LD->getMMOMetadataForSubAccess());
12158
12159 SmallVector<SDValue, 8> Vals;
12160 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12161 unsigned ShiftIntoIdx =
12162 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
12163 SDValue ShiftAmount = DAG.getShiftAmountConstant(
12164 Val: ShiftIntoIdx * SrcEltVT.getSizeInBits(), VT: LoadVT, DL: SL);
12165 SDValue ShiftedElt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: LoadVT, N1: Load, N2: ShiftAmount);
12166 SDValue Elt =
12167 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: LoadVT, N1: ShiftedElt, N2: SrcEltBitMask);
12168 SDValue Scalar = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: SrcEltVT, Operand: Elt);
12169
12170 if (ExtType != ISD::NON_EXTLOAD) {
12171 unsigned ExtendOp = ISD::getExtForLoadExtType(IsFP: false, ExtType);
12172 Scalar = DAG.getNode(Opcode: ExtendOp, DL: SL, VT: DstEltVT, Operand: Scalar);
12173 }
12174
12175 Vals.push_back(Elt: Scalar);
12176 }
12177
12178 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
12179 return std::make_pair(x&: Value, y: Load.getValue(R: 1));
12180 }
12181
12182 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
12183 assert(SrcEltVT.isByteSized());
12184
12185 SmallVector<SDValue, 8> Vals;
12186 SmallVector<SDValue, 8> LoadChains;
12187
12188 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12189 SDValue ScalarLoad = DAG.getExtLoad(
12190 ExtType, dl: SL, VT: DstEltVT, Chain, Ptr: BasePTR,
12191 PtrInfo: LD->getPointerInfo().getWithOffset(O: Idx * Stride), MemVT: SrcEltVT,
12192 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
12193 Metadata: LD->getMMOMetadataForSubAccess());
12194
12195 BasePTR = DAG.getObjectPtrOffset(SL, Ptr: BasePTR, Offset: TypeSize::getFixed(ExactSize: Stride));
12196
12197 Vals.push_back(Elt: ScalarLoad.getValue(R: 0));
12198 LoadChains.push_back(Elt: ScalarLoad.getValue(R: 1));
12199 }
12200
12201 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: LoadChains);
12202 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
12203
12204 return std::make_pair(x&: Value, y&: NewChain);
12205}
12206
12207SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
12208 SelectionDAG &DAG) const {
12209 SDLoc SL(ST);
12210
12211 SDValue Chain = ST->getChain();
12212 SDValue BasePtr = ST->getBasePtr();
12213 SDValue Value = ST->getValue();
12214 EVT StVT = ST->getMemoryVT();
12215
12216 if (StVT.isScalableVector())
12217 report_fatal_error(reason: "Cannot scalarize scalable vector stores");
12218
12219 // The type of the data we want to save
12220 EVT RegVT = Value.getValueType();
12221 EVT RegSclVT = RegVT.getScalarType();
12222
12223 // The type of data as saved in memory.
12224 EVT MemSclVT = StVT.getScalarType();
12225
12226 unsigned NumElem = StVT.getVectorNumElements();
12227
12228 // A vector must always be stored in memory as-is, i.e. without any padding
12229 // between the elements, since various code depend on it, e.g. in the
12230 // handling of a bitcast of a vector type to int, which may be done with a
12231 // vector store followed by an integer load. A vector that does not have
12232 // elements that are byte-sized must therefore be stored as an integer
12233 // built out of the extracted vector elements.
12234 if (!MemSclVT.isByteSized()) {
12235 unsigned NumBits = StVT.getSizeInBits();
12236 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
12237
12238 SDValue CurrVal = DAG.getConstant(Val: 0, DL: SL, VT: IntVT);
12239
12240 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12241 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
12242 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MemSclVT, Operand: Elt);
12243 SDValue ExtElt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: IntVT, Operand: Trunc);
12244 unsigned ShiftIntoIdx =
12245 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
12246 SDValue ShiftAmount =
12247 DAG.getConstant(Val: ShiftIntoIdx * MemSclVT.getSizeInBits(), DL: SL, VT: IntVT);
12248 SDValue ShiftedElt =
12249 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT, N1: ExtElt, N2: ShiftAmount);
12250 CurrVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: CurrVal, N2: ShiftedElt);
12251 }
12252
12253 return DAG.getStore(Chain, dl: SL, Val: CurrVal, Ptr: BasePtr, PtrInfo: ST->getPointerInfo(),
12254 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12255 Metadata: ST->getMMOMetadataForSubAccess());
12256 }
12257
12258 // Store Stride in bytes
12259 unsigned Stride = MemSclVT.getSizeInBits() / 8;
12260 assert(Stride && "Zero stride!");
12261 // Extract each of the elements from the original vector and save them into
12262 // memory individually.
12263 SmallVector<SDValue, 8> Stores;
12264 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12265 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
12266
12267 SDValue Ptr =
12268 DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Idx * Stride));
12269
12270 // This scalar TruncStore may be illegal, but we legalize it later.
12271 SDValue Store = DAG.getTruncStore(
12272 Chain, dl: SL, Val: Elt, Ptr, PtrInfo: ST->getPointerInfo().getWithOffset(O: Idx * Stride),
12273 SVT: MemSclVT, Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12274 Metadata: ST->getMMOMetadataForSubAccess());
12275
12276 Stores.push_back(Elt: Store);
12277 }
12278
12279 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: Stores);
12280}
12281
12282std::pair<SDValue, SDValue>
12283TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
12284 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
12285 "unaligned indexed loads not implemented!");
12286 SDValue Chain = LD->getChain();
12287 SDValue Ptr = LD->getBasePtr();
12288 EVT VT = LD->getValueType(ResNo: 0);
12289 EVT LoadedVT = LD->getMemoryVT();
12290 SDLoc dl(LD);
12291 auto &MF = DAG.getMachineFunction();
12292
12293 if (VT.isFloatingPoint() || VT.isVector()) {
12294 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadedVT.getSizeInBits());
12295 if (isTypeLegal(VT: intVT) && isTypeLegal(VT: LoadedVT)) {
12296 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: intVT) &&
12297 LoadedVT.isVector()) {
12298 // Scalarize the load and let the individual components be handled.
12299 return scalarizeVectorLoad(LD, DAG);
12300 }
12301
12302 // Expand to a (misaligned) integer load of the same size,
12303 // then bitconvert to floating point or vector.
12304 SDValue newLoad = DAG.getLoad(VT: intVT, dl, Chain, Ptr,
12305 MMO: LD->getMemOperand());
12306 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoadedVT, Operand: newLoad);
12307 if (LoadedVT != VT)
12308 Result = DAG.getNode(Opcode: VT.isFloatingPoint() ? ISD::FP_EXTEND :
12309 ISD::ANY_EXTEND, DL: dl, VT, Operand: Result);
12310
12311 return std::make_pair(x&: Result, y: newLoad.getValue(R: 1));
12312 }
12313
12314 // Copy the value to a (aligned) stack slot using (unaligned) integer
12315 // loads and stores, then do a (aligned) load from the stack slot.
12316 MVT RegVT = getRegisterType(Context&: *DAG.getContext(), VT: intVT);
12317 unsigned LoadedBytes = LoadedVT.getStoreSize();
12318 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12319 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
12320
12321 // Make sure the stack slot is also aligned for the register type.
12322 SDValue StackBase = DAG.CreateStackTemporary(VT1: LoadedVT, VT2: RegVT);
12323 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackBase.getNode())->getIndex();
12324 SmallVector<SDValue, 8> Stores;
12325 SDValue StackPtr = StackBase;
12326 unsigned Offset = 0;
12327
12328 EVT PtrVT = Ptr.getValueType();
12329 EVT StackPtrVT = StackPtr.getValueType();
12330
12331 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12332 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12333
12334 // Do all but one copies using the full register width.
12335 for (unsigned i = 1; i < NumRegs; i++) {
12336 // Load one integer register's worth from the original location.
12337 SDValue Load = DAG.getLoad(
12338 VT: RegVT, dl, Chain, Ptr, PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset),
12339 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
12340 Metadata: LD->getMMOMetadataForSubAccess());
12341 // Follow the load with a store to the stack slot. Remember the store.
12342 Stores.push_back(Elt: DAG.getStore(
12343 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12344 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset)));
12345 // Increment the pointers.
12346 Offset += RegBytes;
12347
12348 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12349 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12350 }
12351
12352 // The last copy may be partial. Do an extending load.
12353 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12354 BitWidth: 8 * (LoadedBytes - Offset));
12355 SDValue Load = DAG.getExtLoad(
12356 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain, Ptr,
12357 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT, Alignment: LD->getBaseAlign(),
12358 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getMMOMetadataForSubAccess());
12359 // Follow the load with a store to the stack slot. Remember the store.
12360 // On big-endian machines this requires a truncating store to ensure
12361 // that the bits end up in the right place.
12362 Stores.push_back(Elt: DAG.getTruncStore(
12363 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12364 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), SVT: MemVT));
12365
12366 // The order of the stores doesn't matter - say it with a TokenFactor.
12367 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12368
12369 // Finally, perform the original load only redirected to the stack slot.
12370 Load = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT, Chain: TF, Ptr: StackBase,
12371 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0),
12372 MemVT: LoadedVT);
12373
12374 // Callers expect a MERGE_VALUES node.
12375 return std::make_pair(x&: Load, y&: TF);
12376 }
12377
12378 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
12379 "Unaligned load of unsupported type.");
12380
12381 // Compute the new VT that is half the size of the old one. This is an
12382 // integer MVT.
12383 unsigned NumBits = LoadedVT.getSizeInBits();
12384 EVT NewLoadedVT;
12385 NewLoadedVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits/2);
12386 NumBits >>= 1;
12387
12388 Align Alignment = LD->getBaseAlign();
12389 unsigned IncrementSize = NumBits / 8;
12390 ISD::LoadExtType HiExtType = LD->getExtensionType();
12391
12392 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
12393 if (HiExtType == ISD::NON_EXTLOAD)
12394 HiExtType = ISD::ZEXTLOAD;
12395
12396 // Load the value in two parts
12397 SDValue Lo, Hi;
12398 if (DAG.getDataLayout().isLittleEndian()) {
12399 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12400 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12401 Metadata: LD->getMMOMetadataForSubAccess());
12402
12403 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12404 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr,
12405 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12406 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12407 Metadata: LD->getMMOMetadataForSubAccess());
12408 } else {
12409 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12410 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12411 Metadata: LD->getMMOMetadataForSubAccess());
12412
12413 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12414 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
12415 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12416 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12417 Metadata: LD->getMMOMetadataForSubAccess());
12418 }
12419
12420 // aggregate the two parts
12421 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: NumBits, VT, DL: dl);
12422 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: ShiftAmount);
12423 Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Result, N2: Lo);
12424
12425 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
12426 N2: Hi.getValue(R: 1));
12427
12428 return std::make_pair(x&: Result, y&: TF);
12429}
12430
12431SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
12432 SelectionDAG &DAG) const {
12433 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
12434 "unaligned indexed stores not implemented!");
12435 SDValue Chain = ST->getChain();
12436 SDValue Ptr = ST->getBasePtr();
12437 SDValue Val = ST->getValue();
12438 EVT VT = Val.getValueType();
12439 Align Alignment = ST->getBaseAlign();
12440 auto &MF = DAG.getMachineFunction();
12441 EVT StoreMemVT = ST->getMemoryVT();
12442
12443 SDLoc dl(ST);
12444 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
12445 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
12446 if (isTypeLegal(VT: intVT)) {
12447 if (!isOperationLegalOrCustom(Op: ISD::STORE, VT: intVT) &&
12448 StoreMemVT.isVector()) {
12449 // Scalarize the store and let the individual components be handled.
12450 SDValue Result = scalarizeVectorStore(ST, DAG);
12451 return Result;
12452 }
12453 // Expand to a bitconvert of the value to the integer type of the
12454 // same size, then a (misaligned) int store.
12455 // FIXME: Does not handle truncating floating point stores!
12456 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: intVT, Operand: Val);
12457 Result = DAG.getStore(Chain, dl, Val: Result, Ptr, PtrInfo: ST->getPointerInfo(),
12458 Alignment, MMOFlags: ST->getMemOperand()->getFlags(),
12459 Metadata: ST->getMMOMetadataForSubAccess());
12460 return Result;
12461 }
12462 // Do a (aligned) store to a stack slot, then copy from the stack slot
12463 // to the final destination using (unaligned) integer loads and stores.
12464 MVT RegVT = getRegisterType(
12465 Context&: *DAG.getContext(),
12466 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreMemVT.getSizeInBits()));
12467 EVT PtrVT = Ptr.getValueType();
12468 unsigned StoredBytes = StoreMemVT.getStoreSize();
12469 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12470 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
12471
12472 // Make sure the stack slot is also aligned for the register type.
12473 SDValue StackPtr = DAG.CreateStackTemporary(VT1: StoreMemVT, VT2: RegVT);
12474 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
12475
12476 // Perform the original store, only redirected to the stack slot.
12477 SDValue Store = DAG.getTruncStore(
12478 Chain, dl, Val, Ptr: StackPtr,
12479 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0), SVT: StoreMemVT);
12480
12481 EVT StackPtrVT = StackPtr.getValueType();
12482
12483 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12484 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12485 SmallVector<SDValue, 8> Stores;
12486 unsigned Offset = 0;
12487
12488 // Do all but one copies using the full register width.
12489 for (unsigned i = 1; i < NumRegs; i++) {
12490 // Load one integer register's worth from the stack slot.
12491 SDValue Load = DAG.getLoad(
12492 VT: RegVT, dl, Chain: Store, Ptr: StackPtr,
12493 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset));
12494 // Store it to the final location. Remember the store.
12495 Stores.push_back(Elt: DAG.getStore(
12496 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12497 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), Alignment: ST->getBaseAlign(),
12498 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess()));
12499 // Increment the pointers.
12500 Offset += RegBytes;
12501 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12502 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12503 }
12504
12505 // The last store may be partial. Do a truncating store. On big-endian
12506 // machines this requires an extending load from the stack slot to ensure
12507 // that the bits are in the right place.
12508 EVT LoadMemVT =
12509 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: 8 * (StoredBytes - Offset));
12510
12511 // Load from the stack slot.
12512 SDValue Load = DAG.getExtLoad(
12513 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain: Store, Ptr: StackPtr,
12514 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), MemVT: LoadMemVT);
12515
12516 Stores.push_back(
12517 Elt: DAG.getTruncStore(Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12518 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), SVT: LoadMemVT,
12519 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12520 Metadata: ST->getMMOMetadataForSubAccess()));
12521 // The order of the stores doesn't matter - say it with a TokenFactor.
12522 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12523 return Result;
12524 }
12525
12526 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
12527 "Unaligned store of unknown type.");
12528 // Get the half-size VT
12529 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
12530 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
12531 unsigned IncrementSize = NumBits / 8;
12532
12533 // Divide the stored value in two parts.
12534 SDValue ShiftAmount =
12535 DAG.getShiftAmountConstant(Val: NumBits, VT: Val.getValueType(), DL: dl);
12536 SDValue Lo = Val;
12537 // If Val is a constant, replace the upper bits with 0. The SRL will constant
12538 // fold and not use the upper bits. A smaller constant may be easier to
12539 // materialize.
12540 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Lo); C && !C->isOpaque())
12541 Lo = DAG.getNode(
12542 Opcode: ISD::AND, DL: dl, VT, N1: Lo,
12543 N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: NumBits), DL: dl,
12544 VT));
12545 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Val, N2: ShiftAmount);
12546
12547 // Store the two parts
12548 SDValue Store1, Store2;
12549 Store1 = DAG.getTruncStore(
12550 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Lo : Hi, Ptr,
12551 PtrInfo: ST->getPointerInfo(), SVT: NewStoredVT, Alignment,
12552 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess());
12553
12554 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12555 Store2 = DAG.getTruncStore(
12556 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
12557 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize), SVT: NewStoredVT, Alignment,
12558 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess());
12559
12560 SDValue Result =
12561 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
12562 return Result;
12563}
12564
12565SDValue
12566TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
12567 const SDLoc &DL, EVT DataVT,
12568 SelectionDAG &DAG,
12569 bool IsCompressedMemory) const {
12570 SDValue Increment;
12571 EVT AddrVT = Addr.getValueType();
12572 EVT MaskVT = Mask.getValueType();
12573 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
12574 "Incompatible types of Data and Mask");
12575 if (IsCompressedMemory) {
12576 // Incrementing the pointer according to number of '1's in the mask.
12577 if (DataVT.isScalableVector()) {
12578 EVT MaskExtVT = MaskVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
12579 SDValue MaskExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MaskExtVT, Operand: Mask);
12580 Increment = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: MaskExt);
12581 } else {
12582 EVT MaskIntVT =
12583 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MaskVT.getSizeInBits());
12584 SDValue MaskInIntReg = DAG.getBitcast(VT: MaskIntVT, V: Mask);
12585 if (MaskIntVT.getSizeInBits() < 32) {
12586 MaskInIntReg =
12587 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: MaskInIntReg);
12588 MaskIntVT = MVT::i32;
12589 }
12590 Increment = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MaskIntVT, Operand: MaskInIntReg);
12591 }
12592 // Scale is an element size in bytes.
12593 SDValue Scale = DAG.getConstant(Val: DataVT.getScalarSizeInBits() / 8, DL,
12594 VT: AddrVT);
12595 Increment = DAG.getZExtOrTrunc(Op: Increment, DL, VT: AddrVT);
12596 Increment = DAG.getNode(Opcode: ISD::MUL, DL, VT: AddrVT, N1: Increment, N2: Scale);
12597 } else
12598 Increment = DAG.getTypeSize(DL, VT: AddrVT, TS: DataVT.getStoreSize());
12599
12600 return DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: Addr, N2: Increment);
12601}
12602
12603static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
12604 EVT VecVT, const SDLoc &DL,
12605 ElementCount SubEC) {
12606 assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
12607 "Cannot index a scalable vector within a fixed-width vector");
12608 ElementCount NElts = VecVT.getVectorElementCount();
12609 EVT IdxVT = Idx.getValueType();
12610
12611 // Attempt to prove all accessed lanes sit within VecVT.
12612 ElementCount IdxAsEC;
12613 if (sd_match(N: Idx, P: m_ElementCount(V&: IdxAsEC))) {
12614 if (SubEC.isScalar() && ElementCount::isKnownLT(LHS: IdxAsEC, RHS: NElts))
12615 return Idx;
12616 // Calculate end index as TypeSize to remove possibility of overflow.
12617 if (IdxAsEC.isScalable() == SubEC.isScalable() &&
12618 TypeSize::isKnownLE(LHS: TypeSize(IdxAsEC) + TypeSize(SubEC),
12619 RHS: TypeSize(NElts)))
12620 return Idx;
12621 }
12622
12623 // Can we mask Idx rather than clamp it.
12624 if (SubEC.isScalar() && NElts.isPowerOf2()) {
12625 SDValue EC = DAG.getElementCount(DL, VT: IdxVT, EC: NElts);
12626 SDValue Mask =
12627 DAG.getNode(Opcode: ISD::SUB, DL, VT: IdxVT, N1: EC, N2: DAG.getConstant(Val: 1, DL, VT: IdxVT));
12628 return DAG.getNode(Opcode: ISD::AND, DL, VT: IdxVT, N1: Idx, N2: Mask);
12629 }
12630
12631 unsigned SubOpcode =
12632 ElementCount::isKnownLE(LHS: SubEC, RHS: NElts) ? ISD::SUB : ISD::USUBSAT;
12633 SDValue MaxSafeIndex =
12634 DAG.getNode(Opcode: SubOpcode, DL, VT: IdxVT, N1: DAG.getElementCount(DL, VT: IdxVT, EC: NElts),
12635 N2: DAG.getElementCount(DL, VT: IdxVT, EC: SubEC));
12636 return DAG.getNode(Opcode: ISD::UMIN, DL, VT: IdxVT, N1: Idx, N2: MaxSafeIndex);
12637}
12638
12639SDValue
12640TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr,
12641 EVT VecVT, SDValue Index,
12642 const SDNodeFlags PtrArithFlags) const {
12643 return getVectorSubVecPointer(
12644 DAG, VecPtr, VecVT,
12645 SubVecVT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(), NumElements: 1),
12646 Index, PtrArithFlags);
12647}
12648
12649SDValue
12650TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr,
12651 EVT VecVT, EVT SubVecVT, SDValue Index,
12652 const SDNodeFlags PtrArithFlags) const {
12653 SDLoc dl(Index);
12654 // Make sure the index type is big enough to compute in.
12655 Index = DAG.getZExtOrTrunc(Op: Index, DL: dl, VT: VecPtr.getValueType());
12656
12657 EVT EltVT = VecVT.getVectorElementType();
12658
12659 // Calculate the element offset and add it to the pointer.
12660 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
12661 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
12662 "Converting bits to bytes lost precision");
12663 assert(SubVecVT.getVectorElementType() == EltVT &&
12664 "Sub-vector must be a vector with matching element type");
12665 assert((SubVecVT.getVectorElementCount().isScalar() ||
12666 isa<ConstantSDNode>(Index)) &&
12667 "Subvector Index must be constant!");
12668
12669 // An out-of-range index only makes the vector operation return poison, but
12670 // a load/store through the pointer computed below would be immediate UB, so
12671 // freeze the index before clamping it into range.
12672 if (!DAG.isGuaranteedNotToBePoison(Op: Index))
12673 Index = DAG.getFreeze(V: Index);
12674
12675 EVT IdxVT = Index.getValueType();
12676 if (SubVecVT.isScalableVector())
12677 Index =
12678 DAG.getVScale(DL: dl, VT: IdxVT, MulImm: cast<ConstantSDNode>(Val&: Index)->getAPIntValue());
12679
12680 Index = clampDynamicVectorIndex(DAG, Idx: Index, VecVT, DL: dl,
12681 SubEC: SubVecVT.getVectorElementCount());
12682 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12683 N2: DAG.getConstant(Val: EltSize, DL: dl, VT: IdxVT));
12684 return DAG.getMemBasePlusOffset(Base: VecPtr, Offset: Index, DL: dl, Flags: PtrArithFlags);
12685}
12686
12687//===----------------------------------------------------------------------===//
12688// Implementation of Emulated TLS Model
12689//===----------------------------------------------------------------------===//
12690
12691SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
12692 SelectionDAG &DAG) const {
12693 // Access to address of TLS varialbe xyz is lowered to a function call:
12694 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
12695 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12696 PointerType *VoidPtrType = PointerType::get(C&: *DAG.getContext(), AddressSpace: 0);
12697 SDLoc dl(GA);
12698
12699 ArgListTy Args;
12700 const GlobalValue *GV =
12701 cast<GlobalValue>(Val: GA->getGlobal()->stripPointerCastsAndAliases());
12702 SmallString<32> NameString("__emutls_v.");
12703 NameString += GV->getName();
12704 StringRef EmuTlsVarName(NameString);
12705 const GlobalVariable *EmuTlsVar =
12706 GV->getParent()->getNamedGlobal(Name: EmuTlsVarName);
12707 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
12708 Args.emplace_back(args: DAG.getGlobalAddress(GV: EmuTlsVar, DL: dl, VT: PtrVT), args&: VoidPtrType);
12709
12710 SDValue EmuTlsGetAddr = DAG.getExternalSymbol(Sym: "__emutls_get_address", VT: PtrVT);
12711
12712 TargetLowering::CallLoweringInfo CLI(DAG);
12713 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
12714 CLI.setLibCallee(CC: CallingConv::C, ResultType: VoidPtrType, Target: EmuTlsGetAddr, ArgsList: std::move(Args));
12715 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
12716
12717 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
12718 // At last for X86 targets, maybe good for other targets too?
12719 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
12720 MFI.setAdjustsStack(true); // Is this only for X86 target?
12721 MFI.setHasCalls(true);
12722
12723 assert((GA->getOffset() == 0) &&
12724 "Emulated TLS must have zero offset in GlobalAddressSDNode");
12725 return CallResult.first;
12726}
12727
12728SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
12729 SelectionDAG &DAG) const {
12730 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
12731 if (!isCtlzFast())
12732 return SDValue();
12733 ISD::CondCode CC = Op.getOperandCondCode(i: 2);
12734 SDLoc dl(Op);
12735 if (isNullConstant(V: Op.getOperand(i: 1)) && CC == ISD::SETEQ) {
12736 EVT VT = Op.getOperand(i: 0).getValueType();
12737 SDValue Zext = Op.getOperand(i: 0);
12738 if (VT.bitsLT(VT: MVT::i32)) {
12739 VT = MVT::i32;
12740 Zext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Op.getOperand(i: 0));
12741 }
12742 unsigned Log2b = Log2_32(Value: VT.getSizeInBits());
12743 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Zext);
12744 SDValue Scc = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Clz,
12745 N2: DAG.getConstant(Val: Log2b, DL: dl, VT: MVT::i32));
12746 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Scc);
12747 }
12748 return SDValue();
12749}
12750
12751SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
12752 SDValue Op0 = Node->getOperand(Num: 0);
12753 SDValue Op1 = Node->getOperand(Num: 1);
12754 EVT VT = Op0.getValueType();
12755 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12756 unsigned Opcode = Node->getOpcode();
12757 SDLoc DL(Node);
12758
12759 // If both sign bits are zero, flip UMIN/UMAX <-> SMIN/SMAX if legal.
12760 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(MinMaxOpc: Opcode);
12761 if (isOperationLegal(Op: AltOpcode, VT) && DAG.SignBitIsZero(Op: Op0) &&
12762 DAG.SignBitIsZero(Op: Op1))
12763 return DAG.getNode(Opcode: AltOpcode, DL, VT, N1: Op0, N2: Op1);
12764
12765 // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
12766 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(V: Op1, AllowUndefs: true) && BoolVT == VT &&
12767 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12768 Op0 = DAG.getFreeze(V: Op0);
12769 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
12770 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12771 N2: DAG.getSetCC(DL, VT, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ));
12772 }
12773
12774 // umin(x,y) -> sub(x,usubsat(x,y))
12775 // TODO: Missing freeze(Op0)?
12776 if (Opcode == ISD::UMIN && isOperationLegal(Op: ISD::SUB, VT) &&
12777 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12778 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12779 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op0, N2: Op1));
12780 }
12781
12782 // umax(x,y) -> add(x,usubsat(y,x))
12783 // TODO: Missing freeze(Op0)?
12784 if (Opcode == ISD::UMAX && isOperationLegal(Op: ISD::ADD, VT) &&
12785 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12786 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0,
12787 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op1, N2: Op0));
12788 }
12789
12790 // FIXME: Should really try to split the vector in case it's legal on a
12791 // subvector.
12792 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12793 return DAG.UnrollVectorOp(N: Node);
12794
12795 // Attempt to find an existing SETCC node that we can reuse.
12796 // TODO: Do we need a generic doesSETCCNodeExist?
12797 // TODO: Missing freeze(Op0)/freeze(Op1)?
12798 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
12799 ISD::CondCode PrefCommuteCC,
12800 ISD::CondCode AltCommuteCC) {
12801 SDVTList BoolVTList = DAG.getVTList(VT: BoolVT);
12802 for (ISD::CondCode CC : {PrefCC, AltCC}) {
12803 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12804 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12805 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12806 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12807 }
12808 }
12809 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
12810 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12811 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12812 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12813 return DAG.getSelect(DL, VT, Cond, LHS: Op1, RHS: Op0);
12814 }
12815 }
12816 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: PrefCC);
12817 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12818 };
12819
12820 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
12821 // -> Y = (A < B) ? B : A
12822 // -> Y = (A >= B) ? A : B
12823 // -> Y = (A <= B) ? B : A
12824 switch (Opcode) {
12825 case ISD::SMAX:
12826 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
12827 case ISD::SMIN:
12828 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
12829 case ISD::UMAX:
12830 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
12831 case ISD::UMIN:
12832 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
12833 }
12834
12835 llvm_unreachable("How did we get here?");
12836}
12837
12838SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
12839 unsigned Opcode = Node->getOpcode();
12840 SDValue LHS = Node->getOperand(Num: 0);
12841 SDValue RHS = Node->getOperand(Num: 1);
12842 EVT VT = LHS.getValueType();
12843 SDLoc dl(Node);
12844
12845 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
12846 assert(VT.isInteger() && "Expected operands to be integers");
12847
12848 // usub.sat(a, b) -> umax(a, b) - b
12849 if (Opcode == ISD::USUBSAT && isOperationLegal(Op: ISD::UMAX, VT)) {
12850 SDValue Max = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT, N1: LHS, N2: RHS);
12851 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: RHS);
12852 }
12853
12854 // usub.sat(a, 1) -> sub(a, zext(a != 0))
12855 // Prefer this on targets without legal/cost-effective overflow-carry nodes.
12856 if (Opcode == ISD::USUBSAT && isOneOrOneSplat(V: RHS) &&
12857 !isOperationLegalOrCustom(Op: ISD::USUBO_CARRY, VT)) {
12858 LHS = DAG.getFreeze(V: LHS);
12859 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12860 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12861 SDValue IsNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETNE);
12862 SDValue Subtrahend = DAG.getBoolExtOrTrunc(Op: IsNonZero, SL: dl, VT, OpVT: BoolVT);
12863 Subtrahend =
12864 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Subtrahend, N2: DAG.getConstant(Val: 1, DL: dl, VT));
12865 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: Subtrahend);
12866 }
12867
12868 // uadd.sat(a, b) -> umin(a, ~b) + b
12869 if (Opcode == ISD::UADDSAT && isOperationLegal(Op: ISD::UMIN, VT)) {
12870 SDValue InvRHS = DAG.getNOT(DL: dl, Val: RHS, VT);
12871 SDValue Min = DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: LHS, N2: InvRHS);
12872 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Min, N2: RHS);
12873 }
12874
12875 unsigned OverflowOp;
12876 switch (Opcode) {
12877 case ISD::SADDSAT:
12878 OverflowOp = ISD::SADDO;
12879 break;
12880 case ISD::UADDSAT:
12881 OverflowOp = ISD::UADDO;
12882 break;
12883 case ISD::SSUBSAT:
12884 OverflowOp = ISD::SSUBO;
12885 break;
12886 case ISD::USUBSAT:
12887 OverflowOp = ISD::USUBO;
12888 break;
12889 default:
12890 llvm_unreachable("Expected method to receive signed or unsigned saturation "
12891 "addition or subtraction node.");
12892 }
12893
12894 // FIXME: Should really try to split the vector in case it's legal on a
12895 // subvector.
12896 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12897 return DAG.UnrollVectorOp(N: Node);
12898
12899 unsigned BitWidth = LHS.getScalarValueSizeInBits();
12900 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12901 SDValue Result = DAG.getNode(Opcode: OverflowOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12902 SDValue SumDiff = Result.getValue(R: 0);
12903 SDValue Overflow = Result.getValue(R: 1);
12904 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12905 SDValue AllOnes = DAG.getAllOnesConstant(DL: dl, VT);
12906
12907 if (Opcode == ISD::UADDSAT) {
12908 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12909 // (LHS + RHS) | OverflowMask
12910 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12911 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SumDiff, N2: OverflowMask);
12912 }
12913 // Overflow ? 0xffff.... : (LHS + RHS)
12914 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: AllOnes, RHS: SumDiff);
12915 }
12916
12917 if (Opcode == ISD::USUBSAT) {
12918 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12919 // (LHS - RHS) & ~OverflowMask
12920 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12921 SDValue Not = DAG.getNOT(DL: dl, Val: OverflowMask, VT);
12922 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SumDiff, N2: Not);
12923 }
12924 // Overflow ? 0 : (LHS - RHS)
12925 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Zero, RHS: SumDiff);
12926 }
12927
12928 assert((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
12929 "Expected signed saturating add/sub opcode");
12930
12931 const APInt MinVal = APInt::getSignedMinValue(numBits: BitWidth);
12932 const APInt MaxVal = APInt::getSignedMaxValue(numBits: BitWidth);
12933
12934 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
12935 KnownBits KnownRHS = DAG.computeKnownBits(Op: RHS);
12936
12937 // If either of the operand signs are known, then they are guaranteed to
12938 // only saturate in one direction. If non-negative they will saturate
12939 // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
12940 //
12941 // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
12942 // sign of 'y' has to be flipped.
12943
12944 bool LHSIsNonNegative = KnownLHS.isNonNegative();
12945 bool RHSIsNonNegative =
12946 Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() : KnownRHS.isNegative();
12947 if (LHSIsNonNegative || RHSIsNonNegative) {
12948 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12949 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: SumDiff);
12950 }
12951
12952 bool LHSIsNegative = KnownLHS.isNegative();
12953 bool RHSIsNegative =
12954 Opcode == ISD::SADDSAT ? KnownRHS.isNegative() : KnownRHS.isNonNegative();
12955 if (LHSIsNegative || RHSIsNegative) {
12956 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12957 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMin, RHS: SumDiff);
12958 }
12959
12960 // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
12961 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12962 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SumDiff,
12963 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT));
12964 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Shift, N2: SatMin);
12965 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: SumDiff);
12966}
12967
12968SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
12969 unsigned Opcode = Node->getOpcode();
12970 SDValue LHS = Node->getOperand(Num: 0);
12971 SDValue RHS = Node->getOperand(Num: 1);
12972 EVT VT = LHS.getValueType();
12973 EVT ResVT = Node->getValueType(ResNo: 0);
12974 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12975 SDLoc dl(Node);
12976
12977 auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT);
12978 auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT);
12979 SDValue IsLT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: LTPredicate);
12980 SDValue IsGT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: GTPredicate);
12981
12982 // We can't perform arithmetic on i1 values. Extending them would
12983 // probably result in worse codegen, so let's just use two selects instead.
12984 // Some targets are also just better off using selects rather than subtraction
12985 // because one of the conditions can be merged with one of the selects.
12986 // And finally, if we don't know the contents of high bits of a boolean value
12987 // we can't perform any arithmetic either.
12988 if (preferSelectsOverBooleanArithmetic(VT) ||
12989 BoolVT.getScalarSizeInBits() == 1 ||
12990 getBooleanContents(Type: BoolVT) == UndefinedBooleanContent) {
12991 SDValue SelectZeroOrOne =
12992 DAG.getSelect(DL: dl, VT: ResVT, Cond: IsGT, LHS: DAG.getConstant(Val: 1, DL: dl, VT: ResVT),
12993 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ResVT));
12994 return DAG.getSelect(DL: dl, VT: ResVT, Cond: IsLT, LHS: DAG.getAllOnesConstant(DL: dl, VT: ResVT),
12995 RHS: SelectZeroOrOne);
12996 }
12997
12998 if (getBooleanContents(Type: BoolVT) == ZeroOrNegativeOneBooleanContent)
12999 std::swap(a&: IsGT, b&: IsLT);
13000 return DAG.getSExtOrTrunc(Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: BoolVT, N1: IsGT, N2: IsLT), DL: dl,
13001 VT: ResVT);
13002}
13003
13004SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
13005 unsigned Opcode = Node->getOpcode();
13006 bool IsSigned = Opcode == ISD::SSHLSAT;
13007 SDValue LHS = Node->getOperand(Num: 0);
13008 SDValue RHS = Node->getOperand(Num: 1);
13009 EVT VT = LHS.getValueType();
13010 SDLoc dl(Node);
13011
13012 assert((Node->getOpcode() == ISD::SSHLSAT ||
13013 Node->getOpcode() == ISD::USHLSAT) &&
13014 "Expected a SHLSAT opcode");
13015 assert(VT.isInteger() && "Expected operands to be integers");
13016
13017 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
13018 return DAG.UnrollVectorOp(N: Node);
13019
13020 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
13021
13022 unsigned BW = VT.getScalarSizeInBits();
13023 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13024 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: RHS);
13025 SDValue Orig =
13026 DAG.getNode(Opcode: IsSigned ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: Result, N2: RHS);
13027
13028 SDValue SatVal;
13029 if (IsSigned) {
13030 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: BW), DL: dl, VT);
13031 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: BW), DL: dl, VT);
13032 SDValue Cond =
13033 DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT);
13034 SatVal = DAG.getSelect(DL: dl, VT, Cond, LHS: SatMin, RHS: SatMax);
13035 } else {
13036 SatVal = DAG.getConstant(Val: APInt::getMaxValue(numBits: BW), DL: dl, VT);
13037 }
13038 SDValue Cond = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Orig, Cond: ISD::SETNE);
13039 return DAG.getSelect(DL: dl, VT, Cond, LHS: SatVal, RHS: Result);
13040}
13041
13042void TargetLowering::forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl,
13043 bool Signed, SDValue &Lo, SDValue &Hi,
13044 SDValue LHS, SDValue RHS,
13045 SDValue HiLHS, SDValue HiRHS) const {
13046 EVT VT = LHS.getValueType();
13047 assert(RHS.getValueType() == VT && "Mismatching operand types");
13048
13049 assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
13050 assert((!Signed || !HiLHS) &&
13051 "Signed flag should only be set when HiLHS and RiRHS are null");
13052
13053 // We'll expand the multiplication by brute force because we have no other
13054 // options. This is a trivially-generalized version of the code from
13055 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
13056 // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
13057 // sign bits while calculating the Hi half.
13058 unsigned Bits = VT.getScalarSizeInBits();
13059 unsigned HalfBits = Bits / 2;
13060 SDValue Mask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: Bits, loBitsSet: HalfBits), DL: dl, VT);
13061 SDValue LL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Mask);
13062 SDValue RL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: RHS, N2: Mask);
13063
13064 SDValue T = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RL);
13065 SDValue TL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: Mask);
13066
13067 SDValue Shift = DAG.getShiftAmountConstant(Val: HalfBits, VT, DL: dl);
13068 // This is always an unsigned shift.
13069 SDValue TH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: T, N2: Shift);
13070
13071 unsigned ShiftOpc = Signed ? ISD::SRA : ISD::SRL;
13072 SDValue LH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: LHS, N2: Shift);
13073 SDValue RH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: RHS, N2: Shift);
13074
13075 SDValue U =
13076 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RL), N2: TH);
13077 SDValue UL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: U, N2: Mask);
13078 SDValue UH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: U, N2: Shift);
13079
13080 SDValue V =
13081 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RH), N2: UL);
13082 SDValue VH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: V, N2: Shift);
13083
13084 Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: TL,
13085 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: Shift));
13086
13087 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RH),
13088 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: UH, N2: VH));
13089
13090 // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
13091 // the products to Hi.
13092 if (HiLHS) {
13093 SDValue RHLL = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: HiRHS, N2: LHS);
13094 SDValue RLLH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: RHS, N2: HiLHS);
13095 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Hi,
13096 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: RHLL, N2: RLLH));
13097 }
13098}
13099
13100void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
13101 bool Signed, const SDValue LHS,
13102 const SDValue RHS, SDValue &Lo,
13103 SDValue &Hi) const {
13104 EVT VT = LHS.getValueType();
13105 assert(RHS.getValueType() == VT && "Mismatching operand types");
13106 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13107 // We can fall back to a libcall with an illegal type for the MUL if we
13108 // have a libcall big enough.
13109 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
13110 if (WideVT == MVT::i16)
13111 LC = RTLIB::MUL_I16;
13112 else if (WideVT == MVT::i32)
13113 LC = RTLIB::MUL_I32;
13114 else if (WideVT == MVT::i64)
13115 LC = RTLIB::MUL_I64;
13116 else if (WideVT == MVT::i128)
13117 LC = RTLIB::MUL_I128;
13118
13119 RTLIB::LibcallImpl LibcallImpl = DAG.getLibcalls().getLibcallImpl(Call: LC);
13120 if (LibcallImpl == RTLIB::Unsupported) {
13121 forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
13122 return;
13123 }
13124
13125 // The libcall takes and returns WideVT values as packed VT halves. This only
13126 // works if VT is legal and WideVT isn't. Otherwise, emit a wide multiply and
13127 // let it legalise normally, possibly into a wider libcall.
13128 if (isTypeLegal(VT: WideVT) || !isTypeLegal(VT)) {
13129 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT,
13130 N1: DAG.getExtOrTrunc(IsSigned: Signed, Op: LHS, DL: dl, VT: WideVT),
13131 N2: DAG.getExtOrTrunc(IsSigned: Signed, Op: RHS, DL: dl, VT: WideVT));
13132 SDValue Shift =
13133 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits(), VT: WideVT, DL: dl);
13134 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Mul);
13135 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13136 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT, N1: Mul, N2: Shift));
13137 return;
13138 }
13139
13140 SDValue HiLHS, HiRHS;
13141 if (Signed) {
13142 // The high part is obtained by SRA'ing all but one of the bits of low
13143 // part.
13144 unsigned LoSize = VT.getFixedSizeInBits();
13145 SDValue Shift = DAG.getShiftAmountConstant(Val: LoSize - 1, VT, DL: dl);
13146 HiLHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS, N2: Shift);
13147 HiRHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: RHS, N2: Shift);
13148 } else {
13149 HiLHS = DAG.getConstant(Val: 0, DL: dl, VT);
13150 HiRHS = DAG.getConstant(Val: 0, DL: dl, VT);
13151 }
13152
13153 // Attempt a libcall.
13154 SDValue Ret;
13155 TargetLowering::MakeLibCallOptions CallOptions;
13156 CallOptions.setIsSigned(Signed);
13157 CallOptions.setIsPostTypeLegalization(true);
13158 if (shouldSplitFunctionArgumentsAsLittleEndian(DL: DAG.getDataLayout())) {
13159 // Halves of WideVT are packed into registers in different order
13160 // depending on platform endianness. This is usually handled by
13161 // the C calling convention, but we can't defer to it in
13162 // the legalizer.
13163 SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
13164 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
13165 } else {
13166 SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
13167 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
13168 }
13169 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
13170 "Ret value is a collection of constituent nodes holding result.");
13171 if (DAG.getDataLayout().isLittleEndian()) {
13172 // Same as above.
13173 Lo = Ret.getOperand(i: 0);
13174 Hi = Ret.getOperand(i: 1);
13175 } else {
13176 Lo = Ret.getOperand(i: 1);
13177 Hi = Ret.getOperand(i: 0);
13178 }
13179}
13180
13181SDValue
13182TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
13183 assert((Node->getOpcode() == ISD::SMULFIX ||
13184 Node->getOpcode() == ISD::UMULFIX ||
13185 Node->getOpcode() == ISD::SMULFIXSAT ||
13186 Node->getOpcode() == ISD::UMULFIXSAT) &&
13187 "Expected a fixed point multiplication opcode");
13188
13189 SDLoc dl(Node);
13190 SDValue LHS = Node->getOperand(Num: 0);
13191 SDValue RHS = Node->getOperand(Num: 1);
13192 EVT VT = LHS.getValueType();
13193 unsigned Scale = Node->getConstantOperandVal(Num: 2);
13194 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
13195 Node->getOpcode() == ISD::UMULFIXSAT);
13196 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
13197 Node->getOpcode() == ISD::SMULFIXSAT);
13198 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13199 unsigned VTSize = VT.getScalarSizeInBits();
13200
13201 if (!Scale) {
13202 // [us]mul.fix(a, b, 0) -> mul(a, b)
13203 if (!Saturating) {
13204 if (isOperationLegalOrCustom(Op: ISD::MUL, VT))
13205 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13206 } else if (Signed && isOperationLegalOrCustom(Op: ISD::SMULO, VT)) {
13207 SDValue Result =
13208 DAG.getNode(Opcode: ISD::SMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
13209 SDValue Product = Result.getValue(R: 0);
13210 SDValue Overflow = Result.getValue(R: 1);
13211 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13212
13213 APInt MinVal = APInt::getSignedMinValue(numBits: VTSize);
13214 APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize);
13215 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
13216 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
13217 // Xor the inputs, if resulting sign bit is 0 the product will be
13218 // positive, else negative.
13219 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
13220 SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT);
13221 Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax);
13222 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product);
13223 } else if (!Signed && isOperationLegalOrCustom(Op: ISD::UMULO, VT)) {
13224 SDValue Result =
13225 DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
13226 SDValue Product = Result.getValue(R: 0);
13227 SDValue Overflow = Result.getValue(R: 1);
13228
13229 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
13230 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
13231 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product);
13232 }
13233 }
13234
13235 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
13236 "Expected scale to be less than the number of bits if signed or at "
13237 "most the number of bits if unsigned.");
13238 assert(LHS.getValueType() == RHS.getValueType() &&
13239 "Expected both operands to be the same type");
13240
13241 // Select the saturated value when Cond0 <CC> Cond1, keeping it vectorized:
13242 // SELECT_CC is scalarized for vector types, so build SETCC + VSELECT there.
13243 auto getSaturatingSelect = [&](SDValue Cond0, SDValue Cond1, SDValue Sat,
13244 SDValue Val, ISD::CondCode CC) {
13245 if (VT.isVector())
13246 return DAG.getSelect(DL: dl, VT, Cond: DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Cond0, RHS: Cond1, Cond: CC),
13247 LHS: Sat, RHS: Val);
13248 return DAG.getSelectCC(DL: dl, LHS: Cond0, RHS: Cond1, True: Sat, False: Val, Cond: CC);
13249 };
13250
13251 // Get the upper and lower bits of the result.
13252 SDValue Lo, Hi;
13253 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13254 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
13255 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13256 if (isOperationLegalOrCustom(Op: LoHiOp, VT)) {
13257 SDValue Result = DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
13258 Lo = Result.getValue(R: 0);
13259 Hi = Result.getValue(R: 1);
13260 } else if (isOperationLegalOrCustom(Op: HiOp, VT)) {
13261 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13262 Hi = DAG.getNode(Opcode: HiOp, DL: dl, VT, N1: LHS, N2: RHS);
13263 } else if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
13264 // Try for a multiplication using a wider type.
13265 unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
13266 SDValue LHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: LHS);
13267 SDValue RHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: RHS);
13268 SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHSExt, N2: RHSExt);
13269 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Res);
13270 SDValue Shifted =
13271 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: WideVT, N1: Res,
13272 N2: DAG.getShiftAmountConstant(Val: VTSize, VT: WideVT, DL: dl));
13273 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Shifted);
13274 } else if (VT.isVector()) {
13275 return SDValue();
13276 } else {
13277 forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
13278 }
13279
13280 if (Scale == VTSize)
13281 // Result is just the top half since we'd be shifting by the width of the
13282 // operand. Overflow impossible so this works for both UMULFIX and
13283 // UMULFIXSAT.
13284 return Hi;
13285
13286 // The result will need to be shifted right by the scale since both operands
13287 // are scaled. The result is given to us in 2 halves, so we only want part of
13288 // both in the result.
13289 SDValue Result = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: Hi, N2: Lo,
13290 N3: DAG.getShiftAmountConstant(Val: Scale, VT, DL: dl));
13291 if (!Saturating)
13292 return Result;
13293
13294 if (!Signed) {
13295 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
13296 // widened multiplication) aren't all zeroes.
13297
13298 // Saturate to max if ((Hi >> Scale) != 0),
13299 // which is the same as if (Hi > ((1 << Scale) - 1))
13300 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
13301 SDValue LowMask =
13302 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale), DL: dl, VT);
13303 return getSaturatingSelect(Hi, LowMask, DAG.getConstant(Val: MaxVal, DL: dl, VT),
13304 Result, ISD::SETUGT);
13305 }
13306
13307 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
13308 // widened multiplication) aren't all ones or all zeroes.
13309
13310 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: VTSize), DL: dl, VT);
13311 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: VTSize), DL: dl, VT);
13312
13313 if (Scale == 0) {
13314 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Lo,
13315 N2: DAG.getShiftAmountConstant(Val: VTSize - 1, VT, DL: dl));
13316 SDValue Overflow = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Hi, RHS: Sign, Cond: ISD::SETNE);
13317 // Saturated to SatMin if wide product is negative, and SatMax if wide
13318 // product is positive ...
13319 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13320 SDValue ResultIfOverflow =
13321 getSaturatingSelect(Hi, Zero, SatMin, SatMax, ISD::SETLT);
13322 // ... but only if we overflowed.
13323 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: ResultIfOverflow, RHS: Result);
13324 }
13325
13326 // We handled Scale==0 above so all the bits to examine is in Hi.
13327
13328 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
13329 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
13330 SDValue LowMask =
13331 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale - 1), DL: dl, VT);
13332 // Saturate to min if (Hi >> (Scale - 1)) < -1),
13333 // which is the same as if (HI < (-1 << (Scale - 1))
13334 SDValue HighMask = DAG.getConstant(
13335 Val: APInt::getHighBitsSet(numBits: VTSize, hiBitsSet: VTSize - Scale + 1), DL: dl, VT);
13336 Result = getSaturatingSelect(Hi, LowMask, SatMax, Result, ISD::SETGT);
13337 Result = getSaturatingSelect(Hi, HighMask, SatMin, Result, ISD::SETLT);
13338 return Result;
13339}
13340
13341SDValue
13342TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
13343 SDValue LHS, SDValue RHS,
13344 unsigned Scale, SelectionDAG &DAG) const {
13345 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
13346 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
13347 "Expected a fixed point division opcode");
13348
13349 EVT VT = LHS.getValueType();
13350 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
13351 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
13352 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13353
13354 // If there is enough room in the type to upscale the LHS or downscale the
13355 // RHS before the division, we can perform it in this type without having to
13356 // resize. For signed operations, the LHS headroom is the number of
13357 // redundant sign bits, and for unsigned ones it is the number of zeroes.
13358 // The headroom for the RHS is the number of trailing zeroes.
13359 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(Op: LHS) - 1
13360 : DAG.computeKnownBits(Op: LHS).countMinLeadingZeros();
13361 unsigned RHSTrail = DAG.computeKnownBits(Op: RHS).countMinTrailingZeros();
13362
13363 // For signed saturating operations, we need to be able to detect true integer
13364 // division overflow; that is, when you have MIN / -EPS. However, this
13365 // is undefined behavior and if we emit divisions that could take such
13366 // values it may cause undesired behavior (arithmetic exceptions on x86, for
13367 // example).
13368 // Avoid this by requiring an extra bit so that we never get this case.
13369 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
13370 // signed saturating division, we need to emit a whopping 32-bit division.
13371 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
13372 return SDValue();
13373
13374 unsigned LHSShift = std::min(a: LHSLead, b: Scale);
13375 unsigned RHSShift = Scale - LHSShift;
13376
13377 // At this point, we know that if we shift the LHS up by LHSShift and the
13378 // RHS down by RHSShift, we can emit a regular division with a final scaling
13379 // factor of Scale.
13380
13381 if (LHSShift)
13382 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS,
13383 N2: DAG.getShiftAmountConstant(Val: LHSShift, VT, DL: dl));
13384 if (RHSShift)
13385 RHS = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: RHS,
13386 N2: DAG.getShiftAmountConstant(Val: RHSShift, VT, DL: dl));
13387
13388 SDValue Quot;
13389 if (Signed) {
13390 // For signed operations, if the resulting quotient is negative and the
13391 // remainder is nonzero, subtract 1 from the quotient to round towards
13392 // negative infinity.
13393 SDValue Rem;
13394 // FIXME: Ideally we would always produce an SDIVREM here, but if the
13395 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
13396 // we couldn't just form a libcall, but the type legalizer doesn't do it.
13397 if (isTypeLegal(VT) &&
13398 isOperationLegalOrCustom(Op: ISD::SDIVREM, VT)) {
13399 Quot = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl,
13400 VTList: DAG.getVTList(VT1: VT, VT2: VT),
13401 N1: LHS, N2: RHS);
13402 Rem = Quot.getValue(R: 1);
13403 Quot = Quot.getValue(R: 0);
13404 } else {
13405 Quot = DAG.getNode(Opcode: ISD::SDIV, DL: dl, VT,
13406 N1: LHS, N2: RHS);
13407 Rem = DAG.getNode(Opcode: ISD::SREM, DL: dl, VT,
13408 N1: LHS, N2: RHS);
13409 }
13410 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13411 SDValue RemNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Rem, RHS: Zero, Cond: ISD::SETNE);
13412 SDValue LHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETLT);
13413 SDValue RHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13414 SDValue QuotNeg = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: BoolVT, N1: LHSNeg, N2: RHSNeg);
13415 SDValue Sub1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Quot,
13416 N2: DAG.getConstant(Val: 1, DL: dl, VT));
13417 Quot = DAG.getSelect(DL: dl, VT,
13418 Cond: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolVT, N1: RemNonZero, N2: QuotNeg),
13419 LHS: Sub1, RHS: Quot);
13420 } else
13421 Quot = DAG.getNode(Opcode: ISD::UDIV, DL: dl, VT,
13422 N1: LHS, N2: RHS);
13423
13424 return Quot;
13425}
13426
13427void TargetLowering::expandUADDSUBO(
13428 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13429 SDLoc dl(Node);
13430 SDValue LHS = Node->getOperand(Num: 0);
13431 SDValue RHS = Node->getOperand(Num: 1);
13432 bool IsAdd = Node->getOpcode() == ISD::UADDO;
13433
13434 // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
13435 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
13436 if (isOperationLegalOrCustom(Op: OpcCarry, VT: Node->getValueType(ResNo: 0))) {
13437 SDValue CarryIn = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 1));
13438 SDValue NodeCarry = DAG.getNode(Opcode: OpcCarry, DL: dl, VTList: Node->getVTList(),
13439 Ops: { LHS, RHS, CarryIn });
13440 Result = SDValue(NodeCarry.getNode(), 0);
13441 Overflow = SDValue(NodeCarry.getNode(), 1);
13442 return;
13443 }
13444
13445 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13446 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13447
13448 EVT ResultType = Node->getValueType(ResNo: 1);
13449 EVT SetCCType = getSetCCResultType(
13450 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13451 SDValue SetCC;
13452 if (IsAdd && isOneConstant(V: RHS)) {
13453 // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
13454 // the live range of X. We assume comparing with 0 is cheap.
13455 // The general case (X + C) < C is not necessarily beneficial. Although we
13456 // reduce the live range of X, we may introduce the materialization of
13457 // constant C.
13458 SetCC =
13459 DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result,
13460 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETEQ);
13461 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
13462 // Special case: uaddo X, -1 overflows if X != 0.
13463 SetCC =
13464 DAG.getSetCC(DL: dl, VT: SetCCType, LHS,
13465 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETNE);
13466 } else {
13467 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
13468 SetCC = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result, RHS: LHS, Cond: CC);
13469 }
13470 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13471}
13472
13473void TargetLowering::expandSADDSUBO(
13474 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13475 SDLoc dl(Node);
13476 SDValue LHS = Node->getOperand(Num: 0);
13477 SDValue RHS = Node->getOperand(Num: 1);
13478 bool IsAdd = Node->getOpcode() == ISD::SADDO;
13479
13480 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13481 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13482
13483 EVT ResultType = Node->getValueType(ResNo: 1);
13484 EVT OType = getSetCCResultType(
13485 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13486
13487 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
13488 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
13489 if (isOperationLegal(Op: OpcSat, VT: LHS.getValueType())) {
13490 SDValue Sat = DAG.getNode(Opcode: OpcSat, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS);
13491 SDValue SetCC = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Sat, Cond: ISD::SETNE);
13492 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13493 return;
13494 }
13495
13496 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
13497
13498 if (IsAdd) {
13499 // For an addition, the result should be less than one of the operands (LHS)
13500 // if and only if the other operand (RHS) is negative, otherwise there will
13501 // be overflow.
13502 SDValue ResultLowerThanLHS =
13503 DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: LHS, Cond: ISD::SETLT);
13504 SDValue RHSNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13505 Overflow = DAG.getBoolExtOrTrunc(
13506 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: RHSNegative, N2: ResultLowerThanLHS), SL: dl,
13507 VT: ResultType, OpVT: ResultType);
13508 } else {
13509 // For subtraction, overflow occurs when the signed comparison of operands
13510 // doesn't match the sign of the result.
13511 SDValue LHSLessThanRHS = DAG.getSetCC(DL: dl, VT: OType, LHS, RHS, Cond: ISD::SETLT);
13512 SDValue ResultNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Zero, Cond: ISD::SETLT);
13513 Overflow = DAG.getBoolExtOrTrunc(
13514 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: LHSLessThanRHS, N2: ResultNegative), SL: dl,
13515 VT: ResultType, OpVT: ResultType);
13516 }
13517}
13518
13519bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
13520 SDValue &Overflow, SelectionDAG &DAG) const {
13521 SDLoc dl(Node);
13522 EVT VT = Node->getValueType(ResNo: 0);
13523 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13524 SDValue LHS = Node->getOperand(Num: 0);
13525 SDValue RHS = Node->getOperand(Num: 1);
13526 bool isSigned = Node->getOpcode() == ISD::SMULO;
13527
13528 // For power-of-two multiplications we can use a simpler shift expansion.
13529 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
13530 const APInt &C = RHSC->getAPIntValue();
13531 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
13532 if (C.isPowerOf2()) {
13533 // smulo(x, signed_min) is same as umulo(x, signed_min).
13534 bool UseArithShift = isSigned && !C.isMinSignedValue();
13535 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: C.logBase2(), VT, DL: dl);
13536 Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: ShiftAmt);
13537 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT,
13538 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL,
13539 DL: dl, VT, N1: Result, N2: ShiftAmt),
13540 RHS: LHS, Cond: ISD::SETNE);
13541 return true;
13542 }
13543 }
13544
13545 SDValue BottomHalf;
13546 SDValue TopHalf;
13547 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13548
13549 static const unsigned Ops[2][3] =
13550 { { ISD::UMUL_LOHI, ISD::MULHU, ISD::ZERO_EXTEND },
13551 { ISD::SMUL_LOHI, ISD::MULHS, ISD::SIGN_EXTEND }};
13552 if (isOperationLegalOrCustom(Op: Ops[isSigned][0], VT)) {
13553 BottomHalf = DAG.getNode(Opcode: Ops[isSigned][0], DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS,
13554 N2: RHS);
13555 TopHalf = BottomHalf.getValue(R: 1);
13556 } else if (isOperationLegalOrCustom(Op: Ops[isSigned][1], VT)) {
13557 BottomHalf = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13558 TopHalf = DAG.getNode(Opcode: Ops[isSigned][1], DL: dl, VT, N1: LHS, N2: RHS);
13559 } else if (isTypeLegal(VT: WideVT)) {
13560 LHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: LHS);
13561 RHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: RHS);
13562 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS);
13563 BottomHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Mul);
13564 SDValue ShiftAmt =
13565 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits(), VT: WideVT, DL: dl);
13566 TopHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13567 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT, N1: Mul, N2: ShiftAmt));
13568 } else {
13569 if (VT.isVector())
13570 return false;
13571
13572 forceExpandWideMUL(DAG, dl, Signed: isSigned, LHS, RHS, Lo&: BottomHalf, Hi&: TopHalf);
13573 }
13574
13575 Result = BottomHalf;
13576 if (isSigned) {
13577 SDValue ShiftAmt = DAG.getShiftAmountConstant(
13578 Val: VT.getScalarSizeInBits() - 1, VT: BottomHalf.getValueType(), DL: dl);
13579 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: BottomHalf, N2: ShiftAmt);
13580 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf, RHS: Sign, Cond: ISD::SETNE);
13581 } else {
13582 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf,
13583 RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
13584 }
13585
13586 // Truncate the result if SetCC returns a larger type than needed.
13587 EVT RType = Node->getValueType(ResNo: 1);
13588 if (RType.bitsLT(VT: Overflow.getValueType()))
13589 Overflow = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RType, Operand: Overflow);
13590
13591 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
13592 "Unexpected result type for S/UMULO legalization");
13593 return true;
13594}
13595
13596SDValue TargetLowering::expandMULH(SDNode *Node, SelectionDAG &DAG) const {
13597 SDLoc dl(Node);
13598 EVT VT = Node->getValueType(ResNo: 0);
13599 SDValue LHS = Node->getOperand(Num: 0);
13600 SDValue RHS = Node->getOperand(Num: 1);
13601 bool IsSigned = Node->getOpcode() == ISD::MULHS;
13602
13603 // Use MUL_LOHI if legal/custom for the original type.
13604 unsigned LoHiOp = IsSigned ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13605 if (isOperationLegalOrCustom(Op: LoHiOp, VT))
13606 return DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS).getValue(R: 1);
13607
13608 // Use a wide multiply if available.
13609 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13610 if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
13611 unsigned BW = VT.getScalarSizeInBits();
13612 LHS = DAG.getExtOrTrunc(IsSigned, Op: LHS, DL: dl, VT: WideVT);
13613 RHS = DAG.getExtOrTrunc(IsSigned, Op: RHS, DL: dl, VT: WideVT);
13614 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13615 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT,
13616 N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS),
13617 N2: DAG.getShiftAmountConstant(Val: BW, VT: WideVT, DL: dl)));
13618 }
13619
13620 // Let fixed-length vectors be scalarised by the caller.
13621 // Expand everything else with a wide multiply.
13622 if (!VT.isFixedLengthVector()) {
13623 SDValue Lo, Hi;
13624 forceExpandWideMUL(DAG, dl, Signed: IsSigned, LHS, RHS, Lo, Hi);
13625 return Hi;
13626 }
13627
13628 return SDValue();
13629}
13630
13631SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
13632 SDLoc dl(Node);
13633 ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13634 SDValue Op = Node->getOperand(Num: 0);
13635 SDNodeFlags Flags = Node->getFlags();
13636 EVT VT = Op.getValueType();
13637
13638 // Try to use a shuffle reduction for power of two vectors.
13639 if (VT.isPow2VectorType()) {
13640 // See if the reduction opcode is safe to use with widened types.
13641 bool WidenSrc = false;
13642 switch (Node->getOpcode()) {
13643 case ISD::VECREDUCE_FADD:
13644 case ISD::VECREDUCE_FMUL:
13645 case ISD::VECREDUCE_ADD:
13646 case ISD::VECREDUCE_MUL:
13647 case ISD::VECREDUCE_AND:
13648 case ISD::VECREDUCE_OR:
13649 case ISD::VECREDUCE_XOR:
13650 case ISD::VECREDUCE_SMAX:
13651 case ISD::VECREDUCE_SMIN:
13652 case ISD::VECREDUCE_UMAX:
13653 case ISD::VECREDUCE_UMIN:
13654 WidenSrc = VT.isFixedLengthVector();
13655 break;
13656 }
13657
13658 while (VT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
13659 EVT HalfVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
13660 if (!isOperationLegalOrCustom(Op: BaseOpcode, VT: HalfVT)) {
13661 if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
13662 // Attempt to widen the source vectors to a legal op.
13663 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: HalfVT);
13664 if (WideVT.isVector() &&
13665 WideVT.getScalarType() == HalfVT.getScalarType() &&
13666 WideVT.getVectorNumElements() >= HalfVT.getVectorNumElements() &&
13667 isOperationLegalOrCustom(Op: BaseOpcode, VT: WideVT)) {
13668 SDValue Lo, Hi;
13669 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13670 Lo = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Lo, Idx: 0);
13671 Hi = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Hi, Idx: 0);
13672 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: WideVT, N1: Lo, N2: Hi, Flags);
13673 Op = DAG.getExtractSubvector(DL: dl, VT: HalfVT, Vec: Op, Idx: 0);
13674 VT = HalfVT;
13675 continue;
13676 }
13677 }
13678 break;
13679 }
13680
13681 SDValue Lo, Hi;
13682 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13683 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: HalfVT, N1: Lo, N2: Hi, Flags);
13684 VT = HalfVT;
13685
13686 // Stop if splitting is enough to make the reduction legal.
13687 if (isOperationLegalOrCustom(Op: Node->getOpcode(), VT: HalfVT))
13688 return DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op,
13689 Flags);
13690 }
13691 }
13692
13693 if (VT.isScalableVector())
13694 reportFatalInternalError(
13695 reason: "Expanding reductions for scalable vectors is undefined.");
13696
13697 EVT EltVT = VT.getVectorElementType();
13698 unsigned NumElts = VT.getVectorNumElements();
13699
13700 SmallVector<SDValue, 8> Ops;
13701 DAG.ExtractVectorElements(Op, Args&: Ops, Start: 0, Count: NumElts);
13702
13703 SDValue Res = Ops[0];
13704 for (unsigned i = 1; i < NumElts; i++)
13705 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13706
13707 // Result type may be wider than element type.
13708 if (EltVT != Node->getValueType(ResNo: 0))
13709 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res);
13710 return Res;
13711}
13712
13713SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
13714 SDLoc dl(Node);
13715 SDValue AccOp = Node->getOperand(Num: 0);
13716 SDValue VecOp = Node->getOperand(Num: 1);
13717 SDNodeFlags Flags = Node->getFlags();
13718
13719 EVT VT = VecOp.getValueType();
13720 EVT EltVT = VT.getVectorElementType();
13721
13722 if (VT.isScalableVector())
13723 report_fatal_error(
13724 reason: "Expanding reductions for scalable vectors is undefined.");
13725
13726 unsigned NumElts = VT.getVectorNumElements();
13727
13728 SmallVector<SDValue, 8> Ops;
13729 DAG.ExtractVectorElements(Op: VecOp, Args&: Ops, Start: 0, Count: NumElts);
13730
13731 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13732
13733 SDValue Res = AccOp;
13734 for (unsigned i = 0; i < NumElts; i++)
13735 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13736
13737 return Res;
13738}
13739
13740bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
13741 SelectionDAG &DAG) const {
13742 EVT VT = Node->getValueType(ResNo: 0);
13743 SDLoc dl(Node);
13744 bool isSigned = Node->getOpcode() == ISD::SREM;
13745 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
13746 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
13747 SDValue Dividend = Node->getOperand(Num: 0);
13748 SDValue Divisor = Node->getOperand(Num: 1);
13749 if (isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
13750 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
13751 Result = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Dividend, N2: Divisor).getValue(R: 1);
13752 return true;
13753 }
13754 if (isOperationLegalOrCustom(Op: DivOpc, VT)) {
13755 // X % Y -> X-X/Y*Y
13756 SDValue Divide = DAG.getNode(Opcode: DivOpc, DL: dl, VT, N1: Dividend, N2: Divisor);
13757 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Divide, N2: Divisor);
13758 Result = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
13759 return true;
13760 }
13761 return false;
13762}
13763
13764SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
13765 SelectionDAG &DAG) const {
13766 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
13767 SDLoc dl(SDValue(Node, 0));
13768 SDValue Src = Node->getOperand(Num: 0);
13769
13770 // DstVT is the result type, while SatVT is the size to which we saturate
13771 EVT SrcVT = Src.getValueType();
13772 EVT DstVT = Node->getValueType(ResNo: 0);
13773
13774 EVT SatVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
13775 unsigned SatWidth = SatVT.getScalarSizeInBits();
13776 unsigned DstWidth = DstVT.getScalarSizeInBits();
13777 assert(SatWidth <= DstWidth &&
13778 "Expected saturation width smaller than result width");
13779
13780 // Determine minimum and maximum integer values and their corresponding
13781 // floating-point values.
13782 APInt MinInt, MaxInt;
13783 if (IsSigned) {
13784 MinInt = APInt::getSignedMinValue(numBits: SatWidth).sext(width: DstWidth);
13785 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth).sext(width: DstWidth);
13786 } else {
13787 MinInt = APInt::getMinValue(numBits: SatWidth).zext(width: DstWidth);
13788 MaxInt = APInt::getMaxValue(numBits: SatWidth).zext(width: DstWidth);
13789 }
13790
13791 // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
13792 // libcall emission cannot handle this. Large result types will fail.
13793 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
13794 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f32, Operand: Src);
13795 SrcVT = Src.getValueType();
13796 }
13797
13798 const fltSemantics &Sem = SrcVT.getFltSemantics();
13799 APFloat MinFloat(Sem);
13800 APFloat MaxFloat(Sem);
13801
13802 APFloat::opStatus MinStatus =
13803 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
13804 APFloat::opStatus MaxStatus =
13805 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
13806 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
13807 !(MaxStatus & APFloat::opStatus::opInexact);
13808
13809 SDValue MinFloatNode = DAG.getConstantFP(Val: MinFloat, DL: dl, VT: SrcVT);
13810 SDValue MaxFloatNode = DAG.getConstantFP(Val: MaxFloat, DL: dl, VT: SrcVT);
13811
13812 // If the integer bounds are exactly representable as floats and min/max are
13813 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
13814 // of comparisons and selects.
13815 auto EmitMinMax = [&](unsigned MinOpcode, unsigned MaxOpcode,
13816 bool MayPropagateNaN) {
13817 bool MinMaxLegal = isOperationLegalOrCustom(Op: MinOpcode, VT: SrcVT) &&
13818 isOperationLegalOrCustom(Op: MaxOpcode, VT: SrcVT);
13819 if (!MinMaxLegal)
13820 return SDValue();
13821
13822 SDValue Clamped = Src;
13823
13824 // Clamp Src by MinFloat from below. If !MayPropagateNaN and Src is NaN
13825 // then the result is MinFloat.
13826 Clamped = DAG.getNode(Opcode: MaxOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MinFloatNode);
13827 // Clamp by MaxFloat from above. If !MayPropagateNaN then NaN cannot occur.
13828 Clamped = DAG.getNode(Opcode: MinOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MaxFloatNode);
13829 // Convert clamped value to integer.
13830 SDValue FpToInt = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
13831 DL: dl, VT: DstVT, Operand: Clamped);
13832
13833 // If !MayPropagateNan and the conversion is unsigned case we're done,
13834 // because we mapped NaN to MinFloat, which will cast to zero.
13835 if (!MayPropagateNaN && !IsSigned)
13836 return FpToInt;
13837
13838 // Otherwise, select 0 if Src is NaN.
13839 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13840 EVT SetCCVT =
13841 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13842 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13843 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: FpToInt);
13844 };
13845 if (AreExactFloatBounds) {
13846 if (SDValue Res = EmitMinMax(ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
13847 /*MayPropagateNaN=*/false))
13848 return Res;
13849 // These may propagate NaN for sNaN operands.
13850 if (SDValue Res =
13851 EmitMinMax(ISD::FMINNUM, ISD::FMAXNUM, /*MayPropagateNaN=*/true))
13852 return Res;
13853 // These always propagate NaN.
13854 if (SDValue Res =
13855 EmitMinMax(ISD::FMINIMUM, ISD::FMAXIMUM, /*MayPropagateNaN=*/true))
13856 return Res;
13857 }
13858
13859 SDValue MinIntNode = DAG.getConstant(Val: MinInt, DL: dl, VT: DstVT);
13860 SDValue MaxIntNode = DAG.getConstant(Val: MaxInt, DL: dl, VT: DstVT);
13861
13862 // Result of direct conversion. The assumption here is that the operation is
13863 // non-trapping and it's fine to apply it to an out-of-range value if we
13864 // select it away later.
13865 SDValue FpToInt =
13866 DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL: dl, VT: DstVT, Operand: Src);
13867
13868 SDValue Select = FpToInt;
13869
13870 EVT SetCCVT =
13871 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13872
13873 // If Src ULT MinFloat, select MinInt. In particular, this also selects
13874 // MinInt if Src is NaN.
13875 SDValue ULT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MinFloatNode, Cond: ISD::SETULT);
13876 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: ULT, LHS: MinIntNode, RHS: Select);
13877 // If Src OGT MaxFloat, select MaxInt.
13878 SDValue OGT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MaxFloatNode, Cond: ISD::SETOGT);
13879 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: OGT, LHS: MaxIntNode, RHS: Select);
13880
13881 // In the unsigned case we are done, because we mapped NaN to MinInt, which
13882 // is already zero.
13883 if (!IsSigned)
13884 return Select;
13885
13886 // Otherwise, select 0 if Src is NaN.
13887 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13888 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13889 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: Select);
13890}
13891
13892SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op,
13893 const SDLoc &dl,
13894 SelectionDAG &DAG) const {
13895 EVT OperandVT = Op.getValueType();
13896 if (OperandVT.getScalarType() == ResultVT.getScalarType())
13897 return Op;
13898 EVT ResultIntVT = ResultVT.changeTypeToInteger();
13899 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13900 // can induce double-rounding which may alter the results. We can
13901 // correct for this using a trick explained in: Boldo, Sylvie, and
13902 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13903 // World Congress. 2005.
13904 SDValue Narrow = DAG.getFPExtendOrRound(Op, DL: dl, VT: ResultVT);
13905 SDValue NarrowAsWide = DAG.getFPExtendOrRound(Op: Narrow, DL: dl, VT: OperandVT);
13906
13907 // We can keep the narrow value as-is if narrowing was exact (no
13908 // rounding error), the wide value was NaN (the narrow value is also
13909 // NaN and should be preserved) or if we rounded to the odd value.
13910 SDValue NarrowBits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultIntVT, Operand: Narrow);
13911 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResultIntVT);
13912 SDValue NegativeOne = DAG.getAllOnesConstant(DL: dl, VT: ResultIntVT);
13913 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: One);
13914 EVT ResultIntVTCCVT = getSetCCResultType(
13915 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: And.getValueType());
13916 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ResultIntVT);
13917 // The result is already odd so we don't need to do anything.
13918 SDValue AlreadyOdd = DAG.getSetCC(DL: dl, VT: ResultIntVTCCVT, LHS: And, RHS: Zero, Cond: ISD::SETNE);
13919
13920 EVT WideSetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
13921 VT: Op.getValueType());
13922 // We keep results which are exact, odd or NaN.
13923 SDValue KeepNarrow =
13924 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: Op, RHS: NarrowAsWide, Cond: ISD::SETUEQ);
13925 KeepNarrow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: WideSetCCVT, N1: KeepNarrow, N2: AlreadyOdd);
13926 // We morally performed a round-down if AbsNarrow is smaller than
13927 // AbsWide.
13928 SDValue AbsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: Op);
13929 SDValue AbsNarrowAsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: NarrowAsWide);
13930 SDValue NarrowIsRd =
13931 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: AbsWide, RHS: AbsNarrowAsWide, Cond: ISD::SETOGT);
13932 // If the narrow value is odd or exact, pick it.
13933 // Otherwise, narrow is even and corresponds to either the rounded-up
13934 // or rounded-down value. If narrow is the rounded-down value, we want
13935 // the rounded-up value as it will be odd.
13936 SDValue Adjust = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: NarrowIsRd, LHS: One, RHS: NegativeOne);
13937 SDValue Adjusted = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: Adjust);
13938 Op = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: KeepNarrow, LHS: NarrowBits, RHS: Adjusted);
13939 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultVT, Operand: Op);
13940}
13941
13942SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const {
13943 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
13944 SDValue Op = Node->getOperand(Num: 0);
13945 EVT VT = Node->getValueType(ResNo: 0);
13946 SDLoc dl(Node);
13947 if (VT.getScalarType() == MVT::bf16) {
13948 if (Node->getConstantOperandVal(Num: 1) == 1) {
13949 return DAG.getNode(Opcode: ISD::FP_TO_BF16, DL: dl, VT, Operand: Node->getOperand(Num: 0));
13950 }
13951 EVT OperandVT = Op.getValueType();
13952
13953 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13954 // can induce double-rounding which may alter the results. We can
13955 // correct for this using a trick explained in: Boldo, Sylvie, and
13956 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13957 // World Congress. 2005.
13958 EVT F32 = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
13959 EVT I32 = F32.changeTypeToInteger();
13960 Op = expandRoundInexactToOdd(ResultVT: F32, Op, dl, DAG);
13961 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: I32, Operand: Op);
13962
13963 // Factor in the contribution of the low 16 bits.
13964 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: I32);
13965 SDValue Lsb = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13966 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13967 Lsb = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: I32, N1: Lsb, N2: One);
13968 SDValue RoundingBias =
13969 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Lsb, N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: I32));
13970 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Op, N2: RoundingBias);
13971
13972 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
13973 // 0x80000000.
13974 if (!Node->getFlags().hasNoNaNs() &&
13975 !DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 0))) {
13976 // Conversions should set NaN's quiet bit. This also prevents NaNs from
13977 // turning into infinities.
13978 SDValue NaN =
13979 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: I32, N1: Op, N2: DAG.getConstant(Val: 0x400000, DL: dl, VT: I32));
13980
13981 SDValue IsNaN = DAG.getSetCC(
13982 DL: dl,
13983 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT),
13984 LHS: Node->getOperand(Num: 0), RHS: Node->getOperand(Num: 0), Cond: ISD::SETUO);
13985 Add = DAG.getSelect(DL: dl, VT: I32, Cond: IsNaN, LHS: NaN, RHS: Add);
13986 }
13987
13988 // Now that we have rounded, shift the bits into position.
13989 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Add,
13990 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13991 EVT I16 = I32.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i16);
13992 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: I16, Operand: Op);
13993 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
13994 }
13995 return SDValue();
13996}
13997
13998void TargetLowering::expandVectorInterleave(SDNode *Node,
13999 SmallVectorImpl<SDValue> &Results,
14000 SelectionDAG &DAG) const {
14001 assert(Node->getOpcode() == ISD::VECTOR_INTERLEAVE && "Unexpected opcode");
14002 EVT VT = Node->getValueType(ResNo: 0);
14003 unsigned Factor = Node->getNumOperands();
14004 SDLoc DL(Node);
14005
14006 if (Factor > 2 && Factor % 2 == 0) {
14007 SmallVector<EVT> HalfVTs(Factor / 2, VT);
14008 SmallVector<SDValue, 8> LOps, ROps;
14009 // Interleave so we have 2 factors per result:
14010 // a0a1 b0b1 c0c1 d0d1 -> [a0c0 b0d0] [a1c1 b1d1]
14011 for (unsigned I = 0; I < Factor / 2; I++) {
14012 SDValue Interleave =
14013 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: {VT, VT},
14014 Ops: {Node->getOperand(Num: I), Node->getOperand(Num: I + Factor / 2)});
14015 LOps.push_back(Elt: Interleave.getValue(R: 0));
14016 ROps.push_back(Elt: Interleave.getValue(R: 1));
14017 }
14018 // Interleave at Factor/2:
14019 // [a0c0 b0d0] [a1c1 b1d1] -> a0b0 c0d0 a1b1 c1d1
14020 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: HalfVTs, Ops: LOps);
14021 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: HalfVTs, Ops: ROps);
14022 for (unsigned I = 0; I < Factor / 2; I++)
14023 Results.push_back(Elt: L.getValue(R: I));
14024 for (unsigned I = 0; I < Factor / 2; I++)
14025 Results.push_back(Elt: R.getValue(R: I));
14026 return;
14027 }
14028
14029 if (!VT.isFixedLengthVector())
14030 return;
14031
14032 unsigned NumElts = VT.getVectorNumElements();
14033 EVT EltVT = VT.getVectorElementType();
14034
14035 // A factor of two needs only same-width, two-input shuffles.
14036 if (Factor == 2) {
14037 SmallVector<int, 8> Mask = createInterleaveMask(VF: NumElts, NumVecs: Factor);
14038 for (unsigned I = 0; I != Factor; ++I)
14039 Results.push_back(Elt: DAG.getVectorShuffle(
14040 VT, dl: DL, N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
14041 Mask: ArrayRef<int>(Mask).slice(N: I * NumElts, M: NumElts)));
14042 return;
14043 }
14044
14045 SmallVector<SDValue> InterleavedElts;
14046 InterleavedElts.reserve(N: Factor * NumElts);
14047 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14048 for (unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
14049 InterleavedElts.push_back(Elt: DAG.getExtractVectorElt(
14050 DL, VT: EltVT, Vec: Node->getOperand(Num: OperandIdx), Idx: EltIdx));
14051
14052 for (unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx)
14053 Results.push_back(Elt: DAG.getBuildVector(
14054 VT, DL, Ops: ArrayRef(InterleavedElts).slice(N: ResultIdx * NumElts, M: NumElts)));
14055}
14056
14057void TargetLowering::expandVectorDeinterleave(SDNode *Node,
14058 SmallVectorImpl<SDValue> &Results,
14059 SelectionDAG &DAG) const {
14060 assert(Node->getOpcode() == ISD::VECTOR_DEINTERLEAVE && "Unexpected opcode");
14061 EVT VT = Node->getValueType(ResNo: 0);
14062 unsigned Factor = Node->getNumOperands();
14063 SDLoc DL(Node);
14064
14065 if (Factor > 2 && Factor % 2 == 0) {
14066 SmallVector<SDValue, 8> Ops(Node->ops());
14067 SmallVector<EVT> HalfVTs(Factor / 2, VT);
14068 // Deinterleave at Factor/2 so each result contains two factors interleaved:
14069 // a0b0 c0d0 a1b1 c1d1 -> [a0c0 b0d0] [a1c1 b1d1]
14070 SDValue L = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: HalfVTs,
14071 Ops: ArrayRef(Ops).take_front(N: Factor / 2));
14072 SDValue R = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: HalfVTs,
14073 Ops: ArrayRef(Ops).take_back(N: Factor / 2));
14074 Results.resize(N: Factor);
14075 // Deinterleave the 2 factors out:
14076 // [a0c0 a1c1] [b0d0 b1d1] -> a0a1 b0b1 c0c1 d0d1
14077 for (unsigned I = 0; I < Factor / 2; ++I) {
14078 SDValue Deinterleave = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: {VT, VT},
14079 Ops: {L.getValue(R: I), R.getValue(R: I)});
14080 Results[I] = Deinterleave.getValue(R: 0);
14081 Results[I + Factor / 2] = Deinterleave.getValue(R: 1);
14082 }
14083 return;
14084 }
14085
14086 if (!VT.isFixedLengthVector())
14087 return;
14088
14089 unsigned NumElts = VT.getVectorNumElements();
14090 EVT EltVT = VT.getVectorElementType();
14091
14092 // A factor of two needs only same-width, two-input shuffles.
14093 if (Factor == 2) {
14094 for (unsigned I = 0; I != Factor; ++I)
14095 Results.push_back(
14096 Elt: DAG.getVectorShuffle(VT, dl: DL, N1: Node->getOperand(Num: 0), N2: Node->getOperand(Num: 1),
14097 Mask: createStrideMask(Start: I, Stride: Factor, VF: NumElts)));
14098 return;
14099 }
14100
14101 SmallVector<SDValue> ConcatElts;
14102 ConcatElts.reserve(N: Factor * NumElts);
14103 for (unsigned OperandIdx = 0; OperandIdx != Factor; ++OperandIdx)
14104 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14105 ConcatElts.push_back(Elt: DAG.getExtractVectorElt(
14106 DL, VT: EltVT, Vec: Node->getOperand(Num: OperandIdx), Idx: EltIdx));
14107
14108 SmallVector<SDValue> DeinterleavedElts;
14109 DeinterleavedElts.reserve(N: Factor * NumElts);
14110 for (unsigned ResultIdx = 0; ResultIdx != Factor; ++ResultIdx) {
14111 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx)
14112 DeinterleavedElts.push_back(Elt: ConcatElts[ResultIdx + EltIdx * Factor]);
14113 Results.push_back(Elt: DAG.getBuildVector(
14114 VT, DL,
14115 Ops: ArrayRef(DeinterleavedElts).slice(N: ResultIdx * NumElts, M: NumElts)));
14116 }
14117}
14118
14119SDValue TargetLowering::expandVectorSplice(SDNode *Node,
14120 SelectionDAG &DAG) const {
14121 assert((Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT ||
14122 Node->getOpcode() == ISD::VECTOR_SPLICE_RIGHT) &&
14123 "Unexpected opcode!");
14124 assert((Node->getValueType(0).isScalableVector() ||
14125 !isa<ConstantSDNode>(Node->getOperand(2))) &&
14126 "Fixed length vector types with constant offsets expected to use "
14127 "SHUFFLE_VECTOR!");
14128
14129 EVT VT = Node->getValueType(ResNo: 0);
14130 SDValue V1 = Node->getOperand(Num: 0);
14131 SDValue V2 = Node->getOperand(Num: 1);
14132 SDValue Offset = Node->getOperand(Num: 2);
14133 SDLoc DL(Node);
14134
14135 // Expand through memory thusly:
14136 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
14137 // Store V1, Ptr
14138 // Store V2, Ptr + sizeof(V1)
14139 // if (VECTOR_SPLICE_LEFT)
14140 // Ptr = Ptr + (Offset * sizeof(VT.Elt))
14141 // else
14142 // Ptr = Ptr + sizeof(V1) - (Offset * size(VT.Elt))
14143 // Res = Load Ptr
14144
14145 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
14146
14147 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
14148 EC: VT.getVectorElementCount() * 2);
14149 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
14150 EVT PtrVT = StackPtr.getValueType();
14151 auto &MF = DAG.getMachineFunction();
14152 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
14153 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
14154
14155 // Store the lo part of CONCAT_VECTORS(V1, V2)
14156 SDValue StoreV1 =
14157 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, PtrInfo, Alignment);
14158 // Store the hi part of CONCAT_VECTORS(V1, V2)
14159 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
14160 SDValue StackPtr2 = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: VTBytes);
14161 SDValue StoreV2 =
14162 DAG.getStore(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, PtrInfo, Alignment);
14163
14164 // NOTE: TrailingBytes must be clamped so as not to read outside of V1:V2.
14165 SDValue EltByteSize =
14166 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
14167 Offset = DAG.getZExtOrTrunc(Op: Offset, DL, VT: PtrVT);
14168 SDValue TrailingBytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: Offset, N2: EltByteSize);
14169
14170 TrailingBytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: VTBytes);
14171
14172 if (Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT)
14173 StackPtr = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TrailingBytes, DL);
14174 else
14175 StackPtr = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
14176
14177 // Load the spliced result
14178 return DAG.getLoad(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr,
14179 PtrInfo: MachinePointerInfo::getUnknownStack(MF), Alignment);
14180}
14181
14182SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
14183 SelectionDAG &DAG) const {
14184 SDLoc DL(Node);
14185 SDValue Vec = Node->getOperand(Num: 0);
14186 SDValue Mask = Node->getOperand(Num: 1);
14187 SDValue Passthru = Node->getOperand(Num: 2);
14188
14189 EVT VecVT = Vec.getValueType();
14190 EVT ScalarVT = VecVT.getScalarType();
14191 EVT MaskVT = Mask.getValueType();
14192 EVT MaskScalarVT = MaskVT.getScalarType();
14193
14194 // Needs to be handled by targets that have scalable vector types.
14195 if (VecVT.isScalableVector())
14196 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
14197
14198 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
14199 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment);
14200 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
14201 MachinePointerInfo PtrInfo =
14202 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
14203
14204 MVT PositionVT = getVectorIdxTy(DL: DAG.getDataLayout());
14205 SDValue Chain = DAG.getEntryNode();
14206 SDValue OutPos = DAG.getConstant(Val: 0, DL, VT: PositionVT);
14207
14208 bool HasPassthru = !Passthru.isUndef();
14209
14210 // If we have a passthru vector, store it on the stack, overwrite the matching
14211 // positions and then re-write the last element that was potentially
14212 // overwritten even though mask[i] = false.
14213 if (HasPassthru)
14214 Chain = DAG.getStore(Chain, dl: DL, Val: Passthru, Ptr: StackPtr, PtrInfo, Alignment);
14215
14216 SDValue LastWriteVal;
14217 APInt PassthruSplatVal;
14218 bool IsSplatPassthru = ISD::isConstantSplatVector(V: Passthru, SplatValue&: PassthruSplatVal);
14219
14220 if (IsSplatPassthru) {
14221 // As we do not know which position we wrote to last, we cannot simply
14222 // access that index from the passthru vector. So we first check if passthru
14223 // is a splat vector, to use any element ...
14224 LastWriteVal = DAG.getConstant(Val: PassthruSplatVal, DL, VT: ScalarVT);
14225 } else if (HasPassthru) {
14226 // ... if it is not a splat vector, we need to get the passthru value at
14227 // position = popcount(mask) and re-load it from the stack before it is
14228 // overwritten in the loop below.
14229 EVT PopcountVT = ScalarVT.changeTypeToInteger();
14230 SDValue Popcount = DAG.getNode(
14231 Opcode: ISD::TRUNCATE, DL,
14232 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1), Operand: Mask);
14233 Popcount = DAG.getNode(
14234 Opcode: ISD::ZERO_EXTEND, DL,
14235 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: PopcountVT),
14236 Operand: Popcount);
14237 Popcount = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: PopcountVT, Operand: Popcount);
14238 SDValue LastElmtPtr =
14239 getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Popcount);
14240 LastWriteVal = DAG.getLoad(
14241 VT: ScalarVT, dl: DL, Chain, Ptr: LastElmtPtr,
14242 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
14243 Chain = LastWriteVal.getValue(R: 1);
14244 }
14245
14246 unsigned NumElms = VecVT.getVectorNumElements();
14247 for (unsigned I = 0; I < NumElms; I++) {
14248 SDValue ValI = DAG.getExtractVectorElt(DL, VT: ScalarVT, Vec, Idx: I);
14249 SDValue OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
14250 Chain = DAG.getStore(
14251 Chain, dl: DL, Val: ValI, Ptr: OutPtr,
14252 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
14253
14254 // Get the mask value and add it to the current output position. This
14255 // either increments by 1 if MaskI is true or adds 0 otherwise.
14256 // Freeze in case we have poison/undef mask entries.
14257 SDValue MaskI = DAG.getExtractVectorElt(DL, VT: MaskScalarVT, Vec: Mask, Idx: I);
14258 MaskI = DAG.getFreeze(V: MaskI);
14259 MaskI = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: MaskI);
14260 MaskI = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PositionVT, Operand: MaskI);
14261 OutPos = DAG.getNode(Opcode: ISD::ADD, DL, VT: PositionVT, N1: OutPos, N2: MaskI);
14262
14263 if (HasPassthru && I == NumElms - 1) {
14264 SDValue EndOfVector =
14265 DAG.getConstant(Val: VecVT.getVectorNumElements() - 1, DL, VT: PositionVT);
14266 SDValue AllLanesSelected =
14267 DAG.getSetCC(DL, VT: MVT::i1, LHS: OutPos, RHS: EndOfVector, Cond: ISD::CondCode::SETUGT);
14268 OutPos = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PositionVT, N1: OutPos, N2: EndOfVector);
14269 OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
14270
14271 // Re-write the last ValI if all lanes were selected. Otherwise,
14272 // overwrite the last write it with the passthru value.
14273 LastWriteVal = DAG.getSelect(DL, VT: ScalarVT, Cond: AllLanesSelected, LHS: ValI,
14274 RHS: LastWriteVal, Flags: SDNodeFlags::Unpredictable);
14275 Chain = DAG.getStore(
14276 Chain, dl: DL, Val: LastWriteVal, Ptr: OutPtr,
14277 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
14278 }
14279 }
14280
14281 return DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo, Alignment);
14282}
14283
14284SDValue TargetLowering::expandVECTOR_SHUFFLE_VAR(SDNode *Node,
14285 SelectionDAG &DAG) const {
14286 SDLoc DL(Node);
14287 SDValue V = Node->getOperand(Num: 0);
14288 SDValue Mask = Node->getOperand(Num: 1);
14289
14290 EVT VecVT = Node->getValueType(ResNo: 0);
14291 EVT ScalarVT = VecVT.getScalarType();
14292
14293 // Needs to be handled by targets that have scalable vector types.
14294 if (VecVT.isScalableVector())
14295 reportFatalUsageError(
14296 reason: "Cannot expand vector_shuffle_var for scalable vectors.");
14297
14298 if (VecVT.getVectorNumElements() == 2) {
14299 // Select between V and V with its lanes swapped.
14300 EVT IdxVT = VecVT.changeVectorElementTypeToInteger();
14301 SDValue Idx = DAG.getZExtOrTrunc(Op: Mask, DL, VT: IdxVT);
14302 EVT CCVT =
14303 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IdxVT);
14304 SDValue Keep =
14305 DAG.getSetCC(DL, VT: CCVT, LHS: Idx, RHS: DAG.getStepVector(DL, ResVT: IdxVT), Cond: ISD::SETEQ);
14306 SDValue Swapped = DAG.getVectorShuffle(VT: VecVT, dl: DL, N1: V, N2: V, Mask: {1, 0});
14307 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Keep, N2: V, N3: Swapped);
14308 }
14309
14310 // Res[i] = V[Mask[i]]. Out-of-range extracts are undefined, as required.
14311 MVT IdxVT = getVectorIdxTy(DL: DAG.getDataLayout());
14312 SmallVector<SDValue, 16> Elts;
14313 DAG.ExtractVectorElements(Op: Mask, Args&: Elts);
14314 for (SDValue &Elt : Elts) {
14315 SDValue Idx = DAG.getZExtOrTrunc(Op: Elt, DL, VT: IdxVT);
14316 Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ScalarVT, N1: V, N2: Idx);
14317 }
14318 return DAG.getBuildVector(VT: VecVT, DL, Ops: Elts);
14319}
14320
14321SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
14322 SDLoc DL(Node);
14323 EVT VT = Node->getValueType(ResNo: 0);
14324 SDValue Op = Node->getOperand(Num: 0);
14325 ElementCount EC = Op.getValueType().getVectorElementCount();
14326
14327 bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
14328 auto [Mask, StepVec] = getLegalMaskAndStepVector(Mask: Op, ZeroIsPoison, DL, DAG);
14329
14330 // No legal step vector: split mask in half and recombine results.
14331 // LoNumElts uses the non-poison CTTZ_ELTS so its result is well-defined
14332 // (== LoNumElts when no active lane), allowing the SETNE comparison.
14333 // Result: (ResLo != LoNumElts) ? ResLo : (LoNumElts + ResHi)
14334 if (!StepVec) {
14335 EVT ResVT = Node->getValueType(ResNo: 0);
14336 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Op, DL);
14337 SDValue LoNumElts = DAG.getElementCount(
14338 DL, VT: ResVT, EC: MaskLo.getValueType().getVectorElementCount());
14339 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: MaskLo);
14340 SDValue ResHi = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Operand: MaskHi);
14341 SDValue ResLoNotNumElts = DAG.getSetCC(
14342 DL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ResVT),
14343 LHS: ResLo, RHS: LoNumElts, Cond: ISD::SETNE);
14344 // Per LangRef, ResVT must be wide enough to hold the total element count,
14345 // so the sum cannot wrap as an unsigned add. NSW is not guaranteed since
14346 // the count is only required to fit unsigned.
14347 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: LoNumElts, N2: ResHi,
14348 Flags: SDNodeFlags::NoUnsignedWrap);
14349 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotNumElts, LHS: ResLo, RHS: Sum);
14350 }
14351
14352 EVT StepVecVT = StepVec.getValueType();
14353 EVT StepVT = StepVecVT.getVectorElementType();
14354
14355 // Promote the scalar result type early to avoid redundant zexts.
14356 if (getTypeAction(VT: StepVT.getSimpleVT()) == TypePromoteInteger)
14357 StepVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVT);
14358
14359 SDValue VL = DAG.getElementCount(DL, VT: StepVT, EC);
14360 SDValue SplatVL = DAG.getSplat(VT: StepVecVT, DL, Op: VL);
14361 StepVec = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVecVT, N1: SplatVL, N2: StepVec);
14362 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
14363 SDValue Select = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
14364 SDValue Max = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL,
14365 VT: StepVecVT.getVectorElementType(), Operand: Select);
14366 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVT, N1: VL,
14367 N2: DAG.getZExtOrTrunc(Op: Max, DL, VT: StepVT));
14368
14369 return DAG.getZExtOrTrunc(Op: Sub, DL, VT);
14370}
14371
14372SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
14373 SDLoc DL(N);
14374 SDValue Source = N->getOperand(Num: 0);
14375 SDValue Needle = N->getOperand(Num: 1);
14376 SDValue Mask = N->getOperand(Num: 2);
14377 EVT SourceVT = Source.getValueType();
14378 EVT NeedleVT = Needle.getValueType();
14379 EVT ResVT = N->getValueType(ResNo: 0);
14380 EVT CmpVT =
14381 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SourceVT);
14382
14383 assert(NeedleVT.isFixedLengthVector() && "Needle must be a fixed vector");
14384
14385 SDValue Ret = DAG.getConstant(Val: 0, DL, VT: CmpVT);
14386 EVT NeedleEltVT = NeedleVT.getVectorElementType();
14387 for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
14388 SDValue Splat;
14389 if (NeedleVT == SourceVT) {
14390 // Prefer a shuffle over scalar extracts + splat for fixed vectors.
14391 Splat = DAG.getVectorShuffle(
14392 VT: SourceVT, dl: DL, N1: Needle, N2: DAG.getUNDEF(VT: SourceVT),
14393 Mask: SmallVector<int>(NeedleVT.getVectorNumElements(), I));
14394 } else {
14395 SDValue NeedleElt = DAG.getExtractVectorElt(DL, VT: NeedleEltVT, Vec: Needle, Idx: I);
14396 Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: SourceVT, Operand: NeedleElt);
14397 }
14398 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Source, RHS: Splat, Cond: ISD::SETEQ);
14399 Ret = DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Ret, N2: Cmp);
14400 }
14401
14402 EVT UseVT = ResVT;
14403 // If the result is immediately truncated, only extend to that type (to avoid
14404 // unnecessary sign/zero extends).
14405 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TRUNCATE)
14406 UseVT = N->user_begin()->getValueType(ResNo: 0);
14407
14408 Mask = DAG.getBoolExtOrTrunc(Op: Mask, SL: DL, VT: UseVT, OpVT: Mask.getValueType());
14409 Ret = DAG.getBoolExtOrTrunc(Op: Ret, SL: DL, VT: UseVT, OpVT: Ret.getValueType());
14410
14411 Ret = DAG.getNode(Opcode: ISD::AND, DL, VT: UseVT, N1: Ret, N2: Mask);
14412 if (UseVT != ResVT)
14413 Ret = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ResVT, Operand: Ret);
14414 return Ret;
14415}
14416
14417SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
14418 SelectionDAG &DAG) const {
14419 SDLoc DL(N);
14420 SDValue Acc = N->getOperand(Num: 0);
14421 SDValue MulLHS = N->getOperand(Num: 1);
14422 SDValue MulRHS = N->getOperand(Num: 2);
14423 EVT AccVT = Acc.getValueType();
14424 EVT MulOpVT = MulLHS.getValueType();
14425
14426 EVT ExtMulOpVT =
14427 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccVT.getVectorElementType(),
14428 EC: MulOpVT.getVectorElementCount());
14429
14430 unsigned ExtOpcLHS, ExtOpcRHS;
14431 switch (N->getOpcode()) {
14432 default:
14433 llvm_unreachable("Unexpected opcode");
14434 case ISD::PARTIAL_REDUCE_UMLA:
14435 ExtOpcLHS = ExtOpcRHS = ISD::ZERO_EXTEND;
14436 break;
14437 case ISD::PARTIAL_REDUCE_SMLA:
14438 ExtOpcLHS = ExtOpcRHS = ISD::SIGN_EXTEND;
14439 break;
14440 case ISD::PARTIAL_REDUCE_SUMLA:
14441 ExtOpcLHS = ISD::SIGN_EXTEND;
14442 ExtOpcRHS = ISD::ZERO_EXTEND;
14443 break;
14444 case ISD::PARTIAL_REDUCE_FMLA:
14445 ExtOpcLHS = ExtOpcRHS = ISD::FP_EXTEND;
14446 break;
14447 }
14448
14449 // A wide partial reduction is built from a ladder of narrower ones, a rung
14450 // at a time, each halving the element count and doubling the width.
14451 unsigned Opc = N->getOpcode();
14452 ElementCount MulEC = MulOpVT.getVectorElementCount();
14453 ElementCount AccEC = AccVT.getVectorElementCount();
14454 unsigned CountRatio =
14455 MulEC.hasKnownScalarFactor(RHS: AccEC) ? MulEC.getKnownScalarFactor(RHS: AccEC) : 0;
14456 unsigned WidthRatio =
14457 AccVT.getScalarSizeInBits() / MulOpVT.getScalarSizeInBits();
14458 if (Opc != ISD::PARTIAL_REDUCE_FMLA && CountRatio > 2 && WidthRatio >= 2) {
14459 LLVMContext &Ctx = *DAG.getContext();
14460 EVT ProdVT = MulOpVT.widenIntegerVectorElementType(Context&: Ctx);
14461
14462 // A pure reduction peels one rung and re-enters.
14463 if (llvm::isOneOrOneSplat(V: MulRHS)) {
14464 EVT RungVT = ProdVT.getHalfNumVectorElementsVT(Context&: Ctx);
14465 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc,
14466 N2: DAG.getNode(Opcode: Opc, DL, VT: RungVT,
14467 N1: DAG.getConstant(Val: 0, DL, VT: RungVT), N2: MulLHS,
14468 N3: MulRHS),
14469 N3: DAG.getConstant(Val: 1, DL, VT: RungVT));
14470 }
14471
14472 // A multiply widens the products by one rung, which legalizes back into a
14473 // widening multiply per half, and the ladder re-enters as a plain sum.
14474 SDValue Prod = DAG.getNode(Opcode: ISD::MUL, DL, VT: ProdVT,
14475 N1: DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ProdVT, Operand: MulLHS),
14476 N2: DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ProdVT, Operand: MulRHS));
14477 auto [Lo, Hi] = DAG.SplitVector(N: Prod, DL);
14478 SDValue One = DAG.getConstant(Val: 1, DL, VT: Lo.getValueType());
14479
14480 // The halves meet at the narrowest rung, so the accumulator is added once.
14481 EVT MidVT = Lo.getValueType()
14482 .widenIntegerVectorElementType(Context&: Ctx)
14483 .getHalfNumVectorElementsVT(Context&: Ctx);
14484 if (ElementCount::isKnownLE(LHS: MidVT.getVectorElementCount(), RHS: AccEC))
14485 return DAG.getNode(Opcode: Opc, DL, VT: AccVT,
14486 N1: DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Lo, N3: One), N2: Hi, N3: One);
14487 SDValue Mid =
14488 DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: DAG.getConstant(Val: 0, DL, VT: MidVT), N2: Lo, N3: One);
14489 Mid = DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: Mid, N2: Hi, N3: One);
14490 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Mid, N3: DAG.getConstant(Val: 1, DL, VT: MidVT));
14491 }
14492
14493 if (ExtMulOpVT != MulOpVT) {
14494 MulLHS = DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ExtMulOpVT, Operand: MulLHS);
14495 MulRHS = DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ExtMulOpVT, Operand: MulRHS);
14496 }
14497 SDValue Input = MulLHS;
14498 // Maintain the FP flags (if any).
14499 SDNodeFlags Flags = N->getFlags();
14500 if (N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA) {
14501 if (!llvm::isOneOrOneSplatFP(V: MulRHS))
14502 Input = DAG.getNode(Opcode: ISD::FMUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS, Flags);
14503 } else if (!llvm::isOneOrOneSplat(V: MulRHS)) {
14504 Input = DAG.getNode(Opcode: ISD::MUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
14505 }
14506
14507 unsigned Stride = AccVT.getVectorMinNumElements();
14508 unsigned ScaleFactor = MulOpVT.getVectorMinNumElements() / Stride;
14509
14510 // Collect all of the subvectors
14511 std::deque<SDValue> Subvectors = {Acc};
14512 for (unsigned I = 0; I < ScaleFactor; I++)
14513 Subvectors.push_back(x: DAG.getExtractSubvector(DL, VT: AccVT, Vec: Input, Idx: I * Stride));
14514
14515 unsigned FlatNode =
14516 N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA ? ISD::FADD : ISD::ADD;
14517
14518 // Flatten the subvector tree
14519 while (Subvectors.size() > 1) {
14520 Subvectors.push_back(x: DAG.getNode(Opcode: FlatNode, DL, VT: AccVT,
14521 Ops: {Subvectors[0], Subvectors[1]}, Flags));
14522 Subvectors.pop_front();
14523 Subvectors.pop_front();
14524 }
14525
14526 assert(Subvectors.size() == 1 &&
14527 "There should only be one subvector after tree flattening");
14528
14529 return Subvectors[0];
14530}
14531
14532/// Given a store node \p StoreNode, return true if it is safe to fold that node
14533/// into \p FPNode, which expands to a library call with output pointers.
14534static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
14535 SDNode *FPNode,
14536 unsigned MaxSteps) {
14537 SmallVector<const SDNode *, 8> Worklist;
14538 SmallVector<const SDNode *, 8> DeferredNodes;
14539 SmallPtrSet<const SDNode *, 16> Visited;
14540
14541 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
14542 for (SDValue Op : StoreNode->ops())
14543 if (Op.getNode() != FPNode)
14544 Worklist.push_back(Elt: Op.getNode());
14545
14546 while (!Worklist.empty()) {
14547 const SDNode *Node = Worklist.pop_back_val();
14548 auto [_, Inserted] = Visited.insert(Ptr: Node);
14549 if (!Inserted)
14550 continue;
14551
14552 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
14553 return false;
14554
14555 // Reached the FPNode (would result in a cycle).
14556 // OR Reached CALLSEQ_START (would result in nested call sequences).
14557 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
14558 return false;
14559
14560 if (Node->getOpcode() == ISD::CALLSEQ_END) {
14561 // Defer looking into call sequences (so we can check we're outside one).
14562 // We still need to look through these for the predecessor check.
14563 DeferredNodes.push_back(Elt: Node);
14564 continue;
14565 }
14566
14567 for (SDValue Op : Node->ops())
14568 Worklist.push_back(Elt: Op.getNode());
14569 }
14570
14571 // True if we're outside a call sequence and don't have the FPNode as a
14572 // predecessor. No cycles or nested call sequences possible.
14573 return !SDNode::hasPredecessorHelper(N: FPNode, Visited, Worklist&: DeferredNodes,
14574 MaxSteps);
14575}
14576
14577bool TargetLowering::expandMultipleResultFPLibCall(
14578 SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
14579 SmallVectorImpl<SDValue> &Results,
14580 std::optional<unsigned> CallRetResNo) const {
14581 if (LC == RTLIB::UNKNOWN_LIBCALL)
14582 return false;
14583
14584 RTLIB::LibcallImpl LibcallImpl =
14585 DAG.getLibcalls().getVectorLibcallImplOrMasked(Call: LC);
14586 if (LibcallImpl == RTLIB::Unsupported)
14587 return false;
14588
14589 LLVMContext &Ctx = *DAG.getContext();
14590 EVT VT = Node->getValueType(ResNo: 0);
14591 unsigned NumResults = Node->getNumValues();
14592 unsigned MaxSteps = DAG.getHasPredecessorMaxSteps();
14593
14594 // Find users of the node that store the results (and share input chains). The
14595 // destination pointers can be used instead of creating stack allocations.
14596 SDValue StoresInChain;
14597 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
14598 for (SDNode *User : Node->users()) {
14599 if (!ISD::isNormalStore(N: User))
14600 continue;
14601 auto *ST = cast<StoreSDNode>(Val: User);
14602 SDValue StoreValue = ST->getValue();
14603 unsigned ResNo = StoreValue.getResNo();
14604 // Ensure the store corresponds to an output pointer.
14605 if (CallRetResNo == ResNo)
14606 continue;
14607 // Ensure the store to the default address space and not atomic or volatile.
14608 if (!ST->isSimple() || ST->getAddressSpace() != 0)
14609 continue;
14610 // Ensure all store chains are the same (so they don't alias).
14611 if (StoresInChain && ST->getChain() != StoresInChain)
14612 continue;
14613 // Ensure the store is properly aligned.
14614 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Context&: Ctx);
14615 if (ST->getAlign() <
14616 DAG.getDataLayout().getABITypeAlign(Ty: StoreType->getScalarType()))
14617 continue;
14618 // Avoid:
14619 // 1. Creating cyclic dependencies.
14620 // 2. Expanding the node to a call within a call sequence.
14621 if (!canFoldStoreIntoLibCallOutputPointers(StoreNode: ST, FPNode: Node, MaxSteps))
14622 continue;
14623 ResultStores[ResNo] = ST;
14624 StoresInChain = ST->getChain();
14625 }
14626
14627 ArgListTy Args;
14628
14629 // Pass the arguments.
14630 for (const SDValue &Op : Node->op_values()) {
14631 EVT ArgVT = Op.getValueType();
14632 Type *ArgTy = ArgVT.getTypeForEVT(Context&: Ctx);
14633 Args.emplace_back(args: Op, args&: ArgTy);
14634 }
14635
14636 // Pass the output pointers.
14637 SmallVector<SDValue, 2> ResultPtrs(NumResults);
14638 Type *PointerTy = PointerType::getUnqual(C&: Ctx);
14639 for (auto [ResNo, ST] : llvm::enumerate(First&: ResultStores)) {
14640 if (ResNo == CallRetResNo)
14641 continue;
14642 EVT ResVT = Node->getValueType(ResNo);
14643 SDValue ResultPtr = ST ? ST->getBasePtr() : DAG.CreateStackTemporary(VT: ResVT);
14644 ResultPtrs[ResNo] = ResultPtr;
14645 Args.emplace_back(args&: ResultPtr, args&: PointerTy);
14646 }
14647
14648 SDLoc DL(Node);
14649
14650 if (RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(Impl: LibcallImpl)) {
14651 // Pass the vector mask (if required).
14652 EVT MaskVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
14653 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT);
14654 Args.emplace_back(args&: Mask, args: MaskVT.getTypeForEVT(Context&: Ctx));
14655 }
14656
14657 Type *RetType = CallRetResNo.has_value()
14658 ? Node->getValueType(ResNo: *CallRetResNo).getTypeForEVT(Context&: Ctx)
14659 : Type::getVoidTy(C&: Ctx);
14660 SDValue InChain = StoresInChain ? StoresInChain : DAG.getEntryNode();
14661 SDValue Callee =
14662 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
14663 TargetLowering::CallLoweringInfo CLI(DAG);
14664 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
14665 CC: DAG.getLibcalls().getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetType, Target: Callee,
14666 ArgsList: std::move(Args));
14667
14668 auto [Call, CallChain] = LowerCallTo(CLI);
14669
14670 for (auto [ResNo, ResultPtr] : llvm::enumerate(First&: ResultPtrs)) {
14671 if (ResNo == CallRetResNo) {
14672 Results.push_back(Elt: Call);
14673 continue;
14674 }
14675 MachinePointerInfo PtrInfo;
14676 SDValue LoadResult = DAG.getLoad(VT: Node->getValueType(ResNo), dl: DL, Chain: CallChain,
14677 Ptr: ResultPtr, PtrInfo);
14678 SDValue OutChain = LoadResult.getValue(R: 1);
14679
14680 if (StoreSDNode *ST = ResultStores[ResNo]) {
14681 // Replace store with the library call.
14682 DAG.ReplaceAllUsesOfValueWith(From: SDValue(ST, 0), To: OutChain);
14683 PtrInfo = ST->getPointerInfo();
14684 } else {
14685 PtrInfo = MachinePointerInfo::getFixedStack(
14686 MF&: DAG.getMachineFunction(),
14687 FI: cast<FrameIndexSDNode>(Val&: ResultPtr)->getIndex());
14688 }
14689
14690 Results.push_back(Elt: LoadResult);
14691 }
14692
14693 return true;
14694}
14695
14696bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
14697 SDValue &LHS, SDValue &RHS,
14698 SDValue &CC, bool &NeedInvert,
14699 const SDLoc &dl, SDValue &Chain,
14700 bool IsSignaling) const {
14701 MVT OpVT = LHS.getSimpleValueType();
14702 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
14703 NeedInvert = false;
14704 switch (getCondCodeAction(CC: CCCode, VT: OpVT)) {
14705 default:
14706 llvm_unreachable("Unknown condition code action!");
14707 case TargetLowering::Legal:
14708 // Nothing to do.
14709 break;
14710 case TargetLowering::Expand: {
14711 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(Operation: CCCode);
14712 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14713 std::swap(a&: LHS, b&: RHS);
14714 CC = DAG.getCondCode(Cond: InvCC);
14715 return true;
14716 }
14717 // Swapping operands didn't work. Try inverting the condition.
14718 bool NeedSwap = false;
14719 InvCC = getSetCCInverse(Operation: CCCode, Type: OpVT);
14720 if (!isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14721 // If inverting the condition is not enough, try swapping operands
14722 // on top of it.
14723 InvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
14724 NeedSwap = true;
14725 }
14726 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14727 CC = DAG.getCondCode(Cond: InvCC);
14728 NeedInvert = true;
14729 if (NeedSwap)
14730 std::swap(a&: LHS, b&: RHS);
14731 return true;
14732 }
14733
14734 // Special case: expand i1 comparisons using logical operations.
14735 if (OpVT == MVT::i1) {
14736 SDValue Ret;
14737 switch (CCCode) {
14738 default:
14739 llvm_unreachable("Unknown integer setcc!");
14740 case ISD::SETEQ: // X == Y --> ~(X ^ Y)
14741 Ret = DAG.getNOT(DL: dl, Val: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS),
14742 VT: MVT::i1);
14743 break;
14744 case ISD::SETNE: // X != Y --> (X ^ Y)
14745 Ret = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS);
14746 break;
14747 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
14748 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
14749 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: RHS,
14750 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14751 break;
14752 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
14753 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
14754 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: LHS,
14755 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14756 break;
14757 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
14758 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
14759 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: RHS,
14760 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14761 break;
14762 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
14763 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
14764 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: LHS,
14765 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14766 break;
14767 }
14768
14769 LHS = DAG.getZExtOrTrunc(Op: Ret, DL: dl, VT);
14770 RHS = SDValue();
14771 CC = SDValue();
14772 return true;
14773 }
14774
14775 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
14776 unsigned Opc = 0;
14777 switch (CCCode) {
14778 default:
14779 llvm_unreachable("Don't know how to expand this condition!");
14780 case ISD::SETUO:
14781 if (isCondCodeLegal(CC: ISD::SETUNE, VT: OpVT)) {
14782 CC1 = ISD::SETUNE;
14783 CC2 = ISD::SETUNE;
14784 Opc = ISD::OR;
14785 break;
14786 }
14787 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14788 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
14789 NeedInvert = true;
14790 [[fallthrough]];
14791 case ISD::SETO:
14792 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14793 "If SETO is expanded, SETOEQ must be legal!");
14794 CC1 = ISD::SETOEQ;
14795 CC2 = ISD::SETOEQ;
14796 Opc = ISD::AND;
14797 break;
14798 case ISD::SETONE:
14799 case ISD::SETUEQ:
14800 // If the SETUO or SETO CC isn't legal, we might be able to use
14801 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
14802 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
14803 // the operands.
14804 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14805 if (!isCondCodeLegal(CC: CC2, VT: OpVT) && (isCondCodeLegal(CC: ISD::SETOGT, VT: OpVT) ||
14806 isCondCodeLegal(CC: ISD::SETOLT, VT: OpVT))) {
14807 CC1 = ISD::SETOGT;
14808 CC2 = ISD::SETOLT;
14809 Opc = ISD::OR;
14810 NeedInvert = ((unsigned)CCCode & 0x8U);
14811 break;
14812 }
14813 [[fallthrough]];
14814 case ISD::SETOEQ:
14815 case ISD::SETOGT:
14816 case ISD::SETOGE:
14817 case ISD::SETOLT:
14818 case ISD::SETOLE:
14819 case ISD::SETUNE:
14820 case ISD::SETUGT:
14821 case ISD::SETUGE:
14822 case ISD::SETULT:
14823 case ISD::SETULE:
14824 // If we are floating point, assign and break, otherwise fall through.
14825 if (!OpVT.isInteger()) {
14826 // We can use the 4th bit to tell if we are the unordered
14827 // or ordered version of the opcode.
14828 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14829 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
14830 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
14831 break;
14832 }
14833 // Fallthrough if we are unsigned integer.
14834 [[fallthrough]];
14835 case ISD::SETLE:
14836 case ISD::SETGT:
14837 case ISD::SETGE:
14838 case ISD::SETLT:
14839 case ISD::SETNE:
14840 case ISD::SETEQ:
14841 // If all combinations of inverting the condition and swapping operands
14842 // didn't work then we have no means to expand the condition.
14843 llvm_unreachable("Don't know how to expand this condition!");
14844 }
14845
14846 SDValue SetCC1, SetCC2;
14847 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
14848 // If we aren't the ordered or unorder operation,
14849 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
14850 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC1, Chain, IsSignaling);
14851 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC2, Chain, IsSignaling);
14852 } else {
14853 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
14854 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS: LHS, Cond: CC1, Chain, IsSignaling);
14855 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS: RHS, RHS, Cond: CC2, Chain, IsSignaling);
14856 }
14857 if (Chain)
14858 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: SetCC1.getValue(R: 1),
14859 N2: SetCC2.getValue(R: 1));
14860 LHS = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SetCC1, N2: SetCC2);
14861 RHS = SDValue();
14862 CC = SDValue();
14863 return true;
14864 }
14865 }
14866 return false;
14867}
14868
14869SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
14870 SelectionDAG &DAG) const {
14871 EVT VT = Node->getValueType(ResNo: 0);
14872 // Despite its documentation, GetSplitDestVTs will assert if VT cannot be
14873 // split into two equal parts.
14874 if (!VT.isVector() || !VT.getVectorElementCount().isKnownMultipleOf(RHS: 2))
14875 return SDValue();
14876
14877 // Restrict expansion to cases where both parts can be concatenated.
14878 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
14879 if (LoVT != HiVT || !isTypeLegal(VT: LoVT))
14880 return SDValue();
14881
14882 SDLoc DL(Node);
14883 unsigned Opcode = Node->getOpcode();
14884
14885 // Don't expand if the result is likely to be unrolled anyway.
14886 if (!isOperationLegalOrCustomOrPromote(Op: Opcode, VT: LoVT))
14887 return SDValue();
14888
14889 SmallVector<SDValue, 4> LoOps, HiOps;
14890 for (const SDValue &V : Node->op_values()) {
14891 if (!V.getValueType().isVector()) {
14892 // Scalar operands pass through to both halves unchanged.
14893 LoOps.push_back(Elt: V);
14894 HiOps.push_back(Elt: V);
14895 continue;
14896 }
14897 auto [Lo, Hi] = DAG.SplitVector(N: V, DL, LoVT, HiVT);
14898 LoOps.push_back(Elt: Lo);
14899 HiOps.push_back(Elt: Hi);
14900 }
14901
14902 bool IsStrict = Node->isStrictFPOpcode();
14903 SDVTList LoVTs =
14904 IsStrict ? DAG.getVTList(VT1: LoVT, VT2: MVT::Other) : DAG.getVTList(VT: LoVT);
14905 SDVTList HiVTs =
14906 IsStrict ? DAG.getVTList(VT1: HiVT, VT2: MVT::Other) : DAG.getVTList(VT: HiVT);
14907 SDValue SplitOpLo = DAG.getNode(Opcode, DL, VTList: LoVTs, Ops: LoOps, Flags: Node->getFlags());
14908 SDValue SplitOpHi = DAG.getNode(Opcode, DL, VTList: HiVTs, Ops: HiOps, Flags: Node->getFlags());
14909 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: SplitOpLo, N2: SplitOpHi);
14910
14911 if (!IsStrict)
14912 return Res;
14913
14914 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
14915 N1: SplitOpLo.getValue(R: 1), N2: SplitOpHi.getValue(R: 1));
14916 return DAG.getMergeValues(Ops: {Res, Chain}, dl: DL);
14917}
14918
14919SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
14920 const SDLoc &DL,
14921 EVT InVecVT, SDValue EltNo,
14922 LoadSDNode *OriginalLoad,
14923 SelectionDAG &DAG) const {
14924 assert(OriginalLoad->isSimple());
14925
14926 EVT VecEltVT = InVecVT.getVectorElementType();
14927
14928 // If the vector element type is not a multiple of a byte then we are unable
14929 // to correctly compute an address to load only the extracted element as a
14930 // scalar.
14931 if (!VecEltVT.isByteSized())
14932 return SDValue();
14933
14934 ISD::LoadExtType ExtTy =
14935 ResultVT.bitsGT(VT: VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
14936 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: VecEltVT))
14937 return SDValue();
14938
14939 std::optional<unsigned> ByteOffset;
14940 Align Alignment = OriginalLoad->getAlign();
14941 MachinePointerInfo MPI;
14942 if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo)) {
14943 int Elt = ConstEltNo->getZExtValue();
14944 ByteOffset = VecEltVT.getSizeInBits() * Elt / 8;
14945 MPI = OriginalLoad->getPointerInfo().getWithOffset(O: *ByteOffset);
14946 Alignment = commonAlignment(A: Alignment, Offset: *ByteOffset);
14947 } else {
14948 // Discard the pointer info except the address space because the memory
14949 // operand can't represent this new access since the offset is variable.
14950 MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
14951 Alignment = commonAlignment(A: Alignment, Offset: VecEltVT.getSizeInBits() / 8);
14952 }
14953
14954 if (!shouldReduceLoadWidth(Load: OriginalLoad, ExtTy, NewVT: VecEltVT, ByteOffset))
14955 return SDValue();
14956
14957 unsigned IsFast = 0;
14958 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: VecEltVT,
14959 AddrSpace: OriginalLoad->getAddressSpace(), Alignment,
14960 Flags: OriginalLoad->getMemOperand()->getFlags(), Fast: &IsFast) ||
14961 !IsFast)
14962 return SDValue();
14963
14964 // The original DAG loaded the entire vector from memory, so arithmetic
14965 // within it must be inbounds.
14966 SDValue NewPtr = getInboundsVectorElementPointer(
14967 DAG, VecPtr: OriginalLoad->getBasePtr(), VecVT: InVecVT, Index: EltNo);
14968
14969 // We are replacing a vector load with a scalar load. The new load must have
14970 // identical memory op ordering to the original.
14971 SDValue Load;
14972 if (ResultVT.bitsGT(VT: VecEltVT)) {
14973 // If the result type of vextract is wider than the load, then issue an
14974 // extending load instead.
14975 ISD::LoadExtType ExtType =
14976 isLoadLegal(ValVT: ResultVT, MemVT: VecEltVT, Alignment,
14977 AddrSpace: OriginalLoad->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false)
14978 ? ISD::ZEXTLOAD
14979 : ISD::EXTLOAD;
14980 Load = DAG.getExtLoad(ExtType, dl: DL, VT: ResultVT, Chain: OriginalLoad->getChain(),
14981 Ptr: NewPtr, PtrInfo: MPI, MemVT: VecEltVT, Alignment,
14982 MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14983 Metadata: OriginalLoad->getMMOMetadataForSubAccess());
14984 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14985 } else {
14986 // The result type is narrower or the same width as the vector element
14987 Load = DAG.getLoad(VT: VecEltVT, dl: DL, Chain: OriginalLoad->getChain(), Ptr: NewPtr, PtrInfo: MPI,
14988 Alignment, MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14989 Metadata: OriginalLoad->getMMOMetadataForSubAccess());
14990 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14991 if (ResultVT.bitsLT(VT: VecEltVT))
14992 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: Load);
14993 else
14994 Load = DAG.getBitcast(VT: ResultVT, V: Load);
14995 }
14996
14997 return Load;
14998}
14999
15000// Set type id for call site info and metadata 'call_target'.
15001// We are filtering for:
15002// a) The call-graph-section use case that wants to know about indirect
15003// calls, or
15004// b) We want to annotate indirect calls.
15005void TargetLowering::setTypeIdForCallsiteInfo(
15006 const CallBase *CB, MachineFunction &MF,
15007 MachineFunction::CallSiteInfo &CSInfo) const {
15008 if (CB && CB->isIndirectCall() &&
15009 (MF.getTarget().Options.EmitCallGraphSection ||
15010 MF.getTarget().Options.EmitCallSiteInfo))
15011 CSInfo = MachineFunction::CallSiteInfo(*CB);
15012}
15013