1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/Analysis/VectorUtils.h"
17#include "llvm/CodeGen/Analysis.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/CodeGenCommonISel.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/SDPatternMatch.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/GlobalVariable.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCExpr.h"
33#include "llvm/Support/DivisionByConstantInfo.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Target/TargetMachine.h"
38#include <cctype>
39#include <deque>
40#include <list>
41using namespace llvm;
42using namespace llvm::SDPatternMatch;
43
44/// NOTE: The TargetMachine owns TLOF.
45TargetLowering::TargetLowering(const TargetMachine &tm,
46 const TargetSubtargetInfo &STI)
47 : TargetLoweringBase(tm, STI) {}
48
49// Define the virtual destructor out-of-line for build efficiency.
50TargetLowering::~TargetLowering() = default;
51
52const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
53 return nullptr;
54}
55
56bool TargetLowering::isPositionIndependent() const {
57 return getTargetMachine().isPositionIndependent();
58}
59
60/// Check whether a given call node is in tail position within its function. If
61/// so, it sets Chain to the input chain of the tail call.
62bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
63 SDValue &Chain) const {
64 const Function &F = DAG.getMachineFunction().getFunction();
65
66 // First, check if tail calls have been disabled in this function.
67 if (F.getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
68 return false;
69
70 // Conservatively require the attributes of the call to match those of
71 // the return. Ignore following attributes because they don't affect the
72 // call sequence.
73 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
74 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
75 Attribute::DereferenceableOrNull, Attribute::NoAlias,
76 Attribute::NonNull, Attribute::NoUndef,
77 Attribute::Range, Attribute::NoFPClass})
78 CallerAttrs.removeAttribute(Val: Attr);
79
80 if (CallerAttrs.hasAttributes())
81 return false;
82
83 // It's not safe to eliminate the sign / zero extension of the return value.
84 if (CallerAttrs.contains(A: Attribute::ZExt) ||
85 CallerAttrs.contains(A: Attribute::SExt))
86 return false;
87
88 // Check if the only use is a function return node.
89 return isUsedByReturnOnly(Node, Chain);
90}
91
92bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
93 const uint32_t *CallerPreservedMask,
94 const SmallVectorImpl<CCValAssign> &ArgLocs,
95 const SmallVectorImpl<SDValue> &OutVals) const {
96 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
97 const CCValAssign &ArgLoc = ArgLocs[I];
98 if (!ArgLoc.isRegLoc())
99 continue;
100 MCRegister Reg = ArgLoc.getLocReg();
101 // Only look at callee saved registers.
102 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg: Reg))
103 continue;
104 // Check that we pass the value used for the caller.
105 // (We look for a CopyFromReg reading a virtual register that is used
106 // for the function live-in value of register Reg)
107 SDValue Value = OutVals[I];
108 if (Value->getOpcode() == ISD::AssertZext)
109 Value = Value.getOperand(i: 0);
110 if (Value->getOpcode() != ISD::CopyFromReg)
111 return false;
112 Register ArgReg = cast<RegisterSDNode>(Val: Value->getOperand(Num: 1))->getReg();
113 if (MRI.getLiveInPhysReg(VReg: ArgReg) != Reg)
114 return false;
115 }
116 return true;
117}
118
119static bool paramHasAttr(const CallBase &Call, unsigned ArgIdx,
120 Attribute::AttrKind Kind) {
121 return Call.paramHasAttr(ArgNo: ArgIdx, Kind);
122}
123
124static bool paramHasAttr(const AttributeList &Attrs, unsigned ArgIdx,
125 Attribute::AttrKind Kind) {
126 return Attrs.hasParamAttr(ArgNo: ArgIdx, Kind);
127}
128
129static MaybeAlign getParamStackAlign(const CallBase &Call, unsigned ArgIdx) {
130 return Call.getParamStackAlign(ArgNo: ArgIdx);
131}
132
133static MaybeAlign getParamStackAlign(const AttributeList &Attrs,
134 unsigned ArgIdx) {
135 return Attrs.getParamStackAlignment(ArgNo: ArgIdx);
136}
137
138static MaybeAlign getParamAlign(const CallBase &Call, unsigned ArgIdx) {
139 return Call.getParamAlign(ArgNo: ArgIdx);
140}
141
142static MaybeAlign getParamAlign(const AttributeList &Attrs, unsigned ArgIdx) {
143 return Attrs.getParamAlignment(ArgNo: ArgIdx);
144}
145
146/// Set CallLoweringInfo attribute flags based on a call instruction
147/// and called function attributes.
148template <typename SourceT>
149static void setArgListEntryAttributes(TargetLoweringBase::ArgListEntry &Entry,
150 const SourceT &Src, unsigned ArgIdx) {
151 Entry.IsSExt = paramHasAttr(Src, ArgIdx, Attribute::SExt);
152 Entry.IsZExt = paramHasAttr(Src, ArgIdx, Attribute::ZExt);
153 Entry.IsNoExt = paramHasAttr(Src, ArgIdx, Attribute::NoExt);
154 Entry.IsInReg = paramHasAttr(Src, ArgIdx, Attribute::InReg);
155 Entry.IsSRet = paramHasAttr(Src, ArgIdx, Attribute::StructRet);
156 Entry.IsNest = paramHasAttr(Src, ArgIdx, Attribute::Nest);
157 Entry.IsByVal = paramHasAttr(Src, ArgIdx, Attribute::ByVal);
158 Entry.IsPreallocated = paramHasAttr(Src, ArgIdx, Attribute::Preallocated);
159 Entry.IsInAlloca = paramHasAttr(Src, ArgIdx, Attribute::InAlloca);
160 Entry.IsReturned = paramHasAttr(Src, ArgIdx, Attribute::Returned);
161 Entry.IsSwiftSelf = paramHasAttr(Src, ArgIdx, Attribute::SwiftSelf);
162 Entry.IsSwiftAsync = paramHasAttr(Src, ArgIdx, Attribute::SwiftAsync);
163 Entry.IsSwiftError = paramHasAttr(Src, ArgIdx, Attribute::SwiftError);
164 Entry.Alignment = getParamStackAlign(Src, ArgIdx);
165 Entry.IndirectType = nullptr;
166 assert(Entry.IsByVal + Entry.IsPreallocated + Entry.IsInAlloca +
167 Entry.IsSRet <=
168 1 &&
169 "multiple ABI attributes?");
170 if (Entry.IsByVal) {
171 Entry.IndirectType = Src.getParamByValType(ArgIdx);
172 if (!Entry.Alignment)
173 Entry.Alignment = getParamAlign(Src, ArgIdx);
174 }
175 if (Entry.IsPreallocated)
176 Entry.IndirectType = Src.getParamPreallocatedType(ArgIdx);
177 if (Entry.IsInAlloca)
178 Entry.IndirectType = Src.getParamInAllocaType(ArgIdx);
179 if (Entry.IsSRet)
180 Entry.IndirectType = Src.getParamStructRetType(ArgIdx);
181}
182
183TargetLoweringBase::ArgListEntry::ArgListEntry(SDValue Node, Type *Ty,
184 Attribute::AttrKind Attr)
185 : ArgListEntry(nullptr, Node, Ty) {
186 if (Attr == Attribute::SExt)
187 IsSExt = true;
188 else if (Attr == Attribute::ZExt)
189 IsZExt = true;
190 else if (Attr != Attribute::None)
191 llvm_unreachable("Unexpected attribute left unhandled.");
192}
193
194void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
195 unsigned ArgIdx) {
196 setArgListEntryAttributes(Entry&: *this, Src: *Call, ArgIdx);
197}
198
199void TargetLoweringBase::ArgListEntry::setAttributes(const AttributeList &Attrs,
200 unsigned ArgIdx) {
201 setArgListEntryAttributes(Entry&: *this, Src: Attrs, ArgIdx);
202}
203
204TargetLowering::ArgListTy
205TargetLowering::getArgListForFunctionType(FunctionType *FuncTy,
206 const AttributeList &FuncAttrs,
207 ArrayRef<SDValue> Ops) {
208 // TODO: This assumes each parameter maps to exactly one operand node, which
209 // does not hold when an argument requires type splitting.
210 assert(Ops.size() == FuncTy->getNumParams() &&
211 "argument count does not match the function type");
212 ArgListTy Args;
213 Args.reserve(n: Ops.size());
214 for (unsigned I = 0, E = FuncTy->getNumParams(); I != E; ++I) {
215 ArgListEntry Entry(Ops[I], FuncTy->getParamType(i: I));
216 Entry.setAttributes(Attrs: FuncAttrs, ArgIdx: I);
217 Args.push_back(x: Entry);
218 }
219 return Args;
220}
221
222/// Generate a libcall taking the given operands as arguments and returning a
223/// result of type RetVT.
224std::pair<SDValue, SDValue>
225TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl,
226 EVT RetVT, ArrayRef<SDValue> Ops,
227 MakeLibCallOptions CallOptions, const SDLoc &dl,
228 SDValue InChain) const {
229 if (LibcallImpl == RTLIB::Unsupported)
230 reportFatalInternalError(reason: "unsupported library call operation");
231
232 if (!InChain)
233 InChain = DAG.getEntryNode();
234
235 TargetLowering::ArgListTy Args;
236 Args.reserve(n: Ops.size());
237
238 ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
239 for (unsigned i = 0; i < Ops.size(); ++i) {
240 SDValue NewOp = Ops[i];
241 Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
242 ? OpsTypeOverrides[i]
243 : NewOp.getValueType().getTypeForEVT(Context&: *DAG.getContext());
244 TargetLowering::ArgListEntry Entry(NewOp, Ty);
245 if (CallOptions.IsSoften)
246 Entry.OrigTy =
247 CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(Context&: *DAG.getContext());
248
249 Entry.IsSExt =
250 shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned: CallOptions.IsSigned);
251 Entry.IsZExt = !Entry.IsSExt;
252
253 if (CallOptions.IsSoften &&
254 !shouldExtendTypeInLibCall(Type: CallOptions.OpsVTBeforeSoften[i])) {
255 Entry.IsSExt = Entry.IsZExt = false;
256 }
257 Args.push_back(x: Entry);
258 }
259
260 SDValue Callee =
261 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
262
263 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
264 Type *OrigRetTy = RetTy;
265 TargetLowering::CallLoweringInfo CLI(DAG);
266 bool signExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: CallOptions.IsSigned);
267 bool zeroExtend = !signExtend;
268
269 if (CallOptions.IsSoften) {
270 OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(Context&: *DAG.getContext());
271 if (!shouldExtendTypeInLibCall(Type: CallOptions.RetVTBeforeSoften))
272 signExtend = zeroExtend = false;
273 }
274
275 CLI.setDebugLoc(dl)
276 .setChain(InChain)
277 .setLibCallee(CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetTy, OrigResultType: OrigRetTy,
278 Target: Callee, ArgsList: std::move(Args))
279 .setNoReturn(CallOptions.DoesNotReturn)
280 .setDiscardResult(!CallOptions.IsReturnValueUsed)
281 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
282 .setSExtResult(signExtend)
283 .setZExtResult(zeroExtend);
284 return LowerCallTo(CLI);
285}
286
287bool TargetLowering::findOptimalMemOpLowering(
288 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
289 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
290 const AttributeList &FuncAttributes, EVT *LargestVT) const {
291 EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);
292
293 if (VT == MVT::Other) {
294 // Use the largest integer type whose alignment constraints are satisfied.
295 VT = MVT::LAST_INTEGER_VALUETYPE;
296 if (Op.isFixedDstAlign()) {
297 bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
298 while (VT != MVT::i8) {
299 unsigned VTSize = VT.getSizeInBits() / 8;
300 bool DstOk =
301 Op.getDstAlign() >= VTSize ||
302 allowsMisalignedMemoryAccesses(VT, AddrSpace: DstAS, Alignment: Op.getDstAlign());
303 bool SrcOk =
304 !LoadsFromSrc || Op.getSrcAlign() >= VTSize ||
305 allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcAS, Alignment: Op.getSrcAlign());
306 if (DstOk && SrcOk)
307 break;
308 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
309 }
310 }
311 assert(VT.isInteger());
312
313 // Find the largest legal integer type.
314 MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
315 while (!isTypeLegal(VT: LVT))
316 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
317 assert(LVT.isInteger());
318
319 // If the type we've chosen is larger than the largest legal integer type
320 // then use the largest legal type.
321 if (VT.bitsGT(VT: LVT))
322 VT = LVT;
323 }
324
325 unsigned NumMemOps = 0;
326 uint64_t Size = Op.size();
327 while (Size) {
328 unsigned VTSize = VT.getSizeInBits() / 8;
329 while (VTSize > Size) {
330 // For now, only use non-vector load / store's for the left-over pieces.
331 EVT NewVT = VT;
332 unsigned NewVTSize;
333
334 bool Found = false;
335 if (VT.isVector() || VT.isFloatingPoint()) {
336 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
337 if (isOperationLegalOrCustom(Op: ISD::STORE, VT: NewVT) &&
338 isSafeMemOpType(NewVT.getSimpleVT()))
339 Found = true;
340 else if (NewVT == MVT::i64 &&
341 isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::f64) &&
342 isSafeMemOpType(MVT::f64)) {
343 // i64 is usually not legal on 32-bit targets, but f64 may be.
344 NewVT = MVT::f64;
345 Found = true;
346 }
347 }
348
349 if (!Found) {
350 do {
351 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
352 if (NewVT == MVT::i8)
353 break;
354 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
355 }
356 NewVTSize = NewVT.getSizeInBits() / 8;
357
358 // If the new VT cannot cover all of the remaining bits, then consider
359 // issuing a (or a pair of) unaligned and overlapping load / store.
360 unsigned Fast;
361 if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
362 allowsMisalignedMemoryAccesses(
363 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
364 Flags: MachineMemOperand::MONone, &Fast) &&
365 Fast)
366 VTSize = Size;
367 else {
368 VT = NewVT;
369 VTSize = NewVTSize;
370 }
371 }
372
373 if (++NumMemOps > Limit)
374 return false;
375
376 MemOps.push_back(x: VT);
377 Size -= VTSize;
378 }
379
380 return true;
381}
382
383/// Soften the operands of a comparison. This code is shared among BR_CC,
384/// SELECT_CC, and SETCC handlers.
385void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
386 SDValue &NewLHS, SDValue &NewRHS,
387 ISD::CondCode &CCCode,
388 const SDLoc &dl, const SDValue OldLHS,
389 const SDValue OldRHS) const {
390 SDValue Chain;
391 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: dl, OldLHS,
392 OldRHS, Chain);
393}
394
395/// Select the libcall and the condition code to test its result against 0 for
396/// an ordered floating-point compare. \p BoolLC is the boolean helper (result
397/// is 0/1). \p TriStateLC is the per-predicate three-way helper and \p
398/// GenericLC the generic single-symbol three-way helper (both return -1/0/1,
399/// tested against 0 with \p TriStateCC). The boolean form is preferred, then
400/// the per-predicate three-way, then the generic three-way.
401static std::pair<RTLIB::Libcall, ISD::CondCode>
402selectFPCmpLibcall(const LibcallLoweringInfo &Libcalls, RTLIB::Libcall BoolLC,
403 RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC,
404 ISD::CondCode TriStateCC) {
405 if (Libcalls.getLibcallImpl(Call: BoolLC) != RTLIB::Unsupported)
406 return {BoolLC, ISD::SETNE};
407 if (Libcalls.getLibcallImpl(Call: TriStateLC) != RTLIB::Unsupported)
408 return {TriStateLC, TriStateCC};
409 return {GenericLC, TriStateCC};
410}
411
412void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
413 SDValue &NewLHS, SDValue &NewRHS,
414 ISD::CondCode &CCCode,
415 const SDLoc &dl, const SDValue OldLHS,
416 const SDValue OldRHS,
417 SDValue &Chain,
418 bool IsSignaling) const {
419 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
420 // not supporting it. We can update this code when libgcc provides such
421 // functions.
422
423 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
424 && "Unsupported setcc type!");
425
426 // Expand into one or more soft-fp libcall(s).
427 ISD::CondCode OrigCCCode = CCCode;
428 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
429 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
430 bool ShouldInvertCC = false;
431
432 // Expand a compare libcall family name (e.g. OEQ, FCMP3_PRED_OEQ) to the
433 // RTLIB::Libcall for VT.
434#define FP_CMP_LIBCALL(BASE) \
435 RTLIB::getFPLibCall(VT, RTLIB::BASE##_F32, RTLIB::BASE##_F64, \
436 RTLIB::UNKNOWN_LIBCALL, RTLIB::BASE##_F128, \
437 RTLIB::BASE##_PPCF128)
438
439 switch (CCCode) {
440 case ISD::SETEQ:
441 case ISD::SETOEQ:
442 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
443 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
444 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
445 break;
446 case ISD::SETNE:
447 case ISD::SETUNE:
448 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
449 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(UNE), FP_CMP_LIBCALL(FCMP3_PRED_UNE),
450 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETNE);
451 // Some ABIs (e.g. AEABI) provide neither a not-equal nor a three-way
452 // compare; obtain not-equal (UNE = !OEQ) by inverting ordered-equal.
453 if (DAG.getLibcalls().getLibcallImpl(Call: LC1) == RTLIB::Unsupported) {
454 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
455 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ),
456 FP_CMP_LIBCALL(FCMP3_PRED_OEQ), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
457 ShouldInvertCC = true;
458 }
459 break;
460 case ISD::SETGE:
461 case ISD::SETOGE:
462 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
463 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE), FP_CMP_LIBCALL(FCMP3_PRED_OGE),
464 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
465 break;
466 case ISD::SETLT:
467 case ISD::SETOLT:
468 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
469 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT), FP_CMP_LIBCALL(FCMP3_PRED_OLT),
470 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
471 break;
472 case ISD::SETLE:
473 case ISD::SETOLE:
474 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
475 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE), FP_CMP_LIBCALL(FCMP3_PRED_OLE),
476 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
477 break;
478 case ISD::SETGT:
479 case ISD::SETOGT:
480 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
481 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT), FP_CMP_LIBCALL(FCMP3_PRED_OGT),
482 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
483 break;
484 case ISD::SETO:
485 ShouldInvertCC = true;
486 [[fallthrough]];
487 case ISD::SETUO:
488 // Unordered is a boolean everywhere (__unordXf2 returns 0/1).
489 LC1 = FP_CMP_LIBCALL(UO);
490 CC1 = ISD::SETNE;
491 break;
492 case ISD::SETONE:
493 // SETONE = O && UNE
494 ShouldInvertCC = true;
495 [[fallthrough]];
496 case ISD::SETUEQ:
497 LC1 = FP_CMP_LIBCALL(UO);
498 CC1 = ISD::SETNE;
499 std::tie(args&: LC2, args&: CC2) = selectFPCmpLibcall(
500 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
501 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
502 break;
503 default:
504 // Invert CC for unordered comparisons, handled by the ordered inverse.
505 ShouldInvertCC = true;
506 switch (CCCode) {
507 case ISD::SETULT:
508 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
509 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE),
510 FP_CMP_LIBCALL(FCMP3_PRED_OGE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
511 break;
512 case ISD::SETULE:
513 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
514 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT),
515 FP_CMP_LIBCALL(FCMP3_PRED_OGT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
516 break;
517 case ISD::SETUGT:
518 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
519 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE),
520 FP_CMP_LIBCALL(FCMP3_PRED_OLE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
521 break;
522 case ISD::SETUGE:
523 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
524 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT),
525 FP_CMP_LIBCALL(FCMP3_PRED_OLT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
526 break;
527 default:
528 llvm_unreachable("Do not know how to soften this setcc!");
529 }
530 }
531
532#undef FP_CMP_LIBCALL
533
534 // Use the target specific return value for comparison lib calls.
535 EVT RetVT = getCmpLibcallReturnType();
536 SDValue Ops[2] = {NewLHS, NewRHS};
537 TargetLowering::MakeLibCallOptions CallOptions;
538 EVT OpsVT[2] = { OldLHS.getValueType(),
539 OldRHS.getValueType() };
540 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
541
542 auto ReportNoLibcall = [&]() {
543 DAG.getContext()->emitError(
544 ErrorStr: Twine("no libcall available to soften floating-point ") +
545 ISD::getCondCodeName(Operation: OrigCCCode) + " compare with type " +
546 VT.getEVTString());
547 NewLHS = DAG.getPOISON(VT: RetVT);
548 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
549 CCCode = ISD::SETNE;
550 };
551
552 // Check availability before makeLibCall, which fatally errors otherwise.
553 RTLIB::LibcallImpl LC1Impl = DAG.getLibcalls().getLibcallImpl(Call: LC1);
554 if (LC1Impl == RTLIB::Unsupported) {
555 ReportNoLibcall();
556 return;
557 }
558
559 auto Call = makeLibCall(DAG, LibcallImpl: LC1Impl, RetVT, Ops, CallOptions, dl, InChain: Chain);
560 NewLHS = Call.first;
561 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
562
563 CCCode = CC1;
564 if (ShouldInvertCC) {
565 assert(RetVT.isInteger());
566 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
567 }
568
569 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
570 // Update Chain.
571 Chain = Call.second;
572 } else {
573 RTLIB::LibcallImpl LC2Impl = DAG.getLibcalls().getLibcallImpl(Call: LC2);
574 if (LC2Impl == RTLIB::Unsupported) {
575 ReportNoLibcall();
576 return;
577 }
578
579 assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
580 "unordered call should be simple boolean");
581
582 EVT SetCCVT =
583 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: RetVT);
584 if (getBooleanContents(Type: RetVT) == ZeroOrOneBooleanContent) {
585 NewLHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: RetVT, N1: Call.first,
586 N2: DAG.getValueType(MVT::i1));
587 }
588
589 SDValue Tmp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NewLHS, RHS: NewRHS, Cond: CCCode);
590 auto Call2 = makeLibCall(DAG, LibcallImpl: LC2Impl, RetVT, Ops, CallOptions, dl, InChain: Chain);
591 CCCode = CC2;
592 if (ShouldInvertCC)
593 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
594 NewLHS = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Call2.first, RHS: NewRHS, Cond: CCCode);
595 if (Chain)
596 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Call.second,
597 N2: Call2.second);
598 NewLHS = DAG.getNode(Opcode: ShouldInvertCC ? ISD::AND : ISD::OR, DL: dl,
599 VT: Tmp.getValueType(), N1: Tmp, N2: NewLHS);
600 NewRHS = SDValue();
601 }
602}
603
604/// Return the entry encoding for a jump table in the current function. The
605/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
606unsigned TargetLowering::getJumpTableEncoding() const {
607 // In non-pic modes, just use the address of a block.
608 if (!isPositionIndependent())
609 return MachineJumpTableInfo::EK_BlockAddress;
610
611 // Otherwise, use a label difference.
612 return MachineJumpTableInfo::EK_LabelDifference32;
613}
614
615SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
616 SelectionDAG &DAG) const {
617 return Table;
618}
619
620/// This returns the relocation base for the given PIC jumptable, the same as
621/// getPICJumpTableRelocBase, but as an MCExpr.
622const MCExpr *
623TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
624 unsigned JTI,MCContext &Ctx) const{
625 // The normal PIC reloc base is the label at the start of the jump table.
626 return MCSymbolRefExpr::create(Symbol: MF->getJTISymbol(JTI, Ctx), Ctx);
627}
628
629SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
630 SDValue Addr, int JTI,
631 SelectionDAG &DAG) const {
632 SDValue Chain = Value;
633 // Jump table debug info is only needed if CodeView is enabled.
634 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF()) {
635 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
636 }
637 return DAG.getNode(Opcode: ISD::BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
638}
639
640bool
641TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
642 const TargetMachine &TM = getTargetMachine();
643 const GlobalValue *GV = GA->getGlobal();
644
645 // If the address is not even local to this DSO we will have to load it from
646 // a got and then add the offset.
647 if (!TM.shouldAssumeDSOLocal(GV))
648 return false;
649
650 // If the code is position independent we will have to add a base register.
651 if (isPositionIndependent())
652 return false;
653
654 // Otherwise we can do it.
655 return true;
656}
657
658//===----------------------------------------------------------------------===//
659// Optimization Methods
660//===----------------------------------------------------------------------===//
661
662/// If the specified instruction has a constant integer operand and there are
663/// bits set in that constant that are not demanded, then clear those bits and
664/// return true.
665bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
666 const APInt &DemandedBits,
667 const APInt &DemandedElts,
668 TargetLoweringOpt &TLO) const {
669 SDLoc DL(Op);
670 unsigned Opcode = Op.getOpcode();
671
672 // Early-out if we've ended up calling an undemanded node, leave this to
673 // constant folding.
674 if (DemandedBits.isZero() || DemandedElts.isZero())
675 return false;
676
677 // Do target-specific constant optimization.
678 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
679 return TLO.New.getNode();
680
681 // FIXME: ISD::SELECT, ISD::SELECT_CC
682 switch (Opcode) {
683 default:
684 break;
685 case ISD::XOR:
686 case ISD::AND:
687 case ISD::OR: {
688 auto *Op1C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
689 if (!Op1C || Op1C->isOpaque())
690 return false;
691
692 // If this is a 'not' op, don't touch it because that's a canonical form.
693 const APInt &C = Op1C->getAPIntValue();
694 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(RHS: C))
695 return false;
696
697 if (!C.isSubsetOf(RHS: DemandedBits)) {
698 EVT VT = Op.getValueType();
699 SDValue NewC = TLO.DAG.getConstant(Val: DemandedBits & C, DL, VT);
700 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 0), N2: NewC,
701 Flags: Op->getFlags());
702 return TLO.CombineTo(O: Op, N: NewOp);
703 }
704
705 break;
706 }
707 }
708
709 return false;
710}
711
712bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
713 const APInt &DemandedBits,
714 TargetLoweringOpt &TLO) const {
715 EVT VT = Op.getValueType();
716 APInt DemandedElts = VT.isVector()
717 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
718 : APInt(1, 1);
719 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
720}
721
722/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
723/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
724/// but it could be generalized for targets with other types of implicit
725/// widening casts.
726bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
727 const APInt &DemandedBits,
728 TargetLoweringOpt &TLO) const {
729 assert(Op.getNumOperands() == 2 &&
730 "ShrinkDemandedOp only supports binary operators!");
731 assert(Op.getNode()->getNumValues() == 1 &&
732 "ShrinkDemandedOp only supports nodes with one result!");
733
734 EVT VT = Op.getValueType();
735 SelectionDAG &DAG = TLO.DAG;
736 SDLoc dl(Op);
737
738 // Early return, as this function cannot handle vector types.
739 if (VT.isVector())
740 return false;
741
742 assert(Op.getOperand(0).getValueType().getScalarSizeInBits() == BitWidth &&
743 Op.getOperand(1).getValueType().getScalarSizeInBits() == BitWidth &&
744 "ShrinkDemandedOp only supports operands that have the same size!");
745
746 // Don't do this if the node has another user, which may require the
747 // full value.
748 if (!Op.getNode()->hasOneUse())
749 return false;
750
751 // Search for the smallest integer type with free casts to and from
752 // Op's type. For expedience, just check power-of-2 integer types.
753 unsigned DemandedSize = DemandedBits.getActiveBits();
754 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
755 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
756 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SmallVTBits);
757 if (isTruncateFree(Val: Op, VT2: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT)) {
758 // We found a type with free casts.
759
760 // If the operation has the 'disjoint' flag, then the
761 // operands on the new node are also disjoint.
762 SDNodeFlags Flags(Op->getFlags().hasDisjoint() ? SDNodeFlags::Disjoint
763 : SDNodeFlags::None);
764 unsigned Opcode = Op.getOpcode();
765 if (Opcode == ISD::PTRADD) {
766 // It isn't a ptradd anymore if it doesn't operate on the entire
767 // pointer.
768 Opcode = ISD::ADD;
769 }
770 SDValue X = DAG.getNode(
771 Opcode, DL: dl, VT: SmallVT,
772 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
773 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 1)), Flags);
774 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
775 SDValue Z = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: X);
776 return TLO.CombineTo(O: Op, N: Z);
777 }
778 }
779 return false;
780}
781
782bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
783 DAGCombinerInfo &DCI) const {
784 SelectionDAG &DAG = DCI.DAG;
785 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
786 !DCI.isBeforeLegalizeOps());
787 KnownBits Known;
788
789 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
790 if (Simplified) {
791 DCI.AddToWorklist(N: Op.getNode());
792 DCI.CommitTargetLoweringOpt(TLO);
793 }
794 return Simplified;
795}
796
797bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
798 const APInt &DemandedElts,
799 DAGCombinerInfo &DCI) const {
800 SelectionDAG &DAG = DCI.DAG;
801 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
802 !DCI.isBeforeLegalizeOps());
803 KnownBits Known;
804
805 bool Simplified =
806 SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
807 if (Simplified) {
808 DCI.AddToWorklist(N: Op.getNode());
809 DCI.CommitTargetLoweringOpt(TLO);
810 }
811 return Simplified;
812}
813
814bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
815 KnownBits &Known,
816 TargetLoweringOpt &TLO,
817 unsigned Depth,
818 bool AssumeSingleUse) const {
819 EVT VT = Op.getValueType();
820
821 // Since the number of lanes in a scalable vector is unknown at compile time,
822 // we track one bit which is implicitly broadcast to all lanes. This means
823 // that all lanes in a scalable vector are considered demanded.
824 APInt DemandedElts = VT.isFixedLengthVector()
825 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
826 : APInt(1, 1);
827 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
828 AssumeSingleUse);
829}
830
831// TODO: Under what circumstances can we create nodes? Constant folding?
832SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
833 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
834 SelectionDAG &DAG, unsigned Depth) const {
835 EVT VT = Op.getValueType();
836
837 // Limit search depth.
838 if (Depth >= SelectionDAG::MaxRecursionDepth)
839 return SDValue();
840
841 // Ignore UNDEFs.
842 if (Op.isUndef())
843 return SDValue();
844
845 // Not demanding any bits/elts from Op.
846 if (DemandedBits == 0 || DemandedElts == 0)
847 return DAG.getUNDEF(VT);
848
849 bool IsLE = DAG.getDataLayout().isLittleEndian();
850 unsigned NumElts = DemandedElts.getBitWidth();
851 unsigned BitWidth = DemandedBits.getBitWidth();
852 KnownBits LHSKnown, RHSKnown;
853 switch (Op.getOpcode()) {
854 case ISD::Constant: {
855 const APInt &Value = Op->getAsAPIntVal();
856 if (!Value.isZero() && (Value & DemandedBits).isZero())
857 return DAG.getConstant(Val: 0, DL: SDLoc(Op), VT);
858 break;
859 }
860 case ISD::BITCAST: {
861 if (VT.isScalableVector())
862 return SDValue();
863
864 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
865 EVT SrcVT = Src.getValueType();
866 EVT DstVT = Op.getValueType();
867 if (SrcVT == DstVT)
868 return Src;
869
870 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
871 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
872 if (NumSrcEltBits == NumDstEltBits)
873 if (SDValue V = SimplifyMultipleUseDemandedBits(
874 Op: Src, DemandedBits, DemandedElts, DAG, Depth: Depth + 1))
875 return DAG.getBitcast(VT: DstVT, V);
876
877 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
878 unsigned Scale = NumDstEltBits / NumSrcEltBits;
879 unsigned NumSrcElts = SrcVT.getVectorNumElements();
880 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
881 for (unsigned i = 0; i != Scale; ++i) {
882 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
883 unsigned BitOffset = EltOffset * NumSrcEltBits;
884 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
885 }
886 // Recursive calls below may turn not demanded elements into poison, so we
887 // need to demand all smaller source elements that maps to a demanded
888 // destination element.
889 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
890
891 if (SDValue V = SimplifyMultipleUseDemandedBits(
892 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
893 return DAG.getBitcast(VT: DstVT, V);
894 }
895
896 // TODO - bigendian once we have test coverage.
897 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
898 unsigned Scale = NumSrcEltBits / NumDstEltBits;
899 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
900 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
901 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
902 for (unsigned i = 0; i != NumElts; ++i)
903 if (DemandedElts[i]) {
904 unsigned Offset = (i % Scale) * NumDstEltBits;
905 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
906 DemandedSrcElts.setBit(i / Scale);
907 }
908
909 if (SDValue V = SimplifyMultipleUseDemandedBits(
910 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
911 return DAG.getBitcast(VT: DstVT, V);
912 }
913
914 break;
915 }
916 case ISD::SCALAR_TO_VECTOR: {
917 if (!VT.isScalableVector() && !DemandedElts[0])
918 return DAG.getPOISON(VT);
919 break;
920 }
921 case ISD::AND: {
922 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
923 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
924
925 // If all of the demanded bits are known 1 on one side, return the other.
926 // These bits cannot contribute to the result of the 'and' in this
927 // context.
928 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero | RHSKnown.One))
929 return Op.getOperand(i: 0);
930 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.One))
931 return Op.getOperand(i: 1);
932 break;
933 }
934 case ISD::OR: {
935 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
936 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
937
938 // If all of the demanded bits are known zero on one side, return the
939 // other. These bits cannot contribute to the result of the 'or' in this
940 // context.
941 if (DemandedBits.isSubsetOf(RHS: LHSKnown.One | RHSKnown.Zero))
942 return Op.getOperand(i: 0);
943 if (DemandedBits.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
944 return Op.getOperand(i: 1);
945 break;
946 }
947 case ISD::XOR: {
948 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
949 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
950
951 // If all of the demanded bits are known zero on one side, return the
952 // other.
953 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero))
954 return Op.getOperand(i: 0);
955 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero))
956 return Op.getOperand(i: 1);
957 break;
958 }
959 case ISD::ADD:
960 case ISD::MUL:
961 case ISD::SMIN:
962 case ISD::SMAX:
963 case ISD::UMIN:
964 case ISD::UMAX: {
965 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 1),
966 DemandedElts, OperandNo: 1, Depth: Depth + 1))
967 return Op.getOperand(i: 0);
968
969 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 0),
970 DemandedElts, OperandNo: 0, Depth: Depth + 1))
971 return Op.getOperand(i: 1);
972 break;
973 }
974 case ISD::SHL: {
975 // If we are only demanding sign bits then we can use the shift source
976 // directly.
977 if (std::optional<unsigned> MaxSA =
978 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
979 SDValue Op0 = Op.getOperand(i: 0);
980 unsigned ShAmt = *MaxSA;
981 unsigned NumSignBits =
982 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
983 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
984 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
985 return Op0;
986 }
987 break;
988 }
989 case ISD::SRL: {
990 // If we are only demanding sign bits then we can use the shift source
991 // directly.
992 if (std::optional<unsigned> MaxSA =
993 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
994 SDValue Op0 = Op.getOperand(i: 0);
995 unsigned ShAmt = *MaxSA;
996 // Must already be signbits in DemandedBits bounds, and can't demand any
997 // shifted in zeroes.
998 if (DemandedBits.countl_zero() >= ShAmt) {
999 unsigned NumSignBits =
1000 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
1001 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
1002 return Op0;
1003 }
1004 }
1005 break;
1006 }
1007 case ISD::SETCC: {
1008 SDValue Op0 = Op.getOperand(i: 0);
1009 SDValue Op1 = Op.getOperand(i: 1);
1010 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1011 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1012 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1013 // -1, we may be able to bypass the setcc.
1014 if (DemandedBits.isSignMask() &&
1015 Op0.getScalarValueSizeInBits() == BitWidth &&
1016 getBooleanContents(Type: Op0.getValueType()) ==
1017 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1018 // If we're testing X < 0, then this compare isn't needed - just use X!
1019 // FIXME: We're limiting to integer types here, but this should also work
1020 // if we don't care about FP signed-zero. The use of SETLT with FP means
1021 // that we don't care about NaNs.
1022 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1023 (isNullConstant(V: Op1) || ISD::isBuildVectorAllZeros(N: Op1.getNode())))
1024 return Op0;
1025 }
1026 break;
1027 }
1028 case ISD::SIGN_EXTEND_INREG: {
1029 // If none of the extended bits are demanded, eliminate the sextinreg.
1030 SDValue Op0 = Op.getOperand(i: 0);
1031 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
1032 unsigned ExBits = ExVT.getScalarSizeInBits();
1033 if (DemandedBits.getActiveBits() <= ExBits &&
1034 shouldRemoveRedundantExtend(Op))
1035 return Op0;
1036 // If the input is already sign extended, just drop the extension.
1037 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
1038 if (NumSignBits >= (BitWidth - ExBits + 1))
1039 return Op0;
1040 break;
1041 }
1042 case ISD::ANY_EXTEND_VECTOR_INREG:
1043 case ISD::SIGN_EXTEND_VECTOR_INREG:
1044 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1045 if (VT.isScalableVector())
1046 return SDValue();
1047
1048 // If we only want the lowest element and none of extended bits, then we can
1049 // return the bitcasted source vector.
1050 SDValue Src = Op.getOperand(i: 0);
1051 EVT SrcVT = Src.getValueType();
1052 EVT DstVT = Op.getValueType();
1053 if (IsLE && DemandedElts == 1 &&
1054 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
1055 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
1056 return DAG.getBitcast(VT: DstVT, V: Src);
1057 }
1058 break;
1059 }
1060 case ISD::INSERT_VECTOR_ELT: {
1061 if (VT.isScalableVector())
1062 return SDValue();
1063
1064 // If we don't demand the inserted element, return the base vector.
1065 SDValue Vec = Op.getOperand(i: 0);
1066 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1067 EVT VecVT = Vec.getValueType();
1068 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements()) &&
1069 !DemandedElts[CIdx->getZExtValue()])
1070 return Vec;
1071 break;
1072 }
1073 case ISD::INSERT_SUBVECTOR: {
1074 if (VT.isScalableVector())
1075 return SDValue();
1076
1077 SDValue Vec = Op.getOperand(i: 0);
1078 SDValue Sub = Op.getOperand(i: 1);
1079 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1080 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1081 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1082 // If we don't demand the inserted subvector, return the base vector.
1083 if (DemandedSubElts == 0)
1084 return Vec;
1085 break;
1086 }
1087 case ISD::VECTOR_SHUFFLE: {
1088 assert(!VT.isScalableVector());
1089 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1090
1091 // If all the demanded elts are from one operand and are inline,
1092 // then we can use the operand directly.
1093 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
1094 for (unsigned i = 0; i != NumElts; ++i) {
1095 int M = ShuffleMask[i];
1096 if (M < 0 || !DemandedElts[i])
1097 continue;
1098 AllUndef = false;
1099 IdentityLHS &= (M == (int)i);
1100 IdentityRHS &= ((M - NumElts) == i);
1101 }
1102
1103 if (AllUndef)
1104 return DAG.getUNDEF(VT: Op.getValueType());
1105 if (IdentityLHS)
1106 return Op.getOperand(i: 0);
1107 if (IdentityRHS)
1108 return Op.getOperand(i: 1);
1109 break;
1110 }
1111 default:
1112 // TODO: Probably okay to remove after audit; here to reduce change size
1113 // in initial enablement patch for scalable vectors
1114 if (VT.isScalableVector())
1115 return SDValue();
1116
1117 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
1118 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
1119 Op, DemandedBits, DemandedElts, DAG, Depth))
1120 return V;
1121 break;
1122 }
1123 return SDValue();
1124}
1125
1126SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
1127 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
1128 unsigned Depth) const {
1129 EVT VT = Op.getValueType();
1130 // Since the number of lanes in a scalable vector is unknown at compile time,
1131 // we track one bit which is implicitly broadcast to all lanes. This means
1132 // that all lanes in a scalable vector are considered demanded.
1133 APInt DemandedElts = VT.isFixedLengthVector()
1134 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1135 : APInt(1, 1);
1136 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1137 Depth);
1138}
1139
1140SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
1141 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
1142 unsigned Depth) const {
1143 APInt DemandedBits = APInt::getAllOnes(numBits: Op.getScalarValueSizeInBits());
1144 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1145 Depth);
1146}
1147
1148// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
1149// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
1150static SDValue combineShiftToAVG(SDValue Op,
1151 TargetLowering::TargetLoweringOpt &TLO,
1152 const TargetLowering &TLI,
1153 const APInt &DemandedBits,
1154 const APInt &DemandedElts, unsigned Depth) {
1155 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1156 "SRL or SRA node is required here!");
1157 // Is the right shift using an immediate value of 1?
1158 ConstantSDNode *N1C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
1159 if (!N1C || !N1C->isOne())
1160 return SDValue();
1161
1162 // We are looking for an avgfloor
1163 // add(ext, ext)
1164 // or one of these as a avgceil
1165 // add(add(ext, ext), 1)
1166 // add(add(ext, 1), ext)
1167 // add(ext, add(ext, 1))
1168 SDValue Add = Op.getOperand(i: 0);
1169 if (Add.getOpcode() != ISD::ADD)
1170 return SDValue();
1171
1172 SDValue ExtOpA = Add.getOperand(i: 0);
1173 SDValue ExtOpB = Add.getOperand(i: 1);
1174 SDValue Add2;
1175 auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
1176 ConstantSDNode *ConstOp;
1177 if ((ConstOp = isConstOrConstSplat(N: Op2, DemandedElts)) &&
1178 ConstOp->isOne()) {
1179 ExtOpA = Op1;
1180 ExtOpB = Op3;
1181 Add2 = A;
1182 return true;
1183 }
1184 if ((ConstOp = isConstOrConstSplat(N: Op3, DemandedElts)) &&
1185 ConstOp->isOne()) {
1186 ExtOpA = Op1;
1187 ExtOpB = Op2;
1188 Add2 = A;
1189 return true;
1190 }
1191 return false;
1192 };
1193 bool IsCeil =
1194 (ExtOpA.getOpcode() == ISD::ADD &&
1195 MatchOperands(ExtOpA.getOperand(i: 0), ExtOpA.getOperand(i: 1), ExtOpB, ExtOpA)) ||
1196 (ExtOpB.getOpcode() == ISD::ADD &&
1197 MatchOperands(ExtOpB.getOperand(i: 0), ExtOpB.getOperand(i: 1), ExtOpA, ExtOpB));
1198
1199 // If the shift is signed (sra):
1200 // - Needs >= 2 sign bit for both operands.
1201 // - Needs >= 2 zero bits.
1202 // If the shift is unsigned (srl):
1203 // - Needs >= 1 zero bit for both operands.
1204 // - Needs 1 demanded bit zero and >= 2 sign bits.
1205 SelectionDAG &DAG = TLO.DAG;
1206 unsigned ShiftOpc = Op.getOpcode();
1207 bool IsSigned = false;
1208 unsigned KnownBits;
1209 unsigned NumSignedA = DAG.ComputeNumSignBits(Op: ExtOpA, DemandedElts, Depth);
1210 unsigned NumSignedB = DAG.ComputeNumSignBits(Op: ExtOpB, DemandedElts, Depth);
1211 unsigned NumSigned = std::min(a: NumSignedA, b: NumSignedB) - 1;
1212 unsigned NumZeroA =
1213 DAG.computeKnownBits(Op: ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
1214 unsigned NumZeroB =
1215 DAG.computeKnownBits(Op: ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
1216 unsigned NumZero = std::min(a: NumZeroA, b: NumZeroB);
1217
1218 switch (ShiftOpc) {
1219 default:
1220 llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
1221 case ISD::SRA: {
1222 if (NumZero >= 2 && NumSigned < NumZero) {
1223 IsSigned = false;
1224 KnownBits = NumZero;
1225 break;
1226 }
1227 if (NumSigned >= 1) {
1228 IsSigned = true;
1229 KnownBits = NumSigned;
1230 break;
1231 }
1232 return SDValue();
1233 }
1234 case ISD::SRL: {
1235 if (NumZero >= 1 && NumSigned < NumZero) {
1236 IsSigned = false;
1237 KnownBits = NumZero;
1238 break;
1239 }
1240 if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
1241 IsSigned = true;
1242 KnownBits = NumSigned;
1243 break;
1244 }
1245 return SDValue();
1246 }
1247 }
1248
1249 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
1250 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);
1251
1252 // Find the smallest power-2 type that is legal for this vector size and
1253 // operation, given the original type size and the number of known sign/zero
1254 // bits.
1255 EVT VT = Op.getValueType();
1256 unsigned MinWidth =
1257 std::max<unsigned>(a: VT.getScalarSizeInBits() - KnownBits, b: 8);
1258 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_ceil(Value: MinWidth));
1259 if (NVT.getScalarSizeInBits() > VT.getScalarSizeInBits())
1260 return SDValue();
1261 if (VT.isVector())
1262 NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT, EC: VT.getVectorElementCount());
1263 if (TLO.LegalTypes() && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT)) {
1264 // If we could not transform, and (both) adds are nuw/nsw, we can use the
1265 // larger type size to do the transform.
1266 if (TLO.LegalOperations() && !TLI.isOperationLegal(Op: AVGOpc, VT))
1267 return SDValue();
1268 if (DAG.willNotOverflowAdd(IsSigned, N0: Add.getOperand(i: 0),
1269 N1: Add.getOperand(i: 1)) &&
1270 (!Add2 || DAG.willNotOverflowAdd(IsSigned, N0: Add2.getOperand(i: 0),
1271 N1: Add2.getOperand(i: 1))))
1272 NVT = VT;
1273 else
1274 return SDValue();
1275 }
1276
1277 // Don't create a AVGFLOOR node with a scalar constant unless its legal as
1278 // this is likely to stop other folds (reassociation, value tracking etc.)
1279 if (!IsCeil && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT) &&
1280 (isa<ConstantSDNode>(Val: ExtOpA) || isa<ConstantSDNode>(Val: ExtOpB)))
1281 return SDValue();
1282
1283 SDLoc DL(Op);
1284 SDValue ResultAVG =
1285 DAG.getNode(Opcode: AVGOpc, DL, VT: NVT, N1: DAG.getExtOrTrunc(IsSigned, Op: ExtOpA, DL, VT: NVT),
1286 N2: DAG.getExtOrTrunc(IsSigned, Op: ExtOpB, DL, VT: NVT));
1287 return DAG.getExtOrTrunc(IsSigned, Op: ResultAVG, DL, VT);
1288}
1289
1290/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
1291/// result of Op are ever used downstream. If we can use this information to
1292/// simplify Op, create a new simplified DAG node and return true, returning the
1293/// original and new nodes in Old and New. Otherwise, analyze the expression and
1294/// return a mask of Known bits for the expression (used to simplify the
1295/// caller). The Known bits may only be accurate for those bits in the
1296/// OriginalDemandedBits and OriginalDemandedElts.
1297bool TargetLowering::SimplifyDemandedBits(
1298 SDValue Op, const APInt &OriginalDemandedBits,
1299 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
1300 unsigned Depth, bool AssumeSingleUse) const {
1301 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
1302 assert(Op.getScalarValueSizeInBits() == BitWidth &&
1303 "Mask size mismatches value type size!");
1304
1305 // Don't know anything.
1306 Known = KnownBits(BitWidth);
1307
1308 EVT VT = Op.getValueType();
1309 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
1310 unsigned NumElts = OriginalDemandedElts.getBitWidth();
1311 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
1312 "Unexpected vector size");
1313
1314 APInt DemandedBits = OriginalDemandedBits;
1315 APInt DemandedElts = OriginalDemandedElts;
1316 SDLoc dl(Op);
1317
1318 // Undef operand.
1319 if (Op.isUndef())
1320 return false;
1321
1322 // We can't simplify target constants.
1323 if (Op.getOpcode() == ISD::TargetConstant)
1324 return false;
1325
1326 if (Op.getOpcode() == ISD::Constant) {
1327 // We know all of the bits for a constant!
1328 Known = KnownBits::makeConstant(C: Op->getAsAPIntVal());
1329 return false;
1330 }
1331
1332 if (Op.getOpcode() == ISD::ConstantFP) {
1333 // We know all of the bits for a floating point constant!
1334 Known = KnownBits::makeConstant(
1335 C: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF().bitcastToAPInt());
1336 return false;
1337 }
1338
1339 // Other users may use these bits.
1340 bool HasMultiUse = false;
1341 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1342 if (Depth >= SelectionDAG::MaxRecursionDepth) {
1343 // Limit search depth.
1344 return false;
1345 }
1346 // Allow multiple uses, just set the DemandedBits/Elts to all bits.
1347 DemandedBits = APInt::getAllOnes(numBits: BitWidth);
1348 DemandedElts = APInt::getAllOnes(numBits: NumElts);
1349 HasMultiUse = true;
1350 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1351 // Not demanding any bits/elts from Op.
1352 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
1353 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
1354 // Limit search depth.
1355 return false;
1356 }
1357
1358 KnownBits Known2;
1359 switch (Op.getOpcode()) {
1360 case ISD::SCALAR_TO_VECTOR: {
1361 if (VT.isScalableVector())
1362 return false;
1363 if (!DemandedElts[0])
1364 return TLO.CombineTo(O: Op, N: TLO.DAG.getPOISON(VT));
1365
1366 KnownBits SrcKnown;
1367 SDValue Src = Op.getOperand(i: 0);
1368 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1369 APInt SrcDemandedBits = DemandedBits.zext(width: SrcBitWidth);
1370 if (SimplifyDemandedBits(Op: Src, DemandedBits: SrcDemandedBits, Known&: SrcKnown, TLO, Depth: Depth + 1))
1371 return true;
1372
1373 // Upper elements are poison, so only get the knownbits if we just demand
1374 // the bottom element.
1375 if (DemandedElts == 1)
1376 Known = SrcKnown.anyextOrTrunc(BitWidth);
1377 break;
1378 }
1379 case ISD::BUILD_VECTOR:
1380 // Collect the known bits that are shared by every demanded element.
1381 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1382 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1383 return false; // Don't fall through, will infinitely loop.
1384 case ISD::SPLAT_VECTOR: {
1385 SDValue Scl = Op.getOperand(i: 0);
1386 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: Scl.getValueSizeInBits());
1387 KnownBits KnownScl;
1388 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1389 return true;
1390
1391 // Implicitly truncate the bits to match the official semantics of
1392 // SPLAT_VECTOR.
1393 Known = KnownScl.trunc(BitWidth);
1394 break;
1395 }
1396 case ISD::FREEZE: {
1397 SDValue N0 = Op.getOperand(i: 0);
1398 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
1399 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
1400 return TLO.CombineTo(O: Op, N: N0);
1401 break;
1402 }
1403 case ISD::LOAD: {
1404 auto *LD = cast<LoadSDNode>(Val&: Op);
1405 if (getTargetConstantFromLoad(LD)) {
1406 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1407 return false; // Don't fall through, will infinitely loop.
1408 }
1409 if (ISD::isZEXTLoad(N: Op.getNode()) && Op.getResNo() == 0) {
1410 // If this is a ZEXTLoad and we are looking at the loaded value.
1411 EVT MemVT = LD->getMemoryVT();
1412 unsigned MemBits = MemVT.getScalarSizeInBits();
1413 Known.Zero.setBitsFrom(MemBits);
1414 return false; // Don't fall through, will infinitely loop.
1415 }
1416 break;
1417 }
1418 case ISD::INSERT_VECTOR_ELT: {
1419 if (VT.isScalableVector())
1420 return false;
1421 SDValue Vec = Op.getOperand(i: 0);
1422 SDValue Scl = Op.getOperand(i: 1);
1423 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1424 EVT VecVT = Vec.getValueType();
1425
1426 // If index isn't constant, assume we need all vector elements AND the
1427 // inserted element.
1428 APInt DemandedVecElts(DemandedElts);
1429 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements())) {
1430 unsigned Idx = CIdx->getZExtValue();
1431 DemandedVecElts.clearBit(BitPosition: Idx);
1432
1433 // Inserted element is not required.
1434 if (!DemandedElts[Idx])
1435 return TLO.CombineTo(O: Op, N: Vec);
1436 }
1437
1438 KnownBits KnownScl;
1439 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1440 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: NumSclBits);
1441 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1442 return true;
1443
1444 Known = KnownScl.anyextOrTrunc(BitWidth);
1445
1446 KnownBits KnownVec;
1447 if (SimplifyDemandedBits(Op: Vec, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedVecElts, Known&: KnownVec, TLO,
1448 Depth: Depth + 1))
1449 return true;
1450
1451 if (!!DemandedVecElts)
1452 Known = Known.intersectWith(RHS: KnownVec);
1453
1454 return false;
1455 }
1456 case ISD::INSERT_SUBVECTOR: {
1457 if (VT.isScalableVector())
1458 return false;
1459 // Demand any elements from the subvector and the remainder from the src its
1460 // inserted into.
1461 SDValue Src = Op.getOperand(i: 0);
1462 SDValue Sub = Op.getOperand(i: 1);
1463 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1464 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1465 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1466 APInt DemandedSrcElts = DemandedElts;
1467 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
1468
1469 KnownBits KnownSub, KnownSrc;
1470 if (SimplifyDemandedBits(Op: Sub, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts, Known&: KnownSub, TLO,
1471 Depth: Depth + 1))
1472 return true;
1473 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known&: KnownSrc, TLO,
1474 Depth: Depth + 1))
1475 return true;
1476
1477 Known.setAllConflict();
1478 if (!!DemandedSubElts)
1479 Known = Known.intersectWith(RHS: KnownSub);
1480 if (!!DemandedSrcElts)
1481 Known = Known.intersectWith(RHS: KnownSrc);
1482
1483 // Attempt to avoid multi-use src if we don't need anything from it.
1484 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1485 !DemandedSrcElts.isAllOnes()) {
1486 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1487 Op: Sub, DemandedBits, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
1488 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1489 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1490 if (NewSub || NewSrc) {
1491 NewSub = NewSub ? NewSub : Sub;
1492 NewSrc = NewSrc ? NewSrc : Src;
1493 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: NewSrc, N2: NewSub,
1494 N3: Op.getOperand(i: 2));
1495 return TLO.CombineTo(O: Op, N: NewOp);
1496 }
1497 }
1498 break;
1499 }
1500 case ISD::EXTRACT_SUBVECTOR: {
1501 if (VT.isScalableVector())
1502 return false;
1503 // Offset the demanded elts by the subvector index.
1504 SDValue Src = Op.getOperand(i: 0);
1505 if (Src.getValueType().isScalableVector())
1506 break;
1507 uint64_t Idx = Op.getConstantOperandVal(i: 1);
1508 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1509 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
1510
1511 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known, TLO,
1512 Depth: Depth + 1))
1513 return true;
1514
1515 // Attempt to avoid multi-use src if we don't need anything from it.
1516 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1517 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1518 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1519 if (DemandedSrc) {
1520 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc,
1521 N2: Op.getOperand(i: 1));
1522 return TLO.CombineTo(O: Op, N: NewOp);
1523 }
1524 }
1525 break;
1526 }
1527 case ISD::CONCAT_VECTORS: {
1528 if (VT.isScalableVector())
1529 return false;
1530 Known.setAllConflict();
1531 EVT SubVT = Op.getOperand(i: 0).getValueType();
1532 unsigned NumSubVecs = Op.getNumOperands();
1533 unsigned NumSubElts = SubVT.getVectorNumElements();
1534 for (unsigned i = 0; i != NumSubVecs; ++i) {
1535 APInt DemandedSubElts =
1536 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
1537 if (SimplifyDemandedBits(Op: Op.getOperand(i), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts,
1538 Known&: Known2, TLO, Depth: Depth + 1))
1539 return true;
1540 // Known bits are shared by every demanded subvector element.
1541 if (!!DemandedSubElts)
1542 Known = Known.intersectWith(RHS: Known2);
1543 }
1544 break;
1545 }
1546 case ISD::VECTOR_SHUFFLE: {
1547 assert(!VT.isScalableVector());
1548 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1549
1550 // Collect demanded elements from shuffle operands..
1551 APInt DemandedLHS, DemandedRHS;
1552 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: ShuffleMask, DemandedElts, DemandedLHS,
1553 DemandedRHS))
1554 break;
1555
1556 if (!!DemandedLHS || !!DemandedRHS) {
1557 SDValue Op0 = Op.getOperand(i: 0);
1558 SDValue Op1 = Op.getOperand(i: 1);
1559
1560 Known.setAllConflict();
1561 if (!!DemandedLHS) {
1562 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedLHS, Known&: Known2, TLO,
1563 Depth: Depth + 1))
1564 return true;
1565 Known = Known.intersectWith(RHS: Known2);
1566 }
1567 if (!!DemandedRHS) {
1568 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedRHS, Known&: Known2, TLO,
1569 Depth: Depth + 1))
1570 return true;
1571 Known = Known.intersectWith(RHS: Known2);
1572 }
1573
1574 // Attempt to avoid multi-use ops if we don't need anything from them.
1575 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1576 Op: Op0, DemandedBits, DemandedElts: DemandedLHS, DAG&: TLO.DAG, Depth: Depth + 1);
1577 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1578 Op: Op1, DemandedBits, DemandedElts: DemandedRHS, DAG&: TLO.DAG, Depth: Depth + 1);
1579 if (DemandedOp0 || DemandedOp1) {
1580 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1581 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1582 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, N1: Op0, N2: Op1, Mask: ShuffleMask);
1583 return TLO.CombineTo(O: Op, N: NewOp);
1584 }
1585 }
1586 break;
1587 }
1588 case ISD::AND: {
1589 SDValue Op0 = Op.getOperand(i: 0);
1590 SDValue Op1 = Op.getOperand(i: 1);
1591
1592 // If the RHS is a constant, check to see if the LHS would be zero without
1593 // using the bits from the RHS. Below, we use knowledge about the RHS to
1594 // simplify the LHS, here we're using information from the LHS to simplify
1595 // the RHS.
1596 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: Op1, DemandedElts)) {
1597 // Do not increment Depth here; that can cause an infinite loop.
1598 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth);
1599 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1600 if ((LHSKnown.Zero & DemandedBits) ==
1601 (~RHSC->getAPIntValue() & DemandedBits))
1602 return TLO.CombineTo(O: Op, N: Op0);
1603
1604 // If any of the set bits in the RHS are known zero on the LHS, shrink
1605 // the constant.
1606 if (ShrinkDemandedConstant(Op, DemandedBits: ~LHSKnown.Zero & DemandedBits,
1607 DemandedElts, TLO))
1608 return true;
1609
1610 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1611 // constant, but if this 'and' is only clearing bits that were just set by
1612 // the xor, then this 'and' can be eliminated by shrinking the mask of
1613 // the xor. For example, for a 32-bit X:
1614 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1615 if (isBitwiseNot(V: Op0) && Op0.hasOneUse() &&
1616 LHSKnown.One == ~RHSC->getAPIntValue()) {
1617 SDValue Xor = TLO.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1618 return TLO.CombineTo(O: Op, N: Xor);
1619 }
1620 }
1621
1622 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2 (or zero).
1623 SDValue X, Y;
1624 if (sd_match(N: Op,
1625 P: m_And(L: m_Value(N&: Y),
1626 R: m_OneUse(P: m_AnyOf(preds: m_Add(L: m_Value(N&: X), R: m_Deferred(V&: Y)),
1627 preds: m_Sub(L: m_Value(N&: X), R: m_Deferred(V&: Y)))))) &&
1628 TLO.DAG.isKnownToBeAPowerOfTwo(Val: Y, DemandedElts, /*OrZero=*/true)) {
1629 return TLO.CombineTo(
1630 O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TLO.DAG.getNOT(DL: dl, Val: X, VT), N2: Y));
1631 }
1632
1633 // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
1634 // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
1635 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
1636 (Op0.getOperand(i: 0).isUndef() ||
1637 ISD::isBuildVectorOfConstantSDNodes(N: Op0.getOperand(i: 0).getNode())) &&
1638 Op0->hasOneUse()) {
1639 unsigned NumSubElts =
1640 Op0.getOperand(i: 1).getValueType().getVectorNumElements();
1641 unsigned SubIdx = Op0.getConstantOperandVal(i: 2);
1642 APInt DemandedSub =
1643 APInt::getBitsSet(numBits: NumElts, loBit: SubIdx, hiBit: SubIdx + NumSubElts);
1644 KnownBits KnownSubMask =
1645 TLO.DAG.computeKnownBits(Op: Op1, DemandedElts: DemandedSub & DemandedElts, Depth: Depth + 1);
1646 if (DemandedBits.isSubsetOf(RHS: KnownSubMask.One)) {
1647 SDValue NewAnd =
1648 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1649 SDValue NewInsert =
1650 TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: NewAnd,
1651 N2: Op0.getOperand(i: 1), N3: Op0.getOperand(i: 2));
1652 return TLO.CombineTo(O: Op, N: NewInsert);
1653 }
1654 }
1655
1656 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1657 Depth: Depth + 1))
1658 return true;
1659 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.Zero & DemandedBits, OriginalDemandedElts: DemandedElts,
1660 Known&: Known2, TLO, Depth: Depth + 1))
1661 return true;
1662
1663 // If all of the demanded bits are known one on one side, return the other.
1664 // These bits cannot contribute to the result of the 'and'.
1665 if (DemandedBits.isSubsetOf(RHS: Known2.Zero | Known.One))
1666 return TLO.CombineTo(O: Op, N: Op0);
1667 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.One))
1668 return TLO.CombineTo(O: Op, N: Op1);
1669 // If all of the demanded bits in the inputs are known zeros, return zero.
1670 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1671 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: dl, VT));
1672 // If the RHS is a constant, see if we can simplify it.
1673 if (ShrinkDemandedConstant(Op, DemandedBits: ~Known2.Zero & DemandedBits, DemandedElts,
1674 TLO))
1675 return true;
1676 // If the operation can be done in a smaller type, do so.
1677 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1678 return true;
1679
1680 // Attempt to avoid multi-use ops if we don't need anything from them.
1681 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1682 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1683 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1684 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1685 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1686 if (DemandedOp0 || DemandedOp1) {
1687 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1688 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1689 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1690 return TLO.CombineTo(O: Op, N: NewOp);
1691 }
1692 }
1693
1694 Known &= Known2;
1695 break;
1696 }
1697 case ISD::OR: {
1698 SDValue Op0 = Op.getOperand(i: 0);
1699 SDValue Op1 = Op.getOperand(i: 1);
1700 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1701 Depth: Depth + 1)) {
1702 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1703 return true;
1704 }
1705
1706 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.One & DemandedBits, OriginalDemandedElts: DemandedElts,
1707 Known&: Known2, TLO, Depth: Depth + 1)) {
1708 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1709 return true;
1710 }
1711
1712 // If all of the demanded bits are known zero on one side, return the other.
1713 // These bits cannot contribute to the result of the 'or'.
1714 if (DemandedBits.isSubsetOf(RHS: Known2.One | Known.Zero))
1715 return TLO.CombineTo(O: Op, N: Op0);
1716 if (DemandedBits.isSubsetOf(RHS: Known.One | Known2.Zero))
1717 return TLO.CombineTo(O: Op, N: Op1);
1718 // If the RHS is a constant, see if we can simplify it.
1719 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1720 return true;
1721 // If the operation can be done in a smaller type, do so.
1722 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1723 return true;
1724
1725 // Attempt to avoid multi-use ops if we don't need anything from them.
1726 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1727 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1728 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1729 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1730 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1731 if (DemandedOp0 || DemandedOp1) {
1732 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1733 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1734 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1735 return TLO.CombineTo(O: Op, N: NewOp);
1736 }
1737 }
1738
1739 // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
1740 // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
1741 SDValue X, Y, C1, C2;
1742 if (sd_match(N: Op, P: m_Or(L: m_OneUse(P: m_And(L: m_Value(N&: X), R: m_Value(N&: C1))),
1743 R: m_OneUse(P: m_And(L: m_Or(L: m_Deferred(V&: X), R: m_Value(N&: Y)),
1744 R: m_Value(N&: C2)))))) {
1745 if (SDValue C12 =
1746 TLO.DAG.FoldConstantArithmetic(Opcode: ISD::OR, DL: dl, VT, Ops: {C1, C2})) {
1747 SDValue MaskX = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: C12);
1748 SDValue MaskY = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Y, N2: C2);
1749 return TLO.CombineTo(O: Op,
1750 N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: MaskX, N2: MaskY));
1751 }
1752 }
1753
1754 Known |= Known2;
1755 break;
1756 }
1757 case ISD::XOR: {
1758 SDValue Op0 = Op.getOperand(i: 0);
1759 SDValue Op1 = Op.getOperand(i: 1);
1760
1761 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1762 Depth: Depth + 1))
1763 return true;
1764 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
1765 Depth: Depth + 1))
1766 return true;
1767
1768 // If all of the demanded bits are known zero on one side, return the other.
1769 // These bits cannot contribute to the result of the 'xor'.
1770 if (DemandedBits.isSubsetOf(RHS: Known.Zero))
1771 return TLO.CombineTo(O: Op, N: Op0);
1772 if (DemandedBits.isSubsetOf(RHS: Known2.Zero))
1773 return TLO.CombineTo(O: Op, N: Op1);
1774 // If the operation can be done in a smaller type, do so.
1775 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1776 return true;
1777
1778 // If all of the unknown bits are known to be zero on one side or the other
1779 // turn this into an *inclusive* or.
1780 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1781 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1782 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op0, N2: Op1));
1783
1784 ConstantSDNode *C = isConstOrConstSplat(N: Op1, DemandedElts);
1785 if (C) {
1786 // If one side is a constant, and all of the set bits in the constant are
1787 // also known set on the other side, turn this into an AND, as we know
1788 // the bits will be cleared.
1789 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1790 // NB: it is okay if more bits are known than are requested
1791 if (C->getAPIntValue() == Known2.One) {
1792 SDValue ANDC =
1793 TLO.DAG.getConstant(Val: ~C->getAPIntValue() & DemandedBits, DL: dl, VT);
1794 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0, N2: ANDC));
1795 }
1796
1797 // If the RHS is a constant, see if we can change it. Don't alter a -1
1798 // constant because that's a 'not' op, and that is better for combining
1799 // and codegen.
1800 if (!C->isAllOnes() && DemandedBits.isSubsetOf(RHS: C->getAPIntValue())) {
1801 // We're flipping all demanded bits. Flip the undemanded bits too.
1802 SDValue New = TLO.DAG.getNOT(DL: dl, Val: Op0, VT);
1803 return TLO.CombineTo(O: Op, N: New);
1804 }
1805
1806 unsigned Op0Opcode = Op0.getOpcode();
1807 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
1808 if (ConstantSDNode *ShiftC =
1809 isConstOrConstSplat(N: Op0.getOperand(i: 1), DemandedElts)) {
1810 // Don't crash on an oversized shift. We can not guarantee that a
1811 // bogus shift has been simplified to undef.
1812 if (ShiftC->getAPIntValue().ult(RHS: BitWidth)) {
1813 uint64_t ShiftAmt = ShiftC->getZExtValue();
1814 APInt Ones = APInt::getAllOnes(numBits: BitWidth);
1815 Ones = Op0Opcode == ISD::SHL ? Ones.shl(shiftAmt: ShiftAmt)
1816 : Ones.lshr(shiftAmt: ShiftAmt);
1817 if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
1818 isDesirableToCommuteXorWithShift(N: Op.getNode())) {
1819 // If the xor constant is a demanded mask, do a 'not' before the
1820 // shift:
1821 // xor (X << ShiftC), XorC --> (not X) << ShiftC
1822 // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
1823 SDValue Not = TLO.DAG.getNOT(DL: dl, Val: Op0.getOperand(i: 0), VT);
1824 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op0Opcode, DL: dl, VT, N1: Not,
1825 N2: Op0.getOperand(i: 1)));
1826 }
1827 }
1828 }
1829 }
1830 }
1831
1832 // If we can't turn this into a 'not', try to shrink the constant.
1833 if (!C || !C->isAllOnes())
1834 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1835 return true;
1836
1837 // Attempt to avoid multi-use ops if we don't need anything from them.
1838 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1839 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1840 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1841 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1842 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1843 if (DemandedOp0 || DemandedOp1) {
1844 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1845 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1846 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1847 return TLO.CombineTo(O: Op, N: NewOp);
1848 }
1849 }
1850
1851 Known ^= Known2;
1852 break;
1853 }
1854 case ISD::SELECT:
1855 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1856 Known, TLO, Depth: Depth + 1))
1857 return true;
1858 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1859 Known&: Known2, TLO, Depth: Depth + 1))
1860 return true;
1861
1862 // If the operands are constants, see if we can simplify them.
1863 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1864 return true;
1865
1866 // Only known if known in both the LHS and RHS.
1867 Known = Known.intersectWith(RHS: Known2);
1868 break;
1869 case ISD::VSELECT:
1870 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1871 Known, TLO, Depth: Depth + 1))
1872 return true;
1873 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1874 Known&: Known2, TLO, Depth: Depth + 1))
1875 return true;
1876
1877 // Only known if known in both the LHS and RHS.
1878 Known = Known.intersectWith(RHS: Known2);
1879 break;
1880 case ISD::SELECT_CC:
1881 if (SimplifyDemandedBits(Op: Op.getOperand(i: 3), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1882 Known, TLO, Depth: Depth + 1))
1883 return true;
1884 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1885 Known&: Known2, TLO, Depth: Depth + 1))
1886 return true;
1887
1888 // If the operands are constants, see if we can simplify them.
1889 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1890 return true;
1891
1892 // Only known if known in both the LHS and RHS.
1893 Known = Known.intersectWith(RHS: Known2);
1894 break;
1895 case ISD::SETCC: {
1896 SDValue Op0 = Op.getOperand(i: 0);
1897 SDValue Op1 = Op.getOperand(i: 1);
1898 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1899 // If we're testing X < 0, X >= 0, X <= -1 or X > -1
1900 // (X is of integer type) then we only need the sign mask of the previous
1901 // result
1902 if (Op1.getValueType().isInteger() &&
1903 (((CC == ISD::SETLT || CC == ISD::SETGE) && isNullOrNullSplat(V: Op1)) ||
1904 ((CC == ISD::SETLE || CC == ISD::SETGT) &&
1905 isAllOnesOrAllOnesSplat(V: Op1)))) {
1906 KnownBits KnownOp0;
1907 if (SimplifyDemandedBits(
1908 Op: Op0, OriginalDemandedBits: APInt::getSignMask(BitWidth: Op0.getScalarValueSizeInBits()),
1909 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1))
1910 return true;
1911 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1912 // width as the setcc result, and (3) the result of a setcc conforms to 0
1913 // or -1, we may be able to bypass the setcc.
1914 if (DemandedBits.isSignMask() &&
1915 Op0.getScalarValueSizeInBits() == BitWidth &&
1916 getBooleanContents(Type: Op0.getValueType()) ==
1917 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1918 // If we remove a >= 0 or > -1 (for integers), we need to introduce a
1919 // NOT Operation
1920 if (CC == ISD::SETGE || CC == ISD::SETGT) {
1921 SDLoc DL(Op);
1922 EVT VT = Op0.getValueType();
1923 SDValue NotOp0 = TLO.DAG.getNOT(DL, Val: Op0, VT);
1924 return TLO.CombineTo(O: Op, N: NotOp0);
1925 }
1926 return TLO.CombineTo(O: Op, N: Op0);
1927 }
1928 }
1929 if (getBooleanContents(Type: Op0.getValueType()) ==
1930 TargetLowering::ZeroOrOneBooleanContent &&
1931 BitWidth > 1)
1932 Known.Zero.setBitsFrom(1);
1933 break;
1934 }
1935 case ISD::SHL: {
1936 SDValue Op0 = Op.getOperand(i: 0);
1937 SDValue Op1 = Op.getOperand(i: 1);
1938 EVT ShiftVT = Op1.getValueType();
1939
1940 if (std::optional<unsigned> KnownSA =
1941 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
1942 unsigned ShAmt = *KnownSA;
1943 if (ShAmt == 0)
1944 return TLO.CombineTo(O: Op, N: Op0);
1945
1946 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1947 // single shift. We can do this if the bottom bits (which are shifted
1948 // out) are never demanded.
1949 // TODO - support non-uniform vector amounts.
1950 if (Op0.getOpcode() == ISD::SRL) {
1951 if (!DemandedBits.intersects(RHS: APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ShAmt))) {
1952 if (std::optional<unsigned> InnerSA =
1953 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
1954 unsigned C1 = *InnerSA;
1955 unsigned Opc = ISD::SHL;
1956 int Diff = ShAmt - C1;
1957 if (Diff < 0) {
1958 Diff = -Diff;
1959 Opc = ISD::SRL;
1960 }
1961 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
1962 return TLO.CombineTo(
1963 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
1964 }
1965 }
1966 }
1967
1968 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1969 // are not demanded. This will likely allow the anyext to be folded away.
1970 // TODO - support non-uniform vector amounts.
1971 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1972 SDValue InnerOp = Op0.getOperand(i: 0);
1973 EVT InnerVT = InnerOp.getValueType();
1974 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1975 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1976 isTypeDesirableForOp(ISD::SHL, VT: InnerVT)) {
1977 SDValue NarrowShl = TLO.DAG.getNode(
1978 Opcode: ISD::SHL, DL: dl, VT: InnerVT, N1: InnerOp,
1979 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: InnerVT, DL: dl));
1980 return TLO.CombineTo(
1981 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
1982 }
1983
1984 // Repeat the SHL optimization above in cases where an extension
1985 // intervenes: (shl (anyext (shr x, c1)), c2) to
1986 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1987 // aren't demanded (as above) and that the shifted upper c1 bits of
1988 // x aren't demanded.
1989 // TODO - support non-uniform vector amounts.
1990 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
1991 InnerOp.hasOneUse()) {
1992 if (std::optional<unsigned> SA2 = TLO.DAG.getValidShiftAmount(
1993 V: InnerOp, DemandedElts, Depth: Depth + 2)) {
1994 unsigned InnerShAmt = *SA2;
1995 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1996 DemandedBits.getActiveBits() <=
1997 (InnerBits - InnerShAmt + ShAmt) &&
1998 DemandedBits.countr_zero() >= ShAmt) {
1999 SDValue NewSA =
2000 TLO.DAG.getConstant(Val: ShAmt - InnerShAmt, DL: dl, VT: ShiftVT);
2001 SDValue NewExt = TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT,
2002 Operand: InnerOp.getOperand(i: 0));
2003 return TLO.CombineTo(
2004 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: NewExt, N2: NewSA));
2005 }
2006 }
2007 }
2008 }
2009
2010 APInt InDemandedMask = DemandedBits.lshr(shiftAmt: ShAmt);
2011 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2012 Depth: Depth + 1)) {
2013 // Disable the nsw and nuw flags. We can no longer guarantee that we
2014 // won't wrap after simplification.
2015 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2016 return true;
2017 }
2018 Known <<= ShAmt;
2019 // low bits known zero.
2020 Known.Zero.setLowBits(ShAmt);
2021
2022 // Attempt to avoid multi-use ops if we don't need anything from them.
2023 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2024 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2025 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2026 if (DemandedOp0) {
2027 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2028 return TLO.CombineTo(O: Op, N: NewOp);
2029 }
2030 }
2031
2032 // TODO: Can we merge this fold with the one below?
2033 // Try shrinking the operation as long as the shift amount will still be
2034 // in range.
2035 if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() &&
2036 Op.getNode()->hasOneUse()) {
2037 // Search for the smallest integer type with free casts to and from
2038 // Op's type. For expedience, just check power-of-2 integer types.
2039 unsigned DemandedSize = DemandedBits.getActiveBits();
2040 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
2041 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
2042 EVT SmallVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: SmallVTBits);
2043 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: SmallVT) &&
2044 isTypeDesirableForOp(ISD::SHL, VT: SmallVT) &&
2045 isTruncateFree(FromVT: VT, ToVT: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT) &&
2046 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: SmallVT))) {
2047 assert(DemandedSize <= SmallVTBits &&
2048 "Narrowed below demanded bits?");
2049 // We found a type with free casts.
2050 SDValue NarrowShl = TLO.DAG.getNode(
2051 Opcode: ISD::SHL, DL: dl, VT: SmallVT,
2052 N1: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
2053 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: SmallVT, DL: dl));
2054 return TLO.CombineTo(
2055 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
2056 }
2057 }
2058 }
2059
2060 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2061 // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
2062 // Only do this if we demand the upper half so the knownbits are correct.
2063 unsigned HalfWidth = BitWidth / 2;
2064 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
2065 DemandedBits.countLeadingOnes() >= HalfWidth) {
2066 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: HalfWidth);
2067 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2068 isTypeDesirableForOp(ISD::SHL, VT: HalfVT) &&
2069 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2070 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: HalfVT))) {
2071 // If we're demanding the upper bits at all, we must ensure
2072 // that the upper bits of the shift result are known to be zero,
2073 // which is equivalent to the narrow shift being NUW.
2074 if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
2075 bool IsNSW = Known.countMinSignBits() > HalfWidth;
2076 SDNodeFlags Flags;
2077 Flags.setNoSignedWrap(IsNSW);
2078 Flags.setNoUnsignedWrap(IsNUW);
2079 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2080 SDValue NewShiftAmt =
2081 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2082 SDValue NewShift = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HalfVT, N1: NewOp,
2083 N2: NewShiftAmt, Flags);
2084 SDValue NewExt =
2085 TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift);
2086 return TLO.CombineTo(O: Op, N: NewExt);
2087 }
2088 }
2089 }
2090 } else {
2091 // This is a variable shift, so we can't shift the demand mask by a known
2092 // amount. But if we are not demanding high bits, then we are not
2093 // demanding those bits from the pre-shifted operand either.
2094 if (unsigned CTLZ = DemandedBits.countl_zero()) {
2095 APInt DemandedFromOp(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - CTLZ));
2096 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedFromOp, OriginalDemandedElts: DemandedElts, Known, TLO,
2097 Depth: Depth + 1)) {
2098 // Disable the nsw and nuw flags. We can no longer guarantee that we
2099 // won't wrap after simplification.
2100 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2101 return true;
2102 }
2103 Known.resetAll();
2104 }
2105 }
2106
2107 // If we are only demanding sign bits then we can use the shift source
2108 // directly.
2109 if (std::optional<unsigned> MaxSA =
2110 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2111 unsigned ShAmt = *MaxSA;
2112 unsigned NumSignBits =
2113 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2114 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
2115 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
2116 return TLO.CombineTo(O: Op, N: Op0);
2117 }
2118 break;
2119 }
2120 case ISD::SRL: {
2121 SDValue Op0 = Op.getOperand(i: 0);
2122 SDValue Op1 = Op.getOperand(i: 1);
2123 EVT ShiftVT = Op1.getValueType();
2124
2125 if (std::optional<unsigned> KnownSA =
2126 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2127 unsigned ShAmt = *KnownSA;
2128 if (ShAmt == 0)
2129 return TLO.CombineTo(O: Op, N: Op0);
2130
2131 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
2132 // single shift. We can do this if the top bits (which are shifted out)
2133 // are never demanded.
2134 // TODO - support non-uniform vector amounts.
2135 if (Op0.getOpcode() == ISD::SHL) {
2136 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2137 if (std::optional<unsigned> InnerSA =
2138 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2139 unsigned C1 = *InnerSA;
2140 unsigned Opc = ISD::SRL;
2141 int Diff = ShAmt - C1;
2142 if (Diff < 0) {
2143 Diff = -Diff;
2144 Opc = ISD::SHL;
2145 }
2146 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
2147 return TLO.CombineTo(
2148 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
2149 }
2150 }
2151 }
2152
2153 // If this is (srl (sra X, C1), ShAmt), see if we can combine this into a
2154 // single sra. We can do this if the top bits are never demanded.
2155 if (Op0.getOpcode() == ISD::SRA && Op0.hasOneUse()) {
2156 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2157 if (std::optional<unsigned> InnerSA =
2158 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2159 unsigned C1 = *InnerSA;
2160 // Clamp the combined shift amount if it exceeds the bit width.
2161 unsigned Combined = std::min(a: C1 + ShAmt, b: BitWidth - 1);
2162 SDValue NewSA = TLO.DAG.getConstant(Val: Combined, DL: dl, VT: ShiftVT);
2163 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT,
2164 N1: Op0.getOperand(i: 0), N2: NewSA));
2165 }
2166 }
2167 }
2168
2169 APInt InDemandedMask = (DemandedBits << ShAmt);
2170
2171 // If the shift is exact, then it does demand the low bits (and knows that
2172 // they are zero).
2173 if (Op->getFlags().hasExact())
2174 InDemandedMask.setLowBits(ShAmt);
2175
2176 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2177 // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
2178 if ((BitWidth % 2) == 0 && !VT.isVector()) {
2179 APInt HiBits = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth / 2);
2180 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: BitWidth / 2);
2181 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2182 isTypeDesirableForOp(ISD::SRL, VT: HalfVT) &&
2183 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2184 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT: HalfVT)) &&
2185 ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
2186 TLO.DAG.MaskedValueIsZero(Op: Op0, Mask: HiBits))) {
2187 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2188 SDValue NewShiftAmt =
2189 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2190 SDValue NewShift =
2191 TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HalfVT, N1: NewOp, N2: NewShiftAmt);
2192 return TLO.CombineTo(
2193 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift));
2194 }
2195 }
2196
2197 // Compute the new bits that are at the top now.
2198 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2199 Depth: Depth + 1))
2200 return true;
2201 Known >>= ShAmt;
2202 // High bits known zero.
2203 Known.Zero.setHighBits(ShAmt);
2204
2205 // Attempt to avoid multi-use ops if we don't need anything from them.
2206 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2207 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2208 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2209 if (DemandedOp0) {
2210 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2211 return TLO.CombineTo(O: Op, N: NewOp);
2212 }
2213 }
2214 } else {
2215 // Use generic knownbits computation as it has support for non-uniform
2216 // shift amounts.
2217 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2218 }
2219
2220 // If we are only demanding sign bits then we can use the shift source
2221 // directly.
2222 if (std::optional<unsigned> MaxSA =
2223 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2224 unsigned ShAmt = *MaxSA;
2225 // Must already be signbits in DemandedBits bounds, and can't demand any
2226 // shifted in zeroes.
2227 if (DemandedBits.countl_zero() >= ShAmt) {
2228 unsigned NumSignBits =
2229 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2230 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
2231 return TLO.CombineTo(O: Op, N: Op0);
2232 }
2233 }
2234
2235 // Try to match AVG patterns (after shift simplification).
2236 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2237 DemandedElts, Depth: Depth + 1))
2238 return TLO.CombineTo(O: Op, N: AVG);
2239
2240 break;
2241 }
2242 case ISD::SRA: {
2243 SDValue Op0 = Op.getOperand(i: 0);
2244 SDValue Op1 = Op.getOperand(i: 1);
2245 EVT ShiftVT = Op1.getValueType();
2246
2247 // If we only want bits that already match the signbit then we don't need
2248 // to shift.
2249 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
2250 if (TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1) >=
2251 NumHiDemandedBits)
2252 return TLO.CombineTo(O: Op, N: Op0);
2253
2254 // If this is an arithmetic shift right and only the low-bit is set, we can
2255 // always convert this into a logical shr, even if the shift amount is
2256 // variable. The low bit of the shift cannot be an input sign bit unless
2257 // the shift amount is >= the size of the datatype, which is undefined.
2258 if (DemandedBits.isOne())
2259 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2260
2261 if (std::optional<unsigned> KnownSA =
2262 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2263 unsigned ShAmt = *KnownSA;
2264 if (ShAmt == 0)
2265 return TLO.CombineTo(O: Op, N: Op0);
2266
2267 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
2268 // supports sext_inreg.
2269 if (Op0.getOpcode() == ISD::SHL) {
2270 if (std::optional<unsigned> InnerSA =
2271 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2272 unsigned LowBits = BitWidth - ShAmt;
2273 EVT ExtVT = VT.changeElementType(
2274 Context&: *TLO.DAG.getContext(),
2275 EltVT: EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: LowBits));
2276
2277 if (*InnerSA == ShAmt) {
2278 if (!TLO.LegalOperations() ||
2279 getOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: ExtVT) == Legal)
2280 return TLO.CombineTo(
2281 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT,
2282 N1: Op0.getOperand(i: 0),
2283 N2: TLO.DAG.getValueType(ExtVT)));
2284
2285 // Even if we can't convert to sext_inreg, we might be able to
2286 // remove this shift pair if the input is already sign extended.
2287 unsigned NumSignBits =
2288 TLO.DAG.ComputeNumSignBits(Op: Op0.getOperand(i: 0), DemandedElts);
2289 if (NumSignBits > ShAmt)
2290 return TLO.CombineTo(O: Op, N: Op0.getOperand(i: 0));
2291 }
2292 }
2293 }
2294
2295 APInt InDemandedMask = (DemandedBits << ShAmt);
2296
2297 // If the shift is exact, then it does demand the low bits (and knows that
2298 // they are zero).
2299 if (Op->getFlags().hasExact())
2300 InDemandedMask.setLowBits(ShAmt);
2301
2302 // If any of the demanded bits are produced by the sign extension, we also
2303 // demand the input sign bit.
2304 if (DemandedBits.countl_zero() < ShAmt)
2305 InDemandedMask.setSignBit();
2306
2307 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2308 Depth: Depth + 1))
2309 return true;
2310 Known >>= ShAmt;
2311
2312 // If the input sign bit is known to be zero, or if none of the top bits
2313 // are demanded, turn this into an unsigned shift right.
2314 if (Known.Zero[BitWidth - ShAmt - 1] ||
2315 DemandedBits.countl_zero() >= ShAmt) {
2316 SDNodeFlags Flags;
2317 Flags.setExact(Op->getFlags().hasExact());
2318 return TLO.CombineTo(
2319 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1, Flags));
2320 }
2321
2322 int Log2 = DemandedBits.exactLogBase2();
2323 if (Log2 >= 0) {
2324 // The bit must come from the sign.
2325 SDValue NewSA = TLO.DAG.getConstant(Val: BitWidth - 1 - Log2, DL: dl, VT: ShiftVT);
2326 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: NewSA));
2327 }
2328
2329 if (Known.One[BitWidth - ShAmt - 1])
2330 // New bits are known one.
2331 Known.One.setHighBits(ShAmt);
2332
2333 // Attempt to avoid multi-use ops if we don't need anything from them.
2334 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2335 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2336 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2337 if (DemandedOp0) {
2338 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2339 return TLO.CombineTo(O: Op, N: NewOp);
2340 }
2341 }
2342 }
2343
2344 // Try to match AVG patterns (after shift simplification).
2345 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2346 DemandedElts, Depth: Depth + 1))
2347 return TLO.CombineTo(O: Op, N: AVG);
2348
2349 break;
2350 }
2351 case ISD::FSHL:
2352 case ISD::FSHR: {
2353 SDValue Op0 = Op.getOperand(i: 0);
2354 SDValue Op1 = Op.getOperand(i: 1);
2355 SDValue Op2 = Op.getOperand(i: 2);
2356 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
2357
2358 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op2, DemandedElts)) {
2359 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2360
2361 // For fshl, 0-shift returns the 1st arg.
2362 // For fshr, 0-shift returns the 2nd arg.
2363 if (Amt == 0) {
2364 if (SimplifyDemandedBits(Op: IsFSHL ? Op0 : Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2365 Known, TLO, Depth: Depth + 1))
2366 return true;
2367 break;
2368 }
2369
2370 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
2371 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
2372 APInt Demanded0 = DemandedBits.lshr(shiftAmt: IsFSHL ? Amt : (BitWidth - Amt));
2373 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
2374 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2375 Depth: Depth + 1))
2376 return true;
2377 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: Demanded1, OriginalDemandedElts: DemandedElts, Known, TLO,
2378 Depth: Depth + 1))
2379 return true;
2380
2381 Known2 <<= (IsFSHL ? Amt : (BitWidth - Amt));
2382 Known >>= (IsFSHL ? (BitWidth - Amt) : Amt);
2383 Known = Known.unionWith(RHS: Known2);
2384
2385 // Attempt to avoid multi-use ops if we don't need anything from them.
2386 if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
2387 !DemandedElts.isAllOnes()) {
2388 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2389 Op: Op0, DemandedBits: Demanded0, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2390 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2391 Op: Op1, DemandedBits: Demanded1, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2392 if (DemandedOp0 || DemandedOp1) {
2393 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2394 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2395 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedOp0,
2396 N2: DemandedOp1, N3: Op2);
2397 return TLO.CombineTo(O: Op, N: NewOp);
2398 }
2399 }
2400 }
2401
2402 if (isPowerOf2_32(Value: BitWidth)) {
2403 // Fold FSHR(Op0,Op1,Op2) -> SRL(Op1,Op2)
2404 // iff we're guaranteed not to use Op0.
2405 // TODO: Add FSHL equivalent?
2406 if (!IsFSHL && !DemandedBits.isAllOnes() &&
2407 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT))) {
2408 KnownBits KnownAmt =
2409 TLO.DAG.computeKnownBits(Op: Op2, DemandedElts, Depth: Depth + 1);
2410 unsigned MaxShiftAmt =
2411 KnownAmt.getMaxValue().getLimitedValue(Limit: BitWidth - 1);
2412 // Check we don't demand any shifted bits outside Op1.
2413 if (DemandedBits.countl_zero() >= MaxShiftAmt) {
2414 EVT AmtVT = Op2.getValueType();
2415 SDValue NewAmt =
2416 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Op2,
2417 N2: TLO.DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
2418 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op1, N2: NewAmt);
2419 return TLO.CombineTo(O: Op, N: NewOp);
2420 }
2421 }
2422
2423 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2424 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
2425 if (SimplifyDemandedBits(Op: Op2, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2426 Depth: Depth + 1))
2427 return true;
2428 }
2429 break;
2430 }
2431 case ISD::ROTL:
2432 case ISD::ROTR: {
2433 SDValue Op0 = Op.getOperand(i: 0);
2434 SDValue Op1 = Op.getOperand(i: 1);
2435 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
2436
2437 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
2438 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1))
2439 return TLO.CombineTo(O: Op, N: Op0);
2440
2441 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op1, DemandedElts)) {
2442 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2443 unsigned RevAmt = BitWidth - Amt;
2444
2445 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
2446 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
2447 APInt Demanded0 = DemandedBits.rotr(rotateAmt: IsROTL ? Amt : RevAmt);
2448 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2449 Depth: Depth + 1))
2450 return true;
2451
2452 // rot*(x, 0) --> x
2453 if (Amt == 0)
2454 return TLO.CombineTo(O: Op, N: Op0);
2455
2456 // See if we don't demand either half of the rotated bits.
2457 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT)) &&
2458 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2459 Op1 = TLO.DAG.getConstant(Val: IsROTL ? Amt : RevAmt, DL: dl, VT: Op1.getValueType());
2460 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: Op1));
2461 }
2462 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT)) &&
2463 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2464 Op1 = TLO.DAG.getConstant(Val: IsROTL ? RevAmt : Amt, DL: dl, VT: Op1.getValueType());
2465 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2466 }
2467 }
2468
2469 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2470 if (isPowerOf2_32(Value: BitWidth)) {
2471 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
2472 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2473 Depth: Depth + 1))
2474 return true;
2475 }
2476 break;
2477 }
2478 case ISD::SMIN:
2479 case ISD::SMAX:
2480 case ISD::UMIN:
2481 case ISD::UMAX: {
2482 unsigned Opc = Op.getOpcode();
2483 SDValue Op0 = Op.getOperand(i: 0);
2484 SDValue Op1 = Op.getOperand(i: 1);
2485
2486 // If we're only demanding signbits, then we can simplify to OR/AND node.
2487 unsigned BitOp =
2488 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
2489 unsigned NumSignBits =
2490 std::min(a: TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1),
2491 b: TLO.DAG.ComputeNumSignBits(Op: Op1, DemandedElts, Depth: Depth + 1));
2492 unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
2493 if (NumSignBits >= NumDemandedUpperBits)
2494 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: BitOp, DL: SDLoc(Op), VT, N1: Op0, N2: Op1));
2495
2496 // Check if one arg is always less/greater than (or equal) to the other arg.
2497 KnownBits Known0 = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2498 KnownBits Known1 = TLO.DAG.computeKnownBits(Op: Op1, DemandedElts, Depth: Depth + 1);
2499 switch (Opc) {
2500 case ISD::SMIN:
2501 if (std::optional<bool> IsSLE = KnownBits::sle(LHS: Known0, RHS: Known1))
2502 return TLO.CombineTo(O: Op, N: *IsSLE ? Op0 : Op1);
2503 if (std::optional<bool> IsSLT = KnownBits::slt(LHS: Known0, RHS: Known1))
2504 return TLO.CombineTo(O: Op, N: *IsSLT ? Op0 : Op1);
2505 Known = KnownBits::smin(LHS: Known0, RHS: Known1);
2506 break;
2507 case ISD::SMAX:
2508 if (std::optional<bool> IsSGE = KnownBits::sge(LHS: Known0, RHS: Known1))
2509 return TLO.CombineTo(O: Op, N: *IsSGE ? Op0 : Op1);
2510 if (std::optional<bool> IsSGT = KnownBits::sgt(LHS: Known0, RHS: Known1))
2511 return TLO.CombineTo(O: Op, N: *IsSGT ? Op0 : Op1);
2512 Known = KnownBits::smax(LHS: Known0, RHS: Known1);
2513 break;
2514 case ISD::UMIN:
2515 if (std::optional<bool> IsULE = KnownBits::ule(LHS: Known0, RHS: Known1))
2516 return TLO.CombineTo(O: Op, N: *IsULE ? Op0 : Op1);
2517 if (std::optional<bool> IsULT = KnownBits::ult(LHS: Known0, RHS: Known1))
2518 return TLO.CombineTo(O: Op, N: *IsULT ? Op0 : Op1);
2519 Known = KnownBits::umin(LHS: Known0, RHS: Known1);
2520 break;
2521 case ISD::UMAX:
2522 if (std::optional<bool> IsUGE = KnownBits::uge(LHS: Known0, RHS: Known1))
2523 return TLO.CombineTo(O: Op, N: *IsUGE ? Op0 : Op1);
2524 if (std::optional<bool> IsUGT = KnownBits::ugt(LHS: Known0, RHS: Known1))
2525 return TLO.CombineTo(O: Op, N: *IsUGT ? Op0 : Op1);
2526 Known = KnownBits::umax(LHS: Known0, RHS: Known1);
2527 break;
2528 }
2529 break;
2530 }
2531 case ISD::BITREVERSE: {
2532 SDValue Src = Op.getOperand(i: 0);
2533 APInt DemandedSrcBits = DemandedBits.reverseBits();
2534 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2535 Depth: Depth + 1))
2536 return true;
2537 Known = Known2.reverseBits();
2538 break;
2539 }
2540 case ISD::BSWAP: {
2541 SDValue Src = Op.getOperand(i: 0);
2542
2543 // If the only bits demanded come from one byte of the bswap result,
2544 // just shift the input byte into position to eliminate the bswap.
2545 unsigned NLZ = DemandedBits.countl_zero();
2546 unsigned NTZ = DemandedBits.countr_zero();
2547
2548 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
2549 // we need all the bits down to bit 8. Likewise, round NLZ. If we
2550 // have 14 leading zeros, round to 8.
2551 NLZ = alignDown(Value: NLZ, Align: 8);
2552 NTZ = alignDown(Value: NTZ, Align: 8);
2553 // If we need exactly one byte, we can do this transformation.
2554 if (BitWidth - NLZ - NTZ == 8) {
2555 // Replace this with either a left or right shift to get the byte into
2556 // the right place.
2557 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
2558 if (!TLO.LegalOperations() || isOperationLegal(Op: ShiftOpcode, VT)) {
2559 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2560 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
2561 SDValue NewOp = TLO.DAG.getNode(Opcode: ShiftOpcode, DL: dl, VT, N1: Src, N2: ShAmt);
2562 return TLO.CombineTo(O: Op, N: NewOp);
2563 }
2564 }
2565
2566 APInt DemandedSrcBits = DemandedBits.byteSwap();
2567 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2568 Depth: Depth + 1))
2569 return true;
2570 Known = Known2.byteSwap();
2571 break;
2572 }
2573 case ISD::CTPOP: {
2574 // If only 1 bit is demanded, replace with PARITY as long as we're before
2575 // op legalization.
2576 // FIXME: Limit to scalars for now.
2577 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
2578 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT,
2579 Operand: Op.getOperand(i: 0)));
2580
2581 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2582 break;
2583 }
2584 case ISD::PDEP: {
2585 SDValue Op0 = Op.getOperand(i: 0);
2586 SDValue Op1 = Op.getOperand(i: 1);
2587
2588 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2589 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
2590
2591 // If the demanded bits has leading zeroes, we don't demand those from the
2592 // mask.
2593 if (SimplifyDemandedBits(Op: Op1, DemandedBits: LoMask, Known, TLO, Depth: Depth + 1))
2594 return true;
2595
2596 // The number of possible 1s in the mask determines the number of LSBs of
2597 // operand 0 used. Undemanded bits from the mask don't matter so filter
2598 // them before counting.
2599 KnownBits Known2;
2600 uint64_t Count = (~Known.Zero & LoMask).popcount();
2601 APInt DemandedMask(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: Count));
2602 if (SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, Known&: Known2, TLO, Depth: Depth + 1))
2603 return true;
2604
2605 // Zeroes are retained from the mask, but not ones.
2606 Known.One.clearAllBits();
2607 // The result will have at least as many trailing zeros as the non-mask
2608 // operand since bits can only map to the same or higher bit position.
2609 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
2610 break;
2611 }
2612 case ISD::SIGN_EXTEND_INREG: {
2613 SDValue Op0 = Op.getOperand(i: 0);
2614 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2615 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2616
2617 // If we only care about the highest bit, don't bother shifting right.
2618 if (DemandedBits.isSignMask()) {
2619 unsigned MinSignedBits =
2620 TLO.DAG.ComputeMaxSignificantBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2621 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2622 // However if the input is already sign extended we expect the sign
2623 // extension to be dropped altogether later and do not simplify.
2624 if (!AlreadySignExtended) {
2625 // Compute the correct shift amount type, which must be getShiftAmountTy
2626 // for scalar types after legalization.
2627 SDValue ShiftAmt =
2628 TLO.DAG.getShiftAmountConstant(Val: BitWidth - ExVTBits, VT, DL: dl);
2629 return TLO.CombineTo(O: Op,
2630 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: ShiftAmt));
2631 }
2632 }
2633
2634 // If none of the extended bits are demanded, eliminate the sextinreg.
2635 if (DemandedBits.getActiveBits() <= ExVTBits)
2636 return TLO.CombineTo(O: Op, N: Op0);
2637
2638 APInt InputDemandedBits = DemandedBits.getLoBits(numBits: ExVTBits);
2639
2640 // Since the sign extended bits are demanded, we know that the sign
2641 // bit is demanded.
2642 InputDemandedBits.setBit(ExVTBits - 1);
2643
2644 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InputDemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
2645 Depth: Depth + 1))
2646 return true;
2647
2648 // If the sign bit of the input is known set or clear, then we know the
2649 // top bits of the result.
2650
2651 // If the input sign bit is known zero, convert this into a zero extension.
2652 if (Known.Zero[ExVTBits - 1])
2653 return TLO.CombineTo(O: Op, N: TLO.DAG.getZeroExtendInReg(Op: Op0, DL: dl, VT: ExVT));
2654
2655 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ExVTBits);
2656 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
2657 Known.One.setBitsFrom(ExVTBits);
2658 Known.Zero &= Mask;
2659 } else { // Input sign bit unknown
2660 Known.Zero &= Mask;
2661 Known.One &= Mask;
2662 }
2663 break;
2664 }
2665 case ISD::BUILD_PAIR: {
2666 EVT HalfVT = Op.getOperand(i: 0).getValueType();
2667 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
2668
2669 APInt MaskLo = DemandedBits.getLoBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2670 APInt MaskHi = DemandedBits.getHiBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2671
2672 KnownBits KnownLo, KnownHi;
2673
2674 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: MaskLo, Known&: KnownLo, TLO, Depth: Depth + 1))
2675 return true;
2676
2677 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: MaskHi, Known&: KnownHi, TLO, Depth: Depth + 1))
2678 return true;
2679
2680 Known = KnownHi.concat(Lo: KnownLo);
2681 break;
2682 }
2683 case ISD::ZERO_EXTEND_VECTOR_INREG:
2684 if (VT.isScalableVector())
2685 return false;
2686 [[fallthrough]];
2687 case ISD::ZERO_EXTEND: {
2688 SDValue Src = Op.getOperand(i: 0);
2689 EVT SrcVT = Src.getValueType();
2690 unsigned InBits = SrcVT.getScalarSizeInBits();
2691 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2692 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
2693
2694 // If none of the top bits are demanded, convert this into an any_extend.
2695 if (DemandedBits.getActiveBits() <= InBits) {
2696 // If we only need the non-extended bits of the bottom element
2697 // then we can just bitcast to the result.
2698 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2699 VT.getSizeInBits() == SrcVT.getSizeInBits())
2700 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2701
2702 unsigned Opc =
2703 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2704 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2705 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2706 }
2707
2708 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2709 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2710 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2711 Depth: Depth + 1)) {
2712 Op->dropFlags(Mask: SDNodeFlags::NonNeg);
2713 return true;
2714 }
2715 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2716 Known = Known.zext(BitWidth);
2717
2718 // Attempt to avoid multi-use ops if we don't need anything from them.
2719 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2720 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2721 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2722 break;
2723 }
2724 case ISD::SIGN_EXTEND_VECTOR_INREG:
2725 if (VT.isScalableVector())
2726 return false;
2727 [[fallthrough]];
2728 case ISD::SIGN_EXTEND: {
2729 SDValue Src = Op.getOperand(i: 0);
2730 EVT SrcVT = Src.getValueType();
2731 unsigned InBits = SrcVT.getScalarSizeInBits();
2732 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2733 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
2734
2735 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2736 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2737
2738 // Since some of the sign extended bits are demanded, we know that the sign
2739 // bit is demanded.
2740 InDemandedBits.setBit(InBits - 1);
2741
2742 // If none of the top bits are demanded, convert this into an any_extend.
2743 if (DemandedBits.getActiveBits() <= InBits) {
2744 // If we only need the non-extended bits of the bottom element
2745 // then we can just bitcast to the result.
2746 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2747 VT.getSizeInBits() == SrcVT.getSizeInBits())
2748 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2749
2750 // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
2751 if (getBooleanContents(Type: VT) != ZeroOrNegativeOneBooleanContent ||
2752 TLO.DAG.ComputeNumSignBits(Op: Src, DemandedElts: InDemandedElts, Depth: Depth + 1) !=
2753 InBits) {
2754 unsigned Opc =
2755 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2756 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2757 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2758 }
2759 }
2760
2761 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2762 Depth: Depth + 1))
2763 return true;
2764 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2765
2766 // If the sign bit is known one, the top bits match.
2767 Known = Known.sext(BitWidth);
2768
2769 // If the sign bit is known zero, convert this to a zero extend.
2770 if (Known.isNonNegative()) {
2771 unsigned Opc =
2772 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
2773 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT)) {
2774 SDNodeFlags Flags;
2775 if (!IsVecInReg)
2776 Flags |= SDNodeFlags::NonNeg;
2777 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src, Flags));
2778 }
2779 }
2780
2781 // Attempt to avoid multi-use ops if we don't need anything from them.
2782 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2783 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2784 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2785 break;
2786 }
2787 case ISD::ANY_EXTEND_VECTOR_INREG:
2788 if (VT.isScalableVector())
2789 return false;
2790 [[fallthrough]];
2791 case ISD::ANY_EXTEND: {
2792 SDValue Src = Op.getOperand(i: 0);
2793 EVT SrcVT = Src.getValueType();
2794 unsigned InBits = SrcVT.getScalarSizeInBits();
2795 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2796 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2797
2798 // If we only need the bottom element then we can just bitcast.
2799 // TODO: Handle ANY_EXTEND?
2800 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2801 VT.getSizeInBits() == SrcVT.getSizeInBits())
2802 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2803
2804 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2805 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2806 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2807 Depth: Depth + 1))
2808 return true;
2809 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2810 Known = Known.anyext(BitWidth);
2811
2812 // Attempt to avoid multi-use ops if we don't need anything from them.
2813 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2814 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2815 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2816 break;
2817 }
2818 case ISD::TRUNCATE: {
2819 SDValue Src = Op.getOperand(i: 0);
2820
2821 // Simplify the input, using demanded bit information, and compute the known
2822 // zero/one bits live out.
2823 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2824 APInt TruncMask = DemandedBits.zext(width: OperandBitWidth);
2825 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: TruncMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2826 Depth: Depth + 1)) {
2827 // Disable the nsw and nuw flags. We can no longer guarantee that we
2828 // won't wrap after simplification.
2829 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2830 return true;
2831 }
2832 Known = Known.trunc(BitWidth);
2833
2834 // Attempt to avoid multi-use ops if we don't need anything from them.
2835 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2836 Op: Src, DemandedBits: TruncMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2837 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: NewSrc));
2838
2839 // If the input is only used by this truncate, see if we can shrink it based
2840 // on the known demanded bits.
2841 switch (Src.getOpcode()) {
2842 default:
2843 break;
2844 case ISD::SRL:
2845 // Shrink SRL by a constant if none of the high bits shifted in are
2846 // demanded.
2847 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2848 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2849 // undesirable.
2850 break;
2851
2852 if (Src.getNode()->hasOneUse()) {
2853 if (isTruncateFree(Val: Src, VT2: VT) &&
2854 !isTruncateFree(FromVT: Src.getValueType(), ToVT: VT)) {
2855 // If truncate is only free at trunc(srl), do not turn it into
2856 // srl(trunc). The check is done by first check the truncate is free
2857 // at Src's opcode(srl), then check the truncate is not done by
2858 // referencing sub-register. In test, if both trunc(srl) and
2859 // srl(trunc)'s trunc are free, srl(trunc) performs better. If only
2860 // trunc(srl)'s trunc is free, trunc(srl) is better.
2861 break;
2862 }
2863
2864 std::optional<unsigned> ShAmtC =
2865 TLO.DAG.getValidShiftAmount(V: Src, DemandedElts, Depth: Depth + 2);
2866 if (!ShAmtC || *ShAmtC >= BitWidth)
2867 break;
2868 unsigned ShVal = *ShAmtC;
2869
2870 APInt HighBits =
2871 APInt::getHighBitsSet(numBits: OperandBitWidth, hiBitsSet: OperandBitWidth - BitWidth);
2872 HighBits.lshrInPlace(ShiftAmt: ShVal);
2873 HighBits = HighBits.trunc(width: BitWidth);
2874 if (!(HighBits & DemandedBits)) {
2875 // None of the shifted in bits are needed. Add a truncate of the
2876 // shift input, then shift it.
2877 SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
2878 SDValue NewTrunc =
2879 TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Src.getOperand(i: 0));
2880 return TLO.CombineTo(
2881 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NewTrunc, N2: NewShAmt));
2882 }
2883 }
2884 break;
2885 }
2886
2887 break;
2888 }
2889 case ISD::AssertZext: {
2890 // AssertZext demands all of the high bits, plus any of the low bits
2891 // demanded by its users.
2892 EVT ZVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2893 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ZVT.getSizeInBits());
2894 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: ~InMask | DemandedBits, Known,
2895 TLO, Depth: Depth + 1))
2896 return true;
2897
2898 Known.Zero |= ~InMask;
2899 Known.One &= (~Known.Zero);
2900 break;
2901 }
2902 case ISD::EXTRACT_VECTOR_ELT: {
2903 SDValue Src = Op.getOperand(i: 0);
2904 SDValue Idx = Op.getOperand(i: 1);
2905 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2906 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2907
2908 if (SrcEltCnt.isScalable())
2909 return false;
2910
2911 // Demand the bits from every vector element without a constant index.
2912 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2913 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
2914 if (auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx))
2915 if (CIdx->getAPIntValue().ult(RHS: NumSrcElts))
2916 DemandedSrcElts = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: CIdx->getZExtValue());
2917
2918 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2919 // anything about the extended bits.
2920 APInt DemandedSrcBits = DemandedBits;
2921 if (BitWidth > EltBitWidth)
2922 DemandedSrcBits = DemandedSrcBits.trunc(width: EltBitWidth);
2923
2924 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts, Known&: Known2, TLO,
2925 Depth: Depth + 1))
2926 return true;
2927
2928 // Attempt to avoid multi-use ops if we don't need anything from them.
2929 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2930 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2931 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
2932 SDValue NewOp =
2933 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc, N2: Idx);
2934 return TLO.CombineTo(O: Op, N: NewOp);
2935 }
2936 }
2937
2938 Known = Known2;
2939 if (BitWidth > EltBitWidth)
2940 Known = Known.anyext(BitWidth);
2941 break;
2942 }
2943 case ISD::BITCAST: {
2944 if (VT.isScalableVector())
2945 return false;
2946 SDValue Src = Op.getOperand(i: 0);
2947 EVT SrcVT = Src.getValueType();
2948 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2949
2950 // If this is an FP->Int bitcast and if the sign bit is the only
2951 // thing demanded, turn this into a FGETSIGN.
2952 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2953 DemandedBits == APInt::getSignMask(BitWidth: Op.getValueSizeInBits()) &&
2954 SrcVT.isFloatingPoint()) {
2955 if (isOperationLegalOrCustom(Op: ISD::FGETSIGN, VT)) {
2956 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2957 // place. We expect the SHL to be eliminated by other optimizations.
2958 SDValue Sign = TLO.DAG.getNode(Opcode: ISD::FGETSIGN, DL: dl, VT, Operand: Src);
2959 unsigned ShVal = Op.getValueSizeInBits() - 1;
2960 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
2961 return TLO.CombineTo(O: Op,
2962 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Sign, N2: ShAmt));
2963 }
2964 }
2965
2966 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2967 // Demand the elt/bit if any of the original elts/bits are demanded.
2968 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
2969 unsigned Scale = BitWidth / NumSrcEltBits;
2970 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2971 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
2972 for (unsigned i = 0; i != Scale; ++i) {
2973 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2974 unsigned BitOffset = EltOffset * NumSrcEltBits;
2975 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
2976 }
2977 // Recursive calls below may turn not demanded elements into poison, so we
2978 // need to demand all smaller source elements that maps to a demanded
2979 // destination element.
2980 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
2981
2982 APInt KnownSrcUndef, KnownSrcZero;
2983 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
2984 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
2985 return true;
2986
2987 KnownBits KnownSrcBits;
2988 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
2989 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
2990 return true;
2991 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
2992 // TODO - bigendian once we have test coverage.
2993 unsigned Scale = NumSrcEltBits / BitWidth;
2994 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2995 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
2996 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
2997 for (unsigned i = 0; i != NumElts; ++i)
2998 if (DemandedElts[i]) {
2999 unsigned Offset = (i % Scale) * BitWidth;
3000 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
3001 DemandedSrcElts.setBit(i / Scale);
3002 }
3003
3004 if (SrcVT.isVector()) {
3005 APInt KnownSrcUndef, KnownSrcZero;
3006 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
3007 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
3008 return true;
3009 }
3010
3011 KnownBits KnownSrcBits;
3012 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
3013 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
3014 return true;
3015
3016 // Attempt to avoid multi-use ops if we don't need anything from them.
3017 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
3018 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
3019 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
3020 SDValue NewOp = TLO.DAG.getBitcast(VT, V: DemandedSrc);
3021 return TLO.CombineTo(O: Op, N: NewOp);
3022 }
3023 }
3024 }
3025
3026 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
3027 // recursive call where Known may be useful to the caller.
3028 if (Depth > 0) {
3029 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3030 return false;
3031 }
3032 break;
3033 }
3034 case ISD::MUL:
3035 if (DemandedBits.isPowerOf2()) {
3036 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
3037 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
3038 // odd (has LSB set), then the left-shifted low bit of X is the answer.
3039 unsigned CTZ = DemandedBits.countr_zero();
3040 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
3041 if (C && C->getAPIntValue().countr_zero() == CTZ) {
3042 SDValue AmtC = TLO.DAG.getShiftAmountConstant(Val: CTZ, VT, DL: dl);
3043 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op.getOperand(i: 0), N2: AmtC);
3044 return TLO.CombineTo(O: Op, N: Shl);
3045 }
3046 }
3047 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
3048 // X * X is odd iff X is odd.
3049 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
3050 if (Op.getOperand(i: 0) == Op.getOperand(i: 1) && DemandedBits.ult(RHS: 4)) {
3051 SDValue One = TLO.DAG.getConstant(Val: 1, DL: dl, VT);
3052 SDValue And1 = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op.getOperand(i: 0), N2: One);
3053 return TLO.CombineTo(O: Op, N: And1);
3054 }
3055 [[fallthrough]];
3056 case ISD::PTRADD:
3057 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
3058 break;
3059 // PTRADD behaves like ADD if pointers are represented as integers.
3060 [[fallthrough]];
3061 case ISD::ADD:
3062 case ISD::SUB: {
3063 // Add, Sub, and Mul don't demand any bits in positions beyond that
3064 // of the highest bit demanded of them.
3065 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
3066 SDNodeFlags Flags = Op.getNode()->getFlags();
3067 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
3068 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
3069 KnownBits KnownOp0, KnownOp1;
3070 auto GetDemandedBitsLHSMask = [&](APInt Demanded,
3071 const KnownBits &KnownRHS) {
3072 if (Op.getOpcode() == ISD::MUL)
3073 Demanded.clearHighBits(hiBits: KnownRHS.countMinTrailingZeros());
3074 return Demanded;
3075 };
3076 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: LoMask, OriginalDemandedElts: DemandedElts, Known&: KnownOp1, TLO,
3077 Depth: Depth + 1) ||
3078 SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: GetDemandedBitsLHSMask(LoMask, KnownOp1),
3079 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1) ||
3080 // See if the operation should be performed at a smaller bit width.
3081 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
3082 // Disable the nsw and nuw flags. We can no longer guarantee that we
3083 // won't wrap after simplification.
3084 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
3085 return true;
3086 }
3087
3088 // neg x with only low bit demanded is simply x.
3089 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
3090 isNullConstant(V: Op0))
3091 return TLO.CombineTo(O: Op, N: Op1);
3092
3093 // Attempt to avoid multi-use ops if we don't need anything from them.
3094 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
3095 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
3096 Op: Op0, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3097 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
3098 Op: Op1, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3099 if (DemandedOp0 || DemandedOp1) {
3100 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
3101 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
3102 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1,
3103 Flags: Flags & ~SDNodeFlags::NoWrap);
3104 return TLO.CombineTo(O: Op, N: NewOp);
3105 }
3106 }
3107
3108 // If we have a constant operand, we may be able to turn it into -1 if we
3109 // do not demand the high bits. This can make the constant smaller to
3110 // encode, allow more general folding, or match specialized instruction
3111 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
3112 // is probably not useful (and could be detrimental).
3113 ConstantSDNode *C = isConstOrConstSplat(N: Op1);
3114 APInt HighMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: DemandedBitsLZ);
3115 if (C && !C->isAllOnes() && !C->isOne() &&
3116 (C->getAPIntValue() | HighMask).isAllOnes()) {
3117 SDValue Neg1 = TLO.DAG.getAllOnesConstant(DL: dl, VT);
3118 // Disable the nsw and nuw flags. We can no longer guarantee that we
3119 // won't wrap after simplification.
3120 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Neg1,
3121 Flags: Flags & ~SDNodeFlags::NoWrap);
3122 return TLO.CombineTo(O: Op, N: NewOp);
3123 }
3124
3125 // Match a multiply with a disguised negated-power-of-2 and convert to a
3126 // an equivalent shift-left amount.
3127 // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3128 auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
3129 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
3130 return 0;
3131
3132 // Don't touch opaque constants. Also, ignore zero and power-of-2
3133 // multiplies. Those will get folded later.
3134 ConstantSDNode *MulC = isConstOrConstSplat(N: Mul.getOperand(i: 1));
3135 if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
3136 !MulC->getAPIntValue().isPowerOf2()) {
3137 APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
3138 if (UnmaskedC.isNegatedPowerOf2())
3139 return (-UnmaskedC).logBase2();
3140 }
3141 return 0;
3142 };
3143
3144 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
3145 unsigned ShlAmt) {
3146 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(Val: ShlAmt, VT, DL: dl);
3147 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: X, N2: ShlAmtC);
3148 SDValue Res = TLO.DAG.getNode(Opcode: NT, DL: dl, VT, N1: Y, N2: Shl);
3149 return TLO.CombineTo(O: Op, N: Res);
3150 };
3151
3152 if (isOperationLegalOrCustom(Op: ISD::SHL, VT)) {
3153 if (Op.getOpcode() == ISD::ADD) {
3154 // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3155 if (unsigned ShAmt = getShiftLeftAmt(Op0))
3156 return foldMul(ISD::SUB, Op0.getOperand(i: 0), Op1, ShAmt);
3157 // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
3158 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3159 return foldMul(ISD::SUB, Op1.getOperand(i: 0), Op0, ShAmt);
3160 }
3161 if (Op.getOpcode() == ISD::SUB) {
3162 // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
3163 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3164 return foldMul(ISD::ADD, Op1.getOperand(i: 0), Op0, ShAmt);
3165 }
3166 }
3167
3168 if (Op.getOpcode() == ISD::MUL) {
3169 Known = KnownBits::mul(LHS: KnownOp0, RHS: KnownOp1);
3170 } else { // Op.getOpcode() is either ISD::ADD, ISD::PTRADD, or ISD::SUB.
3171 Known = KnownBits::computeForAddSub(
3172 Add: Op.getOpcode() != ISD::SUB, NSW: Flags.hasNoSignedWrap(),
3173 NUW: Flags.hasNoUnsignedWrap(), LHS: KnownOp0, RHS: KnownOp1);
3174 }
3175 break;
3176 }
3177 case ISD::FABS: {
3178 SDValue Op0 = Op.getOperand(i: 0);
3179 APInt SignMask = APInt::getSignMask(BitWidth);
3180
3181 if (!DemandedBits.intersects(RHS: SignMask))
3182 return TLO.CombineTo(O: Op, N: Op0);
3183
3184 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3185 Depth: Depth + 1))
3186 return true;
3187
3188 if (Known.isNonNegative())
3189 return TLO.CombineTo(O: Op, N: Op0);
3190 if (Known.isNegative())
3191 return TLO.CombineTo(
3192 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3193
3194 Known.Zero |= SignMask;
3195 Known.One &= ~SignMask;
3196
3197 break;
3198 }
3199 case ISD::FCOPYSIGN: {
3200 SDValue Op0 = Op.getOperand(i: 0);
3201 SDValue Op1 = Op.getOperand(i: 1);
3202
3203 unsigned BitWidth0 = Op0.getScalarValueSizeInBits();
3204 unsigned BitWidth1 = Op1.getScalarValueSizeInBits();
3205 APInt SignMask0 = APInt::getSignMask(BitWidth: BitWidth0);
3206 APInt SignMask1 = APInt::getSignMask(BitWidth: BitWidth1);
3207
3208 if (!DemandedBits.intersects(RHS: SignMask0))
3209 return TLO.CombineTo(O: Op, N: Op0);
3210
3211 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~SignMask0 & DemandedBits, OriginalDemandedElts: DemandedElts,
3212 Known, TLO, Depth: Depth + 1) ||
3213 SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: SignMask1, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
3214 Depth: Depth + 1))
3215 return true;
3216
3217 if (Known2.isNonNegative())
3218 return TLO.CombineTo(
3219 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3220
3221 if (Known2.isNegative())
3222 return TLO.CombineTo(
3223 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT,
3224 Operand: TLO.DAG.getNode(Opcode: ISD::FABS, DL: SDLoc(Op0), VT, Operand: Op0)));
3225
3226 Known.Zero &= ~SignMask0;
3227 Known.One &= ~SignMask0;
3228 break;
3229 }
3230 case ISD::FNEG: {
3231 SDValue Op0 = Op.getOperand(i: 0);
3232 APInt SignMask = APInt::getSignMask(BitWidth);
3233
3234 if (!DemandedBits.intersects(RHS: SignMask))
3235 return TLO.CombineTo(O: Op, N: Op0);
3236
3237 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3238 Depth: Depth + 1))
3239 return true;
3240
3241 if (!Known.isSignUnknown()) {
3242 Known.Zero ^= SignMask;
3243 Known.One ^= SignMask;
3244 }
3245
3246 break;
3247 }
3248 default:
3249 // We also ask the target about intrinsics (which could be specific to it).
3250 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3251 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
3252 // TODO: Probably okay to remove after audit; here to reduce change size
3253 // in initial enablement patch for scalable vectors
3254 if (Op.getValueType().isScalableVector())
3255 break;
3256 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
3257 Known, TLO, Depth))
3258 return true;
3259 break;
3260 }
3261
3262 // Just use computeKnownBits to compute output bits.
3263 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3264 break;
3265 }
3266
3267 // If we know the value of all of the demanded bits, return this as a
3268 // constant.
3269 if (!isTargetCanonicalConstantNode(Op) &&
3270 DemandedBits.isSubsetOf(RHS: Known.Zero | Known.One)) {
3271 // Avoid folding to a constant if any OpaqueConstant is involved.
3272 if (llvm::any_of(Range: Op->ops(), P: [](SDValue V) {
3273 auto *C = dyn_cast<ConstantSDNode>(Val&: V);
3274 return C && C->isOpaque();
3275 }))
3276 return false;
3277 if (VT.isInteger())
3278 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: Known.One, DL: dl, VT));
3279 if (VT.isFloatingPoint())
3280 return TLO.CombineTo(
3281 O: Op, N: TLO.DAG.getConstantFP(Val: APFloat(VT.getFltSemantics(), Known.One),
3282 DL: dl, VT));
3283 }
3284
3285 // A multi use 'all demanded elts' simplify failed to find any knownbits.
3286 // Try again just for the original demanded elts.
3287 // Ensure we do this AFTER constant folding above.
3288 if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
3289 Known = TLO.DAG.computeKnownBits(Op, DemandedElts: OriginalDemandedElts, Depth);
3290
3291 return false;
3292}
3293
3294bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
3295 const APInt &DemandedElts,
3296 DAGCombinerInfo &DCI) const {
3297 SelectionDAG &DAG = DCI.DAG;
3298 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
3299 !DCI.isBeforeLegalizeOps());
3300
3301 APInt KnownUndef, KnownZero;
3302 bool Simplified =
3303 SimplifyDemandedVectorElts(Op, DemandedEltMask: DemandedElts, KnownUndef, KnownZero, TLO);
3304 if (Simplified) {
3305 DCI.AddToWorklist(N: Op.getNode());
3306 DCI.CommitTargetLoweringOpt(TLO);
3307 }
3308
3309 return Simplified;
3310}
3311
3312/// Given a vector binary operation and known undefined elements for each input
3313/// operand, compute whether each element of the output is undefined.
3314static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
3315 const APInt &UndefOp0,
3316 const APInt &UndefOp1) {
3317 EVT VT = BO.getValueType();
3318 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
3319 "Vector binop only");
3320
3321 EVT EltVT = VT.getVectorElementType();
3322 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
3323 assert(UndefOp0.getBitWidth() == NumElts &&
3324 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
3325
3326 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
3327 const APInt &UndefVals) {
3328 if (UndefVals[Index])
3329 return DAG.getUNDEF(VT: EltVT);
3330
3331 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
3332 // Try hard to make sure that the getNode() call is not creating temporary
3333 // nodes. Ignore opaque integers because they do not constant fold.
3334 SDValue Elt = BV->getOperand(Num: Index);
3335 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
3336 if (isa<ConstantFPSDNode>(Val: Elt) || Elt.isUndef() || (C && !C->isOpaque()))
3337 return Elt;
3338 }
3339
3340 return SDValue();
3341 };
3342
3343 APInt KnownUndef = APInt::getZero(numBits: NumElts);
3344 for (unsigned i = 0; i != NumElts; ++i) {
3345 // If both inputs for this element are either constant or undef and match
3346 // the element type, compute the constant/undef result for this element of
3347 // the vector.
3348 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
3349 // not handle FP constants. The code within getNode() should be refactored
3350 // to avoid the danger of creating a bogus temporary node here.
3351 SDValue C0 = getUndefOrConstantElt(BO.getOperand(i: 0), i, UndefOp0);
3352 SDValue C1 = getUndefOrConstantElt(BO.getOperand(i: 1), i, UndefOp1);
3353 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
3354 if (DAG.getNode(Opcode: BO.getOpcode(), DL: SDLoc(BO), VT: EltVT, N1: C0, N2: C1).isUndef())
3355 KnownUndef.setBit(i);
3356 }
3357 return KnownUndef;
3358}
3359
3360bool TargetLowering::SimplifyDemandedVectorElts(
3361 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
3362 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
3363 bool AssumeSingleUse) const {
3364 EVT VT = Op.getValueType();
3365 unsigned Opcode = Op.getOpcode();
3366 APInt DemandedElts = OriginalDemandedElts;
3367 unsigned NumElts = DemandedElts.getBitWidth();
3368 assert(VT.isVector() && "Expected vector op");
3369
3370 KnownUndef = KnownZero = APInt::getZero(numBits: NumElts);
3371
3372 if (!shouldSimplifyDemandedVectorElts(Op, TLO))
3373 return false;
3374
3375 // TODO: For now we assume we know nothing about scalable vectors.
3376 if (VT.isScalableVector())
3377 return false;
3378
3379 assert(VT.getVectorNumElements() == NumElts &&
3380 "Mask size mismatches value type element count!");
3381
3382 // Undef operand.
3383 if (Op.isUndef()) {
3384 KnownUndef.setAllBits();
3385 return false;
3386 }
3387
3388 // If Op has other users, assume that all elements are needed.
3389 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
3390 DemandedElts.setAllBits();
3391
3392 // Not demanding any elements from Op.
3393 if (DemandedElts == 0) {
3394 KnownUndef.setAllBits();
3395 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3396 }
3397
3398 // Limit search depth.
3399 if (Depth >= SelectionDAG::MaxRecursionDepth)
3400 return false;
3401
3402 SDLoc DL(Op);
3403 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3404 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
3405
3406 auto TryShrinkBinOp = [&](SDValue Op0, SDValue Op1) {
3407 unsigned ShrunkSize = getPreferredShrunkVectorSizeInBits(Op, DemandedElts);
3408 if (!ShrunkSize)
3409 return false;
3410
3411 assert(ShrunkSize % EltSizeInBits == 0 &&
3412 "Shrunk size not a multiple of element size");
3413 assert(ShrunkSize < VT.getSizeInBits() &&
3414 "Shrunk size must be < original vector size");
3415 assert(ShrunkSize >= EltSizeInBits * DemandedElts.getActiveBits() &&
3416 "Shrunk size must be >= demanded size");
3417
3418 EVT ShrunkVT = VT.changeVectorElementCount(
3419 Context&: *TLO.DAG.getContext(),
3420 EC: ElementCount::getFixed(MinVal: ShrunkSize / EltSizeInBits));
3421 Op0 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op0, Idx: 0);
3422 Op1 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op1, Idx: 0);
3423 SDValue NewOp =
3424 TLO.DAG.getNode(Opcode, DL, VT: ShrunkVT, N1: Op0, N2: Op1, Flags: Op->getFlags());
3425 return TLO.CombineTo(
3426 O: Op, N: TLO.DAG.getInsertSubvector(DL, Vec: TLO.DAG.getUNDEF(VT), SubVec: NewOp, Idx: 0));
3427 };
3428
3429 // Helper for demanding the specified elements and all the bits of both binary
3430 // operands.
3431 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
3432 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op: Op0, DemandedElts,
3433 DAG&: TLO.DAG, Depth: Depth + 1);
3434 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op: Op1, DemandedElts,
3435 DAG&: TLO.DAG, Depth: Depth + 1);
3436 if (NewOp0 || NewOp1) {
3437 SDValue NewOp =
3438 TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, N1: NewOp0 ? NewOp0 : Op0,
3439 N2: NewOp1 ? NewOp1 : Op1, Flags: Op->getFlags());
3440 return TLO.CombineTo(O: Op, N: NewOp);
3441 }
3442
3443 if (TryShrinkBinOp(Op0, Op1))
3444 return true;
3445
3446 return false;
3447 };
3448
3449 switch (Opcode) {
3450 case ISD::SCALAR_TO_VECTOR: {
3451 if (!DemandedElts[0])
3452 return TLO.CombineTo(O: Op, N: TLO.DAG.getPOISON(VT));
3453 // Upper elements are poison, not undef - don't mark them as KnownUndef.
3454 break;
3455 }
3456 case ISD::BITCAST: {
3457 SDValue Src = Op.getOperand(i: 0);
3458 EVT SrcVT = Src.getValueType();
3459
3460 if (!SrcVT.isVector()) {
3461 // TODO - bigendian once we have test coverage.
3462 if (IsLE) {
3463 APInt DemandedSrcBits = APInt::getZero(numBits: SrcVT.getSizeInBits());
3464 unsigned EltSize = VT.getScalarSizeInBits();
3465 for (unsigned I = 0; I != NumElts; ++I) {
3466 if (DemandedElts[I]) {
3467 unsigned Offset = I * EltSize;
3468 DemandedSrcBits.setBits(loBit: Offset, hiBit: Offset + EltSize);
3469 }
3470 }
3471 KnownBits Known;
3472 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, Known, TLO, Depth: Depth + 1))
3473 return true;
3474 }
3475 break;
3476 }
3477
3478 // Fast handling of 'identity' bitcasts.
3479 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3480 if (NumSrcElts == NumElts)
3481 return SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedElts, KnownUndef,
3482 KnownZero, TLO, Depth: Depth + 1);
3483
3484 APInt SrcDemandedElts, SrcZero, SrcUndef;
3485
3486 // Bitcast from 'large element' src vector to 'small element' vector, we
3487 // must demand a source element if any DemandedElt maps to it.
3488 if ((NumElts % NumSrcElts) == 0) {
3489 unsigned Scale = NumElts / NumSrcElts;
3490 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3491 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3492 TLO, Depth: Depth + 1))
3493 return true;
3494
3495 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
3496 // of the large element.
3497 // TODO - bigendian once we have test coverage.
3498 if (IsLE) {
3499 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
3500 APInt SrcDemandedBits = APInt::getZero(numBits: SrcEltSizeInBits);
3501 for (unsigned i = 0; i != NumElts; ++i)
3502 if (DemandedElts[i]) {
3503 unsigned Ofs = (i % Scale) * EltSizeInBits;
3504 SrcDemandedBits.setBits(loBit: Ofs, hiBit: Ofs + EltSizeInBits);
3505 }
3506
3507 KnownBits Known;
3508 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: SrcDemandedBits, OriginalDemandedElts: SrcDemandedElts, Known,
3509 TLO, Depth: Depth + 1))
3510 return true;
3511
3512 // The bitcast has split each wide element into a number of
3513 // narrow subelements. We have just computed the Known bits
3514 // for wide elements. See if element splitting results in
3515 // some subelements being zero. Only for demanded elements!
3516 for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3517 if (!Known.Zero.extractBits(numBits: EltSizeInBits, bitPosition: SubElt * EltSizeInBits)
3518 .isAllOnes())
3519 continue;
3520 for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3521 unsigned Elt = Scale * SrcElt + SubElt;
3522 // A wholly-undef source lane is reported as undef below; don't also
3523 // flag it as zero, keeping the undef and zero sets disjoint.
3524 if (DemandedElts[Elt] && !SrcUndef[SrcElt])
3525 KnownZero.setBit(Elt);
3526 }
3527 }
3528 }
3529
3530 // If the src element is zero/undef then all the output elements will be -
3531 // only demanded elements are guaranteed to be correct.
3532 for (unsigned i = 0; i != NumSrcElts; ++i) {
3533 if (SrcDemandedElts[i]) {
3534 if (SrcZero[i])
3535 KnownZero.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3536 if (SrcUndef[i])
3537 KnownUndef.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3538 }
3539 }
3540 }
3541
3542 // Bitcast from 'small element' src vector to 'large element' vector, we
3543 // demand all smaller source elements covered by the larger demanded element
3544 // of this vector.
3545 if ((NumSrcElts % NumElts) == 0) {
3546 unsigned Scale = NumSrcElts / NumElts;
3547 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3548 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3549 TLO, Depth: Depth + 1))
3550 return true;
3551
3552 // If all the src elements covering an output element are zero/undef, then
3553 // the output element will be as well, assuming it was demanded.
3554 for (unsigned i = 0; i != NumElts; ++i) {
3555 if (DemandedElts[i]) {
3556 if (SrcZero.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3557 KnownZero.setBit(i);
3558 if (SrcUndef.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3559 KnownUndef.setBit(i);
3560 }
3561 }
3562 }
3563 break;
3564 }
3565 case ISD::FREEZE: {
3566 SDValue N0 = Op.getOperand(i: 0);
3567 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
3568 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
3569 return TLO.CombineTo(O: Op, N: N0);
3570
3571 // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE
3572 // freeze(op(x, ...)) -> op(freeze(x), ...).
3573 // Don't sink the freeze below SCALAR_TO_VECTOR when the scalar is a load
3574 // of a promoted (wider than the element) type: freeze(load) can never be
3575 // folded away (the loaded value may be poison in memory), and the extra
3576 // freeze node then blocks ISel patterns matching scalar_to_vector of a
3577 // load, e.g. the AArch64 scalar_to_vector(extload) -> ldr b/h forms.
3578 // freeze(scalar_to_vector(load)) is equivalent for the demanded element
3579 // zero, and ISel selects the freeze as a plain copy.
3580 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1) {
3581 SDValue Scalar = N0.getOperand(i: 0);
3582 bool IsPromotedLoad = Scalar.getOpcode() == ISD::LOAD &&
3583 Scalar.getValueType() != VT.getVectorElementType();
3584 if (!IsPromotedLoad)
3585 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT,
3586 Operand: TLO.DAG.getFreeze(V: Scalar)));
3587 }
3588 break;
3589 }
3590 case ISD::BUILD_VECTOR: {
3591 // Check all elements and simplify any unused elements with UNDEF.
3592 if (!DemandedElts.isAllOnes()) {
3593 // Don't simplify BROADCASTS.
3594 if (llvm::any_of(Range: Op->op_values(),
3595 P: [&](SDValue Elt) { return Op.getOperand(i: 0) != Elt; })) {
3596 SmallVector<SDValue, 32> Ops(Op->ops());
3597 bool Updated = false;
3598 for (unsigned i = 0; i != NumElts; ++i) {
3599 if (!DemandedElts[i] && !Ops[i].isUndef()) {
3600 Ops[i] = TLO.DAG.getUNDEF(VT: Ops[0].getValueType());
3601 KnownUndef.setBit(i);
3602 Updated = true;
3603 }
3604 }
3605 if (Updated)
3606 return TLO.CombineTo(O: Op, N: TLO.DAG.getBuildVector(VT, DL, Ops));
3607 }
3608 }
3609 for (unsigned i = 0; i != NumElts; ++i) {
3610 SDValue SrcOp = Op.getOperand(i);
3611 if (SrcOp.isUndef()) {
3612 KnownUndef.setBit(i);
3613 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
3614 (isNullConstant(V: SrcOp) || isNullFPConstant(V: SrcOp))) {
3615 KnownZero.setBit(i);
3616 }
3617 }
3618 break;
3619 }
3620 case ISD::CONCAT_VECTORS: {
3621 EVT SubVT = Op.getOperand(i: 0).getValueType();
3622 unsigned NumSubVecs = Op.getNumOperands();
3623 unsigned NumSubElts = SubVT.getVectorNumElements();
3624 for (unsigned i = 0; i != NumSubVecs; ++i) {
3625 SDValue SubOp = Op.getOperand(i);
3626 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3627 APInt SubUndef, SubZero;
3628 if (SimplifyDemandedVectorElts(Op: SubOp, OriginalDemandedElts: SubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3629 Depth: Depth + 1))
3630 return true;
3631 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: i * NumSubElts);
3632 KnownZero.insertBits(SubBits: SubZero, bitPosition: i * NumSubElts);
3633 }
3634
3635 // Attempt to avoid multi-use ops if we don't need anything from them.
3636 if (!DemandedElts.isAllOnes()) {
3637 bool FoundNewSub = false;
3638 SmallVector<SDValue, 2> DemandedSubOps;
3639 for (unsigned i = 0; i != NumSubVecs; ++i) {
3640 SDValue SubOp = Op.getOperand(i);
3641 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3642 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3643 Op: SubOp, DemandedElts: SubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3644 DemandedSubOps.push_back(Elt: NewSubOp ? NewSubOp : SubOp);
3645 FoundNewSub = NewSubOp ? true : FoundNewSub;
3646 }
3647 if (FoundNewSub) {
3648 SDValue NewOp =
3649 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, Ops: DemandedSubOps);
3650 return TLO.CombineTo(O: Op, N: NewOp);
3651 }
3652 }
3653 break;
3654 }
3655 case ISD::INSERT_SUBVECTOR: {
3656 // Demand any elements from the subvector and the remainder from the src it
3657 // is inserted into.
3658 SDValue Src = Op.getOperand(i: 0);
3659 SDValue Sub = Op.getOperand(i: 1);
3660 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3661 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3662 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3663 APInt DemandedSrcElts = DemandedElts;
3664 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3665
3666 // If none of the sub operand elements are demanded, bypass the insert.
3667 if (!DemandedSubElts)
3668 return TLO.CombineTo(O: Op, N: Src);
3669
3670 APInt SubUndef, SubZero;
3671 if (SimplifyDemandedVectorElts(Op: Sub, OriginalDemandedElts: DemandedSubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3672 Depth: Depth + 1))
3673 return true;
3674
3675 // If none of the src operand elements are demanded, replace it with undef.
3676 if (!DemandedSrcElts && !Src.isUndef())
3677 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT,
3678 N1: TLO.DAG.getUNDEF(VT), N2: Sub,
3679 N3: Op.getOperand(i: 2)));
3680
3681 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef, KnownZero,
3682 TLO, Depth: Depth + 1))
3683 return true;
3684 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: Idx);
3685 KnownZero.insertBits(SubBits: SubZero, bitPosition: Idx);
3686
3687 // Attempt to avoid multi-use ops if we don't need anything from them.
3688 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
3689 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3690 Op: Src, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
3691 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3692 Op: Sub, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3693 if (NewSrc || NewSub) {
3694 NewSrc = NewSrc ? NewSrc : Src;
3695 NewSub = NewSub ? NewSub : Sub;
3696 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3697 N2: NewSub, N3: Op.getOperand(i: 2));
3698 return TLO.CombineTo(O: Op, N: NewOp);
3699 }
3700 }
3701 break;
3702 }
3703 case ISD::EXTRACT_SUBVECTOR: {
3704 // Offset the demanded elts by the subvector index.
3705 SDValue Src = Op.getOperand(i: 0);
3706 if (Src.getValueType().isScalableVector())
3707 break;
3708 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3709 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3710 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3711
3712 APInt SrcUndef, SrcZero;
3713 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3714 Depth: Depth + 1))
3715 return true;
3716 KnownUndef = SrcUndef.extractBits(numBits: NumElts, bitPosition: Idx);
3717 KnownZero = SrcZero.extractBits(numBits: NumElts, bitPosition: Idx);
3718
3719 // Attempt to avoid multi-use ops if we don't need anything from them.
3720 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(Op: Src, DemandedElts: DemandedSrcElts,
3721 DAG&: TLO.DAG, Depth: Depth + 1);
3722 if (NewSrc) {
3723 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3724 N2: Op.getOperand(i: 1));
3725 return TLO.CombineTo(O: Op, N: NewOp);
3726 }
3727 break;
3728 }
3729 case ISD::INSERT_VECTOR_ELT: {
3730 SDValue Vec = Op.getOperand(i: 0);
3731 SDValue Scl = Op.getOperand(i: 1);
3732 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
3733
3734 // For a legal, constant insertion index, if we don't need this insertion
3735 // then strip it, else remove it from the demanded elts.
3736 if (CIdx && CIdx->getAPIntValue().ult(RHS: NumElts)) {
3737 unsigned Idx = CIdx->getZExtValue();
3738 if (!DemandedElts[Idx])
3739 return TLO.CombineTo(O: Op, N: Vec);
3740
3741 APInt DemandedVecElts(DemandedElts);
3742 DemandedVecElts.clearBit(BitPosition: Idx);
3743 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedVecElts, KnownUndef,
3744 KnownZero, TLO, Depth: Depth + 1))
3745 return true;
3746
3747 KnownUndef.setBitVal(BitPosition: Idx, BitValue: Scl.isUndef());
3748
3749 KnownZero.setBitVal(BitPosition: Idx, BitValue: isNullConstant(V: Scl) || isNullFPConstant(V: Scl));
3750 break;
3751 }
3752
3753 APInt VecUndef, VecZero;
3754 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedElts, KnownUndef&: VecUndef, KnownZero&: VecZero, TLO,
3755 Depth: Depth + 1))
3756 return true;
3757 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
3758 break;
3759 }
3760 case ISD::VSELECT: {
3761 SDValue Sel = Op.getOperand(i: 0);
3762 SDValue LHS = Op.getOperand(i: 1);
3763 SDValue RHS = Op.getOperand(i: 2);
3764
3765 // Try to transform the select condition based on the current demanded
3766 // elements.
3767 APInt UndefSel, ZeroSel;
3768 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3769 Depth: Depth + 1))
3770 return true;
3771
3772 // See if we can simplify either vselect operand.
3773 APInt DemandedLHS(DemandedElts);
3774 APInt DemandedRHS(DemandedElts);
3775 APInt UndefLHS, ZeroLHS;
3776 APInt UndefRHS, ZeroRHS;
3777 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3778 Depth: Depth + 1))
3779 return true;
3780 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3781 Depth: Depth + 1))
3782 return true;
3783
3784 KnownUndef = UndefLHS & UndefRHS;
3785 KnownZero = ZeroLHS & ZeroRHS;
3786
3787 // If we know that the selected element is always zero, we don't need the
3788 // select value element.
3789 APInt DemandedSel = DemandedElts & ~KnownZero;
3790 if (DemandedSel != DemandedElts)
3791 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedSel, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3792 Depth: Depth + 1))
3793 return true;
3794
3795 break;
3796 }
3797 case ISD::VECTOR_SHUFFLE: {
3798 SDValue LHS = Op.getOperand(i: 0);
3799 SDValue RHS = Op.getOperand(i: 1);
3800 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
3801
3802 // Collect demanded elements from shuffle operands..
3803 APInt DemandedLHS(NumElts, 0);
3804 APInt DemandedRHS(NumElts, 0);
3805 for (unsigned i = 0; i != NumElts; ++i) {
3806 int M = ShuffleMask[i];
3807 if (M < 0 || !DemandedElts[i])
3808 continue;
3809 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
3810 if (M < (int)NumElts)
3811 DemandedLHS.setBit(M);
3812 else
3813 DemandedRHS.setBit(M - NumElts);
3814 }
3815
3816 // If either side isn't demanded, replace it by UNDEF. We handle this
3817 // explicitly here to also simplify in case of multiple uses (on the
3818 // contrary to the SimplifyDemandedVectorElts calls below).
3819 bool FoldLHS = !DemandedLHS && !LHS.isUndef();
3820 bool FoldRHS = !DemandedRHS && !RHS.isUndef();
3821 if (FoldLHS || FoldRHS) {
3822 LHS = FoldLHS ? TLO.DAG.getUNDEF(VT: LHS.getValueType()) : LHS;
3823 RHS = FoldRHS ? TLO.DAG.getUNDEF(VT: RHS.getValueType()) : RHS;
3824 SDValue NewOp =
3825 TLO.DAG.getVectorShuffle(VT, dl: SDLoc(Op), N1: LHS, N2: RHS, Mask: ShuffleMask);
3826 return TLO.CombineTo(O: Op, N: NewOp);
3827 }
3828
3829 // See if we can simplify either shuffle operand.
3830 APInt UndefLHS, ZeroLHS;
3831 APInt UndefRHS, ZeroRHS;
3832 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3833 Depth: Depth + 1))
3834 return true;
3835 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3836 Depth: Depth + 1))
3837 return true;
3838
3839 // Simplify mask using undef elements from LHS/RHS.
3840 bool Updated = false;
3841 bool IdentityLHS = true, IdentityRHS = true;
3842 SmallVector<int, 32> NewMask(ShuffleMask);
3843 for (unsigned i = 0; i != NumElts; ++i) {
3844 int &M = NewMask[i];
3845 if (M < 0)
3846 continue;
3847 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
3848 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
3849 Updated = true;
3850 M = -1;
3851 }
3852 IdentityLHS &= (M < 0) || (M == (int)i);
3853 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3854 }
3855
3856 // Update legal shuffle masks based on demanded elements if it won't reduce
3857 // to Identity which can cause premature removal of the shuffle mask.
3858 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
3859 SDValue LegalShuffle =
3860 buildLegalVectorShuffle(VT, DL, N0: LHS, N1: RHS, Mask: NewMask, DAG&: TLO.DAG);
3861 if (LegalShuffle)
3862 return TLO.CombineTo(O: Op, N: LegalShuffle);
3863 }
3864
3865 // Propagate undef/zero elements from LHS/RHS.
3866 for (unsigned i = 0; i != NumElts; ++i) {
3867 int M = ShuffleMask[i];
3868 if (M < 0) {
3869 KnownUndef.setBit(i);
3870 } else if (M < (int)NumElts) {
3871 if (UndefLHS[M])
3872 KnownUndef.setBit(i);
3873 if (ZeroLHS[M])
3874 KnownZero.setBit(i);
3875 } else {
3876 if (UndefRHS[M - NumElts])
3877 KnownUndef.setBit(i);
3878 if (ZeroRHS[M - NumElts])
3879 KnownZero.setBit(i);
3880 }
3881 }
3882 break;
3883 }
3884 case ISD::ANY_EXTEND_VECTOR_INREG:
3885 case ISD::SIGN_EXTEND_VECTOR_INREG:
3886 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3887 APInt SrcUndef, SrcZero;
3888 SDValue Src = Op.getOperand(i: 0);
3889 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3890 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3891 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3892 Depth: Depth + 1))
3893 return true;
3894 KnownZero = SrcZero.zextOrTrunc(width: NumElts);
3895 KnownUndef = SrcUndef.zextOrTrunc(width: NumElts);
3896
3897 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
3898 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3899 DemandedSrcElts == 1) {
3900 // aext - if we just need the bottom element then we can bitcast.
3901 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
3902 }
3903
3904 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
3905 // zext(undef) upper bits are guaranteed to be zero.
3906 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
3907 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3908 KnownUndef.clearAllBits();
3909
3910 // zext - if we just need the bottom element then we can mask:
3911 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
3912 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
3913 Op->isOnlyUserOf(N: Src.getNode()) &&
3914 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3915 SDLoc DL(Op);
3916 EVT SrcVT = Src.getValueType();
3917 EVT SrcSVT = SrcVT.getScalarType();
3918
3919 // If we're after type legalization and SrcSVT is not legal, use the
3920 // promoted type for creating constants to avoid creating nodes with
3921 // illegal types.
3922 if (TLO.LegalTypes())
3923 SrcSVT = getLegalTypeToTransformTo(Context&: *TLO.DAG.getContext(), VT: SrcSVT);
3924
3925 SmallVector<SDValue> MaskElts;
3926 MaskElts.push_back(Elt: TLO.DAG.getAllOnesConstant(DL, VT: SrcSVT));
3927 MaskElts.append(NumInputs: NumSrcElts - 1, Elt: TLO.DAG.getConstant(Val: 0, DL, VT: SrcSVT));
3928 SDValue Mask = TLO.DAG.getBuildVector(VT: SrcVT, DL, Ops: MaskElts);
3929 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
3930 Opcode: ISD::AND, DL, VT: SrcVT, Ops: {Src.getOperand(i: 1), Mask})) {
3931 Fold = TLO.DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src.getOperand(i: 0), N2: Fold);
3932 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Fold));
3933 }
3934 }
3935 }
3936 break;
3937 }
3938
3939 // TODO: There are more binop opcodes that could be handled here - MIN,
3940 // MAX, saturated math, etc.
3941 case ISD::ADD: {
3942 SDValue Op0 = Op.getOperand(i: 0);
3943 SDValue Op1 = Op.getOperand(i: 1);
3944 if (Op0 == Op1 && Op->isOnlyUserOf(N: Op0.getNode())) {
3945 APInt UndefLHS, ZeroLHS;
3946 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3947 Depth: Depth + 1, /*AssumeSingleUse*/ true))
3948 return true;
3949 }
3950 [[fallthrough]];
3951 }
3952 case ISD::AVGCEILS:
3953 case ISD::AVGCEILU:
3954 case ISD::AVGFLOORS:
3955 case ISD::AVGFLOORU:
3956 case ISD::OR:
3957 case ISD::XOR:
3958 case ISD::SUB:
3959 case ISD::FADD:
3960 case ISD::FSUB:
3961 case ISD::FMUL:
3962 case ISD::FDIV:
3963 case ISD::FREM:
3964 case ISD::PSEUDO_FMIN:
3965 case ISD::PSEUDO_FMAX: {
3966 SDValue Op0 = Op.getOperand(i: 0);
3967 SDValue Op1 = Op.getOperand(i: 1);
3968
3969 APInt UndefRHS, ZeroRHS;
3970 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3971 Depth: Depth + 1))
3972 return true;
3973 APInt UndefLHS, ZeroLHS;
3974 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3975 Depth: Depth + 1))
3976 return true;
3977
3978 KnownZero = ZeroLHS & ZeroRHS;
3979 KnownUndef = getKnownUndefForVectorBinop(BO: Op, DAG&: TLO.DAG, UndefOp0: UndefLHS, UndefOp1: UndefRHS);
3980
3981 // Attempt to avoid multi-use ops if we don't need anything from them.
3982 // TODO - use KnownUndef to relax the demandedelts?
3983 if (!DemandedElts.isAllOnes())
3984 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3985 return true;
3986 break;
3987 }
3988 case ISD::SHL:
3989 case ISD::SRL:
3990 case ISD::SRA:
3991 case ISD::ROTL:
3992 case ISD::ROTR: {
3993 SDValue Op0 = Op.getOperand(i: 0);
3994 SDValue Op1 = Op.getOperand(i: 1);
3995
3996 APInt UndefRHS, ZeroRHS;
3997 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3998 Depth: Depth + 1))
3999 return true;
4000 APInt UndefLHS, ZeroLHS;
4001 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
4002 Depth: Depth + 1))
4003 return true;
4004
4005 KnownZero = ZeroLHS;
4006 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
4007
4008 // Attempt to avoid multi-use ops if we don't need anything from them.
4009 // TODO - use KnownUndef to relax the demandedelts?
4010 if (!DemandedElts.isAllOnes())
4011 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4012 return true;
4013 break;
4014 }
4015 case ISD::MUL:
4016 case ISD::MULHU:
4017 case ISD::MULHS:
4018 case ISD::AND: {
4019 SDValue Op0 = Op.getOperand(i: 0);
4020 SDValue Op1 = Op.getOperand(i: 1);
4021
4022 APInt SrcUndef, SrcZero;
4023 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
4024 Depth: Depth + 1))
4025 return true;
4026 // FIXME: If we know that a demanded element was zero in Op1 we don't need
4027 // to demand it in Op0 - its guaranteed to be zero. There is however a
4028 // restriction, as we must not make any of the originally demanded elements
4029 // more poisonous. We could reduce amount of elements demanded, but then we
4030 // also need a to inform SimplifyDemandedVectorElts that some elements must
4031 // not be made more poisonous.
4032 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef, KnownZero,
4033 TLO, Depth: Depth + 1))
4034 return true;
4035
4036 KnownUndef &= DemandedElts;
4037 KnownZero &= DemandedElts;
4038
4039 // If every element pair has a zero/undef/poison then just fold to zero.
4040 // fold (and x, undef/poison) -> 0 / (and x, 0) -> 0
4041 // fold (mul x, undef/poison) -> 0 / (mul x, 0) -> 0
4042 if (DemandedElts.isSubsetOf(RHS: SrcZero | KnownZero | SrcUndef | KnownUndef))
4043 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
4044
4045 // If either side has a zero element, then the result element is zero, even
4046 // if the other is an UNDEF.
4047 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
4048 // and then handle 'and' nodes with the rest of the binop opcodes.
4049 KnownZero |= SrcZero;
4050 KnownUndef &= SrcUndef;
4051 KnownUndef &= ~KnownZero;
4052
4053 // Attempt to avoid multi-use ops if we don't need anything from them.
4054 if (!DemandedElts.isAllOnes())
4055 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
4056 return true;
4057 break;
4058 }
4059 case ISD::TRUNCATE:
4060 case ISD::SIGN_EXTEND:
4061 case ISD::ZERO_EXTEND:
4062 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
4063 KnownZero, TLO, Depth: Depth + 1))
4064 return true;
4065
4066 if (!DemandedElts.isAllOnes())
4067 if (SDValue NewOp = SimplifyMultipleUseDemandedVectorElts(
4068 Op: Op.getOperand(i: 0), DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
4069 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, Operand: NewOp));
4070
4071 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
4072 // zext(undef) upper bits are guaranteed to be zero.
4073 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4074 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
4075 KnownUndef.clearAllBits();
4076 }
4077 break;
4078 case ISD::SINT_TO_FP:
4079 case ISD::UINT_TO_FP:
4080 case ISD::FP_TO_SINT:
4081 case ISD::FP_TO_UINT:
4082 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
4083 KnownZero, TLO, Depth: Depth + 1))
4084 return true;
4085 // Don't fall through to generic undef -> undef handling.
4086 return false;
4087 default: {
4088 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
4089 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
4090 KnownZero, TLO, Depth))
4091 return true;
4092 } else {
4093 KnownBits Known;
4094 APInt DemandedBits = APInt::getAllOnes(numBits: EltSizeInBits);
4095 if (SimplifyDemandedBits(Op, OriginalDemandedBits: DemandedBits, OriginalDemandedElts, Known,
4096 TLO, Depth, AssumeSingleUse))
4097 return true;
4098 }
4099 break;
4100 }
4101 }
4102
4103 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
4104
4105 // Constant fold all undef cases.
4106 // TODO: Handle zero cases as well.
4107 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4108 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
4109
4110 return false;
4111}
4112
4113/// Determine which of the bits specified in Mask are known to be either zero or
4114/// one and return them in the Known.
4115void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
4116 KnownBits &Known,
4117 const APInt &DemandedElts,
4118 const SelectionDAG &DAG,
4119 unsigned Depth) const {
4120 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4121 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4122 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4123 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4124 "Should use MaskedValueIsZero if you don't know whether Op"
4125 " is a target node!");
4126 Known.resetAll();
4127}
4128
4129void TargetLowering::computeKnownBitsForTargetInstr(
4130 GISelValueTracking &Analysis, Register R, KnownBits &Known,
4131 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4132 unsigned Depth) const {
4133 Known.resetAll();
4134}
4135
4136void TargetLowering::computeKnownFPClassForTargetInstr(
4137 GISelValueTracking &Analysis, Register R, KnownFPClass &Known,
4138 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4139 unsigned Depth) const {
4140 Known.resetAll();
4141}
4142
4143void TargetLowering::computeKnownBitsForStackObjectPointer(
4144 KnownBits &Known, const MachineFunction &, Align Alignment) const {
4145 // The low bits are known zero if the pointer is aligned.
4146 Known.Zero.setLowBits(Log2(A: Alignment));
4147}
4148
4149SDValue TargetLowering::annotateStackObjectPointer(SDValue Ptr,
4150 SelectionDAG &DAG,
4151 const SDLoc &DL,
4152 Align Alignment) const {
4153 // Materialize leading-zero stack object pointer facts as AssertZext.
4154 // Alignment-derived low zero bits are not represented on the returned DAG
4155 // value here.
4156 EVT PtrVT = Ptr.getValueType();
4157
4158 unsigned RegSize = PtrVT.getScalarSizeInBits();
4159 KnownBits Known(RegSize);
4160 computeKnownBitsForStackObjectPointer(Known, DAG.getMachineFunction(),
4161 Alignment);
4162
4163 unsigned NumZeroBits = Known.countMinLeadingZeros();
4164 if (!NumZeroBits)
4165 return Ptr;
4166
4167 EVT FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
4168 return DAG.getNode(Opcode: ISD::AssertZext, DL, VT: PtrVT, N1: Ptr, N2: DAG.getValueType(FromVT));
4169}
4170
4171Align TargetLowering::computeKnownAlignForTargetInstr(
4172 GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI,
4173 unsigned Depth) const {
4174 return Align(1);
4175}
4176
4177/// This method can be implemented by targets that want to expose additional
4178/// information about sign bits to the DAG Combiner.
4179unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
4180 const APInt &,
4181 const SelectionDAG &,
4182 unsigned Depth) const {
4183 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4184 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4185 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4186 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4187 "Should use ComputeNumSignBits if you don't know whether Op"
4188 " is a target node!");
4189 return 1;
4190}
4191
4192unsigned TargetLowering::computeNumSignBitsForTargetInstr(
4193 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4194 const MachineRegisterInfo &MRI, unsigned Depth) const {
4195 return 1;
4196}
4197
4198bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
4199 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
4200 TargetLoweringOpt &TLO, unsigned Depth) const {
4201 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4202 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4203 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4204 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4205 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
4206 " is a target node!");
4207 return false;
4208}
4209
4210bool TargetLowering::SimplifyDemandedBitsForTargetNode(
4211 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4212 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
4213 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4214 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4215 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4216 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4217 "Should use SimplifyDemandedBits if you don't know whether Op"
4218 " is a target node!");
4219 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
4220 return false;
4221}
4222
4223SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
4224 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4225 SelectionDAG &DAG, unsigned Depth) const {
4226 assert(
4227 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4228 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4229 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4230 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4231 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
4232 " is a target node!");
4233 return SDValue();
4234}
4235
4236SDValue
4237TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
4238 SDValue N1, MutableArrayRef<int> Mask,
4239 SelectionDAG &DAG) const {
4240 bool LegalMask = isShuffleMaskLegal(Mask, VT);
4241 if (!LegalMask) {
4242 std::swap(a&: N0, b&: N1);
4243 ShuffleVectorSDNode::commuteMask(Mask);
4244 LegalMask = isShuffleMaskLegal(Mask, VT);
4245 }
4246
4247 if (!LegalMask)
4248 return SDValue();
4249
4250 return DAG.getVectorShuffle(VT, dl: DL, N1: N0, N2: N1, Mask);
4251}
4252
4253const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
4254 return nullptr;
4255}
4256
4257bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4258 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4259 UndefPoisonKind Kind, unsigned Depth) const {
4260 assert(
4261 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4262 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4263 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4264 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4265 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
4266 " is a target node!");
4267
4268 // If Op can't create undef/poison and none of its operands are undef/poison
4269 // then Op is never undef/poison.
4270 return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, Kind,
4271 /*ConsiderFlags*/ true, Depth) &&
4272 all_of(Range: Op->ops(), P: [&](SDValue V) {
4273 return DAG.isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
4274 });
4275}
4276
4277bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
4278 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4279 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
4280 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4281 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4282 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4283 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4284 "Should use canCreateUndefOrPoison if you don't know whether Op"
4285 " is a target node!");
4286 // Be conservative and return true.
4287 return true;
4288}
4289
4290void TargetLowering::computeKnownFPClassForTargetNode(const SDValue Op,
4291 KnownFPClass &Known,
4292 const APInt &DemandedElts,
4293 const SelectionDAG &DAG,
4294 unsigned Depth) const {
4295 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4296 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4297 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4298 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4299 "Should use computeKnownFPClass if you don't know whether Op"
4300 " is a target node!");
4301}
4302
4303bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
4304 const APInt &DemandedElts,
4305 const SelectionDAG &DAG,
4306 bool SNaN,
4307 unsigned Depth) const {
4308 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4309 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4310 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4311 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4312 "Should use isKnownNeverNaN if you don't know whether Op"
4313 " is a target node!");
4314 return false;
4315}
4316
4317bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
4318 const APInt &DemandedElts,
4319 APInt &UndefElts,
4320 const SelectionDAG &DAG,
4321 unsigned Depth) const {
4322 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4323 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4324 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4325 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4326 "Should use isSplatValue if you don't know whether Op"
4327 " is a target node!");
4328 return false;
4329}
4330
4331// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
4332// work with truncating build vectors and vectors with elements of less than
4333// 8 bits.
4334bool TargetLowering::isConstTrueVal(SDValue N) const {
4335 if (!N)
4336 return false;
4337
4338 unsigned EltWidth;
4339 APInt CVal;
4340 if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
4341 /*AllowTruncation=*/true)) {
4342 CVal = CN->getAPIntValue();
4343 EltWidth = N.getValueType().getScalarSizeInBits();
4344 } else
4345 return false;
4346
4347 // If this is a truncating splat, truncate the splat value.
4348 // Otherwise, we may fail to match the expected values below.
4349 if (EltWidth < CVal.getBitWidth())
4350 CVal = CVal.trunc(width: EltWidth);
4351
4352 switch (getBooleanContents(Type: N.getValueType())) {
4353 case UndefinedBooleanContent:
4354 return CVal[0];
4355 case ZeroOrOneBooleanContent:
4356 return CVal.isOne();
4357 case ZeroOrNegativeOneBooleanContent:
4358 return CVal.isAllOnes();
4359 }
4360
4361 llvm_unreachable("Invalid boolean contents");
4362}
4363
4364bool TargetLowering::isConstFalseVal(SDValue N) const {
4365 if (!N)
4366 return false;
4367
4368 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N);
4369 if (!CN) {
4370 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
4371 if (!BV)
4372 return false;
4373
4374 // Only interested in constant splats, we don't care about undef
4375 // elements in identifying boolean constants and getConstantSplatNode
4376 // returns NULL if all ops are undef;
4377 CN = BV->getConstantSplatNode();
4378 if (!CN)
4379 return false;
4380 }
4381
4382 if (getBooleanContents(Type: N->getValueType(ResNo: 0)) == UndefinedBooleanContent)
4383 return !CN->getAPIntValue()[0];
4384
4385 return CN->isZero();
4386}
4387
4388bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
4389 bool SExt) const {
4390 if (VT == MVT::i1)
4391 return N->isOne();
4392
4393 TargetLowering::BooleanContent Cnt = getBooleanContents(Type: VT);
4394 switch (Cnt) {
4395 case TargetLowering::ZeroOrOneBooleanContent:
4396 // An extended value of 1 is always true, unless its original type is i1,
4397 // in which case it will be sign extended to -1.
4398 return (N->isOne() && !SExt) || (SExt && (N->getValueType(ResNo: 0) != MVT::i1));
4399 case TargetLowering::UndefinedBooleanContent:
4400 case TargetLowering::ZeroOrNegativeOneBooleanContent:
4401 return N->isAllOnes() && SExt;
4402 }
4403 llvm_unreachable("Unexpected enumeration.");
4404}
4405
4406/// This helper function of SimplifySetCC tries to optimize the comparison when
4407/// either operand of the SetCC node is a bitwise-and instruction.
4408SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
4409 ISD::CondCode Cond, const SDLoc &DL,
4410 DAGCombinerInfo &DCI) const {
4411 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
4412 std::swap(a&: N0, b&: N1);
4413
4414 SelectionDAG &DAG = DCI.DAG;
4415 EVT OpVT = N0.getValueType();
4416 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
4417 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4418 return SDValue();
4419
4420 // (X & Y) != 0 --> zextOrTrunc(X & Y)
4421 // iff everything but LSB is known zero:
4422 if (Cond == ISD::SETNE && isNullConstant(V: N1) &&
4423 (getBooleanContents(Type: OpVT) == TargetLowering::UndefinedBooleanContent ||
4424 getBooleanContents(Type: OpVT) == TargetLowering::ZeroOrOneBooleanContent)) {
4425 unsigned NumEltBits = OpVT.getScalarSizeInBits();
4426 APInt UpperBits = APInt::getHighBitsSet(numBits: NumEltBits, hiBitsSet: NumEltBits - 1);
4427 if (DAG.MaskedValueIsZero(Op: N0, Mask: UpperBits))
4428 return DAG.getBoolExtOrTrunc(Op: N0, SL: DL, VT, OpVT);
4429 }
4430
4431 // Try to eliminate a power-of-2 mask constant by converting to a signbit
4432 // test in a narrow type that we can truncate to with no cost. Examples:
4433 // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
4434 // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
4435 // TODO: This conservatively checks for type legality on the source and
4436 // destination types. That may inhibit optimizations, but it also
4437 // allows setcc->shift transforms that may be more beneficial.
4438 auto *AndC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
4439 if (AndC && isNullConstant(V: N1) && AndC->getAPIntValue().isPowerOf2() &&
4440 isTypeLegal(VT: OpVT) && N0.hasOneUse()) {
4441 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
4442 BitWidth: AndC->getAPIntValue().getActiveBits());
4443 if (isTruncateFree(FromVT: OpVT, ToVT: NarrowVT) && isTypeLegal(VT: NarrowVT)) {
4444 SDValue Trunc = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 0), DL, VT: NarrowVT);
4445 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: NarrowVT);
4446 return DAG.getSetCC(DL, VT, LHS: Trunc, RHS: Zero,
4447 Cond: Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT);
4448 }
4449 }
4450
4451 // Match these patterns in any of their permutations:
4452 // (X & Y) == Y
4453 // (X & Y) != Y
4454 SDValue X, Y;
4455 if (N0.getOperand(i: 0) == N1) {
4456 X = N0.getOperand(i: 1);
4457 Y = N0.getOperand(i: 0);
4458 } else if (N0.getOperand(i: 1) == N1) {
4459 X = N0.getOperand(i: 0);
4460 Y = N0.getOperand(i: 1);
4461 } else {
4462 return SDValue();
4463 }
4464
4465 // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
4466 // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
4467 // its liable to create and infinite loop.
4468 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: OpVT);
4469 if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
4470 DAG.isKnownToBeAPowerOfTwo(Val: Y)) {
4471 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
4472 // Note that where Y is variable and is known to have at most one bit set
4473 // (for example, if it is Z & 1) we cannot do this; the expressions are not
4474 // equivalent when Y == 0.
4475 assert(OpVT.isInteger());
4476 Cond = ISD::getSetCCInverse(Operation: Cond, Type: OpVT);
4477 if (DCI.isBeforeLegalizeOps() ||
4478 isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType()))
4479 return DAG.getSetCC(DL, VT, LHS: N0, RHS: Zero, Cond);
4480 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
4481 // If the target supports an 'and-not' or 'and-complement' logic operation,
4482 // try to use that to make a comparison operation more efficient.
4483 // But don't do this transform if the mask is a single bit because there are
4484 // more efficient ways to deal with that case (for example, 'bt' on x86 or
4485 // 'rlwinm' on PPC).
4486
4487 // Bail out if the compare operand that we want to turn into a zero is
4488 // already a zero (otherwise, infinite loop).
4489 if (isNullConstant(V: Y))
4490 return SDValue();
4491
4492 // Transform this into: ~X & Y == 0.
4493 SDValue NotX = DAG.getNOT(DL: SDLoc(X), Val: X, VT: OpVT);
4494 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: NotX, N2: Y);
4495 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: Zero, Cond);
4496 }
4497
4498 return SDValue();
4499}
4500
4501/// This helper function of SimplifySetCC tries to optimize the comparison when
4502/// either operand of the SetCC node is a bitwise-or instruction.
4503/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
4504SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
4505 ISD::CondCode Cond, const SDLoc &DL,
4506 DAGCombinerInfo &DCI) const {
4507 if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
4508 std::swap(a&: N0, b&: N1);
4509
4510 SelectionDAG &DAG = DCI.DAG;
4511 EVT OpVT = N0.getValueType();
4512 if (!N0.hasOneUse() || !OpVT.isInteger() ||
4513 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4514 return SDValue();
4515
4516 // (X | Y) == Y
4517 // (X | Y) != Y
4518 SDValue X;
4519 if (sd_match(N: N0, P: m_Or(L: m_Value(N&: X), R: m_Specific(N: N1))) && hasAndNotCompare(Y: X)) {
4520 // If the target supports an 'and-not' or 'and-complement' logic operation,
4521 // try to use that to make a comparison operation more efficient.
4522
4523 // Bail out if the compare operand that we want to turn into a zero is
4524 // already a zero (otherwise, infinite loop).
4525 if (isNullConstant(V: N1))
4526 return SDValue();
4527
4528 // Transform this into: X & ~Y ==/!= 0.
4529 SDValue NotY = DAG.getNOT(DL: SDLoc(N1), Val: N1, VT: OpVT);
4530 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: X, N2: NotY);
4531 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4532 }
4533
4534 return SDValue();
4535}
4536
4537/// There are multiple IR patterns that could be checking whether certain
4538/// truncation of a signed number would be lossy or not. The pattern which is
4539/// best at IR level, may not lower optimally. Thus, we want to unfold it.
4540/// We are looking for the following pattern: (KeptBits is a constant)
4541/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
4542/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
4543/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
4544/// We will unfold it into the natural trunc+sext pattern:
4545/// ((%x << C) a>> C) dstcond %x
4546/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
4547SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4548 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
4549 const SDLoc &DL) const {
4550 // We must be comparing with a constant.
4551 ConstantSDNode *C1;
4552 if (!(C1 = dyn_cast<ConstantSDNode>(Val&: N1)))
4553 return SDValue();
4554
4555 // N0 should be: add %x, (1 << (KeptBits-1))
4556 if (N0->getOpcode() != ISD::ADD)
4557 return SDValue();
4558
4559 // And we must be 'add'ing a constant.
4560 ConstantSDNode *C01;
4561 if (!(C01 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1))))
4562 return SDValue();
4563
4564 SDValue X = N0->getOperand(Num: 0);
4565 EVT XVT = X.getValueType();
4566
4567 // Validate constants ...
4568
4569 APInt I1 = C1->getAPIntValue();
4570
4571 ISD::CondCode NewCond;
4572 if (Cond == ISD::CondCode::SETULT) {
4573 NewCond = ISD::CondCode::SETEQ;
4574 } else if (Cond == ISD::CondCode::SETULE) {
4575 NewCond = ISD::CondCode::SETEQ;
4576 // But need to 'canonicalize' the constant.
4577 I1 += 1;
4578 } else if (Cond == ISD::CondCode::SETUGT) {
4579 NewCond = ISD::CondCode::SETNE;
4580 // But need to 'canonicalize' the constant.
4581 I1 += 1;
4582 } else if (Cond == ISD::CondCode::SETUGE) {
4583 NewCond = ISD::CondCode::SETNE;
4584 } else
4585 return SDValue();
4586
4587 APInt I01 = C01->getAPIntValue();
4588
4589 auto checkConstants = [&I1, &I01]() -> bool {
4590 // Both of them must be power-of-two, and the constant from setcc is bigger.
4591 return I1.ugt(RHS: I01) && I1.isPowerOf2() && I01.isPowerOf2();
4592 };
4593
4594 if (checkConstants()) {
4595 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
4596 } else {
4597 // What if we invert constants? (and the target predicate)
4598 I1.negate();
4599 I01.negate();
4600 assert(XVT.isInteger());
4601 NewCond = getSetCCInverse(Operation: NewCond, Type: XVT);
4602 if (!checkConstants())
4603 return SDValue();
4604 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
4605 }
4606
4607 // They are power-of-two, so which bit is set?
4608 const unsigned KeptBits = I1.logBase2();
4609 const unsigned KeptBitsMinusOne = I01.logBase2();
4610
4611 // Magic!
4612 if (KeptBits != (KeptBitsMinusOne + 1))
4613 return SDValue();
4614 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
4615
4616 // We don't want to do this in every single case.
4617 SelectionDAG &DAG = DCI.DAG;
4618 if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
4619 return SDValue();
4620
4621 // Unfold into: sext_inreg(%x) cond %x
4622 // Where 'cond' will be either 'eq' or 'ne'.
4623 SDValue SExtInReg = DAG.getNode(
4624 Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: XVT, N1: X,
4625 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: KeptBits)));
4626 return DAG.getSetCC(DL, VT: SCCVT, LHS: SExtInReg, RHS: X, Cond: NewCond);
4627}
4628
4629// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4630SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4631 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4632 DAGCombinerInfo &DCI, const SDLoc &DL) const {
4633 assert(isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->isZero() &&
4634 "Should be a comparison with 0.");
4635 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4636 "Valid only for [in]equality comparisons.");
4637
4638 unsigned NewShiftOpcode;
4639 SDValue X, C, Y;
4640
4641 SelectionDAG &DAG = DCI.DAG;
4642
4643 // Look for '(C l>>/<< Y)'.
4644 auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
4645 // The shift should be one-use.
4646 if (!V.hasOneUse())
4647 return false;
4648 unsigned OldShiftOpcode = V.getOpcode();
4649 switch (OldShiftOpcode) {
4650 case ISD::SHL:
4651 NewShiftOpcode = ISD::SRL;
4652 break;
4653 case ISD::SRL:
4654 NewShiftOpcode = ISD::SHL;
4655 break;
4656 default:
4657 return false; // must be a logical shift.
4658 }
4659 // We should be shifting a constant.
4660 // FIXME: best to use isConstantOrConstantVector().
4661 C = V.getOperand(i: 0);
4662 ConstantSDNode *CC =
4663 isConstOrConstSplat(N: C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4664 if (!CC)
4665 return false;
4666 Y = V.getOperand(i: 1);
4667
4668 ConstantSDNode *XC =
4669 isConstOrConstSplat(N: X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4670 return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
4671 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
4672 };
4673
4674 // LHS of comparison should be an one-use 'and'.
4675 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
4676 return SDValue();
4677
4678 X = N0.getOperand(i: 0);
4679 SDValue Mask = N0.getOperand(i: 1);
4680
4681 // 'and' is commutative!
4682 if (!Match(Mask)) {
4683 std::swap(a&: X, b&: Mask);
4684 if (!Match(Mask))
4685 return SDValue();
4686 }
4687
4688 EVT VT = X.getValueType();
4689
4690 // Produce:
4691 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
4692 SDValue T0 = DAG.getNode(Opcode: NewShiftOpcode, DL, VT, N1: X, N2: Y);
4693 SDValue T1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T0, N2: C);
4694 SDValue T2 = DAG.getSetCC(DL, VT: SCCVT, LHS: T1, RHS: N1C, Cond);
4695 return T2;
4696}
4697
4698/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
4699/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
4700/// handle the commuted versions of these patterns.
4701SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
4702 ISD::CondCode Cond, const SDLoc &DL,
4703 DAGCombinerInfo &DCI) const {
4704 unsigned BOpcode = N0.getOpcode();
4705 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
4706 "Unexpected binop");
4707 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
4708
4709 // (X + Y) == X --> Y == 0
4710 // (X - Y) == X --> Y == 0
4711 // (X ^ Y) == X --> Y == 0
4712 SelectionDAG &DAG = DCI.DAG;
4713 EVT OpVT = N0.getValueType();
4714 SDValue X = N0.getOperand(i: 0);
4715 SDValue Y = N0.getOperand(i: 1);
4716 if (X == N1)
4717 return DAG.getSetCC(DL, VT, LHS: Y, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4718
4719 if (Y != N1)
4720 return SDValue();
4721
4722 // (X + Y) == Y --> X == 0
4723 // (X ^ Y) == Y --> X == 0
4724 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
4725 return DAG.getSetCC(DL, VT, LHS: X, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4726
4727 // The shift would not be valid if the operands are boolean (i1).
4728 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
4729 return SDValue();
4730
4731 // (X - Y) == Y --> X == Y << 1
4732 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: OpVT, DL);
4733 SDValue YShl1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: N1.getValueType(), N1: Y, N2: One);
4734 if (!DCI.isCalledByLegalizer())
4735 DCI.AddToWorklist(N: YShl1.getNode());
4736 return DAG.getSetCC(DL, VT, LHS: X, RHS: YShl1, Cond);
4737}
4738
4739static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
4740 SDValue N0, const APInt &C1,
4741 ISD::CondCode Cond, const SDLoc &dl,
4742 SelectionDAG &DAG) {
4743 // Look through truncs that don't change the value of a ctpop.
4744 // FIXME: Add vector support? Need to be careful with setcc result type below.
4745 SDValue CTPOP = N0;
4746 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
4747 N0.getScalarValueSizeInBits() > Log2_32(Value: N0.getOperand(i: 0).getScalarValueSizeInBits()))
4748 CTPOP = N0.getOperand(i: 0);
4749
4750 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
4751 return SDValue();
4752
4753 EVT CTVT = CTPOP.getValueType();
4754 SDValue CTOp = CTPOP.getOperand(i: 0);
4755
4756 // Expand a power-of-2-or-zero comparison based on ctpop:
4757 // (ctpop x) u< 2 -> (x & x-1) == 0
4758 // (ctpop x) u> 1 -> (x & x-1) != 0
4759 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
4760 // Keep the CTPOP if it is a cheap vector op.
4761 if (CTVT.isVector() && TLI.isCtpopFast(VT: CTVT))
4762 return SDValue();
4763
4764 unsigned CostLimit = TLI.getCustomCtpopCost(VT: CTVT, Cond);
4765 if (C1.ugt(RHS: CostLimit + (Cond == ISD::SETULT)))
4766 return SDValue();
4767 if (C1 == 0 && (Cond == ISD::SETULT))
4768 return SDValue(); // This is handled elsewhere.
4769
4770 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
4771
4772 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4773 SDValue Result = CTOp;
4774 for (unsigned i = 0; i < Passes; i++) {
4775 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: Result, N2: NegOne);
4776 Result = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: Result, N2: Add);
4777 }
4778 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
4779 return DAG.getSetCC(DL: dl, VT, LHS: Result, RHS: DAG.getConstant(Val: 0, DL: dl, VT: CTVT), Cond: CC);
4780 }
4781
4782 // Expand a power-of-2 comparison based on ctpop
4783 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
4784 // Keep the CTPOP if it is cheap.
4785 if (TLI.isCtpopFast(VT: CTVT))
4786 return SDValue();
4787
4788 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: CTVT);
4789 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4790 assert(CTVT.isInteger());
4791 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: CTOp, N2: NegOne);
4792
4793 // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
4794 // check before emitting a potentially unnecessary op.
4795 if (DAG.isKnownNeverZero(Op: CTOp)) {
4796 // (ctpop x) == 1 --> (x & x-1) == 0
4797 // (ctpop x) != 1 --> (x & x-1) != 0
4798 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4799 SDValue RHS = DAG.getSetCC(DL: dl, VT, LHS: And, RHS: Zero, Cond);
4800 return RHS;
4801 }
4802
4803 // (ctpop x) == 1 --> (x ^ x-1) > x-1
4804 // (ctpop x) != 1 --> (x ^ x-1) <= x-1
4805 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4806 ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE;
4807 return DAG.getSetCC(DL: dl, VT, LHS: Xor, RHS: Add, Cond: CmpCond);
4808 }
4809
4810 return SDValue();
4811}
4812
4813static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1,
4814 ISD::CondCode Cond, const SDLoc &dl,
4815 SelectionDAG &DAG) {
4816 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4817 return SDValue();
4818
4819 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4820 if (!C1 || !(C1->isZero() || C1->isAllOnes()))
4821 return SDValue();
4822
4823 auto getRotateSource = [](SDValue X) {
4824 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
4825 return X.getOperand(i: 0);
4826 return SDValue();
4827 };
4828
4829 // Peek through a rotated value compared against 0 or -1:
4830 // (rot X, Y) == 0/-1 --> X == 0/-1
4831 // (rot X, Y) != 0/-1 --> X != 0/-1
4832 if (SDValue R = getRotateSource(N0))
4833 return DAG.getSetCC(DL: dl, VT, LHS: R, RHS: N1, Cond);
4834
4835 // Peek through an 'or' of a rotated value compared against 0:
4836 // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
4837 // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
4838 //
4839 // TODO: Add the 'and' with -1 sibling.
4840 // TODO: Recurse through a series of 'or' ops to find the rotate.
4841 EVT OpVT = N0.getValueType();
4842 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
4843 if (SDValue R = getRotateSource(N0.getOperand(i: 0))) {
4844 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 1));
4845 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4846 }
4847 if (SDValue R = getRotateSource(N0.getOperand(i: 1))) {
4848 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 0));
4849 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4850 }
4851 }
4852
4853 return SDValue();
4854}
4855
4856static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1,
4857 ISD::CondCode Cond, const SDLoc &dl,
4858 SelectionDAG &DAG) {
4859 // If we are testing for all-bits-clear, we might be able to do that with
4860 // less shifting since bit-order does not matter.
4861 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4862 return SDValue();
4863
4864 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4865 if (!C1 || !C1->isZero())
4866 return SDValue();
4867
4868 if (!N0.hasOneUse() ||
4869 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
4870 return SDValue();
4871
4872 unsigned BitWidth = N0.getScalarValueSizeInBits();
4873 auto *ShAmtC = isConstOrConstSplat(N: N0.getOperand(i: 2));
4874 if (!ShAmtC)
4875 return SDValue();
4876
4877 uint64_t ShAmt = ShAmtC->getAPIntValue().urem(RHS: BitWidth);
4878 if (ShAmt == 0)
4879 return SDValue();
4880
4881 // Canonicalize fshr as fshl to reduce pattern-matching.
4882 if (N0.getOpcode() == ISD::FSHR)
4883 ShAmt = BitWidth - ShAmt;
4884
4885 // Match an 'or' with a specific operand 'Other' in either commuted variant.
4886 SDValue X, Y;
4887 auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
4888 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
4889 return false;
4890 if (Or.getOperand(i: 0) == Other) {
4891 X = Or.getOperand(i: 0);
4892 Y = Or.getOperand(i: 1);
4893 return true;
4894 }
4895 if (Or.getOperand(i: 1) == Other) {
4896 X = Or.getOperand(i: 1);
4897 Y = Or.getOperand(i: 0);
4898 return true;
4899 }
4900 return false;
4901 };
4902
4903 EVT OpVT = N0.getValueType();
4904 EVT ShAmtVT = N0.getOperand(i: 2).getValueType();
4905 SDValue F0 = N0.getOperand(i: 0);
4906 SDValue F1 = N0.getOperand(i: 1);
4907 if (matchOr(F0, F1)) {
4908 // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
4909 SDValue NewShAmt = DAG.getConstant(Val: ShAmt, DL: dl, VT: ShAmtVT);
4910 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4911 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4912 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4913 }
4914 if (matchOr(F1, F0)) {
4915 // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
4916 SDValue NewShAmt = DAG.getConstant(Val: BitWidth - ShAmt, DL: dl, VT: ShAmtVT);
4917 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4918 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4919 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4920 }
4921
4922 return SDValue();
4923}
4924
4925/// Try to simplify a setcc built with the specified operands and cc. If it is
4926/// unable to simplify it, return a null SDValue.
4927SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
4928 ISD::CondCode Cond, bool foldBooleans,
4929 DAGCombinerInfo &DCI,
4930 const SDLoc &dl) const {
4931 SelectionDAG &DAG = DCI.DAG;
4932 const DataLayout &Layout = DAG.getDataLayout();
4933 EVT OpVT = N0.getValueType();
4934 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4935
4936 // Constant fold or commute setcc.
4937 if (SDValue Fold = DAG.FoldSetCC(VT, N1: N0, N2: N1, Cond, dl))
4938 return Fold;
4939
4940 bool N0ConstOrSplat =
4941 isConstOrConstSplat(N: N0, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
4942 bool N1ConstOrSplat =
4943 isConstOrConstSplat(N: N1, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
4944
4945 // Canonicalize toward having the constant on the RHS.
4946 // TODO: Handle non-splat vector constants. All undef causes trouble.
4947 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
4948 // infinite loop here when we encounter one.
4949 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Operation: Cond);
4950 if (N0ConstOrSplat && !N1ConstOrSplat &&
4951 (DCI.isBeforeLegalizeOps() ||
4952 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())))
4953 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
4954
4955 // If we have a subtract with the same 2 non-constant operands as this setcc
4956 // -- but in reverse order -- then try to commute the operands of this setcc
4957 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
4958 // instruction on some targets.
4959 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4960 (DCI.isBeforeLegalizeOps() ||
4961 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())) &&
4962 DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N1, N0}) &&
4963 !DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N0, N1}))
4964 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
4965
4966 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
4967 return V;
4968
4969 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
4970 return V;
4971
4972 if (auto *N1C = isConstOrConstSplat(N: N1)) {
4973 const APInt &C1 = N1C->getAPIntValue();
4974
4975 // Optimize some CTPOP cases.
4976 if (SDValue V = simplifySetCCWithCTPOP(TLI: *this, VT, N0, C1, Cond, dl, DAG))
4977 return V;
4978
4979 // For equality to 0 of a no-wrap multiply, decompose and test each op:
4980 // X * Y == 0 --> (X == 0) || (Y == 0)
4981 // X * Y != 0 --> (X != 0) && (Y != 0)
4982 // TODO: This bails out if minsize is set, but if the target doesn't have a
4983 // single instruction multiply for this type, it would likely be
4984 // smaller to decompose.
4985 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4986 N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
4987 (N0->getFlags().hasNoUnsignedWrap() ||
4988 N0->getFlags().hasNoSignedWrap()) &&
4989 !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
4990 SDValue IsXZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
4991 SDValue IsYZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1, Cond);
4992 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
4993 return DAG.getNode(Opcode: LogicOp, DL: dl, VT, N1: IsXZero, N2: IsYZero);
4994 }
4995
4996 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
4997 // equality comparison, then we're just comparing whether X itself is
4998 // zero.
4999 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
5000 N0.getOperand(i: 0).getOpcode() == ISD::CTLZ &&
5001 llvm::has_single_bit<uint32_t>(Value: N0.getScalarValueSizeInBits())) {
5002 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
5003 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5004 ShAmt->getAPIntValue() == Log2_32(Value: N0.getScalarValueSizeInBits())) {
5005 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
5006 // (srl (ctlz x), 5) == 0 -> X != 0
5007 // (srl (ctlz x), 5) != 1 -> X != 0
5008 Cond = ISD::SETNE;
5009 } else {
5010 // (srl (ctlz x), 5) != 0 -> X == 0
5011 // (srl (ctlz x), 5) == 1 -> X == 0
5012 Cond = ISD::SETEQ;
5013 }
5014 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: N0.getValueType());
5015 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0).getOperand(i: 0), RHS: Zero,
5016 Cond);
5017 }
5018 }
5019 }
5020 }
5021
5022 // setcc X, 0, setlt --> X (when X is all sign bits)
5023 // setcc X, 0, setne --> X (when X is all sign bits)
5024 //
5025 // When we know that X has 0 or -1 in each element (or scalar), this
5026 // comparison will produce X. This is only true when boolean contents are
5027 // represented via 0s and -1s.
5028 if (VT == OpVT &&
5029 // Check that the result of setcc is 0 and -1.
5030 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent &&
5031 // Match only for checks X < 0 and X != 0
5032 (Cond == ISD::SETLT || Cond == ISD::SETNE) && isNullOrNullSplat(V: N1) &&
5033 // The identity holds iff we know all sign bits for all lanes.
5034 DAG.ComputeNumSignBits(Op: N0) == N0.getScalarValueSizeInBits())
5035 return N0;
5036
5037 // FIXME: Support vectors.
5038 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5039 const APInt &C1 = N1C->getAPIntValue();
5040
5041 // (zext x) == C --> x == (trunc C)
5042 // (sext x) == C --> x == (trunc C)
5043 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5044 DCI.isBeforeLegalize() && N0->hasOneUse()) {
5045 unsigned MinBits = N0.getValueSizeInBits();
5046 SDValue PreExt;
5047 bool Signed = false;
5048 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
5049 // ZExt
5050 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
5051 PreExt = N0->getOperand(Num: 0);
5052 } else if (N0->getOpcode() == ISD::AND) {
5053 // DAGCombine turns costly ZExts into ANDs
5054 if (auto *C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
5055 if ((C->getAPIntValue()+1).isPowerOf2()) {
5056 MinBits = C->getAPIntValue().countr_one();
5057 PreExt = N0->getOperand(Num: 0);
5058 }
5059 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
5060 // SExt
5061 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
5062 PreExt = N0->getOperand(Num: 0);
5063 Signed = true;
5064 } else if (auto *LN0 = dyn_cast<LoadSDNode>(Val&: N0)) {
5065 // ZEXTLOAD / SEXTLOAD
5066 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
5067 MinBits = LN0->getMemoryVT().getSizeInBits();
5068 PreExt = N0;
5069 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
5070 Signed = true;
5071 MinBits = LN0->getMemoryVT().getSizeInBits();
5072 PreExt = N0;
5073 }
5074 }
5075
5076 // Figure out how many bits we need to preserve this constant.
5077 unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();
5078
5079 // Make sure we're not losing bits from the constant.
5080 if (MinBits > 0 &&
5081 MinBits < C1.getBitWidth() &&
5082 MinBits >= ReqdBits) {
5083 EVT MinVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MinBits);
5084 if (isTypeDesirableForOp(ISD::SETCC, VT: MinVT)) {
5085 // Will get folded away.
5086 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MinVT, Operand: PreExt);
5087 if (MinBits == 1 && C1 == 1)
5088 // Invert the condition.
5089 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i1),
5090 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5091 SDValue C = DAG.getConstant(Val: C1.trunc(width: MinBits), DL: dl, VT: MinVT);
5092 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: C, Cond);
5093 }
5094
5095 // If truncating the setcc operands is not desirable, we can still
5096 // simplify the expression in some cases:
5097 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
5098 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
5099 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
5100 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
5101 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
5102 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
5103 SDValue TopSetCC = N0->getOperand(Num: 0);
5104 unsigned N0Opc = N0->getOpcode();
5105 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
5106 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
5107 TopSetCC.getOpcode() == ISD::SETCC &&
5108 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
5109 (isConstFalseVal(N: N1) ||
5110 isExtendedTrueVal(N: N1C, VT: N0->getValueType(ResNo: 0), SExt))) {
5111
5112 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
5113 (!N1C->isZero() && Cond == ISD::SETNE);
5114
5115 if (!Inverse)
5116 return TopSetCC;
5117
5118 ISD::CondCode InvCond = ISD::getSetCCInverse(
5119 Operation: cast<CondCodeSDNode>(Val: TopSetCC.getOperand(i: 2))->get(),
5120 Type: TopSetCC.getOperand(i: 0).getValueType());
5121 return DAG.getSetCC(DL: dl, VT, LHS: TopSetCC.getOperand(i: 0),
5122 RHS: TopSetCC.getOperand(i: 1),
5123 Cond: InvCond);
5124 }
5125 }
5126 }
5127
5128 // If the LHS is '(and load, const)', the RHS is 0, the test is for
5129 // equality or unsigned, and all 1 bits of the const are in the same
5130 // partial word, see if we can shorten the load.
5131 if (DCI.isBeforeLegalize() &&
5132 !ISD::isSignedIntSetCC(Code: Cond) &&
5133 N0.getOpcode() == ISD::AND && C1 == 0 &&
5134 N0.getNode()->hasOneUse() &&
5135 isa<LoadSDNode>(Val: N0.getOperand(i: 0)) &&
5136 N0.getOperand(i: 0).getNode()->hasOneUse() &&
5137 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5138 auto *Lod = cast<LoadSDNode>(Val: N0.getOperand(i: 0));
5139 APInt bestMask;
5140 unsigned bestWidth = 0, bestOffset = 0;
5141 if (Lod->isSimple() && Lod->isUnindexed() &&
5142 (Lod->getMemoryVT().isByteSized() ||
5143 isPaddedAtMostSignificantBitsWhenStored(VT: Lod->getMemoryVT()))) {
5144 unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
5145 unsigned origWidth = N0.getValueSizeInBits();
5146 unsigned maskWidth = origWidth;
5147 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
5148 // 8 bits, but have to be careful...
5149 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
5150 origWidth = Lod->getMemoryVT().getSizeInBits();
5151 const APInt &Mask = N0.getConstantOperandAPInt(i: 1);
5152 // Only consider power-of-2 widths (and at least one byte) as candiates
5153 // for the narrowed load.
5154 for (unsigned width = 8; width < origWidth; width *= 2) {
5155 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: width);
5156 APInt newMask = APInt::getLowBitsSet(numBits: maskWidth, loBitsSet: width);
5157 // Avoid accessing any padding here for now (we could use memWidth
5158 // instead of origWidth here otherwise).
5159 unsigned maxOffset = origWidth - width;
5160 for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
5161 if (Mask.isSubsetOf(RHS: newMask)) {
5162 unsigned ptrOffset =
5163 Layout.isLittleEndian() ? offset : memWidth - width - offset;
5164 unsigned IsFast = 0;
5165 assert((ptrOffset % 8) == 0 && "Non-Bytealigned pointer offset");
5166 Align NewAlign = commonAlignment(A: Lod->getAlign(), Offset: ptrOffset / 8);
5167 if (shouldReduceLoadWidth(Load: Lod, ExtTy: ISD::NON_EXTLOAD, NewVT: newVT,
5168 ByteOffset: ptrOffset / 8) &&
5169 allowsMemoryAccess(
5170 Context&: *DAG.getContext(), DL: Layout, VT: newVT, AddrSpace: Lod->getAddressSpace(),
5171 Alignment: NewAlign, Flags: Lod->getMemOperand()->getFlags(), Fast: &IsFast) &&
5172 IsFast) {
5173 bestOffset = ptrOffset / 8;
5174 bestMask = Mask.lshr(shiftAmt: offset);
5175 bestWidth = width;
5176 break;
5177 }
5178 }
5179 newMask <<= 8;
5180 }
5181 if (bestWidth)
5182 break;
5183 }
5184 }
5185 if (bestWidth) {
5186 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: bestWidth);
5187 SDValue Ptr = Lod->getBasePtr();
5188 if (bestOffset != 0)
5189 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: bestOffset));
5190 SDValue NewLoad =
5191 DAG.getLoad(VT: newVT, dl, Chain: Lod->getChain(), Ptr,
5192 PtrInfo: Lod->getPointerInfo().getWithOffset(O: bestOffset),
5193 Alignment: Lod->getBaseAlign());
5194 SDValue And =
5195 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: newVT, N1: NewLoad,
5196 N2: DAG.getConstant(Val: bestMask.trunc(width: bestWidth), DL: dl, VT: newVT));
5197 return DAG.getSetCC(DL: dl, VT, LHS: And, RHS: DAG.getConstant(Val: 0LL, DL: dl, VT: newVT), Cond);
5198 }
5199 }
5200
5201 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
5202 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
5203 unsigned InSize = N0.getOperand(i: 0).getValueSizeInBits();
5204
5205 // If the comparison constant has bits in the upper part, the
5206 // zero-extended value could never match.
5207 if (C1.intersects(RHS: APInt::getHighBitsSet(numBits: C1.getBitWidth(),
5208 hiBitsSet: C1.getBitWidth() - InSize))) {
5209 switch (Cond) {
5210 case ISD::SETUGT:
5211 case ISD::SETUGE:
5212 case ISD::SETEQ:
5213 return DAG.getConstant(Val: 0, DL: dl, VT);
5214 case ISD::SETULT:
5215 case ISD::SETULE:
5216 case ISD::SETNE:
5217 return DAG.getConstant(Val: 1, DL: dl, VT);
5218 case ISD::SETGT:
5219 case ISD::SETGE:
5220 // True if the sign bit of C1 is set.
5221 return DAG.getConstant(Val: C1.isNegative(), DL: dl, VT);
5222 case ISD::SETLT:
5223 case ISD::SETLE:
5224 // True if the sign bit of C1 isn't set.
5225 return DAG.getConstant(Val: C1.isNonNegative(), DL: dl, VT);
5226 default:
5227 break;
5228 }
5229 }
5230
5231 // Otherwise, we can perform the comparison with the low bits.
5232 switch (Cond) {
5233 case ISD::SETEQ:
5234 case ISD::SETNE:
5235 case ISD::SETUGT:
5236 case ISD::SETUGE:
5237 case ISD::SETULT:
5238 case ISD::SETULE: {
5239 EVT newVT = N0.getOperand(i: 0).getValueType();
5240 // FIXME: Should use isNarrowingProfitable.
5241 if (DCI.isBeforeLegalizeOps() ||
5242 (isOperationLegal(Op: ISD::SETCC, VT: newVT) &&
5243 isCondCodeLegal(CC: Cond, VT: newVT.getSimpleVT()) &&
5244 isTypeDesirableForOp(ISD::SETCC, VT: newVT))) {
5245 EVT NewSetCCVT = getSetCCResultType(DL: Layout, Context&: *DAG.getContext(), VT: newVT);
5246 SDValue NewConst = DAG.getConstant(Val: C1.trunc(width: InSize), DL: dl, VT: newVT);
5247
5248 SDValue NewSetCC = DAG.getSetCC(DL: dl, VT: NewSetCCVT, LHS: N0.getOperand(i: 0),
5249 RHS: NewConst, Cond);
5250 return DAG.getBoolExtOrTrunc(Op: NewSetCC, SL: dl, VT, OpVT: N0.getValueType());
5251 }
5252 break;
5253 }
5254 default:
5255 break; // todo, be more careful with signed comparisons
5256 }
5257 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5258 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5259 !isSExtCheaperThanZExt(FromTy: cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT(),
5260 ToTy: OpVT)) {
5261 EVT ExtSrcTy = cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT();
5262 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
5263 EVT ExtDstTy = N0.getValueType();
5264 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
5265
5266 // If the constant doesn't fit into the number of bits for the source of
5267 // the sign extension, it is impossible for both sides to be equal.
5268 if (C1.getSignificantBits() > ExtSrcTyBits)
5269 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
5270
5271 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
5272 ExtDstTy != ExtSrcTy && "Unexpected types!");
5273 APInt Imm = APInt::getLowBitsSet(numBits: ExtDstTyBits, loBitsSet: ExtSrcTyBits);
5274 SDValue ZextOp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ExtDstTy, N1: N0.getOperand(i: 0),
5275 N2: DAG.getConstant(Val: Imm, DL: dl, VT: ExtDstTy));
5276 if (!DCI.isCalledByLegalizer())
5277 DCI.AddToWorklist(N: ZextOp.getNode());
5278 // Otherwise, make this a use of a zext.
5279 return DAG.getSetCC(DL: dl, VT, LHS: ZextOp,
5280 RHS: DAG.getConstant(Val: C1 & Imm, DL: dl, VT: ExtDstTy), Cond);
5281 } else if ((N1C->isZero() || N1C->isOne()) &&
5282 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5283 // SETCC (X), [0|1], [EQ|NE] -> X if X is known 0/1. i1 types are
5284 // excluded as they are handled below whilst checking for foldBooleans.
5285 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
5286 isTypeLegal(VT) && VT.bitsLE(VT: N0.getValueType()) &&
5287 (N0.getValueType() == MVT::i1 ||
5288 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5289 DAG.MaskedValueIsZero(
5290 Op: N0, Mask: APInt::getBitsSetFrom(numBits: N0.getValueSizeInBits(), loBit: 1))) {
5291 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
5292 if (TrueWhenTrue)
5293 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: N0);
5294 // Invert the condition.
5295 if (N0.getOpcode() == ISD::SETCC) {
5296 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
5297 CC = ISD::getSetCCInverse(Operation: CC, Type: N0.getOperand(i: 0).getValueType());
5298 if (DCI.isBeforeLegalizeOps() ||
5299 isCondCodeLegal(CC, VT: N0.getOperand(i: 0).getSimpleValueType()))
5300 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond: CC);
5301 }
5302 }
5303
5304 if ((N0.getOpcode() == ISD::XOR ||
5305 (N0.getOpcode() == ISD::AND &&
5306 N0.getOperand(i: 0).getOpcode() == ISD::XOR &&
5307 N0.getOperand(i: 1) == N0.getOperand(i: 0).getOperand(i: 1))) &&
5308 isOneConstant(V: N0.getOperand(i: 1))) {
5309 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
5310 // can only do this if the top bits are known zero.
5311 unsigned BitWidth = N0.getValueSizeInBits();
5312 if (DAG.MaskedValueIsZero(Op: N0,
5313 Mask: APInt::getHighBitsSet(numBits: BitWidth,
5314 hiBitsSet: BitWidth-1))) {
5315 // Okay, get the un-inverted input value.
5316 SDValue Val;
5317 if (N0.getOpcode() == ISD::XOR) {
5318 Val = N0.getOperand(i: 0);
5319 } else {
5320 assert(N0.getOpcode() == ISD::AND &&
5321 N0.getOperand(0).getOpcode() == ISD::XOR);
5322 // ((X^1)&1)^1 -> X & 1
5323 Val = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N0.getValueType(),
5324 N1: N0.getOperand(i: 0).getOperand(i: 0),
5325 N2: N0.getOperand(i: 1));
5326 }
5327
5328 return DAG.getSetCC(DL: dl, VT, LHS: Val, RHS: N1,
5329 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5330 }
5331 } else if (N1C->isOne()) {
5332 SDValue Op0 = N0;
5333 if (Op0.getOpcode() == ISD::TRUNCATE)
5334 Op0 = Op0.getOperand(i: 0);
5335
5336 if ((Op0.getOpcode() == ISD::XOR) &&
5337 Op0.getOperand(i: 0).getOpcode() == ISD::SETCC &&
5338 Op0.getOperand(i: 1).getOpcode() == ISD::SETCC) {
5339 SDValue XorLHS = Op0.getOperand(i: 0);
5340 SDValue XorRHS = Op0.getOperand(i: 1);
5341 // Ensure that the input setccs return an i1 type or 0/1 value.
5342 if (Op0.getValueType() == MVT::i1 ||
5343 (getBooleanContents(Type: XorLHS.getOperand(i: 0).getValueType()) ==
5344 ZeroOrOneBooleanContent &&
5345 getBooleanContents(Type: XorRHS.getOperand(i: 0).getValueType()) ==
5346 ZeroOrOneBooleanContent)) {
5347 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
5348 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
5349 return DAG.getSetCC(DL: dl, VT, LHS: XorLHS, RHS: XorRHS, Cond);
5350 }
5351 }
5352 if (Op0.getOpcode() == ISD::AND && isOneConstant(V: Op0.getOperand(i: 1))) {
5353 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
5354 if (Op0.getValueType().bitsGT(VT))
5355 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5356 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5357 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5358 else if (Op0.getValueType().bitsLT(VT))
5359 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5360 N1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5361 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5362
5363 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5364 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5365 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5366 }
5367 if (Op0.getOpcode() == ISD::AssertZext &&
5368 cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT() == MVT::i1)
5369 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5370 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5371 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5372 }
5373 }
5374
5375 // Given:
5376 // icmp eq/ne (urem %x, %y), 0
5377 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
5378 // icmp eq/ne %x, 0
5379 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
5380 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5381 KnownBits XKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 0));
5382 KnownBits YKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 1));
5383 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
5384 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
5385 }
5386
5387 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
5388 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
5389 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5390 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
5391 N0.getConstantOperandAPInt(i: 1) == OpVT.getScalarSizeInBits() - 1 &&
5392 N1C->isAllOnes()) {
5393 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
5394 RHS: DAG.getConstant(Val: 0, DL: dl, VT: OpVT),
5395 Cond: Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
5396 }
5397
5398 // fold (setcc (trunc x) c) -> (setcc x c)
5399 if (N0.getOpcode() == ISD::TRUNCATE &&
5400 ((N0->getFlags().hasNoUnsignedWrap() && !ISD::isSignedIntSetCC(Code: Cond)) ||
5401 (N0->getFlags().hasNoSignedWrap() &&
5402 !ISD::isUnsignedIntSetCC(Code: Cond))) &&
5403 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5404 EVT NewVT = N0.getOperand(i: 0).getValueType();
5405 SDValue NewConst = DAG.getConstant(
5406 Val: (N0->getFlags().hasNoSignedWrap() && !ISD::isUnsignedIntSetCC(Code: Cond))
5407 ? C1.sext(width: NewVT.getSizeInBits())
5408 : C1.zext(width: NewVT.getSizeInBits()),
5409 DL: dl, VT: NewVT);
5410 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NewConst, Cond);
5411 }
5412
5413 if (SDValue V =
5414 optimizeSetCCOfSignedTruncationCheck(SCCVT: VT, N0, N1, Cond, DCI, DL: dl))
5415 return V;
5416 }
5417
5418 // These simplifications apply to splat vectors as well.
5419 // TODO: Handle more splat vector cases.
5420 if (auto *N1C = isConstOrConstSplat(N: N1)) {
5421 const APInt &C1 = N1C->getAPIntValue();
5422
5423 APInt MinVal, MaxVal;
5424 unsigned OperandBitSize = N1C->getValueType(ResNo: 0).getScalarSizeInBits();
5425 if (ISD::isSignedIntSetCC(Code: Cond)) {
5426 MinVal = APInt::getSignedMinValue(numBits: OperandBitSize);
5427 MaxVal = APInt::getSignedMaxValue(numBits: OperandBitSize);
5428 } else {
5429 MinVal = APInt::getMinValue(numBits: OperandBitSize);
5430 MaxVal = APInt::getMaxValue(numBits: OperandBitSize);
5431 }
5432
5433 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
5434 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
5435 // X >= MIN --> true
5436 if (C1 == MinVal)
5437 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5438
5439 if (!VT.isVector()) { // TODO: Support this for vectors.
5440 // X >= C0 --> X > (C0 - 1)
5441 APInt C = C1 - 1;
5442 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
5443 if ((DCI.isBeforeLegalizeOps() ||
5444 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5445 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5446 isLegalICmpImmediate(C.getSExtValue())))) {
5447 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5448 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5449 Cond: NewCC);
5450 }
5451 }
5452 }
5453
5454 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
5455 // X <= MAX --> true
5456 if (C1 == MaxVal)
5457 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5458
5459 // X <= C0 --> X < (C0 + 1)
5460 if (!VT.isVector()) { // TODO: Support this for vectors.
5461 APInt C = C1 + 1;
5462 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
5463 if ((DCI.isBeforeLegalizeOps() ||
5464 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5465 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5466 isLegalICmpImmediate(C.getSExtValue())))) {
5467 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5468 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5469 Cond: NewCC);
5470 }
5471 }
5472 }
5473
5474 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
5475 if (C1 == MinVal)
5476 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X < MIN --> false
5477
5478 // TODO: Support this for vectors after legalize ops.
5479 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5480 // Canonicalize setlt X, Max --> setne X, Max
5481 if (C1 == MaxVal)
5482 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5483
5484 // If we have setult X, 1, turn it into seteq X, 0
5485 if (C1 == MinVal+1)
5486 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5487 RHS: DAG.getConstant(Val: MinVal, DL: dl, VT: N0.getValueType()),
5488 Cond: ISD::SETEQ);
5489 }
5490 }
5491
5492 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
5493 if (C1 == MaxVal)
5494 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X > MAX --> false
5495
5496 // TODO: Support this for vectors after legalize ops.
5497 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5498 // Canonicalize setgt X, Min --> setne X, Min
5499 if (C1 == MinVal)
5500 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5501
5502 // If we have setugt X, Max-1, turn it into seteq X, Max
5503 if (C1 == MaxVal-1)
5504 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5505 RHS: DAG.getConstant(Val: MaxVal, DL: dl, VT: N0.getValueType()),
5506 Cond: ISD::SETEQ);
5507 }
5508 }
5509
5510 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
5511 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
5512 if (C1.isZero())
5513 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
5514 SCCVT: VT, N0, N1C: N1, Cond, DCI, DL: dl))
5515 return CC;
5516
5517 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
5518 // For example, when high 32-bits of i64 X are known clear:
5519 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
5520 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
5521 bool CmpZero = N1C->isZero();
5522 bool CmpNegOne = N1C->isAllOnes();
5523 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
5524 // Match or(lo,shl(hi,bw/2)) pattern.
5525 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
5526 unsigned EltBits = V.getScalarValueSizeInBits();
5527 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
5528 return false;
5529 SDValue LHS = V.getOperand(i: 0);
5530 SDValue RHS = V.getOperand(i: 1);
5531 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: EltBits / 2);
5532 // Unshifted element must have zero upperbits.
5533 if (RHS.getOpcode() == ISD::SHL &&
5534 isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)) &&
5535 RHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5536 DAG.MaskedValueIsZero(Op: LHS, Mask: HiBits)) {
5537 Lo = LHS;
5538 Hi = RHS.getOperand(i: 0);
5539 return true;
5540 }
5541 if (LHS.getOpcode() == ISD::SHL &&
5542 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
5543 LHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5544 DAG.MaskedValueIsZero(Op: RHS, Mask: HiBits)) {
5545 Lo = RHS;
5546 Hi = LHS.getOperand(i: 0);
5547 return true;
5548 }
5549 return false;
5550 };
5551
5552 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
5553 unsigned EltBits = N0.getScalarValueSizeInBits();
5554 unsigned HalfBits = EltBits / 2;
5555 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: HalfBits);
5556 SDValue LoBits = DAG.getConstant(Val: ~HiBits, DL: dl, VT: OpVT);
5557 SDValue HiMask = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Hi, N2: LoBits);
5558 SDValue NewN0 =
5559 DAG.getNode(Opcode: CmpZero ? ISD::OR : ISD::AND, DL: dl, VT: OpVT, N1: Lo, N2: HiMask);
5560 SDValue NewN1 = CmpZero ? DAG.getConstant(Val: 0, DL: dl, VT: OpVT) : LoBits;
5561 return DAG.getSetCC(DL: dl, VT, LHS: NewN0, RHS: NewN1, Cond);
5562 };
5563
5564 SDValue Lo, Hi;
5565 if (IsConcat(N0, Lo, Hi))
5566 return MergeConcat(Lo, Hi);
5567
5568 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
5569 SDValue Lo0, Lo1, Hi0, Hi1;
5570 if (IsConcat(N0.getOperand(i: 0), Lo0, Hi0) &&
5571 IsConcat(N0.getOperand(i: 1), Lo1, Hi1)) {
5572 return MergeConcat(DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Lo0, N2: Lo1),
5573 DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Hi0, N2: Hi1));
5574 }
5575 }
5576 }
5577 }
5578
5579 // If we have "setcc X, C0", check to see if we can shrink the immediate
5580 // by changing cc.
5581 // TODO: Support this for vectors after legalize ops.
5582 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5583 // SETUGT X, SINTMAX -> SETLT X, 0
5584 // SETUGE X, SINTMIN -> SETLT X, 0
5585 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
5586 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
5587 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5588 RHS: DAG.getConstant(Val: 0, DL: dl, VT: N1.getValueType()),
5589 Cond: ISD::SETLT);
5590
5591 // SETULT X, SINTMIN -> SETGT X, -1
5592 // SETULE X, SINTMAX -> SETGT X, -1
5593 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
5594 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
5595 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5596 RHS: DAG.getAllOnesConstant(DL: dl, VT: N1.getValueType()),
5597 Cond: ISD::SETGT);
5598 }
5599 }
5600
5601 // Back to non-vector simplifications.
5602 // TODO: Can we do these for vector splats?
5603 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5604 const APInt &C1 = N1C->getAPIntValue();
5605 EVT ShValTy = N0.getValueType();
5606
5607 // Fold bit comparisons when we can. This will result in an
5608 // incorrect value when boolean false is negative one, unless
5609 // the bitsize is 1 in which case the false value is the same
5610 // in practice regardless of the representation.
5611 if ((VT.getSizeInBits() == 1 ||
5612 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5613 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5614 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(VT: ShValTy))) &&
5615 N0.getOpcode() == ISD::AND) {
5616 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5617 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
5618 // Perform the xform if the AND RHS is a single bit.
5619 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5620 if (AndRHS->getAPIntValue().isPowerOf2() &&
5621 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5622 return DAG.getNode(
5623 Opcode: ISD::TRUNCATE, DL: dl, VT,
5624 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5625 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5626 }
5627 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5628 // (X & 8) == 8 --> (X & 8) >> 3
5629 // Perform the xform if C1 is a single bit.
5630 unsigned ShCt = C1.logBase2();
5631 if (C1.isPowerOf2() && !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5632 return DAG.getNode(
5633 Opcode: ISD::TRUNCATE, DL: dl, VT,
5634 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5635 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5636 }
5637 }
5638 }
5639 }
5640
5641 if (C1.getSignificantBits() <= 64 &&
5642 !isLegalICmpImmediate(C1.getSExtValue())) {
5643 // (X & -256) == 256 -> (X >> 8) == 1
5644 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5645 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
5646 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5647 const APInt &AndRHSC = AndRHS->getAPIntValue();
5648 if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(RHS: AndRHSC)) {
5649 unsigned ShiftBits = AndRHSC.countr_zero();
5650 if (!shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5651 // If using an unsigned shift doesn't yield a legal compare
5652 // immediate, try using sra instead.
5653 APInt NewC = C1.lshr(shiftAmt: ShiftBits);
5654 if (NewC.getSignificantBits() <= 64 &&
5655 !isLegalICmpImmediate(NewC.getSExtValue())) {
5656 APInt SignedC = C1.ashr(ShiftAmt: ShiftBits);
5657 if (SignedC.getSignificantBits() <= 64 &&
5658 isLegalICmpImmediate(SignedC.getSExtValue())) {
5659 SDValue Shift = DAG.getNode(
5660 Opcode: ISD::SRA, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5661 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5662 SDValue CmpRHS = DAG.getConstant(Val: SignedC, DL: dl, VT: ShValTy);
5663 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5664 }
5665 }
5666 SDValue Shift = DAG.getNode(
5667 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5668 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5669 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5670 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5671 }
5672 }
5673 }
5674 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
5675 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
5676 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
5677 // X < 0x100000000 -> (X >> 32) < 1
5678 // X >= 0x100000000 -> (X >> 32) >= 1
5679 // X <= 0x0ffffffff -> (X >> 32) < 1
5680 // X > 0x0ffffffff -> (X >> 32) >= 1
5681 unsigned ShiftBits;
5682 APInt NewC = C1;
5683 ISD::CondCode NewCond = Cond;
5684 if (AdjOne) {
5685 ShiftBits = C1.countr_one();
5686 NewC = NewC + 1;
5687 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
5688 } else {
5689 ShiftBits = C1.countr_zero();
5690 }
5691 APInt RangeWidth = NewC;
5692 NewC.lshrInPlace(ShiftAmt: ShiftBits);
5693 if (ShiftBits && NewC.getSignificantBits() <= 64 &&
5694 isLegalICmpImmediate(NewC.getSExtValue()) &&
5695 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5696 // If this is an offset range check, try to move the offset after the
5697 // shift to avoid preserving the pre-shift add with a mask.
5698 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse()) {
5699 if (auto *AddC = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
5700 const APInt &AddVal = AddC->getAPIntValue();
5701 if (AddVal.countr_zero() >= ShiftBits) {
5702 APInt RangeLower = -AddVal;
5703 bool Overflow;
5704 (void)RangeLower.uadd_ov(RHS: RangeWidth, Overflow);
5705 if (!RangeWidth.isZero() && !Overflow) {
5706 SDValue Shift = DAG.getNode(
5707 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5708 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5709 APInt Offset = -RangeLower.lshr(shiftAmt: ShiftBits);
5710 SDValue ShiftedAdd =
5711 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ShValTy, N1: Shift,
5712 N2: DAG.getConstant(Val: Offset, DL: dl, VT: ShValTy));
5713 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5714 return DAG.getSetCC(DL: dl, VT, LHS: ShiftedAdd, RHS: CmpRHS, Cond: NewCond);
5715 }
5716 }
5717 }
5718 }
5719 SDValue Shift =
5720 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5721 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5722 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5723 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond: NewCond);
5724 }
5725 }
5726 }
5727 }
5728
5729 if (!isa<ConstantFPSDNode>(Val: N0) && isa<ConstantFPSDNode>(Val: N1)) {
5730 auto *CFP = cast<ConstantFPSDNode>(Val&: N1);
5731 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
5732
5733 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
5734 // constant if knowing that the operand is non-nan is enough. We prefer to
5735 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
5736 // materialize 0.0.
5737 if (Cond == ISD::SETO || Cond == ISD::SETUO)
5738 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N0, Cond);
5739
5740 // setcc (fneg x), C -> setcc swap(pred) x, -C
5741 if (N0.getOpcode() == ISD::FNEG) {
5742 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Operation: Cond);
5743 if (DCI.isBeforeLegalizeOps() ||
5744 isCondCodeLegal(CC: SwapCond, VT: N0.getSimpleValueType())) {
5745 SDValue NegN1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: N0.getValueType(), Operand: N1);
5746 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NegN1, Cond: SwapCond);
5747 }
5748 }
5749
5750 // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
5751 if (isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT: N0.getValueType()) &&
5752 !isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(ResNo: 0))) {
5753 bool IsFabs = N0.getOpcode() == ISD::FABS;
5754 SDValue Op = IsFabs ? N0.getOperand(i: 0) : N0;
5755 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
5756 FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
5757 : (IsFabs ? fcInf : fcPosInf);
5758 if (Cond == ISD::SETUEQ)
5759 Flag |= fcNan;
5760 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: dl, VT, N1: Op,
5761 N2: DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32));
5762 }
5763 }
5764
5765 // If the condition is not legal, see if we can find an equivalent one
5766 // which is legal.
5767 if (!isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType())) {
5768 // If the comparison was an awkward floating-point == or != and one of
5769 // the comparison operands is infinity or negative infinity, convert the
5770 // condition to a less-awkward <= or >=.
5771 if (CFP->getValueAPF().isInfinity()) {
5772 bool IsNegInf = CFP->getValueAPF().isNegative();
5773 ISD::CondCode NewCond = ISD::SETCC_INVALID;
5774 switch (Cond) {
5775 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
5776 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
5777 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
5778 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
5779 default: break;
5780 }
5781 if (NewCond != ISD::SETCC_INVALID &&
5782 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType()))
5783 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5784 }
5785 }
5786 }
5787
5788 if (N0 == N1) {
5789 // The sext(setcc()) => setcc() optimization relies on the appropriate
5790 // constant being emitted.
5791 assert(!N0.getValueType().isInteger() &&
5792 "Integer types should be handled by FoldSetCC");
5793
5794 bool EqTrue = ISD::isTrueWhenEqual(Cond);
5795 unsigned UOF = ISD::getUnorderedFlavor(Cond);
5796 if (UOF == 2) // FP operators that are undefined on NaNs.
5797 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5798 if (UOF == unsigned(EqTrue))
5799 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5800 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
5801 // if it is not already.
5802 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
5803 if (NewCond != Cond &&
5804 (DCI.isBeforeLegalizeOps() ||
5805 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType())))
5806 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5807 }
5808
5809 // ~X > ~Y --> Y > X
5810 // ~X < ~Y --> Y < X
5811 // ~X < C --> X > ~C
5812 // ~X > C --> X < ~C
5813 if ((isSignedIntSetCC(Code: Cond) || isUnsignedIntSetCC(Code: Cond)) &&
5814 N0.getValueType().isInteger()) {
5815 if (isBitwiseNot(V: N0)) {
5816 if (isBitwiseNot(V: N1))
5817 return DAG.getSetCC(DL: dl, VT, LHS: N1.getOperand(i: 0), RHS: N0.getOperand(i: 0), Cond);
5818
5819 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N1) &&
5820 !DAG.isConstantIntBuildVectorOrConstantInt(N: N0.getOperand(i: 0))) {
5821 SDValue Not = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5822 return DAG.getSetCC(DL: dl, VT, LHS: Not, RHS: N0.getOperand(i: 0), Cond);
5823 }
5824 }
5825 }
5826
5827 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5828 N0.getValueType().isInteger()) {
5829 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
5830 N0.getOpcode() == ISD::XOR) {
5831 // Simplify (X+Y) == (X+Z) --> Y == Z
5832 if (N0.getOpcode() == N1.getOpcode()) {
5833 if (N0.getOperand(i: 0) == N1.getOperand(i: 0))
5834 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 1), Cond);
5835 if (N0.getOperand(i: 1) == N1.getOperand(i: 1))
5836 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
5837 if (isCommutativeBinOp(Opcode: N0.getOpcode())) {
5838 // If X op Y == Y op X, try other combinations.
5839 if (N0.getOperand(i: 0) == N1.getOperand(i: 1))
5840 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 0),
5841 Cond);
5842 if (N0.getOperand(i: 1) == N1.getOperand(i: 0))
5843 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 1),
5844 Cond);
5845 }
5846 }
5847
5848 // If RHS is a legal immediate value for a compare instruction, we need
5849 // to be careful about increasing register pressure needlessly.
5850 bool LegalRHSImm = false;
5851
5852 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: N1)) {
5853 if (auto *LHSR = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5854 // Turn (X+C1) == C2 --> X == C2-C1
5855 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
5856 return DAG.getSetCC(
5857 DL: dl, VT, LHS: N0.getOperand(i: 0),
5858 RHS: DAG.getConstant(Val: RHSC->getAPIntValue() - LHSR->getAPIntValue(),
5859 DL: dl, VT: N0.getValueType()),
5860 Cond);
5861
5862 // Turn (X^C1) == C2 --> X == C1^C2
5863 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
5864 return DAG.getSetCC(
5865 DL: dl, VT, LHS: N0.getOperand(i: 0),
5866 RHS: DAG.getConstant(Val: LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
5867 DL: dl, VT: N0.getValueType()),
5868 Cond);
5869 }
5870
5871 // Turn (C1-X) == C2 --> X == C1-C2
5872 if (auto *SUBC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0)))
5873 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
5874 return DAG.getSetCC(
5875 DL: dl, VT, LHS: N0.getOperand(i: 1),
5876 RHS: DAG.getConstant(Val: SUBC->getAPIntValue() - RHSC->getAPIntValue(),
5877 DL: dl, VT: N0.getValueType()),
5878 Cond);
5879
5880 // Could RHSC fold directly into a compare?
5881 if (RHSC->getValueType(ResNo: 0).getSizeInBits() <= 64)
5882 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
5883 }
5884
5885 // (X+Y) == X --> Y == 0 and similar folds.
5886 // Don't do this if X is an immediate that can fold into a cmp
5887 // instruction and X+Y has other uses. It could be an induction variable
5888 // chain, and the transform would increase register pressure.
5889 if (!LegalRHSImm || N0.hasOneUse())
5890 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, DL: dl, DCI))
5891 return V;
5892 }
5893
5894 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
5895 N1.getOpcode() == ISD::XOR)
5896 if (SDValue V = foldSetCCWithBinOp(VT, N0: N1, N1: N0, Cond, DL: dl, DCI))
5897 return V;
5898
5899 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, DL: dl, DCI))
5900 return V;
5901
5902 if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, DL: dl, DCI))
5903 return V;
5904 }
5905
5906 // Fold remainder of division by a constant.
5907 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
5908 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5909 // When division is cheap or optimizing for minimum size,
5910 // fall through to DIVREM creation by skipping this fold.
5911 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
5912 if (N0.getOpcode() == ISD::UREM) {
5913 if (SDValue Folded = buildUREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5914 return Folded;
5915 } else if (N0.getOpcode() == ISD::SREM) {
5916 if (SDValue Folded = buildSREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5917 return Folded;
5918 }
5919 }
5920 }
5921
5922 // Fold away ALL boolean setcc's.
5923 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
5924 SDValue Temp;
5925 switch (Cond) {
5926 default: llvm_unreachable("Unknown integer setcc!");
5927 case ISD::SETEQ: // X == Y -> ~(X^Y)
5928 Temp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
5929 N0 = DAG.getNOT(DL: dl, Val: Temp, VT: OpVT);
5930 if (!DCI.isCalledByLegalizer())
5931 DCI.AddToWorklist(N: Temp.getNode());
5932 break;
5933 case ISD::SETNE: // X != Y --> (X^Y)
5934 N0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
5935 break;
5936 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
5937 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
5938 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
5939 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1, N2: Temp);
5940 if (!DCI.isCalledByLegalizer())
5941 DCI.AddToWorklist(N: Temp.getNode());
5942 break;
5943 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
5944 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
5945 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5946 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: N0, N2: Temp);
5947 if (!DCI.isCalledByLegalizer())
5948 DCI.AddToWorklist(N: Temp.getNode());
5949 break;
5950 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
5951 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
5952 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
5953 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1, N2: Temp);
5954 if (!DCI.isCalledByLegalizer())
5955 DCI.AddToWorklist(N: Temp.getNode());
5956 break;
5957 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
5958 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
5959 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5960 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: N0, N2: Temp);
5961 break;
5962 }
5963 if (VT.getScalarType() != MVT::i1) {
5964 if (!DCI.isCalledByLegalizer())
5965 DCI.AddToWorklist(N: N0.getNode());
5966 // FIXME: If running after legalize, we probably can't do this.
5967 ISD::NodeType ExtendCode = getExtendForContent(Content: getBooleanContents(Type: OpVT));
5968 N0 = DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: N0);
5969 }
5970 return N0;
5971 }
5972
5973 // Fold (setcc (trunc x) (trunc y)) -> (setcc x y)
5974 if (N0.getOpcode() == ISD::TRUNCATE && N1.getOpcode() == ISD::TRUNCATE &&
5975 N0.getOperand(i: 0).getValueType() == N1.getOperand(i: 0).getValueType() &&
5976 ((!ISD::isSignedIntSetCC(Code: Cond) && N0->getFlags().hasNoUnsignedWrap() &&
5977 N1->getFlags().hasNoUnsignedWrap()) ||
5978 (!ISD::isUnsignedIntSetCC(Code: Cond) && N0->getFlags().hasNoSignedWrap() &&
5979 N1->getFlags().hasNoSignedWrap())) &&
5980 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5981 if (VT.getScalarType() == MVT::i1)
5982 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
5983 // For (legal) non vXi1 cases - ensure we adjust the cmp and result types.
5984 EVT OldCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
5985 VT: N0.getValueType());
5986 if (VT == OldCCVT) {
5987 EVT NewCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
5988 VT: N0.getOperand(i: 0).getValueType());
5989 return DAG.getBoolExtOrTrunc(
5990 Op: DAG.getSetCC(DL: dl, VT: NewCCVT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond),
5991 SL: dl, VT, OpVT: N0.getOperand(i: 0).getValueType());
5992 }
5993 }
5994
5995 // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
5996 // TODO: Remove that .isVector() check
5997 if (VT.isVector() && isZeroOrZeroSplat(N: N1) && N0.getOpcode() == ISD::SUB &&
5998 N0->getFlags().hasNoSignedWrap() && ISD::isSignedIntSetCC(Code: Cond)) {
5999 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond);
6000 }
6001
6002 // Could not fold it.
6003 return SDValue();
6004}
6005
6006/// Returns true (and the GlobalValue and the offset) if the node is a
6007/// GlobalAddress + offset.
6008bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
6009 int64_t &Offset) const {
6010
6011 SDNode *N = unwrapAddress(N: SDValue(WN, 0)).getNode();
6012
6013 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(Val: N)) {
6014 GA = GASD->getGlobal();
6015 Offset += GASD->getOffset();
6016 return true;
6017 }
6018
6019 if (N->isAnyAdd()) {
6020 SDValue N1 = N->getOperand(Num: 0);
6021 SDValue N2 = N->getOperand(Num: 1);
6022 if (isGAPlusOffset(WN: N1.getNode(), GA, Offset)) {
6023 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N2)) {
6024 Offset += V->getSExtValue();
6025 return true;
6026 }
6027 } else if (isGAPlusOffset(WN: N2.getNode(), GA, Offset)) {
6028 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N1)) {
6029 Offset += V->getSExtValue();
6030 return true;
6031 }
6032 }
6033 }
6034
6035 return false;
6036}
6037
6038SDValue TargetLowering::PerformDAGCombine(SDNode *N,
6039 DAGCombinerInfo &DCI) const {
6040 // Default implementation: no optimization.
6041 return SDValue();
6042}
6043
6044//===----------------------------------------------------------------------===//
6045// Inline Assembler Implementation Methods
6046//===----------------------------------------------------------------------===//
6047
6048TargetLowering::ConstraintType
6049TargetLowering::getConstraintType(StringRef Constraint) const {
6050 unsigned S = Constraint.size();
6051
6052 if (S == 1) {
6053 switch (Constraint[0]) {
6054 default: break;
6055 case 'r':
6056 return C_RegisterClass;
6057 case 'm': // memory
6058 case 'o': // offsetable
6059 case 'V': // not offsetable
6060 return C_Memory;
6061 case 'p': // Address.
6062 return C_Address;
6063 case 'n': // Simple Integer
6064 case 'E': // Floating Point Constant
6065 case 'F': // Floating Point Constant
6066 return C_Immediate;
6067 case 'i': // Simple Integer or Relocatable Constant
6068 case 's': // Relocatable Constant
6069 case 'X': // Allow ANY value.
6070 case 'I': // Target registers.
6071 case 'J':
6072 case 'K':
6073 case 'L':
6074 case 'M':
6075 case 'N':
6076 case 'O':
6077 case 'P':
6078 case '<':
6079 case '>':
6080 return C_Other;
6081 }
6082 }
6083
6084 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
6085 if (S == 8 && Constraint.substr(Start: 1, N: 6) == "memory") // "{memory}"
6086 return C_Memory;
6087 return C_Register;
6088 }
6089 return C_Unknown;
6090}
6091
6092/// Try to replace an X constraint, which matches anything, with another that
6093/// has more specific requirements based on the type of the corresponding
6094/// operand.
6095const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
6096 if (ConstraintVT.isInteger())
6097 return "r";
6098 if (ConstraintVT.isFloatingPoint())
6099 return "f"; // works for many targets
6100 return nullptr;
6101}
6102
6103SDValue TargetLowering::LowerAsmOutputForConstraint(
6104 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
6105 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
6106 return SDValue();
6107}
6108
6109/// Lower the specified operand into the Ops vector.
6110/// If it is invalid, don't add anything to Ops.
6111void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
6112 StringRef Constraint,
6113 std::vector<SDValue> &Ops,
6114 SelectionDAG &DAG) const {
6115
6116 if (Constraint.size() > 1)
6117 return;
6118
6119 char ConstraintLetter = Constraint[0];
6120 switch (ConstraintLetter) {
6121 default: break;
6122 case 'X': // Allows any operand
6123 case 'i': // Simple Integer or Relocatable Constant
6124 case 'n': // Simple Integer
6125 case 's': { // Relocatable Constant
6126
6127 ConstantSDNode *C;
6128 uint64_t Offset = 0;
6129
6130 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
6131 // etc., since getelementpointer is variadic. We can't use
6132 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
6133 // while in this case the GA may be furthest from the root node which is
6134 // likely an ISD::ADD.
6135 while (true) {
6136 if ((C = dyn_cast<ConstantSDNode>(Val&: Op)) && ConstraintLetter != 's') {
6137 // gcc prints these as sign extended. Sign extend value to 64 bits
6138 // now; without this it would get ZExt'd later in
6139 // ScheduleDAGSDNodes::EmitNode, which is very generic.
6140 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
6141 BooleanContent BCont = getBooleanContents(Type: MVT::i64);
6142 ISD::NodeType ExtOpc =
6143 IsBool ? getExtendForContent(Content: BCont) : ISD::SIGN_EXTEND;
6144 int64_t ExtVal =
6145 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
6146 Ops.push_back(
6147 x: DAG.getTargetConstant(Val: Offset + ExtVal, DL: SDLoc(C), VT: MVT::i64));
6148 return;
6149 }
6150 if (ConstraintLetter != 'n') {
6151 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
6152 Ops.push_back(x: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(Op),
6153 VT: GA->getValueType(ResNo: 0),
6154 offset: Offset + GA->getOffset()));
6155 return;
6156 }
6157 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
6158 Ops.push_back(x: DAG.getTargetBlockAddress(
6159 BA: BA->getBlockAddress(), VT: BA->getValueType(ResNo: 0),
6160 Offset: Offset + BA->getOffset(), TargetFlags: BA->getTargetFlags()));
6161 return;
6162 }
6163 if (isa<BasicBlockSDNode>(Val: Op)) {
6164 Ops.push_back(x: Op);
6165 return;
6166 }
6167 }
6168 const unsigned OpCode = Op.getOpcode();
6169 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
6170 if ((C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 0))))
6171 Op = Op.getOperand(i: 1);
6172 // Subtraction is not commutative.
6173 else if (OpCode == ISD::ADD &&
6174 (C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))))
6175 Op = Op.getOperand(i: 0);
6176 else
6177 return;
6178 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
6179 continue;
6180 }
6181 return;
6182 }
6183 break;
6184 }
6185 }
6186}
6187
6188void TargetLowering::CollectTargetIntrinsicOperands(
6189 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
6190}
6191
6192std::pair<unsigned, const TargetRegisterClass *>
6193TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
6194 StringRef Constraint,
6195 MVT VT) const {
6196 if (!Constraint.starts_with(Prefix: "{"))
6197 return std::make_pair(x: 0u, y: static_cast<TargetRegisterClass *>(nullptr));
6198 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
6199
6200 // Remove the braces from around the name.
6201 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
6202
6203 std::pair<unsigned, const TargetRegisterClass *> R =
6204 std::make_pair(x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
6205
6206 // Figure out which register class contains this reg.
6207 for (const TargetRegisterClass &RC : RI->regclasses()) {
6208 // If none of the value types for this register class are valid, we
6209 // can't use it. For example, 64-bit reg classes on 32-bit targets.
6210 if (!isLegalRC(TRI: *RI, RC))
6211 continue;
6212
6213 for (const MCPhysReg &PR : RC) {
6214 if (RegName.equals_insensitive(RHS: RI->getRegAsmName(Reg: PR))) {
6215 std::pair<unsigned, const TargetRegisterClass *> S =
6216 std::make_pair(x: PR, y: &RC);
6217
6218 // If this register class has the requested value type, return it,
6219 // otherwise keep searching and return the first class found
6220 // if no other is found which explicitly has the requested type.
6221 if (RI->isTypeLegalForClass(RC, T: VT))
6222 return S;
6223 if (!R.second)
6224 R = S;
6225 }
6226 }
6227 }
6228
6229 return R;
6230}
6231
6232//===----------------------------------------------------------------------===//
6233// Constraint Selection.
6234
6235/// Return true of this is an input operand that is a matching constraint like
6236/// "4".
6237bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
6238 assert(!ConstraintCode.empty() && "No known constraint!");
6239 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
6240}
6241
6242/// If this is an input matching constraint, this method returns the output
6243/// operand it matches.
6244unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
6245 assert(!ConstraintCode.empty() && "No known constraint!");
6246 return atoi(nptr: ConstraintCode.c_str());
6247}
6248
6249/// Split up the constraint string from the inline assembly value into the
6250/// specific constraints and their prefixes, and also tie in the associated
6251/// operand values.
6252/// If this returns an empty vector, and if the constraint string itself
6253/// isn't empty, there was an error parsing.
6254TargetLowering::AsmOperandInfoVector
6255TargetLowering::ParseConstraints(const DataLayout &DL,
6256 const TargetRegisterInfo *TRI,
6257 const CallBase &Call) const {
6258 /// Information about all of the constraints.
6259 AsmOperandInfoVector ConstraintOperands;
6260 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
6261 unsigned maCount = 0; // Largest number of multiple alternative constraints.
6262
6263 // Do a prepass over the constraints, canonicalizing them, and building up the
6264 // ConstraintOperands list.
6265 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6266 unsigned ResNo = 0; // ResNo - The result number of the next output.
6267 unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.
6268
6269 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
6270 ConstraintOperands.emplace_back(args: std::move(CI));
6271 AsmOperandInfo &OpInfo = ConstraintOperands.back();
6272
6273 // Update multiple alternative constraint count.
6274 if (OpInfo.multipleAlternatives.size() > maCount)
6275 maCount = OpInfo.multipleAlternatives.size();
6276
6277 OpInfo.ConstraintVT = MVT::Other;
6278
6279 // Compute the value type for each operand.
6280 switch (OpInfo.Type) {
6281 case InlineAsm::isOutput: {
6282 // Indirect outputs just consume an argument.
6283 if (OpInfo.isIndirect) {
6284 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6285 break;
6286 }
6287
6288 // The return value of the call is this value. As such, there is no
6289 // corresponding argument.
6290 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
6291 EVT VT;
6292 if (auto *STy = dyn_cast<StructType>(Val: Call.getType())) {
6293 VT = getAsmOperandValueType(DL, Ty: STy->getElementType(N: ResNo));
6294 } else {
6295 assert(ResNo == 0 && "Asm only has one result!");
6296 VT = getAsmOperandValueType(DL, Ty: Call.getType());
6297 }
6298 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6299 ++ResNo;
6300 break;
6301 }
6302 case InlineAsm::isInput:
6303 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6304 break;
6305 case InlineAsm::isLabel:
6306 OpInfo.CallOperandVal = cast<CallBrInst>(Val: &Call)->getIndirectDest(i: LabelNo);
6307 ++LabelNo;
6308 continue;
6309 case InlineAsm::isClobber:
6310 // Nothing to do.
6311 break;
6312 }
6313
6314 if (OpInfo.CallOperandVal) {
6315 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
6316 if (OpInfo.isIndirect) {
6317 OpTy = Call.getParamElementType(ArgNo);
6318 assert(OpTy && "Indirect operand must have elementtype attribute");
6319 }
6320
6321 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6322 if (StructType *STy = dyn_cast<StructType>(Val: OpTy))
6323 if (STy->getNumElements() == 1)
6324 OpTy = STy->getElementType(N: 0);
6325
6326 // If OpTy is not a single value, it may be a struct/union that we
6327 // can tile with integers.
6328 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6329 unsigned BitSize = DL.getTypeSizeInBits(Ty: OpTy);
6330 switch (BitSize) {
6331 default: break;
6332 case 1:
6333 case 8:
6334 case 16:
6335 case 32:
6336 case 64:
6337 case 128:
6338 OpTy = IntegerType::get(C&: OpTy->getContext(), NumBits: BitSize);
6339 break;
6340 }
6341 }
6342
6343 EVT VT = getAsmOperandValueType(DL, Ty: OpTy, AllowUnknown: true);
6344 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6345 ArgNo++;
6346 }
6347 }
6348
6349 // If we have multiple alternative constraints, select the best alternative.
6350 if (!ConstraintOperands.empty()) {
6351 if (maCount) {
6352 unsigned bestMAIndex = 0;
6353 int bestWeight = -1;
6354 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
6355 int weight = -1;
6356 unsigned maIndex;
6357 // Compute the sums of the weights for each alternative, keeping track
6358 // of the best (highest weight) one so far.
6359 for (maIndex = 0; maIndex < maCount; ++maIndex) {
6360 int weightSum = 0;
6361 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6362 cIndex != eIndex; ++cIndex) {
6363 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6364 if (OpInfo.Type == InlineAsm::isClobber)
6365 continue;
6366
6367 // If this is an output operand with a matching input operand,
6368 // look up the matching input. If their types mismatch, e.g. one
6369 // is an integer, the other is floating point, or their sizes are
6370 // different, flag it as an maCantMatch.
6371 if (OpInfo.hasMatchingInput()) {
6372 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6373 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6374 if ((OpInfo.ConstraintVT.isInteger() !=
6375 Input.ConstraintVT.isInteger()) ||
6376 (OpInfo.ConstraintVT.getSizeInBits() !=
6377 Input.ConstraintVT.getSizeInBits())) {
6378 weightSum = -1; // Can't match.
6379 break;
6380 }
6381 }
6382 }
6383 weight = getMultipleConstraintMatchWeight(info&: OpInfo, maIndex);
6384 if (weight == -1) {
6385 weightSum = -1;
6386 break;
6387 }
6388 weightSum += weight;
6389 }
6390 // Update best.
6391 if (weightSum > bestWeight) {
6392 bestWeight = weightSum;
6393 bestMAIndex = maIndex;
6394 }
6395 }
6396
6397 // Now select chosen alternative in each constraint.
6398 for (AsmOperandInfo &cInfo : ConstraintOperands)
6399 if (cInfo.Type != InlineAsm::isClobber)
6400 cInfo.selectAlternative(index: bestMAIndex);
6401 }
6402 }
6403
6404 // Check and hook up tied operands, choose constraint code to use.
6405 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6406 cIndex != eIndex; ++cIndex) {
6407 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6408
6409 // If this is an output operand with a matching input operand, look up the
6410 // matching input. If their types mismatch, e.g. one is an integer, the
6411 // other is floating point, or their sizes are different, flag it as an
6412 // error.
6413 if (OpInfo.hasMatchingInput()) {
6414 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6415
6416 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6417 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6418 getRegForInlineAsmConstraint(RI: TRI, Constraint: OpInfo.ConstraintCode,
6419 VT: OpInfo.ConstraintVT);
6420 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6421 getRegForInlineAsmConstraint(RI: TRI, Constraint: Input.ConstraintCode,
6422 VT: Input.ConstraintVT);
6423 const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
6424 OpInfo.ConstraintVT.isFloatingPoint();
6425 const bool InOpIsIntOrFP = Input.ConstraintVT.isInteger() ||
6426 Input.ConstraintVT.isFloatingPoint();
6427 if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
6428 (MatchRC.second != InputRC.second)) {
6429 report_fatal_error(reason: "Unsupported asm: input constraint"
6430 " with a matching output constraint of"
6431 " incompatible type!");
6432 }
6433 }
6434 }
6435 }
6436
6437 return ConstraintOperands;
6438}
6439
6440/// Return a number indicating our preference for chosing a type of constraint
6441/// over another, for the purpose of sorting them. Immediates are almost always
6442/// preferrable (when they can be emitted). A higher return value means a
6443/// stronger preference for one constraint type relative to another.
6444/// FIXME: We should prefer registers over memory but doing so may lead to
6445/// unrecoverable register exhaustion later.
6446/// https://github.com/llvm/llvm-project/issues/20571
6447static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) {
6448 switch (CT) {
6449 case TargetLowering::C_Immediate:
6450 case TargetLowering::C_Other:
6451 return 4;
6452 case TargetLowering::C_Memory:
6453 case TargetLowering::C_Address:
6454 return 3;
6455 case TargetLowering::C_RegisterClass:
6456 return 2;
6457 case TargetLowering::C_Register:
6458 return 1;
6459 case TargetLowering::C_Unknown:
6460 return 0;
6461 }
6462 llvm_unreachable("Invalid constraint type");
6463}
6464
6465/// Examine constraint type and operand type and determine a weight value.
6466/// This object must already have been set up with the operand type
6467/// and the current alternative constraint selected.
6468TargetLowering::ConstraintWeight
6469 TargetLowering::getMultipleConstraintMatchWeight(
6470 AsmOperandInfo &info, int maIndex) const {
6471 InlineAsm::ConstraintCodeVector *rCodes;
6472 if (maIndex >= (int)info.multipleAlternatives.size())
6473 rCodes = &info.Codes;
6474 else
6475 rCodes = &info.multipleAlternatives[maIndex].Codes;
6476 ConstraintWeight BestWeight = CW_Invalid;
6477
6478 // Loop over the options, keeping track of the most general one.
6479 for (const std::string &rCode : *rCodes) {
6480 ConstraintWeight weight =
6481 getSingleConstraintMatchWeight(info, constraint: rCode.c_str());
6482 if (weight > BestWeight)
6483 BestWeight = weight;
6484 }
6485
6486 return BestWeight;
6487}
6488
6489/// Examine constraint type and operand type and determine a weight value.
6490/// This object must already have been set up with the operand type
6491/// and the current alternative constraint selected.
6492TargetLowering::ConstraintWeight
6493 TargetLowering::getSingleConstraintMatchWeight(
6494 AsmOperandInfo &info, const char *constraint) const {
6495 ConstraintWeight weight = CW_Invalid;
6496 Value *CallOperandVal = info.CallOperandVal;
6497 // If we don't have a value, we can't do a match,
6498 // but allow it at the lowest weight.
6499 if (!CallOperandVal)
6500 return CW_Default;
6501 // Look at the constraint type.
6502 switch (*constraint) {
6503 case 'i': // immediate integer.
6504 case 'n': // immediate integer with a known value.
6505 if (isa<ConstantInt>(Val: CallOperandVal))
6506 weight = CW_Constant;
6507 break;
6508 case 's': // non-explicit intregal immediate.
6509 if (isa<GlobalValue>(Val: CallOperandVal))
6510 weight = CW_Constant;
6511 break;
6512 case 'E': // immediate float if host format.
6513 case 'F': // immediate float.
6514 if (isa<ConstantFP>(Val: CallOperandVal))
6515 weight = CW_Constant;
6516 break;
6517 case '<': // memory operand with autodecrement.
6518 case '>': // memory operand with autoincrement.
6519 case 'm': // memory operand.
6520 case 'o': // offsettable memory operand
6521 case 'V': // non-offsettable memory operand
6522 weight = CW_Memory;
6523 break;
6524 case 'r': // general register.
6525 case 'g': // general register, memory operand or immediate integer.
6526 // note: Clang converts "g" to "imr".
6527 if (CallOperandVal->getType()->isIntegerTy())
6528 weight = CW_Register;
6529 break;
6530 case 'X': // any operand.
6531 default:
6532 weight = CW_Default;
6533 break;
6534 }
6535 return weight;
6536}
6537
6538/// If there are multiple different constraints that we could pick for this
6539/// operand (e.g. "imr") try to pick the 'best' one.
6540/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
6541/// into seven classes:
6542/// Register -> one specific register
6543/// RegisterClass -> a group of regs
6544/// Memory -> memory
6545/// Address -> a symbolic memory reference
6546/// Immediate -> immediate values
6547/// Other -> magic values (such as "Flag Output Operands")
6548/// Unknown -> something we don't recognize yet and can't handle
6549/// Ideally, we would pick the most specific constraint possible: if we have
6550/// something that fits into a register, we would pick it. The problem here
6551/// is that if we have something that could either be in a register or in
6552/// memory that use of the register could cause selection of *other*
6553/// operands to fail: they might only succeed if we pick memory. Because of
6554/// this the heuristic we use is:
6555///
6556/// 1) If there is an 'other' constraint, and if the operand is valid for
6557/// that constraint, use it. This makes us take advantage of 'i'
6558/// constraints when available.
6559/// 2) Otherwise, pick the most general constraint present. This prefers
6560/// 'm' over 'r', for example.
6561///
6562TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
6563 TargetLowering::AsmOperandInfo &OpInfo) const {
6564 ConstraintGroup Ret;
6565
6566 Ret.reserve(N: OpInfo.Codes.size());
6567 for (StringRef Code : OpInfo.Codes) {
6568 TargetLowering::ConstraintType CType = getConstraintType(Constraint: Code);
6569
6570 // Indirect 'other' or 'immediate' constraints are not allowed.
6571 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6572 CType == TargetLowering::C_Register ||
6573 CType == TargetLowering::C_RegisterClass))
6574 continue;
6575
6576 // Things with matching constraints can only be registers, per gcc
6577 // documentation. This mainly affects "g" constraints.
6578 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
6579 continue;
6580
6581 Ret.emplace_back(Args&: Code, Args&: CType);
6582 }
6583
6584 llvm::stable_sort(Range&: Ret, C: [](ConstraintPair a, ConstraintPair b) {
6585 return getConstraintPiority(CT: a.second) > getConstraintPiority(CT: b.second);
6586 });
6587
6588 return Ret;
6589}
6590
6591/// If we have an immediate, see if we can lower it. Return true if we can,
6592/// false otherwise.
6593static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P,
6594 SDValue Op, SelectionDAG *DAG,
6595 const TargetLowering &TLI) {
6596
6597 assert((P.second == TargetLowering::C_Other ||
6598 P.second == TargetLowering::C_Immediate) &&
6599 "need immediate or other");
6600
6601 if (!Op.getNode())
6602 return false;
6603
6604 std::vector<SDValue> ResultOps;
6605 TLI.LowerAsmOperandForConstraint(Op, Constraint: P.first, Ops&: ResultOps, DAG&: *DAG);
6606 return !ResultOps.empty();
6607}
6608
6609/// Determines the constraint code and constraint type to use for the specific
6610/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
6611void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
6612 SDValue Op,
6613 SelectionDAG *DAG) const {
6614 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
6615
6616 // Single-letter constraints ('r') are very common.
6617 if (OpInfo.Codes.size() == 1) {
6618 OpInfo.ConstraintCode = OpInfo.Codes[0];
6619 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6620 } else {
6621 ConstraintGroup G = getConstraintPreferences(OpInfo);
6622 if (G.empty())
6623 return;
6624
6625 unsigned BestIdx = 0;
6626 for (const unsigned E = G.size();
6627 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
6628 G[BestIdx].second == TargetLowering::C_Immediate);
6629 ++BestIdx) {
6630 if (lowerImmediateIfPossible(P&: G[BestIdx], Op, DAG, TLI: *this))
6631 break;
6632 // If we're out of constraints, just pick the first one.
6633 if (BestIdx + 1 == E) {
6634 BestIdx = 0;
6635 break;
6636 }
6637 }
6638
6639 OpInfo.ConstraintCode = G[BestIdx].first;
6640 OpInfo.ConstraintType = G[BestIdx].second;
6641 }
6642
6643 // 'X' matches anything.
6644 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
6645 // Constants are handled elsewhere. For Functions, the type here is the
6646 // type of the result, which is not what we want to look at; leave them
6647 // alone.
6648 Value *v = OpInfo.CallOperandVal;
6649 if (isa<ConstantInt>(Val: v) || isa<Function>(Val: v)) {
6650 return;
6651 }
6652
6653 if (isa<BasicBlock>(Val: v) || isa<BlockAddress>(Val: v)) {
6654 OpInfo.ConstraintCode = "i";
6655 return;
6656 }
6657
6658 // Otherwise, try to resolve it to something we know about by looking at
6659 // the actual operand type.
6660 if (const char *Repl = LowerXConstraint(ConstraintVT: OpInfo.ConstraintVT)) {
6661 OpInfo.ConstraintCode = Repl;
6662 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6663 }
6664 }
6665}
6666
6667/// Given an exact SDIV by a constant, create a multiplication
6668/// with the multiplicative inverse of the constant.
6669/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6670static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
6671 const SDLoc &dl, SelectionDAG &DAG,
6672 SmallVectorImpl<SDNode *> &Created) {
6673 SDValue Op0 = N->getOperand(Num: 0);
6674 SDValue Op1 = N->getOperand(Num: 1);
6675 EVT VT = N->getValueType(ResNo: 0);
6676 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6677 EVT ShSVT = ShVT.getScalarType();
6678
6679 bool UseSRA = false;
6680 SmallVector<SDValue, 16> Shifts, Factors;
6681
6682 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6683 if (C->isZero())
6684 return false;
6685
6686 EVT CT = C->getValueType(ResNo: 0);
6687 APInt Divisor = C->getAPIntValue();
6688 unsigned Shift = Divisor.countr_zero();
6689 if (Shift) {
6690 Divisor.ashrInPlace(ShiftAmt: Shift);
6691 UseSRA = true;
6692 }
6693 APInt Factor = Divisor.multiplicativeInverse();
6694 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6695 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6696 return true;
6697 };
6698
6699 // Collect all magic values from the build vector.
6700 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildSDIVPattern))
6701 return SDValue();
6702
6703 SDValue Shift, Factor;
6704 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6705 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6706 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6707 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6708 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6709 "Expected matchUnaryPredicate to return one element for scalable "
6710 "vectors");
6711 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6712 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6713 } else {
6714 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6715 Shift = Shifts[0];
6716 Factor = Factors[0];
6717 }
6718
6719 SDValue Res = Op0;
6720 if (UseSRA) {
6721 Res = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6722 Created.push_back(Elt: Res.getNode());
6723 }
6724
6725 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6726}
6727
6728/// Given an exact UDIV by a constant, create a multiplication
6729/// with the multiplicative inverse of the constant.
6730/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6731static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
6732 const SDLoc &dl, SelectionDAG &DAG,
6733 SmallVectorImpl<SDNode *> &Created) {
6734 EVT VT = N->getValueType(ResNo: 0);
6735 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6736 EVT ShSVT = ShVT.getScalarType();
6737
6738 bool UseSRL = false;
6739 SmallVector<SDValue, 16> Shifts, Factors;
6740
6741 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6742 if (C->isZero())
6743 return false;
6744
6745 EVT CT = C->getValueType(ResNo: 0);
6746 APInt Divisor = C->getAPIntValue();
6747 unsigned Shift = Divisor.countr_zero();
6748 if (Shift) {
6749 Divisor.lshrInPlace(ShiftAmt: Shift);
6750 UseSRL = true;
6751 }
6752 // Calculate the multiplicative inverse modulo BW.
6753 APInt Factor = Divisor.multiplicativeInverse();
6754 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6755 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6756 return true;
6757 };
6758
6759 SDValue Op1 = N->getOperand(Num: 1);
6760
6761 // Collect all magic values from the build vector.
6762 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildUDIVPattern))
6763 return SDValue();
6764
6765 SDValue Shift, Factor;
6766 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6767 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6768 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6769 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6770 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6771 "Expected matchUnaryPredicate to return one element for scalable "
6772 "vectors");
6773 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6774 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6775 } else {
6776 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6777 Shift = Shifts[0];
6778 Factor = Factors[0];
6779 }
6780
6781 SDValue Res = N->getOperand(Num: 0);
6782 if (UseSRL) {
6783 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6784 Created.push_back(Elt: Res.getNode());
6785 }
6786
6787 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6788}
6789
6790SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
6791 SelectionDAG &DAG,
6792 SmallVectorImpl<SDNode *> &Created) const {
6793 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6794 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6795 return SDValue(N, 0); // Lower SDIV as SDIV
6796 return SDValue();
6797}
6798
6799SDValue
6800TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
6801 SelectionDAG &DAG,
6802 SmallVectorImpl<SDNode *> &Created) const {
6803 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6804 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6805 return SDValue(N, 0); // Lower SREM as SREM
6806 return SDValue();
6807}
6808
6809/// Build sdiv by power-of-2 with conditional move instructions
6810/// Ref: "Hacker's Delight" by Henry Warren 10-1
6811/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
6812/// bgez x, label
6813/// add x, x, 2**k-1
6814/// label:
6815/// sra res, x, k
6816/// neg res, res (when the divisor is negative)
6817SDValue TargetLowering::buildSDIVPow2WithCMov(
6818 SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
6819 SmallVectorImpl<SDNode *> &Created) const {
6820 unsigned Lg2 = Divisor.countr_zero();
6821 EVT VT = N->getValueType(ResNo: 0);
6822
6823 SDLoc DL(N);
6824 SDValue N0 = N->getOperand(Num: 0);
6825 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
6826 APInt Lg2Mask = APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: Lg2);
6827 SDValue Pow2MinusOne = DAG.getConstant(Val: Lg2Mask, DL, VT);
6828
6829 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
6830 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
6831 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: N0, RHS: Zero, Cond: ISD::SETLT);
6832 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: Pow2MinusOne);
6833 SDValue CMov = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cmp, N2: Add, N3: N0);
6834
6835 Created.push_back(Elt: Cmp.getNode());
6836 Created.push_back(Elt: Add.getNode());
6837 Created.push_back(Elt: CMov.getNode());
6838
6839 // Divide by pow2.
6840 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CMov,
6841 N2: DAG.getShiftAmountConstant(Val: Lg2, VT, DL));
6842
6843 // If we're dividing by a positive value, we're done. Otherwise, we must
6844 // negate the result.
6845 if (Divisor.isNonNegative())
6846 return SRA;
6847
6848 Created.push_back(Elt: SRA.getNode());
6849 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: SRA);
6850}
6851
6852/// Given an ISD::SDIV node expressing a divide by constant,
6853/// return a DAG expression to select that will generate the same value by
6854/// multiplying by a magic number.
6855/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6856SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
6857 bool IsAfterLegalization,
6858 bool IsAfterLegalTypes,
6859 SmallVectorImpl<SDNode *> &Created) const {
6860 SDLoc dl(N);
6861
6862 // If the sdiv has an 'exact' bit we can use a simpler lowering.
6863 if (N->getFlags().hasExact())
6864 return BuildExactSDIV(TLI: *this, N, dl, DAG, Created);
6865
6866 EVT VT = N->getValueType(ResNo: 0);
6867 EVT SVT = VT.getScalarType();
6868 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6869 EVT ShSVT = ShVT.getScalarType();
6870 unsigned EltBits = VT.getScalarSizeInBits();
6871 EVT MulVT;
6872
6873 // Check to see if we can do this.
6874 // FIXME: We should be more aggressive here.
6875 EVT QueryVT = VT;
6876 if (VT.isVector()) {
6877 // If the vector type will be legalized to a vector type with the same
6878 // element type, allow the transform before type legalization if MULHS or
6879 // SMUL_LOHI are supported.
6880 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
6881 if (!QueryVT.isVector() ||
6882 QueryVT.getVectorElementType() != VT.getVectorElementType())
6883 return SDValue();
6884 } else if (!isTypeLegal(VT)) {
6885 // Limit this to simple scalars for now.
6886 if (!VT.isSimple())
6887 return SDValue();
6888
6889 // If this type will be promoted to a large enough type with a legal
6890 // multiply operation, we can go ahead and do this transform.
6891 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
6892 return SDValue();
6893
6894 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6895 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6896 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
6897 return SDValue();
6898 }
6899
6900 bool HasMULHS =
6901 isOperationLegalOrCustom(Op: ISD::MULHS, VT: QueryVT, LegalOnly: IsAfterLegalization);
6902 bool HasSMUL_LOHI =
6903 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
6904
6905 if (isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
6906 // If type twice as wide legal, widen and use a mul plus a shift.
6907 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
6908 // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
6909 // custom lowered. This is very expensive so avoid it at all costs for
6910 // constant divisors.
6911 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::SDIV, VT) &&
6912 isOperationCustom(Op: ISD::SDIVREM, VT: VT.getScalarType())) ||
6913 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
6914 MulVT = WideVT;
6915 }
6916
6917 if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
6918 return SDValue();
6919
6920 // If we're after type legalization and SVT is not legal, use the
6921 // promoted type for creating constants to avoid creating nodes with
6922 // illegal types.
6923 if (IsAfterLegalTypes && VT.isVector()) {
6924 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
6925 if (SVT.bitsLT(VT: VT.getScalarType()))
6926 return SDValue();
6927 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
6928 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
6929 return SDValue();
6930 }
6931 const unsigned SVTBits = SVT.getSizeInBits();
6932
6933 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
6934
6935 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6936 if (C->isZero())
6937 return false;
6938 // Truncate the divisor to the target scalar type in case it was promoted
6939 // during type legalization.
6940 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
6941 SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(D: Divisor);
6942 int NumeratorFactor = 0;
6943 int ShiftMask = -1;
6944
6945 if (Divisor.isOne() || Divisor.isAllOnes()) {
6946 // If d is +1/-1, we just multiply the numerator by +1/-1.
6947 NumeratorFactor = Divisor.getSExtValue();
6948 magics.Magic = 0;
6949 magics.ShiftAmount = 0;
6950 ShiftMask = 0;
6951 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
6952 // If d > 0 and m < 0, add the numerator.
6953 NumeratorFactor = 1;
6954 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
6955 // If d < 0 and m > 0, subtract the numerator.
6956 NumeratorFactor = -1;
6957 }
6958
6959 MagicFactors.push_back(
6960 Elt: DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT));
6961 Factors.push_back(Elt: DAG.getSignedConstant(Val: NumeratorFactor, DL: dl, VT: SVT));
6962 Shifts.push_back(Elt: DAG.getConstant(Val: magics.ShiftAmount, DL: dl, VT: ShSVT));
6963 ShiftMasks.push_back(Elt: DAG.getSignedConstant(Val: ShiftMask, DL: dl, VT: SVT));
6964 return true;
6965 };
6966
6967 SDValue N0 = N->getOperand(Num: 0);
6968 SDValue N1 = N->getOperand(Num: 1);
6969
6970 // Collect the shifts / magic values from each element.
6971 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildSDIVPattern, /*AllowUndefs=*/false,
6972 /*AllowTruncation=*/true))
6973 return SDValue();
6974
6975 SDValue MagicFactor, Factor, Shift, ShiftMask;
6976 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
6977 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
6978 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6979 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6980 ShiftMask = DAG.getBuildVector(VT, DL: dl, Ops: ShiftMasks);
6981 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
6982 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
6983 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
6984 "Expected matchUnaryPredicate to return one element for scalable "
6985 "vectors");
6986 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
6987 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6988 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6989 ShiftMask = DAG.getSplatVector(VT, DL: dl, Op: ShiftMasks[0]);
6990 } else {
6991 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
6992 MagicFactor = MagicFactors[0];
6993 Factor = Factors[0];
6994 Shift = Shifts[0];
6995 ShiftMask = ShiftMasks[0];
6996 }
6997
6998 // Multiply the numerator (operand 0) by the magic value.
6999 auto GetMULHS = [&](SDValue X, SDValue Y) {
7000 if (HasMULHS)
7001 return DAG.getNode(Opcode: ISD::MULHS, DL: dl, VT, N1: X, N2: Y);
7002 if (HasSMUL_LOHI) {
7003 SDValue LoHi =
7004 DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7005 return LoHi.getValue(R: 1);
7006 }
7007
7008 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: X);
7009 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7010 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7011 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7012 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7013 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7014 };
7015
7016 SDValue Q = GetMULHS(N0, MagicFactor);
7017 if (!Q)
7018 return SDValue();
7019
7020 Created.push_back(Elt: Q.getNode());
7021
7022 // (Optionally) Add/subtract the numerator using Factor.
7023 Factor = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: N0, N2: Factor);
7024 Created.push_back(Elt: Factor.getNode());
7025 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: Factor);
7026 Created.push_back(Elt: Q.getNode());
7027
7028 // Shift right algebraic by shift value.
7029 Q = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Q, N2: Shift);
7030 Created.push_back(Elt: Q.getNode());
7031
7032 // Extract the sign bit, mask it and add it to the quotient.
7033 SDValue SignShift = DAG.getConstant(Val: EltBits - 1, DL: dl, VT: ShVT);
7034 SDValue T = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: SignShift);
7035 Created.push_back(Elt: T.getNode());
7036 T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: ShiftMask);
7037 Created.push_back(Elt: T.getNode());
7038 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: T);
7039}
7040
7041/// Given an ISD::UDIV node expressing a divide by constant,
7042/// return a DAG expression to select that will generate the same value by
7043/// multiplying by a magic number.
7044/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
7045SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
7046 bool IsAfterLegalization,
7047 bool IsAfterLegalTypes,
7048 SmallVectorImpl<SDNode *> &Created) const {
7049 SDLoc dl(N);
7050
7051 // If the udiv has an 'exact' bit we can use a simpler lowering.
7052 if (N->getFlags().hasExact())
7053 return BuildExactUDIV(TLI: *this, N, dl, DAG, Created);
7054
7055 EVT VT = N->getValueType(ResNo: 0);
7056 EVT SVT = VT.getScalarType();
7057 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7058 EVT ShSVT = ShVT.getScalarType();
7059 unsigned EltBits = VT.getScalarSizeInBits();
7060 EVT MulVT;
7061
7062 // Check to see if we can do this.
7063 // FIXME: We should be more aggressive here.
7064 EVT QueryVT = VT;
7065 if (VT.isVector()) {
7066 // If the vector type will be legalized to a vector type with the same
7067 // element type, allow the transform before type legalization if MULHU or
7068 // UMUL_LOHI are supported.
7069 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
7070 if (!QueryVT.isVector() ||
7071 QueryVT.getVectorElementType() != VT.getVectorElementType())
7072 return SDValue();
7073 } else if (!isTypeLegal(VT)) {
7074 // Limit this to simple scalars for now.
7075 if (!VT.isSimple())
7076 return SDValue();
7077
7078 // If this type will be promoted to a large enough type with a legal
7079 // multiply operation, we can go ahead and do this transform.
7080 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
7081 return SDValue();
7082
7083 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
7084 if (MulVT.getSizeInBits() < (2 * EltBits) ||
7085 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
7086 return SDValue();
7087 }
7088
7089 bool HasMULHU =
7090 isOperationLegalOrCustom(Op: ISD::MULHU, VT: QueryVT, LegalOnly: IsAfterLegalization);
7091 bool HasUMUL_LOHI =
7092 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
7093
7094 if (isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
7095 // If type twice as wide legal, widen and use a mul plus a shift.
7096 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
7097 // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
7098 // custom lowered. This is very expensive so avoid it at all costs for
7099 // constant divisors.
7100 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::UDIV, VT) &&
7101 isOperationCustom(Op: ISD::UDIVREM, VT: VT.getScalarType())) ||
7102 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
7103 MulVT = WideVT;
7104 }
7105
7106 if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
7107 return SDValue();
7108
7109 SDValue N0 = N->getOperand(Num: 0);
7110 SDValue N1 = N->getOperand(Num: 1);
7111
7112 // Try to use leading zeros of the dividend to reduce the multiplier and
7113 // avoid expensive fixups.
7114 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
7115
7116 // If we're after type legalization and SVT is not legal, use the
7117 // promoted type for creating constants to avoid creating nodes with
7118 // illegal types.
7119 if (IsAfterLegalTypes && VT.isVector()) {
7120 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
7121 if (SVT.bitsLT(VT: VT.getScalarType()))
7122 return SDValue();
7123 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
7124 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
7125 return SDValue();
7126 }
7127 const unsigned SVTBits = SVT.getSizeInBits();
7128
7129 // Allow i32 to be widened to i64 for uncooperative divisors if i64 MULHU or
7130 // UMUL_LOHI is supported.
7131 const EVT WideSVT = MVT::i64;
7132 const bool HasWideMULHU =
7133 VT == MVT::i32 &&
7134 isOperationLegalOrCustom(Op: ISD::MULHU, VT: WideSVT, LegalOnly: IsAfterLegalization);
7135 const bool HasWideUMUL_LOHI =
7136 VT == MVT::i32 &&
7137 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: WideSVT, LegalOnly: IsAfterLegalization);
7138 const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
7139
7140 // For even divisors with a 33-bit magic number, the widened high-multiply
7141 // path is only worthwhile over the even-divisor rewrite on targets that
7142 // zero-extend i32 to i64 for free (e.g. x86-64 and AArch64). Elsewhere (e.g.
7143 // RISC-V) keep the even-divisor rewrite, which avoids the explicit extension.
7144 const bool AllowEvenToWiden = AllowWiden && isZExtFree(FromTy: VT, ToTy: WideSVT);
7145
7146 bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
7147 bool UseWiden = false;
7148 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
7149
7150 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
7151 if (C->isZero())
7152 return false;
7153 // Truncate the divisor to the target scalar type in case it was promoted
7154 // during type legalization.
7155 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
7156
7157 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
7158
7159 // Magic algorithm doesn't work for division by 1. We need to emit a select
7160 // at the end.
7161 if (Divisor.isOne()) {
7162 PreShift = PostShift = DAG.getUNDEF(VT: ShSVT);
7163 MagicFactor = NPQFactor = DAG.getUNDEF(VT: SVT);
7164 } else {
7165 UnsignedDivisionByConstantInfo magics =
7166 UnsignedDivisionByConstantInfo::get(
7167 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()),
7168 /*AllowEvenDivisorOptimization=*/!AllowEvenToWiden,
7169 /*AllowWidenOptimization=*/AllowWiden);
7170
7171 if (magics.Widen) {
7172 UseWiden = true;
7173 MagicFactor = DAG.getConstant(Val: magics.Magic, DL: dl, VT: WideSVT);
7174 } else {
7175 MagicFactor = DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT);
7176 }
7177
7178 assert(magics.PreShift < Divisor.getBitWidth() &&
7179 "We shouldn't generate an undefined shift!");
7180 assert(magics.PostShift < Divisor.getBitWidth() &&
7181 "We shouldn't generate an undefined shift!");
7182 assert((!magics.IsAdd || magics.PreShift == 0) &&
7183 "Unexpected pre-shift");
7184 PreShift = DAG.getConstant(Val: magics.PreShift, DL: dl, VT: ShSVT);
7185 PostShift = DAG.getConstant(Val: magics.PostShift, DL: dl, VT: ShSVT);
7186 NPQFactor = DAG.getConstant(
7187 Val: magics.IsAdd ? APInt::getOneBitSet(numBits: SVTBits, BitNo: EltBits - 1)
7188 : APInt::getZero(numBits: SVTBits),
7189 DL: dl, VT: SVT);
7190 UseNPQ |= magics.IsAdd;
7191 UsePreShift |= magics.PreShift != 0;
7192 UsePostShift |= magics.PostShift != 0;
7193 }
7194
7195 PreShifts.push_back(Elt: PreShift);
7196 MagicFactors.push_back(Elt: MagicFactor);
7197 NPQFactors.push_back(Elt: NPQFactor);
7198 PostShifts.push_back(Elt: PostShift);
7199 return true;
7200 };
7201
7202 // Collect the shifts/magic values from each element.
7203 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildUDIVPattern, /*AllowUndefs=*/false,
7204 /*AllowTruncation=*/true))
7205 return SDValue();
7206
7207 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
7208 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7209 PreShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PreShifts);
7210 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
7211 NPQFactor = DAG.getBuildVector(VT, DL: dl, Ops: NPQFactors);
7212 PostShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PostShifts);
7213 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7214 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
7215 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
7216 "Expected matchUnaryPredicate to return one for scalable vectors");
7217 PreShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PreShifts[0]);
7218 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
7219 NPQFactor = DAG.getSplatVector(VT, DL: dl, Op: NPQFactors[0]);
7220 PostShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PostShifts[0]);
7221 } else {
7222 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7223 PreShift = PreShifts[0];
7224 MagicFactor = MagicFactors[0];
7225 PostShift = PostShifts[0];
7226 }
7227
7228 if (UseWiden) {
7229 // Compute: (WideSVT(x) * MagicFactor) >> WideSVTBits.
7230 SDValue WideN0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: WideSVT, Operand: N0);
7231
7232 // Perform WideSVTxWideSVT -> 2*WideSVT multiplication and extract high
7233 // WideSVT bits
7234 SDValue High;
7235 if (HasWideMULHU) {
7236 High = DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT: WideSVT, N1: WideN0, N2: MagicFactor);
7237 } else {
7238 assert(HasWideUMUL_LOHI);
7239 SDValue LoHi =
7240 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: WideSVT, VT2: WideSVT),
7241 N1: WideN0, N2: MagicFactor);
7242 High = LoHi.getValue(R: 1);
7243 }
7244
7245 Created.push_back(Elt: High.getNode());
7246 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: High);
7247 }
7248
7249 SDValue Q = N0;
7250 if (UsePreShift) {
7251 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PreShift);
7252 Created.push_back(Elt: Q.getNode());
7253 }
7254
7255 auto GetMULHU = [&](SDValue X, SDValue Y) {
7256 if (HasMULHU)
7257 return DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT, N1: X, N2: Y);
7258 if (HasUMUL_LOHI) {
7259 SDValue LoHi =
7260 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7261 return LoHi.getValue(R: 1);
7262 }
7263
7264 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: X);
7265 Y = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7266 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7267 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7268 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7269 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7270 };
7271
7272 // Multiply the numerator (operand 0) by the magic value.
7273 Q = GetMULHU(Q, MagicFactor);
7274 if (!Q)
7275 return SDValue();
7276
7277 Created.push_back(Elt: Q.getNode());
7278
7279 if (UseNPQ) {
7280 SDValue NPQ = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: N0, N2: Q);
7281 Created.push_back(Elt: NPQ.getNode());
7282
7283 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
7284 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
7285 if (VT.isVector())
7286 NPQ = GetMULHU(NPQ, NPQFactor);
7287 else
7288 NPQ = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NPQ, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT));
7289
7290 Created.push_back(Elt: NPQ.getNode());
7291
7292 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: NPQ, N2: Q);
7293 Created.push_back(Elt: Q.getNode());
7294 }
7295
7296 if (UsePostShift) {
7297 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PostShift);
7298 Created.push_back(Elt: Q.getNode());
7299 }
7300
7301 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7302
7303 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
7304 SDValue IsOne = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N1, RHS: One, Cond: ISD::SETEQ);
7305 return DAG.getSelect(DL: dl, VT, Cond: IsOne, LHS: N0, RHS: Q);
7306}
7307
7308/// If all values in Values that *don't* match the predicate are same 'splat'
7309/// value, then replace all values with that splat value.
7310/// Else, if AlternativeReplacement was provided, then replace all values that
7311/// do match predicate with AlternativeReplacement value.
7312static void
7313turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
7314 std::function<bool(SDValue)> Predicate,
7315 SDValue AlternativeReplacement = SDValue()) {
7316 SDValue Replacement;
7317 // Is there a value for which the Predicate does *NOT* match? What is it?
7318 auto SplatValue = llvm::find_if_not(Range&: Values, P: Predicate);
7319 if (SplatValue != Values.end()) {
7320 // Does Values consist only of SplatValue's and values matching Predicate?
7321 if (llvm::all_of(Range&: Values, P: [Predicate, SplatValue](SDValue Value) {
7322 return Value == *SplatValue || Predicate(Value);
7323 })) // Then we shall replace values matching predicate with SplatValue.
7324 Replacement = *SplatValue;
7325 }
7326 if (!Replacement) {
7327 // Oops, we did not find the "baseline" splat value.
7328 if (!AlternativeReplacement)
7329 return; // Nothing to do.
7330 // Let's replace with provided value then.
7331 Replacement = AlternativeReplacement;
7332 }
7333 std::replace_if(first: Values.begin(), last: Values.end(), pred: Predicate, new_value: Replacement);
7334}
7335
7336/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
7337/// where the divisor and comparison target are constants,
7338/// return a DAG expression that will generate the same comparison result
7339/// using only multiplications, additions and shifts/rotations.
7340/// Ref: "Hacker's Delight" 10-17.
7341SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
7342 SDValue CompTargetNode,
7343 ISD::CondCode Cond,
7344 DAGCombinerInfo &DCI,
7345 const SDLoc &DL) const {
7346 SmallVector<SDNode *, 5> Built;
7347 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7348 DCI, DL, Created&: Built)) {
7349 for (SDNode *N : Built)
7350 DCI.AddToWorklist(N);
7351 return Folded;
7352 }
7353
7354 return SDValue();
7355}
7356
7357SDValue
7358TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
7359 SDValue CompTargetNode, ISD::CondCode Cond,
7360 DAGCombinerInfo &DCI, const SDLoc &DL,
7361 SmallVectorImpl<SDNode *> &Created) const {
7362 // fold (seteq/ne (urem N, D), C) ->
7363 // (setule/ugt (rotr (mul (sub N, C), P), K), Q)
7364 // - D must be constant, with D = D0 * 2^K where D0 is odd
7365 // - P is the multiplicative inverse of D0 modulo 2^W
7366 // - Q = floor(((2^W) - 1) / D)
7367 // where W is the width of the common type of N and D.
7368 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7369 "Only applicable for (in)equality comparisons.");
7370
7371 SelectionDAG &DAG = DCI.DAG;
7372
7373 EVT VT = REMNode.getValueType();
7374 EVT SVT = VT.getScalarType();
7375 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7376 EVT ShSVT = ShVT.getScalarType();
7377
7378 // If MUL is unavailable, we cannot proceed in any case.
7379 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7380 return SDValue();
7381
7382 bool ComparingWithAllZeros = true;
7383 bool AllComparisonsWithNonZerosAreTautological = true;
7384 bool HadTautologicalLanes = false;
7385 bool AllLanesAreTautological = true;
7386 bool HadEvenDivisor = false;
7387 bool AllDivisorsArePowerOfTwo = true;
7388 bool HadTautologicalInvertedLanes = false;
7389 SmallVector<SDValue, 16> PAmts, KAmts, QAmts;
7390
7391 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
7392 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7393 if (CDiv->isZero())
7394 return false;
7395
7396 const APInt &D = CDiv->getAPIntValue();
7397 const APInt &Cmp = CCmp->getAPIntValue();
7398
7399 ComparingWithAllZeros &= Cmp.isZero();
7400
7401 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7402 // if C2 is not less than C1, the comparison is always false.
7403 // But we will only be able to produce the comparison that will give the
7404 // opposive tautological answer. So this lane would need to be fixed up.
7405 bool TautologicalInvertedLane = D.ule(RHS: Cmp);
7406 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
7407
7408 // If all lanes are tautological (either all divisors are ones, or divisor
7409 // is not greater than the constant we are comparing with),
7410 // we will prefer to avoid the fold.
7411 bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
7412 HadTautologicalLanes |= TautologicalLane;
7413 AllLanesAreTautological &= TautologicalLane;
7414
7415 // If we are comparing with non-zero, we need'll need to subtract said
7416 // comparison value from the LHS. But there is no point in doing that if
7417 // every lane where we are comparing with non-zero is tautological..
7418 if (!Cmp.isZero())
7419 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
7420
7421 // Decompose D into D0 * 2^K
7422 unsigned K = D.countr_zero();
7423 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7424 APInt D0 = D.lshr(shiftAmt: K);
7425
7426 // D is even if it has trailing zeros.
7427 HadEvenDivisor |= (K != 0);
7428 // D is a power-of-two if D0 is one.
7429 // If all divisors are power-of-two, we will prefer to avoid the fold.
7430 AllDivisorsArePowerOfTwo &= D0.isOne();
7431
7432 // P = inv(D0, 2^W)
7433 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7434 unsigned W = D.getBitWidth();
7435 APInt P = D0.multiplicativeInverse();
7436 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7437
7438 // Q = floor((2^W - 1) u/ D)
7439 // R = ((2^W - 1) u% D)
7440 APInt Q, R;
7441 APInt::udivrem(LHS: APInt::getAllOnes(numBits: W), RHS: D, Quotient&: Q, Remainder&: R);
7442
7443 // If we are comparing with zero, then that comparison constant is okay,
7444 // else it may need to be one less than that.
7445 if (Cmp.ugt(RHS: R))
7446 Q -= 1;
7447
7448 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7449 "We are expecting that K is always less than all-ones for ShSVT");
7450
7451 // If the lane is tautological the result can be constant-folded.
7452 if (TautologicalLane) {
7453 // Set P and K amount to a bogus values so we can try to splat them.
7454 P = 0;
7455 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7456 // And ensure that comparison constant is tautological,
7457 // it will always compare true/false.
7458 Q.setAllBits();
7459 } else {
7460 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7461 }
7462
7463 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7464 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7465 return true;
7466 };
7467
7468 SDValue N = REMNode.getOperand(i: 0);
7469 SDValue D = REMNode.getOperand(i: 1);
7470
7471 // Collect the values from each element.
7472 if (!ISD::matchBinaryPredicate(LHS: D, RHS: CompTargetNode, Match: BuildUREMPattern))
7473 return SDValue();
7474
7475 // If all lanes are tautological, the result can be constant-folded.
7476 if (AllLanesAreTautological)
7477 return SDValue();
7478
7479 // If this is a urem by a powers-of-two, avoid the fold since it can be
7480 // best implemented as a bit test.
7481 if (AllDivisorsArePowerOfTwo)
7482 return SDValue();
7483
7484 SDValue PVal, KVal, QVal;
7485 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7486 if (HadTautologicalLanes) {
7487 // Try to turn PAmts into a splat, since we don't care about the values
7488 // that are currently '0'. If we can't, just keep '0'`s.
7489 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7490 // Try to turn KAmts into a splat, since we don't care about the values
7491 // that are currently '-1'. If we can't, change them to '0'`s.
7492 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7493 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7494 }
7495
7496 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7497 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7498 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7499 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7500 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
7501 "Expected matchBinaryPredicate to return one element for "
7502 "SPLAT_VECTORs");
7503 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7504 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7505 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7506 } else {
7507 PVal = PAmts[0];
7508 KVal = KAmts[0];
7509 QVal = QAmts[0];
7510 }
7511
7512 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
7513 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::SUB, VT))
7514 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
7515 assert(CompTargetNode.getValueType() == N.getValueType() &&
7516 "Expecting that the types on LHS and RHS of comparisons match.");
7517 N = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N, N2: CompTargetNode);
7518 }
7519
7520 // (mul N, P)
7521 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7522 Created.push_back(Elt: Op0.getNode());
7523
7524 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7525 // divisors as a performance improvement, since rotating by 0 is a no-op.
7526 if (HadEvenDivisor) {
7527 // We need ROTR to do this.
7528 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7529 return SDValue();
7530 // UREM: (rotr (mul N, P), K)
7531 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7532 Created.push_back(Elt: Op0.getNode());
7533 }
7534
7535 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
7536 SDValue NewCC =
7537 DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7538 Cond: ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
7539 if (!HadTautologicalInvertedLanes)
7540 return NewCC;
7541
7542 // If any lanes previously compared always-false, the NewCC will give
7543 // always-true result for them, so we need to fixup those lanes.
7544 // Or the other way around for inequality predicate.
7545 assert(VT.isVector() && "Can/should only get here for vectors.");
7546 Created.push_back(Elt: NewCC.getNode());
7547
7548 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7549 // if C2 is not less than C1, the comparison is always false.
7550 // But we have produced the comparison that will give the
7551 // opposive tautological answer. So these lanes would need to be fixed up.
7552 SDValue TautologicalInvertedChannels =
7553 DAG.getSetCC(DL, VT: SETCCVT, LHS: D, RHS: CompTargetNode, Cond: ISD::SETULE);
7554 Created.push_back(Elt: TautologicalInvertedChannels.getNode());
7555
7556 // NOTE: we avoid letting illegal types through even if we're before legalize
7557 // ops – legalization has a hard time producing good code for this.
7558 if (isOperationLegalOrCustom(Op: ISD::VSELECT, VT: SETCCVT)) {
7559 // If we have a vector select, let's replace the comparison results in the
7560 // affected lanes with the correct tautological result.
7561 SDValue Replacement = DAG.getBoolConstant(V: Cond == ISD::SETEQ ? false : true,
7562 DL, VT: SETCCVT, OpVT: SETCCVT);
7563 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: SETCCVT, N1: TautologicalInvertedChannels,
7564 N2: Replacement, N3: NewCC);
7565 }
7566
7567 // Else, we can just invert the comparison result in the appropriate lanes.
7568 //
7569 // NOTE: see the note above VSELECT above.
7570 if (isOperationLegalOrCustom(Op: ISD::XOR, VT: SETCCVT))
7571 return DAG.getNode(Opcode: ISD::XOR, DL, VT: SETCCVT, N1: NewCC,
7572 N2: TautologicalInvertedChannels);
7573
7574 return SDValue(); // Don't know how to lower.
7575}
7576
7577/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
7578/// where the divisor is constant and the comparison target is zero,
7579/// return a DAG expression that will generate the same comparison result
7580/// using only multiplications, additions and shifts/rotations.
7581/// Ref: "Hacker's Delight" 10-17.
7582SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
7583 SDValue CompTargetNode,
7584 ISD::CondCode Cond,
7585 DAGCombinerInfo &DCI,
7586 const SDLoc &DL) const {
7587 SmallVector<SDNode *, 7> Built;
7588 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7589 DCI, DL, Created&: Built)) {
7590 assert(Built.size() <= 7 && "Max size prediction failed.");
7591 for (SDNode *N : Built)
7592 DCI.AddToWorklist(N);
7593 return Folded;
7594 }
7595
7596 return SDValue();
7597}
7598
7599SDValue
7600TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
7601 SDValue CompTargetNode, ISD::CondCode Cond,
7602 DAGCombinerInfo &DCI, const SDLoc &DL,
7603 SmallVectorImpl<SDNode *> &Created) const {
7604 // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
7605 // Fold:
7606 // (seteq/ne (srem N, D), 0)
7607 // To:
7608 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
7609 //
7610 // - D must be constant, with D = D0 * 2^K where D0 is odd
7611 // - P is the multiplicative inverse of D0 modulo 2^W
7612 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
7613 // - Q = floor((2 * A) / (2^K))
7614 // where W is the width of the common type of N and D.
7615 //
7616 // When D is a power of two (and thus D0 is 1), the normal
7617 // formula for A and Q don't apply, because the derivation
7618 // depends on D not dividing 2^(W-1), and thus theorem ZRS
7619 // does not apply. This specifically fails when N = INT_MIN.
7620 //
7621 // Instead, for power-of-two D, we use:
7622 // - A = 0
7623 // | -> No offset needed. We're effectively treating it the same as urem.
7624 // - Q = 2^(W-K) - 1
7625 // |-> Test that the top K bits are zero after rotation
7626 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7627 "Only applicable for (in)equality comparisons.");
7628
7629 SelectionDAG &DAG = DCI.DAG;
7630
7631 EVT VT = REMNode.getValueType();
7632 EVT SVT = VT.getScalarType();
7633 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7634 EVT ShSVT = ShVT.getScalarType();
7635
7636 // If we are after ops legalization, and MUL is unavailable, we can not
7637 // proceed.
7638 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7639 return SDValue();
7640
7641 // TODO: Could support comparing with non-zero too.
7642 ConstantSDNode *CompTarget = isConstOrConstSplat(N: CompTargetNode);
7643 if (!CompTarget || !CompTarget->isZero())
7644 return SDValue();
7645
7646 bool HadOneDivisor = false;
7647 bool AllDivisorsAreOnes = true;
7648 bool HadEvenDivisor = false;
7649 bool AllDivisorsArePowerOfTwo = true;
7650 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
7651
7652 auto BuildSREMPattern = [&](ConstantSDNode *C) {
7653 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7654 if (C->isZero())
7655 return false;
7656
7657 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
7658
7659 // WARNING: this fold is only valid for positive divisors!
7660 // `rem %X, -C` is equivalent to `rem %X, C`
7661 APInt D = C->getAPIntValue().abs();
7662
7663 // If all divisors are ones, we will prefer to avoid the fold.
7664 HadOneDivisor |= D.isOne();
7665 AllDivisorsAreOnes &= D.isOne();
7666
7667 // Decompose D into D0 * 2^K
7668 unsigned K = D.countr_zero();
7669 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7670 APInt D0 = D.lshr(shiftAmt: K);
7671
7672 // D is even if it has trailing zeros.
7673 HadEvenDivisor |= (K != 0);
7674
7675 // D is a power-of-two if D0 is one. This includes INT_MIN.
7676 // If all divisors are power-of-two, we will prefer to avoid the fold.
7677 AllDivisorsArePowerOfTwo &= D0.isOne();
7678
7679 // P = inv(D0, 2^W)
7680 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7681 unsigned W = D.getBitWidth();
7682 APInt P = D0.multiplicativeInverse();
7683 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7684
7685 // A = floor((2^(W - 1) - 1) / D0) & -2^K
7686 APInt A = APInt::getSignedMaxValue(numBits: W).udiv(RHS: D0);
7687 A.clearLowBits(loBits: K);
7688
7689 // Q = floor((2 * A) / (2^K))
7690 APInt Q = (2 * A).udiv(RHS: APInt::getOneBitSet(numBits: W, BitNo: K));
7691
7692 assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
7693 "We are expecting that A is always less than all-ones for SVT");
7694 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7695 "We are expecting that K is always less than all-ones for ShSVT");
7696
7697 // If D was a power of two, apply the alternate constant derivation.
7698 if (D0.isOne()) {
7699 // A = 0
7700 A = APInt(W, 0);
7701 // - Q = 2^(W-K) - 1
7702 Q = APInt::getLowBitsSet(numBits: W, loBitsSet: W - K);
7703 }
7704
7705 // If the divisor is 1 the result can be constant-folded.
7706 if (D.isOne()) {
7707 // Set P, A and K to a bogus values so we can try to splat them.
7708 P = 0;
7709 A.setAllBits();
7710 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7711
7712 // x ?% 1 == 0 <--> true <--> x u<= -1
7713 Q.setAllBits();
7714 } else {
7715 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7716 }
7717
7718 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7719 AAmts.push_back(Elt: DAG.getConstant(Val: A, DL, VT: SVT));
7720 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7721 return true;
7722 };
7723
7724 SDValue N = REMNode.getOperand(i: 0);
7725 SDValue D = REMNode.getOperand(i: 1);
7726
7727 // Collect the values from each element.
7728 if (!ISD::matchUnaryPredicate(Op: D, Match: BuildSREMPattern))
7729 return SDValue();
7730
7731 // If this is a srem by a one, avoid the fold since it can be constant-folded.
7732 if (AllDivisorsAreOnes)
7733 return SDValue();
7734
7735 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
7736 // since it can be best implemented as a bit test.
7737 if (AllDivisorsArePowerOfTwo)
7738 return SDValue();
7739
7740 SDValue PVal, AVal, KVal, QVal;
7741 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7742 if (HadOneDivisor) {
7743 // Try to turn PAmts into a splat, since we don't care about the values
7744 // that are currently '0'. If we can't, just keep '0'`s.
7745 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7746 // Try to turn AAmts into a splat, since we don't care about the
7747 // values that are currently '-1'. If we can't, change them to '0'`s.
7748 turnVectorIntoSplatVector(Values: AAmts, Predicate: isAllOnesConstant,
7749 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: SVT));
7750 // Try to turn KAmts into a splat, since we don't care about the values
7751 // that are currently '-1'. If we can't, change them to '0'`s.
7752 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7753 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7754 }
7755
7756 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7757 AVal = DAG.getBuildVector(VT, DL, Ops: AAmts);
7758 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7759 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7760 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7761 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
7762 QAmts.size() == 1 &&
7763 "Expected matchUnaryPredicate to return one element for scalable "
7764 "vectors");
7765 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7766 AVal = DAG.getSplatVector(VT, DL, Op: AAmts[0]);
7767 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7768 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7769 } else {
7770 assert(isa<ConstantSDNode>(D) && "Expected a constant");
7771 PVal = PAmts[0];
7772 AVal = AAmts[0];
7773 KVal = KAmts[0];
7774 QVal = QAmts[0];
7775 }
7776
7777 // (mul N, P)
7778 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7779 Created.push_back(Elt: Op0.getNode());
7780
7781 // We need ADD to do this.
7782 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ADD, VT))
7783 return SDValue();
7784
7785 // (add (mul N, P), A)
7786 Op0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: AVal);
7787 Created.push_back(Elt: Op0.getNode());
7788
7789 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7790 // divisors as a performance improvement, since rotating by 0 is a no-op.
7791 if (HadEvenDivisor) {
7792 // We need ROTR to do this.
7793 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7794 return SDValue();
7795 // SREM: (rotr (add (mul N, P), A), K)
7796 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7797 Created.push_back(Elt: Op0.getNode());
7798 }
7799
7800 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
7801 return DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7802 Cond: (Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT);
7803}
7804
7805SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
7806 const DenormalMode &Mode,
7807 SDNodeFlags Flags) const {
7808 SDLoc DL(Op);
7809 EVT VT = Op.getValueType();
7810 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7811 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL, VT);
7812
7813 // This is specifically a check for the handling of denormal inputs, not the
7814 // result.
7815 if (Mode.Input == DenormalMode::PreserveSign ||
7816 Mode.Input == DenormalMode::PositiveZero) {
7817 // Test = X == 0.0
7818 return DAG.getSetCC(DL, VT: CCVT, LHS: Op, RHS: FPZero, Cond: ISD::SETEQ, /*Chain=*/{},
7819 /*Signaling=*/IsSignaling: false, Flags);
7820 }
7821
7822 // Testing it with denormal inputs to avoid wrong estimate.
7823 //
7824 // Test = fabs(X) < SmallestNormal
7825 const fltSemantics &FltSem = VT.getFltSemantics();
7826 APFloat SmallestNorm = APFloat::getSmallestNormalized(Sem: FltSem);
7827 SDValue NormC = DAG.getConstantFP(Val: SmallestNorm, DL, VT);
7828 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: Op, Flags);
7829 return DAG.getSetCC(DL, VT: CCVT, LHS: Fabs, RHS: NormC, Cond: ISD::SETLT, /*Chain=*/{},
7830 /*Signaling=*/IsSignaling: false, Flags);
7831}
7832
7833SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
7834 bool LegalOps, bool OptForSize,
7835 NegatibleCost &Cost,
7836 unsigned Depth) const {
7837 // fneg is removable even if it has multiple uses.
7838 if (Op.getOpcode() == ISD::FNEG) {
7839 Cost = NegatibleCost::Cheaper;
7840 return Op.getOperand(i: 0);
7841 }
7842
7843 // Don't recurse exponentially.
7844 if (Depth > SelectionDAG::MaxRecursionDepth)
7845 return SDValue();
7846
7847 // Pre-increment recursion depth for use in recursive calls.
7848 ++Depth;
7849 const SDNodeFlags Flags = Op->getFlags();
7850 EVT VT = Op.getValueType();
7851 unsigned Opcode = Op.getOpcode();
7852
7853 // Don't allow anything with multiple uses unless we know it is free.
7854 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
7855 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
7856 isFPExtFree(DestVT: VT, SrcVT: Op.getOperand(i: 0).getValueType());
7857 if (!IsFreeExtend)
7858 return SDValue();
7859 }
7860
7861 auto RemoveDeadNode = [&](SDValue N) {
7862 if (N && N.getNode()->use_empty())
7863 DAG.RemoveDeadNode(N: N.getNode());
7864 };
7865
7866 SDLoc DL(Op);
7867
7868 // Because getNegatedExpression can delete nodes we need a handle to keep
7869 // temporary nodes alive in case the recursion manages to create an identical
7870 // node.
7871 std::list<HandleSDNode> Handles;
7872
7873 switch (Opcode) {
7874 case ISD::ConstantFP: {
7875 // Don't invert constant FP values after legalization unless the target says
7876 // the negated constant is legal.
7877 bool IsOpLegal =
7878 isOperationLegal(Op: ISD::ConstantFP, VT) ||
7879 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF()), VT,
7880 ForCodeSize: OptForSize);
7881
7882 if (LegalOps && !IsOpLegal)
7883 break;
7884
7885 APFloat V = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
7886 V.changeSign();
7887 SDValue CFP = DAG.getConstantFP(Val: V, DL, VT);
7888
7889 // If we already have the use of the negated floating constant, it is free
7890 // to negate it even it has multiple uses.
7891 if (!Op.hasOneUse() && CFP.use_empty())
7892 break;
7893 Cost = NegatibleCost::Neutral;
7894 return CFP;
7895 }
7896 case ISD::SPLAT_VECTOR: {
7897 // fold splat_vector(fneg(X)) -> splat_vector(-X)
7898 SDValue X = Op.getOperand(i: 0);
7899 if (!isOperationLegal(Op: ISD::SPLAT_VECTOR, VT))
7900 break;
7901
7902 SDValue NegX = getCheaperNegatedExpression(Op: X, DAG, LegalOps, OptForSize);
7903 if (!NegX)
7904 break;
7905 Cost = NegatibleCost::Cheaper;
7906 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: NegX);
7907 }
7908 case ISD::BUILD_VECTOR: {
7909 // Only permit BUILD_VECTOR of constants.
7910 if (llvm::any_of(Range: Op->op_values(), P: [&](SDValue N) {
7911 return !N.isUndef() && !isa<ConstantFPSDNode>(Val: N);
7912 }))
7913 break;
7914
7915 bool IsOpLegal =
7916 (isOperationLegal(Op: ISD::ConstantFP, VT) &&
7917 isOperationLegal(Op: ISD::BUILD_VECTOR, VT)) ||
7918 llvm::all_of(Range: Op->op_values(), P: [&](SDValue N) {
7919 return N.isUndef() ||
7920 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: N)->getValueAPF()), VT,
7921 ForCodeSize: OptForSize);
7922 });
7923
7924 if (LegalOps && !IsOpLegal)
7925 break;
7926
7927 SmallVector<SDValue, 4> Ops;
7928 for (SDValue C : Op->op_values()) {
7929 if (C.isUndef()) {
7930 Ops.push_back(Elt: C);
7931 continue;
7932 }
7933 APFloat V = cast<ConstantFPSDNode>(Val&: C)->getValueAPF();
7934 V.changeSign();
7935 Ops.push_back(Elt: DAG.getConstantFP(Val: V, DL, VT: C.getValueType()));
7936 }
7937 Cost = NegatibleCost::Neutral;
7938 return DAG.getBuildVector(VT, DL, Ops);
7939 }
7940 case ISD::FADD: {
7941 if (!Flags.hasNoSignedZeros())
7942 break;
7943
7944 // After operation legalization, it might not be legal to create new FSUBs.
7945 if (LegalOps && !isOperationLegalOrCustom(Op: ISD::FSUB, VT))
7946 break;
7947 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7948
7949 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
7950 NegatibleCost CostX = NegatibleCost::Expensive;
7951 SDValue NegX =
7952 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
7953 // Prevent this node from being deleted by the next call.
7954 if (NegX)
7955 Handles.emplace_back(args&: NegX);
7956
7957 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
7958 NegatibleCost CostY = NegatibleCost::Expensive;
7959 SDValue NegY =
7960 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
7961
7962 // We're done with the handles.
7963 Handles.clear();
7964
7965 // Negate the X if its cost is less or equal than Y.
7966 if (NegX && (CostX <= CostY)) {
7967 Cost = CostX;
7968 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegX, N2: Y, Flags);
7969 if (NegY != N)
7970 RemoveDeadNode(NegY);
7971 return N;
7972 }
7973
7974 // Negate the Y if it is not expensive.
7975 if (NegY) {
7976 Cost = CostY;
7977 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegY, N2: X, Flags);
7978 if (NegX != N)
7979 RemoveDeadNode(NegX);
7980 return N;
7981 }
7982 break;
7983 }
7984 case ISD::FSUB: {
7985 // We can't turn -(A-B) into B-A when we honor signed zeros.
7986 if (!Flags.hasNoSignedZeros())
7987 break;
7988
7989 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7990 // fold (fneg (fsub 0, Y)) -> Y
7991 if (ConstantFPSDNode *C = isConstOrConstSplatFP(N: X, /*AllowUndefs*/ true))
7992 if (C->isZero()) {
7993 Cost = NegatibleCost::Cheaper;
7994 return Y;
7995 }
7996
7997 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
7998 Cost = NegatibleCost::Neutral;
7999 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: X, Flags);
8000 }
8001 case ISD::FMUL:
8002 case ISD::FDIV: {
8003 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
8004
8005 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
8006 NegatibleCost CostX = NegatibleCost::Expensive;
8007 SDValue NegX =
8008 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8009 // Prevent this node from being deleted by the next call.
8010 if (NegX)
8011 Handles.emplace_back(args&: NegX);
8012
8013 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
8014 NegatibleCost CostY = NegatibleCost::Expensive;
8015 SDValue NegY =
8016 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8017
8018 // We're done with the handles.
8019 Handles.clear();
8020
8021 // Negate the X if its cost is less or equal than Y.
8022 if (NegX && (CostX <= CostY)) {
8023 Cost = CostX;
8024 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, Flags);
8025 if (NegY != N)
8026 RemoveDeadNode(NegY);
8027 return N;
8028 }
8029
8030 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
8031 if (auto *C = isConstOrConstSplatFP(N: Op.getOperand(i: 1)))
8032 if (C->isExactlyValue(V: 2.0) && Op.getOpcode() == ISD::FMUL)
8033 break;
8034
8035 // Negate the Y if it is not expensive.
8036 if (NegY) {
8037 Cost = CostY;
8038 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, Flags);
8039 if (NegX != N)
8040 RemoveDeadNode(NegX);
8041 return N;
8042 }
8043 break;
8044 }
8045 case ISD::FMA:
8046 case ISD::FMULADD:
8047 case ISD::FMAD: {
8048 if (!Flags.hasNoSignedZeros())
8049 break;
8050
8051 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), Z = Op.getOperand(i: 2);
8052 NegatibleCost CostZ = NegatibleCost::Expensive;
8053 SDValue NegZ =
8054 getNegatedExpression(Op: Z, DAG, LegalOps, OptForSize, Cost&: CostZ, Depth);
8055 // Give up if fail to negate the Z.
8056 if (!NegZ)
8057 break;
8058
8059 // Prevent this node from being deleted by the next two calls.
8060 Handles.emplace_back(args&: NegZ);
8061
8062 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
8063 NegatibleCost CostX = NegatibleCost::Expensive;
8064 SDValue NegX =
8065 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
8066 // Prevent this node from being deleted by the next call.
8067 if (NegX)
8068 Handles.emplace_back(args&: NegX);
8069
8070 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
8071 NegatibleCost CostY = NegatibleCost::Expensive;
8072 SDValue NegY =
8073 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
8074
8075 // We're done with the handles.
8076 Handles.clear();
8077
8078 // Negate the X if its cost is less or equal than Y.
8079 if (NegX && (CostX <= CostY)) {
8080 Cost = std::min(a: CostX, b: CostZ);
8081 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, N3: NegZ, Flags);
8082 if (NegY != N)
8083 RemoveDeadNode(NegY);
8084 return N;
8085 }
8086
8087 // Negate the Y if it is not expensive.
8088 if (NegY) {
8089 Cost = std::min(a: CostY, b: CostZ);
8090 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, N3: NegZ, Flags);
8091 if (NegX != N)
8092 RemoveDeadNode(NegX);
8093 return N;
8094 }
8095 break;
8096 }
8097
8098 case ISD::FP_EXTEND:
8099 case ISD::FSIN:
8100 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
8101 OptForSize, Cost, Depth))
8102 return DAG.getNode(Opcode, DL, VT, Operand: NegV);
8103 break;
8104 case ISD::FP_ROUND:
8105 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
8106 OptForSize, Cost, Depth))
8107 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NegV, N2: Op.getOperand(i: 1));
8108 break;
8109 case ISD::SELECT:
8110 case ISD::VSELECT: {
8111 // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
8112 // iff at least one cost is cheaper and the other is neutral/cheaper
8113 SDValue LHS = Op.getOperand(i: 1);
8114 NegatibleCost CostLHS = NegatibleCost::Expensive;
8115 SDValue NegLHS =
8116 getNegatedExpression(Op: LHS, DAG, LegalOps, OptForSize, Cost&: CostLHS, Depth);
8117 if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
8118 RemoveDeadNode(NegLHS);
8119 break;
8120 }
8121
8122 // Prevent this node from being deleted by the next call.
8123 Handles.emplace_back(args&: NegLHS);
8124
8125 SDValue RHS = Op.getOperand(i: 2);
8126 NegatibleCost CostRHS = NegatibleCost::Expensive;
8127 SDValue NegRHS =
8128 getNegatedExpression(Op: RHS, DAG, LegalOps, OptForSize, Cost&: CostRHS, Depth);
8129
8130 // We're done with the handles.
8131 Handles.clear();
8132
8133 if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
8134 (CostLHS != NegatibleCost::Cheaper &&
8135 CostRHS != NegatibleCost::Cheaper)) {
8136 RemoveDeadNode(NegLHS);
8137 RemoveDeadNode(NegRHS);
8138 break;
8139 }
8140
8141 Cost = std::min(a: CostLHS, b: CostRHS);
8142 return DAG.getSelect(DL, VT, Cond: Op.getOperand(i: 0), LHS: NegLHS, RHS: NegRHS);
8143 }
8144 }
8145
8146 return SDValue();
8147}
8148
8149//===----------------------------------------------------------------------===//
8150// Legalization Utilities
8151//===----------------------------------------------------------------------===//
8152
8153bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
8154 SDValue LHS, SDValue RHS,
8155 SmallVectorImpl<SDValue> &Result,
8156 EVT HiLoVT, SelectionDAG &DAG,
8157 MulExpansionKind Kind, SDValue LL,
8158 SDValue LH, SDValue RL, SDValue RH) const {
8159 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
8160 Opcode == ISD::SMUL_LOHI);
8161
8162 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
8163 isOperationLegalOrCustom(Op: ISD::MULHS, VT: HiLoVT);
8164 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
8165 isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT);
8166 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8167 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: HiLoVT);
8168 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8169 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT);
8170
8171 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
8172 return false;
8173
8174 unsigned OuterBitSize = VT.getScalarSizeInBits();
8175 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
8176
8177 // LL, LH, RL, and RH must be either all NULL or all set to a value.
8178 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
8179 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
8180
8181 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
8182 bool Signed) -> bool {
8183 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
8184 SDVTList VTs = DAG.getVTList(VT1: HiLoVT, VT2: HiLoVT);
8185 Lo = DAG.getNode(Opcode: Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, DL: dl, VTList: VTs, N1: L, N2: R);
8186 Hi = Lo.getValue(R: 1);
8187 return true;
8188 }
8189 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
8190 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: L, N2: R);
8191 Hi = DAG.getNode(Opcode: Signed ? ISD::MULHS : ISD::MULHU, DL: dl, VT: HiLoVT, N1: L, N2: R);
8192 return true;
8193 }
8194 return false;
8195 };
8196
8197 SDValue Lo, Hi;
8198
8199 if (!LL.getNode() && !RL.getNode() &&
8200 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8201 LL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LHS);
8202 RL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RHS);
8203 }
8204
8205 if (!LL.getNode())
8206 return false;
8207
8208 APInt HighMask = APInt::getHighBitsSet(numBits: OuterBitSize, hiBitsSet: InnerBitSize);
8209 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask) &&
8210 DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask)) {
8211 // The inputs are both zero-extended.
8212 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
8213 Result.push_back(Elt: Lo);
8214 Result.push_back(Elt: Hi);
8215 if (Opcode != ISD::MUL) {
8216 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8217 Result.push_back(Elt: Zero);
8218 Result.push_back(Elt: Zero);
8219 }
8220 return true;
8221 }
8222 }
8223
8224 if (!VT.isVector() && Opcode == ISD::MUL &&
8225 DAG.ComputeMaxSignificantBits(Op: LHS) <= InnerBitSize &&
8226 DAG.ComputeMaxSignificantBits(Op: RHS) <= InnerBitSize) {
8227 // The input values are both sign-extended.
8228 // TODO non-MUL case?
8229 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
8230 Result.push_back(Elt: Lo);
8231 Result.push_back(Elt: Hi);
8232 return true;
8233 }
8234 }
8235
8236 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
8237 SDValue Shift = DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
8238
8239 if (!LH.getNode() && !RH.getNode() &&
8240 isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
8241 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8242 LH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS, N2: Shift);
8243 LH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LH);
8244 RH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: RHS, N2: Shift);
8245 RH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RH);
8246 }
8247
8248 if (!LH.getNode())
8249 return false;
8250
8251 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
8252 return false;
8253
8254 Result.push_back(Elt: Lo);
8255
8256 if (Opcode == ISD::MUL) {
8257 RH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LL, N2: RH);
8258 LH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LH, N2: RL);
8259 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: RH);
8260 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: LH);
8261 Result.push_back(Elt: Hi);
8262 return true;
8263 }
8264
8265 // Compute the full width result.
8266 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
8267 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
8268 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8269 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
8270 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi);
8271 };
8272
8273 SDValue Next = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8274 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
8275 return false;
8276
8277 // This is effectively the add part of a multiply-add of half-sized operands,
8278 // so it cannot overflow.
8279 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8280
8281 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
8282 return false;
8283
8284 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8285 EVT BoolType = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
8286
8287 bool UseGlue = (isOperationLegalOrCustom(Op: ISD::ADDC, VT) &&
8288 isOperationLegalOrCustom(Op: ISD::ADDE, VT));
8289 if (UseGlue)
8290 Next = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Glue), N1: Next,
8291 N2: Merge(Lo, Hi));
8292 else
8293 Next = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolType), N1: Next,
8294 N2: Merge(Lo, Hi), N3: DAG.getConstant(Val: 0, DL: dl, VT: BoolType));
8295
8296 SDValue Carry = Next.getValue(R: 1);
8297 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8298 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8299
8300 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
8301 return false;
8302
8303 if (UseGlue)
8304 Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::Glue), N1: Hi, N2: Zero,
8305 N3: Carry);
8306 else
8307 Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: BoolType), N1: Hi,
8308 N2: Zero, N3: Carry);
8309
8310 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8311
8312 if (Opcode == ISD::SMUL_LOHI) {
8313 SDValue NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8314 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RL));
8315 Next = DAG.getSelectCC(DL: dl, LHS: LH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8316
8317 NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8318 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LL));
8319 Next = DAG.getSelectCC(DL: dl, LHS: RH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8320 }
8321
8322 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8323 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8324 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8325 return true;
8326}
8327
8328bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
8329 SelectionDAG &DAG, MulExpansionKind Kind,
8330 SDValue LL, SDValue LH, SDValue RL,
8331 SDValue RH) const {
8332 SmallVector<SDValue, 2> Result;
8333 bool Ok = expandMUL_LOHI(Opcode: N->getOpcode(), VT: N->getValueType(ResNo: 0), dl: SDLoc(N),
8334 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Result, HiLoVT,
8335 DAG, Kind, LL, LH, RL, RH);
8336 if (Ok) {
8337 assert(Result.size() == 2);
8338 Lo = Result[0];
8339 Hi = Result[1];
8340 }
8341 return Ok;
8342}
8343
8344// Optimize unsigned division or remainder by constants for types twice as large
8345// as a legal VT.
8346//
8347// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
8348// can be computed
8349// as:
8350// Sum = __builtin_uadd_overflow(Lo, High, &Sum);
8351// Remainder = Sum % Constant;
8352//
8353// If (1 << (BitWidth / 2)) % Constant != 1, we can search for a smaller value
8354// W such that W != (BitWidth / 2) and (1 << W) % Constant == 1. We can break
8355// High:Low into 3 chunks of W bits and compute remainder as
8356// Sum = Chunk0 + Chunk1 + Chunk2;
8357// Remainder = Sum % Constant;
8358//
8359// This is based on "Remainder by Summing Digits" from Hacker's Delight.
8360//
8361// For division, we can compute the remainder using the algorithm described
8362// above, subtract it from the dividend to get an exact multiple of Constant.
8363// Then multiply that exact multiply by the multiplicative inverse modulo
8364// (1 << (BitWidth / 2)) to get the quotient.
8365
8366// If Constant is even, we can shift right the dividend and the divisor by the
8367// number of trailing zeros in Constant before applying the remainder algorithm.
8368// If we're after the quotient, we can subtract this value from the shifted
8369// dividend and multiply by the multiplicative inverse of the shifted divisor.
8370// If we want the remainder, we shift the value left by the number of trailing
8371// zeros and add the bits that were shifted out of the dividend.
8372bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
8373 SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
8374 SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8375 unsigned Opcode = N->getOpcode();
8376 EVT VT = N->getValueType(ResNo: 0);
8377
8378 unsigned BitWidth = Divisor.getBitWidth();
8379 unsigned HBitWidth = BitWidth / 2;
8380 assert(VT.getScalarSizeInBits() == BitWidth &&
8381 HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");
8382
8383 // If the divisor is even, shift it until it becomes odd.
8384 unsigned TrailingZeros = 0;
8385 if (!Divisor[0]) {
8386 TrailingZeros = Divisor.countr_zero();
8387 Divisor.lshrInPlace(ShiftAmt: TrailingZeros);
8388 }
8389
8390 // After removing trailing zeros, the divisor needs to be less than
8391 // (1 << HBitWidth).
8392 APInt HalfMaxPlus1 = APInt::getOneBitSet(numBits: BitWidth, BitNo: HBitWidth);
8393 if (Divisor.uge(RHS: HalfMaxPlus1))
8394 return false;
8395
8396 // Look for the largest chunk width W such that (1 << W) % Divisor == 1 or
8397 // (1 << W) % Divisor == -1.
8398 unsigned BestChunkWidth = 0, AltChunkWidth = 0;
8399 for (unsigned I = HBitWidth, E = HBitWidth / 2; I > E; --I) {
8400 // Skip HBitWidth-1, it doesn't have enough bits for carries.
8401 if (I == HBitWidth - 1)
8402 continue;
8403
8404 APInt Mod = APInt::getOneBitSet(numBits: Divisor.getBitWidth(), BitNo: I).urem(RHS: Divisor);
8405
8406 if (Mod.isOne()) {
8407 BestChunkWidth = I;
8408 break;
8409 }
8410
8411 // We have an alternate strategy for Remainder == Divisor - 1.
8412 // FIXME: Support HBitWidth.
8413 if (I != HBitWidth && Mod == Divisor - 1)
8414 AltChunkWidth = I;
8415 }
8416
8417 bool Alternate = false;
8418 if (!BestChunkWidth) {
8419 if (!AltChunkWidth)
8420 return false;
8421 Alternate = true;
8422 BestChunkWidth = AltChunkWidth;
8423 }
8424
8425 SDLoc dl(N);
8426
8427 assert(!LL == !LH && "Expected both input halves or no input halves!");
8428 if (!LL)
8429 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8430
8431 bool HasFSHR = isOperationLegal(Op: ISD::FSHR, VT: HiLoVT);
8432
8433 auto GetFSHR = [&](SDValue Lo, SDValue Hi, unsigned ShiftAmt) {
8434 assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
8435 if (HasFSHR)
8436 return DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: HiLoVT, N1: Hi, N2: Lo,
8437 N3: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8438 return DAG.getNode(
8439 Opcode: ISD::OR, DL: dl, VT: HiLoVT,
8440 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Lo,
8441 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl)),
8442 N2: DAG.getNode(
8443 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: Hi,
8444 N2: DAG.getShiftAmountConstant(Val: HBitWidth - ShiftAmt, VT: HiLoVT, DL: dl)));
8445 };
8446
8447 // Helper to perform a right shift on a 128-bit value split into two halves.
8448 // Handles shifts >= HBitWidth by moving Hi to Lo and shifting Hi.
8449 auto ShiftRight = [&](SDValue &Lo, SDValue &Hi, unsigned ShiftAmt) {
8450 if (ShiftAmt == 0)
8451 return;
8452 if (ShiftAmt < HBitWidth) {
8453 Lo = GetFSHR(Lo, Hi, ShiftAmt);
8454 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8455 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8456 } else if (ShiftAmt == HBitWidth) {
8457 Lo = Hi;
8458 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8459 } else {
8460 Lo = DAG.getNode(
8461 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8462 N2: DAG.getShiftAmountConstant(Val: ShiftAmt - HBitWidth, VT: HiLoVT, DL: dl));
8463 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8464 }
8465 };
8466
8467 // Shift the input by the number of TrailingZeros in the divisor. The
8468 // shifted out bits will be added to the remainder later.
8469 SDValue PartialRemL, PartialRemH;
8470 if (TrailingZeros && Opcode != ISD::UDIV) {
8471 // Save the shifted off bits if we need the remainder.
8472 if (TrailingZeros < HBitWidth) {
8473 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros);
8474 PartialRemL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LL,
8475 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8476 } else if (TrailingZeros == HBitWidth) {
8477 // All of LL is part of the remainder.
8478 PartialRemL = LL;
8479 } else {
8480 // TrailingZeros > HBitWidth: LL and part of LH are the remainder.
8481 PartialRemL = LL;
8482 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros - HBitWidth);
8483 PartialRemH = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LH,
8484 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8485 }
8486 }
8487
8488 SDValue Sum;
8489 // If BestChunkWidth is HBitWidth add low and high half. If there is a carry
8490 // out, add that to the final sum.
8491 if (BestChunkWidth == HBitWidth) {
8492 assert(!Alternate);
8493 // Shift LH:LL right if there were trailing zeros in the divisor.
8494 ShiftRight(LL, LH, TrailingZeros);
8495
8496 // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
8497 EVT SetCCType =
8498 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: HiLoVT);
8499 if (isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT: HiLoVT)) {
8500 SDVTList VTList = DAG.getVTList(VT1: HiLoVT, VT2: SetCCType);
8501 Sum = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, N1: LL, N2: LH);
8502 Sum = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, N1: Sum,
8503 N2: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT), N3: Sum.getValue(R: 1));
8504 } else {
8505 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: LL, N2: LH);
8506 SDValue Carry = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LL, Cond: ISD::SETULT);
8507 // If the boolean for the target is 0 or 1, we can add the setcc result
8508 // directly.
8509 if (getBooleanContents(Type: HiLoVT) ==
8510 TargetLoweringBase::ZeroOrOneBooleanContent)
8511 Carry = DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: HiLoVT);
8512 else
8513 Carry = DAG.getSelect(DL: dl, VT: HiLoVT, Cond: Carry, LHS: DAG.getConstant(Val: 1, DL: dl, VT: HiLoVT),
8514 RHS: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT));
8515 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum, N2: Carry);
8516 }
8517 } else {
8518 // Otherwise split into multple chunks and add them together. We chose
8519 // BestChunkWidth so that the sum will not overflow.
8520 SDValue Mask = DAG.getConstant(
8521 Val: APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: BestChunkWidth), DL: dl, VT: HiLoVT);
8522
8523 for (unsigned I = 0; I < BitWidth - TrailingZeros; I += BestChunkWidth) {
8524 // If there were trailing zeros in the divisor, increase the shift amount.
8525 unsigned Shift = I + TrailingZeros;
8526 SDValue Chunk;
8527 if (Shift == 0)
8528 Chunk = LL;
8529 else if (Shift >= HBitWidth)
8530 Chunk = DAG.getNode(
8531 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: LH,
8532 N2: DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL: dl));
8533 else
8534 Chunk = GetFSHR(LL, LH, Shift);
8535 // If we're on the last chunk, we don't need an AND.
8536 if (I + BestChunkWidth < BitWidth - TrailingZeros)
8537 Chunk = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: Chunk, N2: Mask);
8538 if (!Sum) {
8539 Sum = Chunk;
8540 } else {
8541 // For Alternate, we need to subtract odd chunks.
8542 unsigned ChunkNum = I / BestChunkWidth;
8543 unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ? ISD::SUB : ISD::ADD;
8544 Sum = DAG.getNode(Opcode: Opc, DL: dl, VT: HiLoVT, N1: Sum, N2: Chunk);
8545 }
8546 }
8547
8548 // For Alternate, the sum may be negative, but we need a positive sum. We
8549 // can increase it by a multiple of the divisor to make it positive. For 3
8550 // chunks the largest negative value is -(2^BestChunkWidth - 1). For 4
8551 // chunks, it's 2*-(2^BestChunkWidth - 1). We know that 2^BestChunkWidth + 1
8552 // is a multiple of the divisor. Add that 1 or 2 times to make the sum
8553 // positive.
8554 if (Alternate) {
8555 unsigned NumChunks = divideCeil(Numerator: BitWidth - TrailingZeros, Denominator: BestChunkWidth);
8556 assert(NumChunks <= 4);
8557
8558 APInt Adjust = APInt::getOneBitSet(numBits: HBitWidth, BitNo: BestChunkWidth);
8559 Adjust.setBit(0);
8560 // If there are 4 chunks, we need to adjust twice.
8561 if (NumChunks == 4)
8562 Adjust <<= 1;
8563 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum,
8564 N2: DAG.getConstant(Val: Adjust, DL: dl, VT: HiLoVT));
8565 }
8566 }
8567
8568 // Perform a HiLoVT urem on the Sum using truncated divisor.
8569 SDValue RemL =
8570 DAG.getNode(Opcode: ISD::UREM, DL: dl, VT: HiLoVT, N1: Sum,
8571 N2: DAG.getConstant(Val: Divisor.trunc(width: HBitWidth), DL: dl, VT: HiLoVT));
8572 SDValue RemH = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8573
8574 if (Opcode != ISD::UREM) {
8575 // If we didn't shift LH/LR earlier, do it now.
8576 if (BestChunkWidth != HBitWidth)
8577 ShiftRight(LL, LH, TrailingZeros);
8578
8579 // Subtract the remainder from the shifted dividend.
8580 SDValue Dividend = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: LL, N2: LH);
8581 SDValue Rem = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: RemL, N2: RemH);
8582
8583 Dividend = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Rem);
8584
8585 // Multiply by the multiplicative inverse of the divisor modulo
8586 // (1 << BitWidth).
8587 APInt MulFactor = Divisor.multiplicativeInverse();
8588
8589 SDValue Quotient = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Dividend,
8590 N2: DAG.getConstant(Val: MulFactor, DL: dl, VT));
8591
8592 // Split the quotient into low and high parts.
8593 SDValue QuotL, QuotH;
8594 std::tie(args&: QuotL, args&: QuotH) = DAG.SplitScalar(N: Quotient, DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8595 Result.push_back(Elt: QuotL);
8596 Result.push_back(Elt: QuotH);
8597 }
8598
8599 if (Opcode != ISD::UDIV) {
8600 // If we shifted the input, shift the remainder left and add the bits we
8601 // shifted off the input.
8602 if (TrailingZeros) {
8603 if (TrailingZeros < HBitWidth) {
8604 // Shift RemH:RemL left by TrailingZeros.
8605 // RemH gets the high bits shifted out of RemL.
8606 RemH = DAG.getNode(
8607 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: RemL,
8608 N2: DAG.getShiftAmountConstant(Val: HBitWidth - TrailingZeros, VT: HiLoVT, DL: dl));
8609 RemL =
8610 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8611 N2: DAG.getShiftAmountConstant(Val: TrailingZeros, VT: HiLoVT, DL: dl));
8612 // OR in the partial remainder.
8613 RemL = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemL, N2: PartialRemL,
8614 Flags: SDNodeFlags::Disjoint);
8615 } else if (TrailingZeros == HBitWidth) {
8616 // Shift left by exactly HBitWidth: RemH becomes RemL, RemL becomes
8617 // PartialRemL.
8618 RemH = RemL;
8619 RemL = PartialRemL;
8620 } else {
8621 // Shift left by more than HBitWidth.
8622 RemH = DAG.getNode(
8623 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8624 N2: DAG.getShiftAmountConstant(Val: TrailingZeros - HBitWidth, VT: HiLoVT, DL: dl));
8625 RemH = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemH, N2: PartialRemH,
8626 Flags: SDNodeFlags::Disjoint);
8627 RemL = PartialRemL;
8628 }
8629 }
8630 Result.push_back(Elt: RemL);
8631 Result.push_back(Elt: RemH);
8632 }
8633
8634 return true;
8635}
8636
8637bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
8638 SDNode *N, const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
8639 EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8640
8641 SDValue N0 = N->getOperand(Num: 0);
8642 EVT VT = N0->getValueType(ResNo: 0);
8643 SDLoc DL{N};
8644
8645 assert(!Divisor.isOne() && "Magic algorithm does not work for division by 1");
8646
8647 // This helper creates a MUL_LOHI of the pair (LL, LH) by a constant.
8648 auto MakeMUL_LOHIByConst = [&](unsigned Opc, SDValue LL, SDValue LH,
8649 const APInt &Const,
8650 SmallVectorImpl<SDValue> &Result) {
8651 SDValue LHS = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT, N1: LL, N2: LH);
8652 SDValue RHS = DAG.getConstant(Val: Const, DL, VT);
8653 auto [RL, RH] = DAG.SplitScalar(N: RHS, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8654 return expandMUL_LOHI(Opcode: Opc, VT, dl: DL, LHS, RHS, Result, HiLoVT, DAG,
8655 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
8656 LL, LH, RL, RH);
8657 };
8658
8659 // This helper creates an ADD/SUB of the pairs (LL, LH) and (RL, RH).
8660 auto MakeAddSubLong = [&](unsigned Opc, SDValue LL, SDValue LH, SDValue RL,
8661 SDValue RH) {
8662 SDValue AddSubNode =
8663 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO : ISD::USUBO, DL,
8664 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LL, N2: RL);
8665 SDValue OutL = AddSubNode.getValue(R: 0);
8666 SDValue Overflow = AddSubNode.getValue(R: 1);
8667 SDValue AddSubWithOverflow =
8668 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, DL,
8669 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LH, N2: RH, N3: Overflow);
8670 SDValue OutH = AddSubWithOverflow.getValue(R: 0);
8671 return std::make_pair(x&: OutL, y&: OutH);
8672 };
8673
8674 // This helper creates a SRL of the pair (LL, LH) by Shift.
8675 auto MakeSRLLong = [&](SDValue LL, SDValue LH, unsigned Shift) {
8676 unsigned HBitWidth = HiLoVT.getScalarSizeInBits();
8677 if (Shift < HBitWidth) {
8678 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift, VT: HiLoVT, DL);
8679 SDValue ResL = DAG.getNode(Opcode: ISD::FSHR, DL, VT: HiLoVT, N1: LH, N2: LL, N3: ShAmt);
8680 SDValue ResH = DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt);
8681 return std::make_pair(x&: ResL, y&: ResH);
8682 }
8683 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HiLoVT);
8684 if (Shift == HBitWidth)
8685 return std::make_pair(x&: LH, y&: Zero);
8686 assert(Shift - HBitWidth < HBitWidth &&
8687 "We shouldn't generate an undefined shift");
8688 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL);
8689 return std::make_pair(x: DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt), y&: Zero);
8690 };
8691
8692 // Knowledge of leading zeros may help to reduce the multiplier.
8693 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
8694
8695 UnsignedDivisionByConstantInfo Magics = UnsignedDivisionByConstantInfo::get(
8696 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
8697
8698 assert(!LL == !LH && "Expected both input halves or no input halves!");
8699 if (!LL)
8700 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N0, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8701 SDValue QL = LL;
8702 SDValue QH = LH;
8703 if (Magics.PreShift != 0)
8704 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PreShift);
8705
8706 SmallVector<SDValue, 4> UMulResult;
8707 if (!MakeMUL_LOHIByConst(ISD::UMUL_LOHI, QL, QH, Magics.Magic, UMulResult))
8708 return false;
8709
8710 QL = UMulResult[2];
8711 QH = UMulResult[3];
8712
8713 if (Magics.IsAdd) {
8714 auto [NPQL, NPQH] = MakeAddSubLong(ISD::SUB, LL, LH, QL, QH);
8715 std::tie(args&: NPQL, args&: NPQH) = MakeSRLLong(NPQL, NPQH, 1);
8716 std::tie(args&: QL, args&: QH) = MakeAddSubLong(ISD::ADD, NPQL, NPQH, QL, QH);
8717 }
8718
8719 if (Magics.PostShift != 0)
8720 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PostShift);
8721
8722 unsigned Opcode = N->getOpcode();
8723 if (Opcode != ISD::UREM) {
8724 Result.push_back(Elt: QL);
8725 Result.push_back(Elt: QH);
8726 }
8727
8728 if (Opcode != ISD::UDIV) {
8729 SmallVector<SDValue, 2> MulResult;
8730 if (!MakeMUL_LOHIByConst(ISD::MUL, QL, QH, Divisor, MulResult))
8731 return false;
8732
8733 assert(MulResult.size() == 2);
8734
8735 auto [RemL, RemH] =
8736 MakeAddSubLong(ISD::SUB, LL, LH, MulResult[0], MulResult[1]);
8737
8738 Result.push_back(Elt: RemL);
8739 Result.push_back(Elt: RemH);
8740 }
8741
8742 return true;
8743}
8744
8745bool TargetLowering::expandDIVREMByConstant(SDNode *N,
8746 SmallVectorImpl<SDValue> &Result,
8747 EVT HiLoVT, SelectionDAG &DAG,
8748 SDValue LL, SDValue LH) const {
8749 unsigned Opcode = N->getOpcode();
8750
8751 // TODO: Support signed division/remainder.
8752 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
8753 return false;
8754 assert(
8755 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
8756 "Unexpected opcode");
8757
8758 auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8759 if (!CN)
8760 return false;
8761
8762 APInt Divisor = CN->getAPIntValue();
8763
8764 // The generated half-width UREM is normally optimized using high multiply.
8765 // If the wide UREM libcall is unavailable, a legal or custom half-width
8766 // UDIVREM can lower it instead.
8767 bool CanDecomposeUREMWithoutMulHi =
8768 Opcode == ISD::UREM &&
8769 getLibcallImpl(Call: RTLIB::getUREM(VT: N->getValueType(ResNo: 0))) ==
8770 RTLIB::Unsupported &&
8771 isOperationLegalOrCustom(Op: ISD::UDIVREM, VT: HiLoVT);
8772 if (!CanDecomposeUREMWithoutMulHi &&
8773 !isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT) &&
8774 !isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT))
8775 return false;
8776
8777 // Prefer the smaller libcall when one is available.
8778 if (DAG.shouldOptForSize() && !CanDecomposeUREMWithoutMulHi)
8779 return false;
8780
8781 // Early out for 0 or 1 divisors.
8782 if (Divisor.ule(RHS: 1))
8783 return false;
8784
8785 if (expandUDIVREMByConstantViaUREMDecomposition(N, Divisor, Result, HiLoVT,
8786 DAG, LL, LH))
8787 return true;
8788
8789 if (expandUDIVREMByConstantViaUMulHiMagic(N, Divisor, Result, HiLoVT, DAG, LL,
8790 LH))
8791 return true;
8792
8793 return false;
8794}
8795
8796// Check that (every element of) Z is undef or not an exact multiple of BW.
8797static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
8798 return ISD::matchUnaryPredicate(
8799 Op: Z,
8800 Match: [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(RHS: BW) != 0; },
8801 /*AllowUndefs=*/true, /*AllowTruncation=*/true);
8802}
8803
8804SDValue TargetLowering::expandFunnelShift(SDNode *Node,
8805 SelectionDAG &DAG) const {
8806 EVT VT = Node->getValueType(ResNo: 0);
8807
8808 if (VT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8809 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8810 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8811 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
8812 return SDValue();
8813
8814 SDValue X = Node->getOperand(Num: 0);
8815 SDValue Y = Node->getOperand(Num: 1);
8816 SDValue Z = Node->getOperand(Num: 2);
8817
8818 unsigned BW = VT.getScalarSizeInBits();
8819 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
8820 SDLoc DL(SDValue(Node, 0));
8821
8822 EVT ShVT = Z.getValueType();
8823
8824 // If a funnel shift in the other direction is more supported, use it.
8825 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
8826 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8827 isOperationLegalOrCustom(Op: RevOpcode, VT) && isPowerOf2_32(Value: BW)) {
8828 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8829 // fshl X, Y, Z -> fshr X, Y, -Z
8830 // fshr X, Y, Z -> fshl X, Y, -Z
8831 Z = DAG.getNegative(Val: Z, DL, VT: ShVT);
8832 } else {
8833 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8834 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8835 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8836 if (IsFSHL) {
8837 Y = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8838 X = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: One);
8839 } else {
8840 X = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8841 Y = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Y, N2: One);
8842 }
8843 Z = DAG.getNOT(DL, Val: Z, VT: ShVT);
8844 }
8845 return DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: Z);
8846 }
8847
8848 SDValue ShX, ShY;
8849 SDValue ShAmt, InvShAmt;
8850 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8851 // fshl: X << C | Y >> (BW - C)
8852 // fshr: X << (BW - C) | Y >> C
8853 // where C = Z % BW is not zero
8854 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8855 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8856 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthC, N2: ShAmt);
8857 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: IsFSHL ? ShAmt : InvShAmt);
8858 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: IsFSHL ? InvShAmt : ShAmt);
8859 } else {
8860 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8861 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8862 SDValue Mask = DAG.getConstant(Val: BW - 1, DL, VT: ShVT);
8863 if (isPowerOf2_32(Value: BW)) {
8864 // Z % BW -> Z & (BW - 1)
8865 ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Z, N2: Mask);
8866 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8867 InvShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: DAG.getNOT(DL, Val: Z, VT: ShVT), N2: Mask);
8868 } else {
8869 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8870 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8871 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Mask, N2: ShAmt);
8872 }
8873
8874 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8875 if (IsFSHL) {
8876 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShAmt);
8877 SDValue ShY1 = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: One);
8878 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShY1, N2: InvShAmt);
8879 } else {
8880 SDValue ShX1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: One);
8881 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShX1, N2: InvShAmt);
8882 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt);
8883 }
8884 }
8885 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShX, N2: ShY);
8886}
8887
8888// TODO: Merge with expandFunnelShift.
8889SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
8890 SelectionDAG &DAG) const {
8891 EVT VT = Node->getValueType(ResNo: 0);
8892 unsigned EltSizeInBits = VT.getScalarSizeInBits();
8893 bool IsLeft = Node->getOpcode() == ISD::ROTL;
8894 SDValue Op0 = Node->getOperand(Num: 0);
8895 SDValue Op1 = Node->getOperand(Num: 1);
8896 SDLoc DL(SDValue(Node, 0));
8897
8898 EVT ShVT = Op1.getValueType();
8899 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: ShVT);
8900
8901 // If a rotate in the other direction is more supported, use it.
8902 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
8903 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8904 isOperationLegalOrCustom(Op: RevRot, VT) && isPowerOf2_32(Value: EltSizeInBits)) {
8905 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8906 return DAG.getNode(Opcode: RevRot, DL, VT, N1: Op0, N2: Sub);
8907 }
8908
8909 if (!AllowVectorOps && VT.isVector() &&
8910 (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8911 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8912 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8913 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT) ||
8914 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT)))
8915 return SDValue();
8916
8917 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
8918 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
8919 SDValue BitWidthMinusOneC = DAG.getConstant(Val: EltSizeInBits - 1, DL, VT: ShVT);
8920 SDValue ShVal;
8921 SDValue HsVal;
8922 if (isPowerOf2_32(Value: EltSizeInBits)) {
8923 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8924 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8925 SDValue NegOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8926 SDValue ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Op1, N2: BitWidthMinusOneC);
8927 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
8928 SDValue HsAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: NegOp1, N2: BitWidthMinusOneC);
8929 HsVal = DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: HsAmt);
8930 } else {
8931 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8932 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8933 SDValue BitWidthC = DAG.getConstant(Val: EltSizeInBits, DL, VT: ShVT);
8934 SDValue ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Op1, N2: BitWidthC);
8935 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
8936 SDValue HsAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthMinusOneC, N2: ShAmt);
8937 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8938 HsVal =
8939 DAG.getNode(Opcode: HsOpc, DL, VT, N1: DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: One), N2: HsAmt);
8940 }
8941 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShVal, N2: HsVal);
8942}
8943
8944/// Check if CLMUL on VT can eventually reach a type with legal CLMUL through
8945/// a chain of halving decompositions (halving element width) and/or vector
8946/// widening (doubling element count). This guides expansion strategy selection:
8947/// if true, the halving/widening path produces better code than bit-by-bit.
8948///
8949/// HalveDepth tracks halving steps only (each creates ~4x more operations).
8950/// Widening steps are cheap (O(1) pad/extract) and don't count.
8951/// Limiting halvings to 2 prevents exponential blowup:
8952/// 1 halving: ~4 sub-CLMULs (good, e.g. v8i16 -> v8i8)
8953/// 2 halvings: ~16 sub-CLMULs (acceptable, e.g. v4i32 -> v4i16 -> v8i8)
8954/// 3 halvings: ~64 sub-CLMULs (worse than bit-by-bit expansion)
8955static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx,
8956 EVT VT, unsigned HalveDepth = 0,
8957 unsigned TotalDepth = 0) {
8958 if (HalveDepth > 2 || TotalDepth > 8 || !VT.isFixedLengthVector())
8959 return false;
8960 if (TLI.isOperationLegalOrCustom(Op: ISD::CLMUL, VT))
8961 return true;
8962 if (!TLI.isTypeLegal(VT))
8963 return false;
8964
8965 unsigned BW = VT.getScalarSizeInBits();
8966
8967 // Halve: halve element width, same element count.
8968 // This is the expensive step -- each halving creates ~4x more operations.
8969 if (BW % 2 == 0) {
8970 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: BW / 2);
8971 EVT HalfVT = VT.changeVectorElementType(Context&: Ctx, EltVT: HalfEltVT);
8972 if (TLI.isTypeLegal(VT: HalfVT) &&
8973 canNarrowCLMULToLegal(TLI, Ctx, VT: HalfVT, HalveDepth: HalveDepth + 1, TotalDepth: TotalDepth + 1))
8974 return true;
8975 }
8976
8977 // Widen: double element count (fixed-width vectors only).
8978 // This is cheap -- just INSERT_SUBVECTOR + EXTRACT_SUBVECTOR.
8979 EVT WideVT = VT.getDoubleNumVectorElementsVT(Context&: Ctx);
8980 if (TLI.isTypeLegal(VT: WideVT) &&
8981 canNarrowCLMULToLegal(TLI, Ctx, VT: WideVT, HalveDepth, TotalDepth: TotalDepth + 1))
8982 return true;
8983
8984 return false;
8985}
8986
8987SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
8988 SDLoc DL(Node);
8989 EVT VT = Node->getValueType(ResNo: 0);
8990 SDValue X = Node->getOperand(Num: 0);
8991 SDValue Y = Node->getOperand(Num: 1);
8992 unsigned BW = VT.getScalarSizeInBits();
8993 unsigned Opcode = Node->getOpcode();
8994 LLVMContext &Ctx = *DAG.getContext();
8995
8996 switch (Opcode) {
8997 case ISD::CLMUL: {
8998 // For vector types, try decomposition strategies that leverage legal
8999 // CLMUL on narrower or wider element types, avoiding the expensive
9000 // bit-by-bit expansion.
9001 if (VT.isVector()) {
9002 // Strategy 1: Halving decomposition to half-element-width CLMUL.
9003 // Applies ExpandIntRes_CLMUL's identity element-wise:
9004 // CLMUL(X, Y) = (Hi << HalfBW) | Lo
9005 // where:
9006 // Lo = CLMUL(XLo, YLo)
9007 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9008 unsigned HalfBW = BW / 2;
9009 if (BW % 2 == 0) {
9010 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: HalfBW);
9011 EVT HalfVT =
9012 EVT::getVectorVT(Context&: Ctx, VT: HalfEltVT, EC: VT.getVectorElementCount());
9013 if (isTypeLegal(VT: HalfVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: HalfVT,
9014 /*HalveDepth=*/1)) {
9015 SDValue ShAmt = DAG.getShiftAmountConstant(Val: HalfBW, VT, DL);
9016
9017 // Extract low and high halves of each element.
9018 SDValue XLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: X);
9019 SDValue XHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
9020 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: ShAmt));
9021 SDValue YLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: Y);
9022 SDValue YHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
9023 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt));
9024
9025 // Lo = CLMUL(XLo, YLo)
9026 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YLo);
9027
9028 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
9029 SDValue LoH = DAG.getNode(Opcode: ISD::CLMULH, DL, VT: HalfVT, N1: XLo, N2: YLo);
9030 SDValue Cross1 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YHi);
9031 SDValue Cross2 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XHi, N2: YLo);
9032 SDValue Cross = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: Cross1, N2: Cross2);
9033 SDValue Hi = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: LoH, N2: Cross);
9034
9035 // Reassemble: Result = ZExt(Lo) | (AnyExt(Hi) << HalfBW)
9036 SDValue LoExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Lo);
9037 SDValue HiExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Hi);
9038 SDValue HiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: HiExt, N2: ShAmt);
9039 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoExt, N2: HiShifted);
9040 }
9041 }
9042
9043 // Strategy 2: Promote to double-element-width CLMUL.
9044 // CLMUL(X, Y) = Trunc(CLMUL(AnyExt(X), AnyExt(Y)))
9045 {
9046 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9047 if (isTypeLegal(VT: ExtVT) && isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT)) {
9048 // If CLMUL on ExtVT is Custom (not Legal), the target may
9049 // scalarize it, costing O(NumElements) scalar ops. The bit-by-bit
9050 // fallback costs O(BW) vectorized iterations. Only widen when
9051 // element count is small enough that scalarization is cheaper.
9052 unsigned NumElts = VT.getVectorMinNumElements();
9053 if (isOperationLegal(Op: ISD::CLMUL, VT: ExtVT) || NumElts < BW) {
9054 SDValue XExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: X);
9055 SDValue YExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: Y);
9056 SDValue Mul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9057 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Mul);
9058 }
9059 }
9060 }
9061
9062 // Strategy 3: Widen element count (pad with undef, do CLMUL on wider
9063 // vector, extract lower result). CLMUL is element-wise, so upper
9064 // (undef) lanes don't affect the lower results.
9065 // e.g. v4i16 => pad to v8i16 => halve to v8i8 PMUL => extract v4i16.
9066 if (auto EC = VT.getVectorElementCount(); EC.isFixed()) {
9067 EVT WideVT = EVT::getVectorVT(Context&: Ctx, VT: VT.getVectorElementType(), EC: EC * 2);
9068 if (isTypeLegal(VT: WideVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: WideVT)) {
9069 SDValue Undef = DAG.getUNDEF(VT: WideVT);
9070 SDValue XWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
9071 N2: X, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9072 SDValue YWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
9073 N2: Y, N3: DAG.getVectorIdxConstant(Val: 0, DL));
9074 SDValue WideRes = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: WideVT, N1: XWide, N2: YWide);
9075 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WideRes,
9076 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9077 }
9078 }
9079 }
9080
9081 // Special case: clmul(X, Y) where Y is a known constant (splat) that forms
9082 // a contiguous block of trailing ones whose length N is a power of two
9083 // (e.g. i8 0xFF, i8 0x0F, ...) or equal to the operand width. In this
9084 // special case, clmul(X, Y) is equivalent to a "parallel prefix XOR" or
9085 // "bitwise parity" operation on X.
9086 //
9087 // Note: This special currently dose NOT apply when the mask is neither a
9088 // power of two nor equal to the operand width because the loop inside
9089 // behaves as if the mask was bit-ceiled, and "undoing" the XOR with parts
9090 // of that CLMUL is a recursive problem (e.g. CLMUL with a 20-bit mask
9091 // requires correction XOR with CLMUL with 12-bit mask).
9092 if (auto *C = isConstOrConstSplat(N: Y, /*AllowUndefs=*/true)) {
9093 const APInt &YVal = C->getAPIntValue();
9094 unsigned N = YVal.countr_one();
9095 if (YVal.isAllOnes() || (YVal.isMask() && isPowerOf2_32(Value: N))) {
9096 SDValue R = X;
9097 for (unsigned I = 1; I < N; I <<= 1) {
9098 SDValue ShAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9099 SDValue Shifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: R, N2: ShAmt);
9100 R = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: R, N2: Shifted);
9101 }
9102 return R;
9103 }
9104 }
9105
9106 // NOTE: If you change this expansion, please update the cost model
9107 // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
9108 // Intrinsic::clmul.
9109
9110 // Strategy 4: multiplication with holes.
9111 //
9112 // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
9113 // do occur, they wind up in a "hole" and are subsequently masked out of the
9114 // result.
9115 //
9116 // https://www.bearssl.org/constanttime.html#ghash-for-gcm describes this
9117 // approach.
9118
9119 // Stride S handles operands up to S·2^S bits using S² multiplies.
9120 //
9121 // * BW <= 8 uses S = 2 (holes of 1 bit)
9122 // * BW <= 24 uses S = 3 (holes of 2 bits)
9123 // * BW <= 64 uses S = 4 (holes of 3 bits)
9124 // * BW <= 160 uses S = 5 (holes of 4 bits)
9125 // * BW <= 384 uses S = 6 (holes of 5 bits)
9126 //
9127 // We distribute the BW bits over S phases:
9128 //
9129 // phase 0 keeps bits: 0, S, 2S, ...
9130 // phase 1 keeps bits: 1, S + 1, 2S + 1, ...
9131 // ...
9132 //
9133 // Each phase has up to n = ceil(BW / S) bits set, and the holes are S-1
9134 // bits wide.
9135 //
9136 // Take BW = 4, S = 2, n = 2. The worst case is a fully populated phase (all
9137 // non-hole bits are set to 1) multiplied by itself, 0b0101 * 0b0101. Each
9138 // set bit of one operand shifts a copy of the other, and we add the copies:
9139 //
9140 // col: 4 3 2 1 0
9141 // 0b0101 << 0: 0 0 1 0 1
9142 // 0b0101 << 2: 1 0 1 0 0
9143 // ----------------- +
9144 // count: 1 0 2 0 1
9145 //
9146 // Counting the number of one-bits in each column gives a triangle: the
9147 // counts climb 1, 2, ..., n and back down (here 1, 2, 1 across the data
9148 // columns). So a column holds at most n one-bits, and that maximum n is
9149 // reached in only one column: the peak. Every other column holds at most n
9150 // - 1 one-bits.
9151 //
9152 // A stack of one-bits in a column turns into carries: column 2 above really
9153 // stores the value 1 + 1 = 2 = n. A column spans S bits, its kept bit
9154 // plus S-1 hole bits, and the count is written from the kept bit upward,
9155 // so any count <= 2^S - 1 stays within the column and never interferes with
9156 // the next data bit S positions up. Every non-peak column holds at most n -
9157 // 1, so they all fit as soon as n - 1 <= 2^S - 1.
9158 //
9159 // That leaves only the peak column. Because both operands set all data
9160 // bits, the triangle peaks at the top of the word at the highest data bit
9161 // still inside BW. Here the count reaches exactly n = 2^S and overflows.
9162 // But its carry lands at bit n*S >= BW, off the top, where it (and the
9163 // whole descending half of the triangle) is truncated.
9164 //
9165 // Hence the holes suffice exactly when n = ceil(BW / S) <= 2^S, i.e. BW <=
9166 // S*2^S.
9167 //
9168 // Here we find the smallest S that satisfies this inequality.
9169 unsigned S = 1;
9170 while (S < 32 && divideCeil(Numerator: BW, Denominator: S) > (1u << S))
9171 ++S;
9172
9173 // The "multiplication with holes" expansion emits S*S MULs, 3*S ANDs,
9174 // S*(S-1) XORs and S-1 ORs.
9175 unsigned HolesCost = S * S + 3 * S + S * (S - 1) + (S - 1);
9176
9177 // Estimate the cost of the naive algorithm.
9178 KnownBits KnownY = DAG.computeKnownBits(Op: Y);
9179 unsigned NaiveCost = 0;
9180 for (unsigned I = 0; I < BW; ++I) {
9181 // The iteration folds away entirely and is free.
9182 if (KnownY.Zero[I])
9183 continue;
9184
9185 // On targets with a fast bit test instruction more instructions are used
9186 // to not need a (potentially expensive) multiplication. See also below.
9187 if (hasBitTest(X: Y, Y: DAG.getShiftAmountConstant(Val: I, VT, DL))) {
9188 // AND + SETCC + SHL + SELECT + XOR.
9189 NaiveCost += 5;
9190 } else {
9191 // AND + MUL + XOR.
9192 NaiveCost += 3;
9193 }
9194 }
9195
9196 // Only use multiplication with holes when it is cheaper, else use the naive
9197 // fallback below.
9198 if (HolesCost < NaiveCost &&
9199 isOperationLegalOrCustom(Op: ISD::MUL, VT: getTypeToTransformTo(Context&: Ctx, VT))) {
9200
9201 // Set a bit every S positions, e.g. for S = 4 this is equivalent to
9202 // 0b...00010001...0001.
9203 APInt MaskVal = APInt::getSplat(NewLen: BW, V: APInt(S, 1));
9204
9205 // Create versions of X and Y that keep only the I-th bit of each S-bit
9206 // slice.
9207 SmallVector<SDValue, 4> M(S), Xp(S), Yp(S);
9208 for (unsigned I = 0; I < S; ++I) {
9209 M[I] = DAG.getConstant(Val: MaskVal.shl(shiftAmt: I), DL, VT);
9210 Xp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: M[I]);
9211 Yp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: M[I]);
9212 }
9213
9214 // Codegens these expressions (S*S multiplications), e.g. for S=4:
9215 //
9216 // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
9217 // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
9218 // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
9219 // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
9220 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9221 for (unsigned I = 0; I < S; ++I) {
9222 SDValue Zi = DAG.getConstant(Val: 0, DL, VT);
9223 for (unsigned J = 0; J < S; ++J) {
9224 unsigned K = (I + S - J) % S;
9225 SDValue P = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Xp[J], N2: Yp[K]);
9226 Zi = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Zi, N2: P);
9227 }
9228
9229 // Keep only the bits belonging to this iteration, and bitwise or it all
9230 // together.
9231 Zi = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Zi, N2: M[I]);
9232 Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Res, N2: Zi, Flags: SDNodeFlags::Disjoint);
9233 }
9234 return Res;
9235 }
9236
9237 // Strategy 5: the naive fallback.
9238 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
9239
9240 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9241 for (unsigned I = 0; I < BW; ++I) {
9242 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9243 SDValue Mask = DAG.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: I), DL, VT);
9244 SDValue YMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: Mask);
9245
9246 // For targets with a fast bit test instruction (e.g., x86 BT) or without
9247 // multiply, use a shift-based expansion to avoid expensive MUL
9248 // instructions.
9249 SDValue Part;
9250 if (!hasBitTest(X: Y, Y: ShiftAmt) &&
9251 isOperationLegalOrCustom(
9252 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
9253 Part = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: YMasked);
9254 } else {
9255 // Canonical bit test: (Y & (1 << I)) != 0
9256 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
9257 SDValue Cond = DAG.getSetCC(DL, VT: SetCCVT, LHS: YMasked, RHS: Zero, Cond: ISD::SETEQ);
9258 SDValue XShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftAmt);
9259 Part = DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: XShifted);
9260 }
9261 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Res, N2: Part);
9262 }
9263 return Res;
9264 }
9265 case ISD::CLMULR:
9266 // If we have CLMUL/CLMULH, merge the shifted results to form CLMULR.
9267 if (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) &&
9268 isOperationLegalOrCustom(Op: ISD::CLMULH, VT)) {
9269 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
9270 SDValue Hi = DAG.getNode(Opcode: ISD::CLMULH, DL, VT, N1: X, N2: Y);
9271 Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo,
9272 N2: DAG.getShiftAmountConstant(Val: BW - 1, VT, DL));
9273 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi,
9274 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9275 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Lo, N2: Hi);
9276 }
9277 [[fallthrough]];
9278 case ISD::CLMULH: {
9279 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9280 // Use bitreverse-based lowering (CLMULR/H = rev(CLMUL(rev,rev)) >> S)
9281 // when any of these hold:
9282 // (a) ZERO_EXTEND to ExtVT or SRL on ExtVT isn't legal.
9283 // (b) CLMUL is legal on VT but not on ExtVT (e.g. v8i8 on AArch64).
9284 // (c) CLMUL on ExtVT isn't legal, but CLMUL on VT can be efficiently
9285 // expanded via halving/widening to reach legal CLMUL. The bitreverse
9286 // path creates CLMUL(VT) which will be expanded efficiently. The
9287 // promote path would create CLMUL(ExtVT) => halving => CLMULH(VT),
9288 // causing a cycle.
9289 // Note: when CLMUL is legal on ExtVT, the zext => CLMUL(ExtVT) => shift
9290 // => trunc path is preferred over the bitreverse path, as it avoids the
9291 // cost of 3 bitreverse operations.
9292 if (!isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT: ExtVT) ||
9293 !isOperationLegalOrCustom(Op: ISD::SRL, VT: ExtVT) ||
9294 (!isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT) &&
9295 (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) ||
9296 canNarrowCLMULToLegal(TLI: *this, Ctx, VT)))) {
9297 SDValue XRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: X);
9298 SDValue YRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: Y);
9299 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: XRev, N2: YRev);
9300 SDValue Res = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: ClMul);
9301 if (Opcode == ISD::CLMULH)
9302 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res,
9303 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9304 return Res;
9305 }
9306 SDValue XExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: X);
9307 SDValue YExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: Y);
9308 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9309 unsigned ShAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
9310 SDValue HiBits = DAG.getNode(Opcode: ISD::SRL, DL, VT: ExtVT, N1: ClMul,
9311 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: ExtVT, DL));
9312 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: HiBits);
9313 }
9314 }
9315 llvm_unreachable("Expected CLMUL, CLMULR, or CLMULH");
9316}
9317
9318SDValue TargetLowering::expandPEXT(SDNode *Node, SelectionDAG &DAG) const {
9319 SDLoc DL(Node);
9320 EVT VT = Node->getValueType(ResNo: 0);
9321 SDValue Val = Node->getOperand(Num: 0);
9322 SDValue Msk = Node->getOperand(Num: 1);
9323 unsigned BW = VT.getScalarSizeInBits();
9324
9325 // Just scalarize if scalar PEXT is legal
9326 if (VT.isVector() && isOperationLegal(Op: ISD::PEXT, VT: VT.getVectorElementType()))
9327 return DAG.UnrollVectorOp(N: Node);
9328
9329 // Hacker's Delight §7-4: Compress, or Generalized Extract
9330 SDValue X = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Val, N2: Msk);
9331 SDValue M = Msk;
9332 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9333 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: M, VT), N2: One);
9334
9335 // Repeatedly compute which bits would shift to the right by an odd amount,
9336 // shift all such bits in parallel using a mask, and double the shift amount.
9337 for (unsigned I = 1; I < BW; I *= 2) {
9338 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9339 SDValue Mp =
9340 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9341 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: M);
9342 SDValue ShiftI = DAG.getShiftAmountConstant(Val: I, VT, DL);
9343 SDValue MvS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mv, N2: ShiftI);
9344 M = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: M, N2: Mv), N2: MvS,
9345 Flags: SDNodeFlags::Disjoint);
9346 SDValue T = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Mv);
9347 SDValue TS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: T, N2: ShiftI);
9348 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: X, N2: T), N2: TS,
9349 Flags: SDNodeFlags::Disjoint);
9350 if (I * 2 < BW)
9351 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9352 }
9353
9354 return X;
9355}
9356
9357SDValue TargetLowering::expandPDEP(SDNode *Node, SelectionDAG &DAG) const {
9358 SDLoc DL(Node);
9359 EVT VT = Node->getValueType(ResNo: 0);
9360 SDValue Val = Node->getOperand(Num: 0);
9361 SDValue Msk = Node->getOperand(Num: 1);
9362 unsigned BW = VT.getScalarSizeInBits();
9363
9364 // Just scalarize if scalar PDEP is legal
9365 if (VT.isVector() && isOperationLegal(Op: ISD::PDEP, VT: VT.getVectorElementType()))
9366 return DAG.UnrollVectorOp(N: Node);
9367
9368 // Hacker's Delight §7-5: Expand, or Generalized Insert.
9369 unsigned LogBW = Log2_32_Ceil(Value: BW);
9370 SmallVector<SDValue, 8> MvArray(LogBW);
9371 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9372 SDValue Mc = Msk;
9373 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: Msk, VT), N2: One);
9374
9375 // First pass: compute move masks for each power of two that a bit moves by.
9376 for (unsigned S = 0; S < LogBW; ++S) {
9377 unsigned ShiftS = 1u << S;
9378 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9379 SDValue Mp =
9380 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9381 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: Mc);
9382 MvArray[S] = Mv;
9383 if (S + 1 < LogBW) {
9384 SDValue McXorMv = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Mc, N2: Mv);
9385 SDValue MvShifted = DAG.getNode(
9386 Opcode: ISD::SRL, DL, VT, N1: Mv, N2: DAG.getShiftAmountConstant(Val: ShiftS, VT, DL));
9387 Mc = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: McXorMv, N2: MvShifted,
9388 Flags: SDNodeFlags::Disjoint);
9389 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9390 }
9391 }
9392
9393 // Second pass: move bits by 32, 16, 8, 4, 2, 1, using masks, in parallel.
9394 // Each pass handles half the shift amount of the previous pass.
9395 SDValue X = Val;
9396 for (int S = (int)LogBW - 1; S >= 0; --S) {
9397 SDValue ShiftSv = DAG.getShiftAmountConstant(Val: 1ull << S, VT, DL);
9398 SDValue T = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftSv);
9399 SDValue UnshiftedBits =
9400 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: DAG.getNOT(DL, Val: MvArray[S], VT));
9401 SDValue ShiftedBits = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T, N2: MvArray[S]);
9402 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: UnshiftedBits, N2: ShiftedBits,
9403 Flags: SDNodeFlags::Disjoint);
9404 }
9405
9406 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Msk);
9407}
9408
9409void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
9410 SelectionDAG &DAG) const {
9411 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
9412 EVT VT = Node->getValueType(ResNo: 0);
9413 unsigned VTBits = VT.getScalarSizeInBits();
9414 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
9415
9416 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
9417 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
9418 SDValue ShOpLo = Node->getOperand(Num: 0);
9419 SDValue ShOpHi = Node->getOperand(Num: 1);
9420 SDValue ShAmt = Node->getOperand(Num: 2);
9421 EVT ShAmtVT = ShAmt.getValueType();
9422 EVT ShAmtCCVT =
9423 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ShAmtVT);
9424 SDLoc dl(Node);
9425
9426 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
9427 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
9428 // away during isel.
9429 SDValue SafeShAmt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9430 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT));
9431 SDValue Tmp1 = IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: ShOpHi,
9432 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT))
9433 : DAG.getConstant(Val: 0, DL: dl, VT);
9434
9435 SDValue Tmp2, Tmp3;
9436 if (IsSHL) {
9437 Tmp2 = DAG.getNode(Opcode: ISD::FSHL, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9438 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: SafeShAmt);
9439 } else {
9440 Tmp2 = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9441 Tmp3 = DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: ShOpHi, N2: SafeShAmt);
9442 }
9443
9444 // If the shift amount is larger or equal than the width of a part we don't
9445 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
9446 // values for large shift amounts.
9447 SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9448 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: ShAmtVT));
9449 SDValue Cond = DAG.getSetCC(DL: dl, VT: ShAmtCCVT, LHS: AndNode,
9450 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShAmtVT), Cond: ISD::SETNE);
9451
9452 if (IsSHL) {
9453 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9454 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9455 } else {
9456 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9457 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9458 }
9459}
9460
9461SDValue TargetLowering::expandFCANONICALIZE(SDNode *Node,
9462 SelectionDAG &DAG) const {
9463 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
9464 // suggested in
9465 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
9466 // It uses strict_fp operations even outside a strict_fp context in order
9467 // to guarantee that the canonicalization is not optimized away by later
9468 // passes. The result chain introduced by that is intentionally ignored
9469 // since no ordering requirement is intended here.
9470 EVT VT = Node->getValueType(ResNo: 0);
9471 SDLoc DL(Node);
9472 SDNodeFlags Flags = Node->getFlags();
9473 Flags.setNoFPExcept(true);
9474 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT);
9475 SDValue Mul =
9476 DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {VT, MVT::Other},
9477 Ops: {DAG.getEntryNode(), Node->getOperand(Num: 0), One}, Flags);
9478 return Mul;
9479}
9480
9481SDValue TargetLowering::expandCONVERT_TO_ARBITRARY_FP(SDNode *Node,
9482 SelectionDAG &DAG) const {
9483 // Expand conversion from a native IEEE float type to an arbitrary FP format
9484 // returning the result as an integer using bit manipulation.
9485 EVT ResVT = Node->getValueType(ResNo: 0);
9486 SDLoc dl(Node);
9487
9488 SDValue FloatVal = Node->getOperand(Num: 0);
9489 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9490 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9491 const auto RoundMode =
9492 static_cast<RoundingMode>(Node->getConstantOperandVal(Num: 2));
9493 const bool Saturate = Node->getConstantOperandVal(Num: 3) != 0;
9494
9495 // Supported destination formats.
9496 switch (Sem) {
9497 case APFloatBase::S_Float8E5M2:
9498 case APFloatBase::S_Float8E4M3FN:
9499 case APFloatBase::S_Float8E5M3FNU:
9500 case APFloatBase::S_Float6E3M2FN:
9501 case APFloatBase::S_Float6E2M3FN:
9502 case APFloatBase::S_Float4E2M1FN:
9503 break;
9504 default:
9505 DAG.getContext()->emitError(ErrorStr: "CONVERT_TO_ARBITRARY_FP: not implemented "
9506 "destination format (semantics enum " +
9507 Twine(SemEnum) + ")");
9508 return SDValue();
9509 }
9510
9511 // Supported rounding modes.
9512 switch (RoundMode) {
9513 case RoundingMode::NearestTiesToEven:
9514 case RoundingMode::TowardZero:
9515 case RoundingMode::TowardPositive:
9516 case RoundingMode::TowardNegative:
9517 case RoundingMode::NearestTiesToAway:
9518 break;
9519 default:
9520 DAG.getContext()->emitError(
9521 ErrorStr: "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
9522 Twine(static_cast<int>(RoundMode)) + ")");
9523 return SDValue();
9524 }
9525
9526 // Destination format parameters.
9527 const fltSemantics &DstSem = APFloatBase::EnumToSemantics(S: Sem);
9528 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9529 const unsigned DstPrecision = APFloat::semanticsPrecision(DstSem);
9530 const unsigned DstMant = DstPrecision - 1;
9531 // Unsigned formats spend no bit on the sign.
9532 const bool DstHasSign = APFloat::semanticsHasSignedRepr(DstSem);
9533 const unsigned DstExpBits = DstBits - (DstHasSign ? 1 : 0) - DstMant;
9534 const int DstBias = 1 - APFloat::semanticsMinExponent(DstSem);
9535 const unsigned DstExpMax = (1U << DstExpBits) - 1;
9536 const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
9537 const fltNonfiniteBehavior DstNFBehavior = DstSem.nonFiniteBehavior;
9538 const fltNanEncoding DstNanEnc = DstSem.nanEncoding;
9539
9540 // Compute the maximum normal exponent for the destination format.
9541 const unsigned DstExpMaxNormal =
9542 DstNFBehavior == fltNonfiniteBehavior::IEEE754 ? DstExpMax - 1
9543 : DstExpMax;
9544
9545 // For NanOnly formats the max exponent field for finite values
9546 // is DstExpMax, but the encoding with exp = DstExpMax and
9547 // mant = all-ones is NaN. So DstExpMaxNormal = DstExpMax, but max
9548 // mantissa at that exponent is DstMantMask - 1 (if NanEnc == AllOnes) to
9549 // avoid the NaN encoding.
9550 uint64_t DstMaxMantAtMaxExp = DstMantMask;
9551 if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9552 DstNanEnc == fltNanEncoding::AllOnes)
9553 DstMaxMantAtMaxExp = DstMantMask - 1;
9554
9555 // Source format parameters.
9556 EVT SrcVT = FloatVal.getValueType();
9557 const fltSemantics &SrcSem = SrcVT.getScalarType().getFltSemantics();
9558 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9559 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9560 const unsigned SrcMant = SrcPrecision - 1;
9561 const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;
9562
9563 // Work in the source integer type. Match the destination shape so the
9564 // expansion stays vector when ResVT is a vector.
9565 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcBits);
9566 EVT IntVT = ResVT.changeElementType(Context&: *DAG.getContext(), EltVT: IntScalarVT);
9567 EVT SetCCVT =
9568 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9569 EVT FPSetCCVT =
9570 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
9571
9572 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9573 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9574
9575 // Bitcast source float to integer to extract the sign bit.
9576 SDValue Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FloatVal);
9577 SDValue SignBit =
9578 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9579 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
9580
9581 // Classify the input.
9582 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT);
9583 SDValue FPInf = DAG.getConstantFP(Val: APFloat::getInf(Sem: SrcSem), DL: dl, VT: SrcVT);
9584 SDValue AbsVal = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: SrcVT, Operand: FloatVal);
9585 SDValue IsNaN = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETUO);
9586 SDValue IsInf = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: AbsVal, RHS: FPInf, Cond: ISD::SETOEQ);
9587 SDValue IsZero = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOEQ);
9588
9589 // Split into a normalized fraction and unbiased exponent. FFREXP normalizes
9590 // source denormals automatically. The result is unspecified for Inf/NaN, but
9591 // those inputs are detected above and override the final result.
9592 EVT FrexpExpScalarVT =
9593 getValueType(DL: DAG.getDataLayout(), Ty: Type::getInt32Ty(C&: *DAG.getContext()));
9594 EVT FrexpExpVT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: FrexpExpScalarVT);
9595 SDValue Frexp =
9596 DAG.getNode(Opcode: ISD::FFREXP, DL: dl, VTList: DAG.getVTList(VT1: SrcVT, VT2: FrexpExpVT), N: FloatVal);
9597 SDValue FrexpFrac = Frexp.getValue(R: 0);
9598 SDValue FrexpExp = Frexp.getValue(R: 1);
9599
9600 SDValue FrexpFracInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FrexpFrac);
9601 SDValue EffSrcMant = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FrexpFracInt,
9602 N2: DAG.getConstant(Val: SrcMantMask, DL: dl, VT: IntVT));
9603
9604 SDValue FrexpExpExt = DAG.getSExtOrTrunc(Op: FrexpExp, DL: dl, VT: IntVT);
9605 SDValue NewExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: FrexpExpExt,
9606 N2: DAG.getConstant(Val: DstBias - 1, DL: dl, VT: IntVT));
9607
9608 // Compute rounding increment given the round bit, sticky bits, and LSB
9609 // of the truncated mantissa.
9610 auto ComputeRoundUp = [&](SDValue RoundBit, SDValue StickyBits,
9611 SDValue LSB) -> SDValue {
9612 switch (RoundMode) {
9613 case RoundingMode::NearestTiesToEven: {
9614 // Round up if round_bit && (sticky || lsb)
9615 SDValue StickyOrLSB = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: StickyBits, N2: LSB);
9616 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyOrLSB);
9617 }
9618 case RoundingMode::TowardZero:
9619 return Zero;
9620 case RoundingMode::TowardPositive: {
9621 // Round up if positive and any truncated bits are set.
9622 SDValue AnyTruncBits =
9623 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9624 SDValue HasTruncBits =
9625 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9626 SDValue IsPositive = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETEQ);
9627 SDValue DoRound =
9628 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsPositive);
9629 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9630 }
9631 case RoundingMode::TowardNegative: {
9632 // Round up if negative and any truncated bits are set (to -Inf).
9633 SDValue AnyTruncBits =
9634 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9635 SDValue HasTruncBits =
9636 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9637 SDValue IsNegative = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETNE);
9638 SDValue DoRound =
9639 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsNegative);
9640 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9641 }
9642 case RoundingMode::NearestTiesToAway:
9643 return RoundBit;
9644 default:
9645 llvm_unreachable("unsupported rounding mode");
9646 }
9647 };
9648
9649 // Round mantissa from SrcMant bits to DstMant bits.
9650 SDValue TruncMant;
9651 SDValue RoundUp;
9652 if (SrcMant > DstMant) {
9653 const unsigned Shift = SrcMant - DstMant;
9654 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: Shift, VT: IntVT, DL: dl);
9655 TruncMant = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ShiftConst);
9656
9657 // Check bit at position Shift - 1 aka the round bit.
9658 SDValue RoundBit;
9659 if (Shift >= 1) {
9660 SDValue RoundBitShift = DAG.getShiftAmountConstant(Val: Shift - 1, VT: IntVT, DL: dl);
9661 SDValue ShiftedMant =
9662 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: RoundBitShift);
9663 RoundBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: ShiftedMant, N2: One);
9664 } else {
9665 RoundBit = Zero;
9666 }
9667
9668 // OR of all bits below the round bit to get sticky bits.
9669 SDValue StickyBits;
9670 if (Shift >= 2) {
9671 uint64_t StickyMask = maskTrailingOnes<uint64_t>(N: Shift - 1);
9672 StickyBits = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: EffSrcMant,
9673 N2: DAG.getConstant(Val: StickyMask, DL: dl, VT: IntVT));
9674 StickyBits = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: StickyBits, RHS: Zero, Cond: ISD::SETNE);
9675 StickyBits = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: StickyBits);
9676 } else {
9677 StickyBits = Zero;
9678 }
9679
9680 // LSB of truncated mantissa.
9681 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: TruncMant, N2: One);
9682
9683 RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
9684 } else {
9685 // If DstMant >= SrcMant, then no rounding needed, just shift left.
9686 SDValue MantShift =
9687 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
9688 TruncMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: MantShift);
9689 RoundUp = Zero;
9690 }
9691
9692 // Apply rounding.
9693 SDValue RoundedMant = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: TruncMant, N2: RoundUp);
9694
9695 // Handle mantissa overflow from rounding.
9696 // If rounded_mant > DstMantMask, carry into exponent.
9697 SDValue MantOverflow =
9698 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: RoundedMant,
9699 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9700 // On overflow: mant = 0, exp += 1.
9701 SDValue AdjMant = DAG.getSelect(DL: dl, VT: IntVT, Cond: MantOverflow, LHS: Zero, RHS: RoundedMant);
9702 SDValue AdjExp =
9703 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: NewExp,
9704 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: MantOverflow));
9705
9706 // Precompute sign shifted to MSB of destination. Unsigned formats have no
9707 // sign bit to merge in.
9708 SDValue SignShifted =
9709 DstHasSign
9710 ? DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
9711 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
9712 : Zero;
9713
9714 // Destination denormal conversion (when new_exp <= 0).
9715 // Shift the mantissa right by 1 - new_exp additional bits and set the
9716 // exponent field to 0.
9717 SDValue ExpIsNeg = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9718 RHS: DAG.getConstant(Val: 1, DL: dl, VT: IntVT), Cond: ISD::SETLT);
9719
9720 SDValue DenormResult;
9721 {
9722 // denorm_shift = 1 - NewExp.
9723 SDValue DenormShift = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: One, N2: NewExp);
9724
9725 // full_src_mant = (1 << SrcMant) | EffSrcMant.
9726 SDValue ImplicitOne =
9727 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One,
9728 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl));
9729 SDValue FullSrcMant =
9730 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ImplicitOne);
9731
9732 // Total right shift = DenormShift + (SrcMant - DstMant).
9733 int64_t MantDelta = static_cast<int64_t>(SrcMant) - DstMant;
9734 SDValue TotalShift =
9735 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormShift,
9736 N2: DAG.getSignedConstant(Val: MantDelta, DL: dl, VT: IntVT));
9737
9738 // Clamp total shift to avoid UB, then truncate denorm mantissa.
9739 EVT ShiftVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
9740 SDValue MaxShift = DAG.getConstant(Val: SrcBits - 1, DL: dl, VT: IntVT);
9741 SDValue ClampedShift =
9742 DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IntVT, N1: TotalShift, N2: MaxShift);
9743 SDValue DenormTruncMant =
9744 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant,
9745 N2: DAG.getZExtOrTrunc(Op: ClampedShift, DL: dl, VT: ShiftVT));
9746
9747 // Rounding for denorm path.
9748 SDValue DenormRoundUp;
9749 {
9750 // Round bit is at position TotalShift - 1 of FullSrcMant.
9751 // Clamp to at least 1 so the subtraction doesn't underflow and create
9752 // shift nodes with invalid shift amounts.
9753 SDValue SafeShift = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT: IntVT, N1: ClampedShift, N2: One);
9754 SDValue RoundBitPos = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: SafeShift, N2: One);
9755 SDValue RoundBitPosAmt = DAG.getZExtOrTrunc(Op: RoundBitPos, DL: dl, VT: ShiftVT);
9756 SDValue DenormRoundBit = DAG.getNode(
9757 Opcode: ISD::AND, DL: dl, VT: IntVT,
9758 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant, N2: RoundBitPosAmt), N2: One);
9759
9760 // Sticky: all bits below round bit.
9761 // sticky_mask = (1 << RoundBitPos) - 1
9762 SDValue StickyMask = DAG.getNode(
9763 Opcode: ISD::SUB, DL: dl, VT: IntVT,
9764 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: RoundBitPosAmt), N2: One);
9765 SDValue DenormStickyBits =
9766 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FullSrcMant, N2: StickyMask);
9767 SDValue HasSticky = DAG.getNode(
9768 Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT,
9769 Operand: DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormStickyBits, RHS: Zero, Cond: ISD::SETNE));
9770
9771 SDValue DenormLSB =
9772 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: One);
9773
9774 DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);
9775
9776 // Only apply rounding if TotalShift >= 1 (i.e., there are bits to round).
9777 SDValue ShiftGEOne =
9778 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ClampedShift, RHS: One, Cond: ISD::SETUGE);
9779 DenormRoundUp = DAG.getSelect(DL: dl, VT: IntVT, Cond: ShiftGEOne, LHS: DenormRoundUp, RHS: Zero);
9780 }
9781
9782 SDValue DenormRoundedMant =
9783 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: DenormRoundUp);
9784
9785 // If rounding caused overflow into the normal range, then we get the
9786 // smallest normal number.
9787 SDValue DenormMantOF =
9788 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormRoundedMant,
9789 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9790 SDValue DenormFinalMant =
9791 DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: Zero, RHS: DenormRoundedMant);
9792 SDValue DenormFinalExp = DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: One, RHS: Zero);
9793
9794 // Assemble: sign | (exp << DstMant) | mant
9795 SDValue DenormExpShifted =
9796 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormFinalExp,
9797 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9798 DenormResult = DAG.getNode(
9799 Opcode: ISD::OR, DL: dl, VT: IntVT,
9800 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
9801 N2: DenormFinalMant);
9802 }
9803
9804 // Exponent overflow detection.
9805 SDValue ExpOF =
9806 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9807 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9808
9809 // Also check if AdjExp == DstExpMaxNormal and mantissa overflow into
9810 // a value that exceeds the max allowed mantissa at that exponent.
9811 SDValue ExpAtMax =
9812 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9813 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETEQ);
9814 SDValue MantExceedsMax =
9815 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjMant,
9816 RHS: DAG.getConstant(Val: DstMaxMantAtMaxExp, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9817 SDValue ExpMantOF =
9818 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: ExpAtMax, N2: MantExceedsMax);
9819 SDValue IsOverflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCVT, N1: ExpOF, N2: ExpMantOF);
9820
9821 // Build overflow result.
9822 SDValue OverflowResult;
9823
9824 if (Saturate) {
9825 // Clamp to max finite value:
9826 // sign | (DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp
9827 uint64_t MaxFinite =
9828 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9829 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9830 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9831 } else if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9832 // Produce infinity.
9833 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9834 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9835 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9836 } else {
9837 // Emit poison if no Inf in format and not saturating.
9838 OverflowResult = DAG.getPOISON(VT: IntVT);
9839 }
9840
9841 // Assemble normal result: sign | (AdjExp << DstMant) | AdjMant
9842 SDValue NormExpShifted =
9843 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: AdjExp,
9844 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9845 SDValue NormResult = DAG.getNode(
9846 Opcode: ISD::OR, DL: dl, VT: IntVT,
9847 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted), N2: AdjMant);
9848
9849 // Build special-value results.
9850 SDValue NaNResult;
9851 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9852 // Produce canonical NaN.
9853 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
9854 NaNResult =
9855 DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | QNaNBit, DL: dl, VT: IntVT);
9856 } else if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9857 DstNanEnc == fltNanEncoding::AllOnes) {
9858 // E4M3FN-style: NaN is exp=all-ones, mant=all-ones.
9859 NaNResult = DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | DstMantMask,
9860 DL: dl, VT: IntVT);
9861 } else {
9862 // NaN -> poison for finite only values.
9863 NaNResult = DAG.getPOISON(VT: IntVT);
9864 }
9865
9866 // Inf handling.
9867 SDValue InfResult;
9868 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9869 // Produce signed infinity.
9870 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9871 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9872 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9873 } else if (Saturate) {
9874 // Inf saturates to max finite.
9875 uint64_t MaxFinite =
9876 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9877 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9878 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9879 } else {
9880 // No Inf and not saturating -> poison.
9881 InfResult = DAG.getPOISON(VT: IntVT);
9882 }
9883
9884 SDValue ZeroResult = SignShifted;
9885
9886 // Final selection in an order: NaN takes priority, then Inf, then Zero.
9887 SDValue FiniteResult =
9888 DAG.getSelect(DL: dl, VT: IntVT, Cond: ExpIsNeg, LHS: DenormResult, RHS: NormResult);
9889 FiniteResult =
9890 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: OverflowResult, RHS: FiniteResult);
9891
9892 SDValue Result = FiniteResult;
9893 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
9894 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
9895
9896 // Negative values are unrepresentable in an unsigned format: clamp to zero
9897 // when saturating, poison otherwise so no select is needed. -0.0 is handled
9898 // by IsZero above. Run before the NaN case so a negative NaN still yields
9899 // NaN.
9900 if (!DstHasSign && Saturate) {
9901 SDValue IsNegative =
9902 DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOLT);
9903 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNegative, LHS: Zero, RHS: Result);
9904 }
9905
9906 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
9907
9908 // Truncate to destination integer type.
9909 return DAG.getZExtOrTrunc(Op: Result, DL: dl, VT: ResVT);
9910}
9911
9912SDValue
9913TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
9914 SelectionDAG &DAG) const {
9915 SDLoc dl(Node);
9916 EVT DstVT = Node->getValueType(ResNo: 0);
9917 EVT DstScalarVT = DstVT.getScalarType();
9918
9919 SDValue IntVal = Node->getOperand(Num: 0);
9920 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9921 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9922
9923 // Supported source formats.
9924 switch (Sem) {
9925 case APFloatBase::S_Float8E5M2:
9926 case APFloatBase::S_Float8E4M3FN:
9927 case APFloatBase::S_Float8E5M3FNU:
9928 case APFloatBase::S_Float6E3M2FN:
9929 case APFloatBase::S_Float6E2M3FN:
9930 case APFloatBase::S_Float4E2M1FN:
9931 break;
9932 default:
9933 DAG.getContext()->emitError(ErrorStr: "CONVERT_FROM_ARBITRARY_FP: not implemented "
9934 "source format (semantics enum " +
9935 Twine(SemEnum) + ")");
9936 return SDValue();
9937 }
9938
9939 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(S: Sem);
9940 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9941 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9942 const unsigned SrcMant = SrcPrecision - 1;
9943 // Unsigned formats spend no bit on the sign.
9944 const bool SrcHasSign = APFloat::semanticsHasSignedRepr(SrcSem);
9945 const unsigned SrcExp = SrcBits - (SrcHasSign ? 1 : 0) - SrcMant;
9946 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
9947 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
9948
9949 // Destination format parameters.
9950 const fltSemantics &DstSem = DstScalarVT.getFltSemantics();
9951 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9952 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
9953 const unsigned DstExpBits = DstBits - DstMant - 1;
9954 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
9955 const int DstBias = 1 - DstMinExp;
9956 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
9957
9958 // Work in an integer type matching the destination float width.
9959 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstBits);
9960 EVT IntVT = IntScalarVT;
9961 if (DstVT.isVector()) {
9962 IntVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IntScalarVT,
9963 EC: DstVT.getVectorElementCount());
9964 } else if (!isTypeLegal(VT: IntScalarVT)) {
9965 // Avoid generating illegal type as there is no other places that'll
9966 // legalize it. Vector types don't have this problem because they
9967 // are subject to LegalizeVectorOps and another type legalization phase
9968 // will follow.
9969 if (getTypeAction(Context&: *DAG.getContext(), VT: IntScalarVT) != TypePromoteInteger) {
9970 // We only know how to handle situations where the legal type is wider.
9971 DAG.getContext()->emitError(
9972 ErrorStr: "CONVERT_FROM_ARBITRARY_FP: the requested integer value type for its "
9973 "legalization is not supported");
9974 return SDValue();
9975 }
9976 IntVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: IntScalarVT);
9977 }
9978
9979 SDValue Src = DAG.getZExtOrTrunc(Op: IntVal, DL: dl, VT: IntVT);
9980
9981 EVT SetCCVT =
9982 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9983
9984 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9985 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9986
9987 // Extract bit fields.
9988 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
9989 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
9990
9991 SDValue MantField = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Src,
9992 N2: DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT));
9993
9994 SDValue ExpField =
9995 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT,
9996 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9997 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl)),
9998 N2: DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT));
9999
10000 // An unsigned source has no sign bit; bit SrcBits - 1 is part of the
10001 // exponent.
10002 SDValue SignShifted =
10003 SrcHasSign
10004 ? DAG.getNode(
10005 Opcode: ISD::SHL, DL: dl, VT: IntVT,
10006 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
10007 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl)),
10008 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
10009 : Zero;
10010
10011 // Classify the input.
10012 SDValue ExpAllOnes = DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT);
10013 SDValue IsExpAllOnes =
10014 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: ExpAllOnes, Cond: ISD::SETEQ);
10015 SDValue IsExpZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: Zero, Cond: ISD::SETEQ);
10016 SDValue IsMantZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETEQ);
10017 SDValue IsMantNonZero =
10018 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETNE);
10019
10020 SDValue IsNaN;
10021 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
10022 IsNaN = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
10023 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
10024 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantNonZero);
10025 } else {
10026 assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
10027 SDValue MantAllOnes = DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT);
10028 SDValue IsMantAllOnes =
10029 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: MantAllOnes, Cond: ISD::SETEQ);
10030 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantAllOnes);
10031 }
10032
10033 SDValue IsInf;
10034 if (NFBehavior == fltNonfiniteBehavior::IEEE754)
10035 IsInf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantZero);
10036 else
10037 IsInf = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
10038
10039 SDValue IsZero = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantZero);
10040 SDValue IsDenorm =
10041 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantNonZero);
10042
10043 // Normal value conversion.
10044 const int BiasAdjust = DstBias - SrcBias;
10045 SDValue NormDstExp = DAG.getNode(
10046 Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: ExpField,
10047 N2: DAG.getConstant(Val: APInt(IntVT.getScalarSizeInBits(), BiasAdjust, true), DL: dl,
10048 VT: IntVT));
10049
10050 SDValue NormDstMant;
10051 if (DstMant > SrcMant) {
10052 SDValue NormDstMantShift =
10053 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
10054 NormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: MantField, N2: NormDstMantShift);
10055 } else {
10056 NormDstMant = MantField;
10057 }
10058
10059 SDValue DstMantShift = DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl);
10060 SDValue NormExpShifted =
10061 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: NormDstExp, N2: DstMantShift);
10062 SDValue NormResult =
10063 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
10064 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted),
10065 N2: NormDstMant);
10066
10067 // With identical exponent biases, denormal values remain denormal and the
10068 // normal conversion's mantissa shift is sufficient.
10069 SDValue DenormResult = NormResult;
10070 if (BiasAdjust != 0) {
10071 const unsigned IntVTBits = IntVT.getScalarSizeInBits();
10072 SDValue LeadingZeros =
10073 DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT: IntVT, Operand: MantField);
10074
10075 const int DenormExpConst =
10076 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
10077 SDValue DenormDstExp = DAG.getNode(
10078 Opcode: ISD::SUB, DL: dl, VT: IntVT,
10079 N1: DAG.getConstant(Val: APInt(IntVTBits, DenormExpConst, true), DL: dl, VT: IntVT),
10080 N2: LeadingZeros);
10081
10082 SDValue MantMSB =
10083 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT,
10084 N1: DAG.getConstant(Val: IntVTBits - 1, DL: dl, VT: IntVT), N2: LeadingZeros);
10085
10086 SDValue LeadingOne = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: MantMSB);
10087 SDValue Frac = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: IntVT, N1: MantField, N2: LeadingOne);
10088
10089 const unsigned ShiftSub = IntVTBits - 1 - DstMant;
10090 SDValue ShiftAmount = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: LeadingZeros,
10091 N2: DAG.getConstant(Val: ShiftSub, DL: dl, VT: IntVT));
10092
10093 SDValue DenormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: Frac, N2: ShiftAmount);
10094
10095 SDValue DenormExpShifted =
10096 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormDstExp, N2: DstMantShift);
10097 DenormResult = DAG.getNode(
10098 Opcode: ISD::OR, DL: dl, VT: IntVT,
10099 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
10100 N2: DenormDstMant);
10101 }
10102
10103 SDValue FiniteResult =
10104 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsDenorm, LHS: DenormResult, RHS: NormResult);
10105
10106 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
10107 SDValue NaNResult =
10108 DAG.getConstant(Val: (DstExpAllOnes << DstMant) | QNaNBit, DL: dl, VT: IntVT);
10109
10110 SDValue InfResult =
10111 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
10112 N2: DAG.getConstant(Val: DstExpAllOnes << DstMant, DL: dl, VT: IntVT));
10113
10114 // A source format may have a larger finite exponent range despite having
10115 // fewer bits, as with Float8E5M3FNU converted to half. Its overflowing finite
10116 // values become infinity. The NaN selection below still takes precedence.
10117 if (APFloat::semanticsMaxExponent(SrcSem) >
10118 APFloat::semanticsMaxExponent(DstSem)) {
10119 SDValue IsOverflow =
10120 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NormDstExp,
10121 RHS: DAG.getConstant(Val: DstExpAllOnes, DL: dl, VT: IntVT), Cond: ISD::SETUGE);
10122 FiniteResult =
10123 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: InfResult, RHS: FiniteResult);
10124 }
10125
10126 SDValue ZeroResult = SignShifted;
10127
10128 SDValue Result = FiniteResult;
10129 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
10130 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
10131 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
10132
10133 if (!DstVT.bitsEq(VT: IntVT)) {
10134 // Store to stack before loading it back.
10135 assert(!IntVT.isVector() && IntVT.bitsGT(DstVT));
10136 // IntScalarVT is the original type that has the same width as DstVT.
10137 Align Alignment = DAG.getReducedAlign(VT: IntScalarVT, /*UseABI=*/false);
10138 SDValue StackPtr =
10139 DAG.CreateStackTemporary(Bytes: IntScalarVT.getStoreSize(), Alignment);
10140 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
10141 MachineFunction &MF = DAG.getMachineFunction();
10142 MachinePointerInfo PtrInfo =
10143 MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
10144 SDValue Store = DAG.getTruncStore(Chain: DAG.getEntryNode(), dl, Val: Result, Ptr: StackPtr,
10145 PtrInfo, SVT: IntScalarVT, Alignment);
10146
10147 SDValue Load = DAG.getLoad(VT: DstVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment);
10148 return DAG.getMergeValues(Ops: {Load, Load.getValue(R: 1)}, dl);
10149 }
10150
10151 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT, Operand: Result);
10152}
10153
10154bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
10155 SelectionDAG &DAG) const {
10156 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10157 SDValue Src = Node->getOperand(Num: OpNo);
10158 EVT SrcVT = Src.getValueType();
10159 EVT DstVT = Node->getValueType(ResNo: 0);
10160 SDLoc dl(SDValue(Node, 0));
10161
10162 // FIXME: Only f32 to i64 conversions are supported.
10163 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
10164 return false;
10165
10166 if (Node->isStrictFPOpcode())
10167 // When a NaN is converted to an integer a trap is allowed. We can't
10168 // use this expansion here because it would eliminate that trap. Other
10169 // traps are also allowed and cannot be eliminated. See
10170 // IEEE 754-2008 sec 5.8.
10171 return false;
10172
10173 // Expand f32 -> i64 conversion
10174 // This algorithm comes from compiler-rt's implementation of fixsfdi:
10175 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
10176 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
10177 EVT IntVT = SrcVT.changeTypeToInteger();
10178 EVT IntShVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
10179
10180 SDValue ExponentMask = DAG.getConstant(Val: 0x7F800000, DL: dl, VT: IntVT);
10181 SDValue ExponentLoBit = DAG.getConstant(Val: 23, DL: dl, VT: IntVT);
10182 SDValue Bias = DAG.getConstant(Val: 127, DL: dl, VT: IntVT);
10183 SDValue SignMask = DAG.getConstant(Val: APInt::getSignMask(BitWidth: SrcEltBits), DL: dl, VT: IntVT);
10184 SDValue SignLowBit = DAG.getConstant(Val: SrcEltBits - 1, DL: dl, VT: IntVT);
10185 SDValue MantissaMask = DAG.getConstant(Val: 0x007FFFFF, DL: dl, VT: IntVT);
10186
10187 SDValue Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: Src);
10188
10189 SDValue ExponentBits = DAG.getNode(
10190 Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: ExponentMask),
10191 N2: DAG.getZExtOrTrunc(Op: ExponentLoBit, DL: dl, VT: IntShVT));
10192 SDValue Exponent = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentBits, N2: Bias);
10193
10194 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: IntVT,
10195 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: SignMask),
10196 N2: DAG.getZExtOrTrunc(Op: SignLowBit, DL: dl, VT: IntShVT));
10197 Sign = DAG.getSExtOrTrunc(Op: Sign, DL: dl, VT: DstVT);
10198
10199 SDValue R = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
10200 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: MantissaMask),
10201 N2: DAG.getConstant(Val: 0x00800000, DL: dl, VT: IntVT));
10202
10203 R = DAG.getZExtOrTrunc(Op: R, DL: dl, VT: DstVT);
10204
10205 R = DAG.getSelectCC(
10206 DL: dl, LHS: Exponent, RHS: ExponentLoBit,
10207 True: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: DstVT, N1: R,
10208 N2: DAG.getZExtOrTrunc(
10209 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: Exponent, N2: ExponentLoBit),
10210 DL: dl, VT: IntShVT)),
10211 False: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: DstVT, N1: R,
10212 N2: DAG.getZExtOrTrunc(
10213 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentLoBit, N2: Exponent),
10214 DL: dl, VT: IntShVT)),
10215 Cond: ISD::SETGT);
10216
10217 SDValue Ret = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: DstVT,
10218 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: R, N2: Sign), N2: Sign);
10219
10220 Result = DAG.getSelectCC(DL: dl, LHS: Exponent, RHS: DAG.getConstant(Val: 0, DL: dl, VT: IntVT),
10221 True: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), False: Ret, Cond: ISD::SETLT);
10222 return true;
10223}
10224
10225bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
10226 SDValue &Chain,
10227 SelectionDAG &DAG) const {
10228 SDLoc dl(SDValue(Node, 0));
10229 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10230 SDValue Src = Node->getOperand(Num: OpNo);
10231
10232 EVT SrcVT = Src.getValueType();
10233 EVT DstVT = Node->getValueType(ResNo: 0);
10234 EVT SetCCVT =
10235 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
10236 EVT DstSetCCVT =
10237 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: DstVT);
10238
10239 // Only expand vector types if we have the appropriate vector bit operations.
10240 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
10241 ISD::FP_TO_SINT;
10242 if (DstVT.isVector() && (!isOperationLegalOrCustom(Op: SIntOpcode, VT: DstVT) ||
10243 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT: SrcVT)))
10244 return false;
10245
10246 // If the maximum float value is smaller then the signed integer range,
10247 // the destination signmask can't be represented by the float, so we can
10248 // just use FP_TO_SINT directly.
10249 const fltSemantics &APFSem = SrcVT.getFltSemantics();
10250 APFloat APF(APFSem, APInt::getZero(numBits: SrcVT.getScalarSizeInBits()));
10251 APInt SignMask = APInt::getSignMask(BitWidth: DstVT.getScalarSizeInBits());
10252 if (APFloat::opOverflow &
10253 APF.convertFromAPInt(Input: SignMask, IsSigned: false, RM: APFloat::rmNearestTiesToEven)) {
10254 if (Node->isStrictFPOpcode()) {
10255 Result = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10256 Ops: { Node->getOperand(Num: 0), Src });
10257 Chain = Result.getValue(R: 1);
10258 } else
10259 Result = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10260 return true;
10261 }
10262
10263 // Don't expand it if there isn't cheap fsub instruction.
10264 if (!isOperationLegalOrCustom(
10265 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, VT: SrcVT))
10266 return false;
10267
10268 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
10269 SDValue Sel;
10270
10271 if (Node->isStrictFPOpcode()) {
10272 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
10273 Chain: Node->getOperand(Num: 0), /*IsSignaling*/ true);
10274 Chain = Sel.getValue(R: 1);
10275 } else {
10276 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT);
10277 }
10278
10279 bool Strict = Node->isStrictFPOpcode() ||
10280 shouldUseStrictFP_TO_INT(FpVT: SrcVT, IntVT: DstVT, /*IsSigned*/ false);
10281
10282 if (Strict) {
10283 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
10284 // signmask then offset (the result of which should be fully representable).
10285 // Sel = Src < 0x8000000000000000
10286 // FltOfs = select Sel, 0, 0x8000000000000000
10287 // IntOfs = select Sel, 0, 0x8000000000000000
10288 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
10289
10290 // TODO: Should any fast-math-flags be set for the FSUB?
10291 SDValue FltOfs = DAG.getSelect(DL: dl, VT: SrcVT, Cond: Sel,
10292 LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
10293 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10294 SDValue IntOfs = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel,
10295 LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT),
10296 RHS: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10297 SDValue SInt;
10298 if (Node->isStrictFPOpcode()) {
10299 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: { SrcVT, MVT::Other },
10300 Ops: { Chain, Src, FltOfs });
10301 SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10302 Ops: { Val.getValue(R: 1), Val });
10303 Chain = SInt.getValue(R: 1);
10304 } else {
10305 SDValue Val = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: FltOfs);
10306 SInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Val);
10307 }
10308 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
10309 } else {
10310 // Expand based on maximum range of FP_TO_SINT:
10311 // True = fp_to_sint(Src)
10312 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
10313 // Result = select (Src < 0x8000000000000000), True, False
10314
10315 SDValue True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10316 // TODO: Should any fast-math-flags be set for the FSUB?
10317 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT,
10318 Operand: DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: Cst));
10319 False = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: False,
10320 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10321 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10322 Result = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel, LHS: True, RHS: False);
10323 }
10324 return true;
10325}
10326
10327bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
10328 SDValue &Chain, SelectionDAG &DAG) const {
10329 // This transform is not correct for converting 0 when rounding mode is set
10330 // to round toward negative infinity which will produce -0.0. So disable
10331 // under strictfp.
10332 if (Node->isStrictFPOpcode())
10333 return false;
10334
10335 SDValue Src = Node->getOperand(Num: 0);
10336 EVT SrcVT = Src.getValueType();
10337 EVT DstVT = Node->getValueType(ResNo: 0);
10338
10339 // If the input is known to be non-negative and SINT_TO_FP is legal then use
10340 // it.
10341 if (Node->getFlags().hasNonNeg() &&
10342 isOperationLegalOrCustom(Op: ISD::SINT_TO_FP, VT: SrcVT)) {
10343 Result =
10344 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Node->getOperand(Num: 0));
10345 return true;
10346 }
10347
10348 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
10349 return false;
10350
10351 // Only expand vector types if we have the appropriate vector bit
10352 // operations.
10353 if (SrcVT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SRL, VT: SrcVT) ||
10354 !isOperationLegalOrCustom(Op: ISD::FADD, VT: DstVT) ||
10355 !isOperationLegalOrCustom(Op: ISD::FSUB, VT: DstVT) ||
10356 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: SrcVT) ||
10357 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: SrcVT)))
10358 return false;
10359
10360 SDLoc dl(SDValue(Node, 0));
10361
10362 // Implementation of unsigned i64 to f64 following the algorithm in
10363 // __floatundidf in compiler_rt. This implementation performs rounding
10364 // correctly in all rounding modes with the exception of converting 0
10365 // when rounding toward negative infinity. In that case the fsub will
10366 // produce -0.0. This will be added to +0.0 and produce -0.0 which is
10367 // incorrect.
10368 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), DL: dl, VT: SrcVT);
10369 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
10370 Val: llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), DL: dl, VT: DstVT);
10371 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), DL: dl, VT: SrcVT);
10372 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), DL: dl, VT: SrcVT);
10373 SDValue HiShift = DAG.getShiftAmountConstant(Val: 32, VT: SrcVT, DL: dl);
10374
10375 SDValue Lo = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Src, N2: LoMask);
10376 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Src, N2: HiShift);
10377 SDValue LoOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Lo, N2: TwoP52);
10378 SDValue HiOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Hi, N2: TwoP84);
10379 SDValue LoFlt = DAG.getBitcast(VT: DstVT, V: LoOr);
10380 SDValue HiFlt = DAG.getBitcast(VT: DstVT, V: HiOr);
10381 SDValue HiSub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: DstVT, N1: HiFlt, N2: TwoP84PlusTwoP52);
10382 Result = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DstVT, N1: LoFlt, N2: HiSub);
10383 return true;
10384}
10385
10386SDValue
10387TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node,
10388 SelectionDAG &DAG) const {
10389 unsigned Opcode = Node->getOpcode();
10390 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
10391 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
10392 "Wrong opcode");
10393
10394 if (Node->getFlags().hasNoNaNs()) {
10395 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
10396 EVT VT = Node->getValueType(ResNo: 0);
10397 if ((!isCondCodeLegal(CC: Pred, VT: VT.getSimpleVT()) ||
10398 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)) &&
10399 VT.isVector())
10400 return SDValue();
10401 SDValue Op1 = Node->getOperand(Num: 0);
10402 SDValue Op2 = Node->getOperand(Num: 1);
10403 return DAG.getSelectCC(DL: SDLoc(Node), LHS: Op1, RHS: Op2, True: Op1, False: Op2, Cond: Pred,
10404 Flags: Node->getFlags());
10405 }
10406
10407 return SDValue();
10408}
10409
10410SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
10411 SelectionDAG &DAG) const {
10412 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node, DAG))
10413 return Expanded;
10414
10415 EVT VT = Node->getValueType(ResNo: 0);
10416 if (VT.isScalableVector())
10417 report_fatal_error(
10418 reason: "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
10419
10420 SDLoc dl(Node);
10421 unsigned NewOp =
10422 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10423
10424 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10425 SDValue Quiet0 = Node->getOperand(Num: 0);
10426 SDValue Quiet1 = Node->getOperand(Num: 1);
10427
10428 if (!Node->getFlags().hasNoNaNs()) {
10429 // Insert canonicalizes if it's possible we need to quiet to get correct
10430 // sNaN behavior.
10431 if (!DAG.isKnownNeverSNaN(Op: Quiet0)) {
10432 Quiet0 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet0,
10433 Flags: Node->getFlags());
10434 }
10435 if (!DAG.isKnownNeverSNaN(Op: Quiet1)) {
10436 Quiet1 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet1,
10437 Flags: Node->getFlags());
10438 }
10439 }
10440
10441 return DAG.getNode(Opcode: NewOp, DL: dl, VT, N1: Quiet0, N2: Quiet1, Flags: Node->getFlags());
10442 }
10443
10444 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
10445 // instead if there are no NaNs.
10446 if (Node->getFlags().hasNoNaNs() ||
10447 (DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 0)) &&
10448 DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 1)))) {
10449 unsigned IEEE2018Op =
10450 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10451 if (isOperationLegalOrCustom(Op: IEEE2018Op, VT))
10452 return DAG.getNode(Opcode: IEEE2018Op, DL: dl, VT, N1: Node->getOperand(Num: 0),
10453 N2: Node->getOperand(Num: 1), Flags: Node->getFlags());
10454 }
10455
10456 if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
10457 return SelCC;
10458
10459 return SDValue();
10460}
10461
10462static SDValue isSpecificZeroAfterMaybeRounding(SelectionDAG &DAG,
10463 const TargetLowering &TLI,
10464 const SDLoc &DL, SDValue Val,
10465 FPClassTest FPClass) {
10466 EVT VT = Val.getValueType();
10467 EVT CCVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10468 EVT IntVT = VT.changeTypeToInteger();
10469 EVT FloatVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
10470 SDValue TestZero = DAG.getTargetConstant(Val: FPClass, DL, VT: MVT::i32);
10471 if (!TLI.isTypeLegal(VT: IntVT) &&
10472 !TLI.isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT))
10473 Val = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: FloatVT, N1: Val,
10474 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
10475 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: CCVT, N1: Val, N2: TestZero);
10476}
10477
10478SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
10479 SelectionDAG &DAG) const {
10480 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node: N, DAG))
10481 return Expanded;
10482
10483 SDLoc DL(N);
10484 SDValue LHS = N->getOperand(Num: 0);
10485 SDValue RHS = N->getOperand(Num: 1);
10486 unsigned Opc = N->getOpcode();
10487 EVT VT = N->getValueType(ResNo: 0);
10488 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10489 bool IsMax = Opc == ISD::FMAXIMUM;
10490 SDNodeFlags Flags = N->getFlags();
10491
10492 // First, implement comparison not propagating NaN. If no native fmin or fmax
10493 // available, use plain select with setcc instead.
10494 SDValue MinMax;
10495 unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
10496 unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;
10497
10498 // FIXME: We should probably define fminnum/fmaxnum variants with correct
10499 // signed zero behavior.
10500 bool MinMaxMustRespectOrderedZero = false;
10501
10502 if (isOperationLegalOrCustom(Op: CompOpcIeee, VT)) {
10503 MinMax = DAG.getNode(Opcode: CompOpcIeee, DL, VT, N1: LHS, N2: RHS, Flags);
10504 MinMaxMustRespectOrderedZero = true;
10505 } else if (isOperationLegalOrCustom(Op: CompOpc, VT)) {
10506 MinMax = DAG.getNode(Opcode: CompOpc, DL, VT, N1: LHS, N2: RHS, Flags);
10507 } else {
10508 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
10509 return DAG.UnrollVectorOp(N);
10510
10511 // NaN (if exists) will be propagated later, so orderness doesn't matter.
10512 SDValue Compare =
10513 DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: IsMax ? ISD::SETOGT : ISD::SETOLT);
10514 MinMax = DAG.getSelect(DL, VT, Cond: Compare, LHS, RHS, Flags);
10515 }
10516
10517 // Propagate any NaN of both operands
10518 if (!N->getFlags().hasNoNaNs() &&
10519 (!DAG.isKnownNeverNaN(Op: RHS) || !DAG.isKnownNeverNaN(Op: LHS))) {
10520 ConstantFP *FPNaN = ConstantFP::get(Context&: *DAG.getContext(),
10521 V: APFloat::getNaN(Sem: VT.getFltSemantics()));
10522 MinMax = DAG.getSelect(DL, VT, Cond: DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETUO),
10523 LHS: DAG.getConstantFP(V: *FPNaN, DL, VT), RHS: MinMax, Flags);
10524 }
10525
10526 // fminimum/fmaximum requires -0.0 less than +0.0
10527 if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
10528 !DAG.isKnownNeverLogicalZero(Op: RHS) && !DAG.isKnownNeverLogicalZero(Op: LHS)) {
10529 SDValue IsEqual = DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETOEQ);
10530 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10531 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10532 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS, Flags);
10533 MinMax = DAG.getSelect(DL, VT, Cond: IsEqual, LHS: RetZero, RHS: MinMax, Flags);
10534 }
10535
10536 return MinMax;
10537}
10538
10539SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
10540 SelectionDAG &DAG) const {
10541 SDLoc DL(Node);
10542 SDValue LHS = Node->getOperand(Num: 0);
10543 SDValue RHS = Node->getOperand(Num: 1);
10544 unsigned Opc = Node->getOpcode();
10545 EVT VT = Node->getValueType(ResNo: 0);
10546 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10547 bool IsMax = Opc == ISD::FMAXIMUMNUM;
10548 SDNodeFlags Flags = Node->getFlags();
10549
10550 unsigned NewOp =
10551 Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10552
10553 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10554 if (!Flags.hasNoNaNs()) {
10555 // Insert canonicalizes if it's possible we need to quiet to get correct
10556 // sNaN behavior.
10557 if (!DAG.isKnownNeverSNaN(Op: LHS)) {
10558 LHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: LHS, Flags);
10559 }
10560 if (!DAG.isKnownNeverSNaN(Op: RHS)) {
10561 RHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: RHS, Flags);
10562 }
10563 }
10564
10565 return DAG.getNode(Opcode: NewOp, DL, VT, N1: LHS, N2: RHS, Flags);
10566 }
10567
10568 // We can use FMINIMUM/FMAXIMUM if there is no NaN, since it has
10569 // same behaviors for all of other cases: +0.0 vs -0.0 included.
10570 if (Flags.hasNoNaNs() ||
10571 (DAG.isKnownNeverNaN(Op: LHS) && DAG.isKnownNeverNaN(Op: RHS))) {
10572 unsigned IEEE2019Op =
10573 Opc == ISD::FMINIMUMNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10574 if (isOperationLegalOrCustom(Op: IEEE2019Op, VT))
10575 return DAG.getNode(Opcode: IEEE2019Op, DL, VT, N1: LHS, N2: RHS, Flags);
10576 }
10577
10578 // FMINNUM/FMAXMUM returns qNaN if either operand is sNaN, and it may return
10579 // either one for +0.0 vs -0.0.
10580 if ((Flags.hasNoNaNs() ||
10581 (DAG.isKnownNeverSNaN(Op: LHS) && DAG.isKnownNeverSNaN(Op: RHS))) &&
10582 (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10583 DAG.isKnownNeverLogicalZero(Op: RHS))) {
10584 unsigned IEEE2008Op = Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM : ISD::FMAXNUM;
10585 if (isOperationLegalOrCustom(Op: IEEE2008Op, VT))
10586 return DAG.getNode(Opcode: IEEE2008Op, DL, VT, N1: LHS, N2: RHS, Flags);
10587 }
10588
10589 if (VT.isVector() &&
10590 (isOperationLegalOrCustomOrPromote(Op: Opc, VT: VT.getVectorElementType()) ||
10591 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)))
10592 return DAG.UnrollVectorOp(N: Node);
10593
10594 // If only one operand is NaN, override it with another operand.
10595 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: LHS)) {
10596 LHS = DAG.getSelectCC(DL, LHS, RHS: LHS, True: RHS, False: LHS, Cond: ISD::SETUO);
10597 }
10598 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: RHS)) {
10599 RHS = DAG.getSelectCC(DL, LHS: RHS, RHS, True: LHS, False: RHS, Cond: ISD::SETUO);
10600 }
10601
10602 // Always prefer RHS if equal.
10603 SDValue MinMax =
10604 DAG.getSelectCC(DL, LHS, RHS, True: LHS, False: RHS, Cond: IsMax ? ISD::SETGT : ISD::SETLT);
10605
10606 // TODO: We need quiet sNaN if strictfp.
10607
10608 // Fixup signed zero behavior.
10609 if (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10610 DAG.isKnownNeverLogicalZero(Op: RHS)) {
10611 return MinMax;
10612 }
10613 SDValue IsZero = DAG.getSetCC(DL, VT: CCVT, LHS: MinMax,
10614 RHS: DAG.getConstantFP(Val: 0.0, DL, VT), Cond: ISD::SETEQ);
10615 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10616 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10617 // It's OK to select from LHS and MinMax, with only one ISD::IS_FPCLASS, as
10618 // we preferred RHS when generate MinMax, if the operands are equal.
10619 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS: MinMax, Flags);
10620 return DAG.getSelect(DL, VT, Cond: IsZero, LHS: RetZero, RHS: MinMax, Flags);
10621}
10622
10623/// Returns a true value if if this FPClassTest can be performed with an ordered
10624/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
10625/// std::nullopt if it cannot be performed as a compare with 0.
10626static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
10627 const fltSemantics &Semantics,
10628 const MachineFunction &MF) {
10629 FPClassTest OrderedMask = Test & ~fcNan;
10630 FPClassTest NanTest = Test & fcNan;
10631 bool IsOrdered = NanTest == fcNone;
10632 bool IsUnordered = NanTest == fcNan;
10633
10634 // Skip cases that are testing for only a qnan or snan.
10635 if (!IsOrdered && !IsUnordered)
10636 return std::nullopt;
10637
10638 if (OrderedMask == fcZero &&
10639 MF.getDenormalMode(FPType: Semantics).Input == DenormalMode::IEEE)
10640 return IsOrdered;
10641 if (OrderedMask == (fcZero | fcSubnormal) &&
10642 MF.getDenormalMode(FPType: Semantics).inputsAreZero())
10643 return IsOrdered;
10644 return std::nullopt;
10645}
10646
10647SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
10648 const FPClassTest OrigTestMask,
10649 SDNodeFlags Flags, const SDLoc &DL,
10650 SelectionDAG &DAG) const {
10651 EVT OperandVT = Op.getValueType();
10652 assert(OperandVT.isFloatingPoint());
10653 FPClassTest Test = OrigTestMask;
10654
10655 // Degenerated cases.
10656 if (Test == fcNone)
10657 return DAG.getBoolConstant(V: false, DL, VT: ResultVT, OpVT: OperandVT);
10658 if (Test == fcAllFlags)
10659 return DAG.getBoolConstant(V: true, DL, VT: ResultVT, OpVT: OperandVT);
10660
10661 // PPC double double is a pair of doubles, of which the higher part determines
10662 // the value class.
10663 if (OperandVT == MVT::ppcf128) {
10664 Op = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::f64, N1: Op,
10665 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
10666 OperandVT = MVT::f64;
10667 }
10668
10669 // Floating-point type properties.
10670 EVT ScalarFloatVT = OperandVT.getScalarType();
10671 const Type *FloatTy = ScalarFloatVT.getTypeForEVT(Context&: *DAG.getContext());
10672 const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
10673 bool IsF80 = (ScalarFloatVT == MVT::f80);
10674
10675 // Some checks can be implemented using float comparisons, if floating point
10676 // exceptions are ignored.
10677 if (Flags.hasNoFPExcept() &&
10678 isOperationLegalOrCustom(Op: ISD::SETCC, VT: OperandVT.getScalarType())) {
10679 FPClassTest FPTestMask = Test;
10680 bool IsInvertedFP = false;
10681
10682 if (FPClassTest InvertedFPCheck =
10683 invertFPClassTestIfSimpler(Test: FPTestMask, UseFCmp: true)) {
10684 FPTestMask = InvertedFPCheck;
10685 IsInvertedFP = true;
10686 }
10687
10688 ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ;
10689 ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ;
10690
10691 // See if we can fold an | fcNan into an unordered compare.
10692 FPClassTest OrderedFPTestMask = FPTestMask & ~fcNan;
10693
10694 // Can't fold the ordered check if we're only testing for snan or qnan
10695 // individually.
10696 if ((FPTestMask & fcNan) != fcNan)
10697 OrderedFPTestMask = FPTestMask;
10698
10699 const bool IsOrdered = FPTestMask == OrderedFPTestMask;
10700
10701 if (std::optional<bool> IsCmp0 =
10702 isFCmpEqualZero(Test: FPTestMask, Semantics, MF: DAG.getMachineFunction());
10703 IsCmp0 && (isCondCodeLegalOrCustom(
10704 CC: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
10705 VT: OperandVT.getScalarType().getSimpleVT()))) {
10706
10707 // If denormals could be implicitly treated as 0, this is not equivalent
10708 // to a compare with 0 since it will also be true for denormals.
10709 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op,
10710 RHS: DAG.getConstantFP(Val: 0.0, DL, VT: OperandVT),
10711 Cond: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
10712 }
10713
10714 if (FPTestMask == fcNan &&
10715 isCondCodeLegalOrCustom(CC: IsInvertedFP ? ISD::SETO : ISD::SETUO,
10716 VT: OperandVT.getScalarType().getSimpleVT()))
10717 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Op,
10718 Cond: IsInvertedFP ? ISD::SETO : ISD::SETUO);
10719
10720 bool IsOrderedInf = FPTestMask == fcInf;
10721 if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) &&
10722 isCondCodeLegalOrCustom(CC: IsOrderedInf ? OrderedCmpOpcode
10723 : UnorderedCmpOpcode,
10724 VT: OperandVT.getScalarType().getSimpleVT()) &&
10725 isOperationLegalOrCustom(Op: ISD::FABS, VT: OperandVT.getScalarType()) &&
10726 (isOperationLegal(Op: ISD::ConstantFP, VT: OperandVT.getScalarType()) ||
10727 (OperandVT.isVector() &&
10728 isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: OperandVT)))) {
10729 // isinf(x) --> fabs(x) == inf
10730 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10731 SDValue Inf =
10732 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10733 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf,
10734 Cond: IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
10735 }
10736
10737 if ((OrderedFPTestMask == fcPosInf || OrderedFPTestMask == fcNegInf) &&
10738 isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedCmpOpcode
10739 : UnorderedCmpOpcode,
10740 VT: OperandVT.getSimpleVT())) {
10741 // isposinf(x) --> x == inf
10742 // isneginf(x) --> x == -inf
10743 // isposinf(x) || nan --> x u== inf
10744 // isneginf(x) || nan --> x u== -inf
10745
10746 SDValue Inf = DAG.getConstantFP(
10747 Val: APFloat::getInf(Sem: Semantics, Negative: OrderedFPTestMask == fcNegInf), DL,
10748 VT: OperandVT);
10749 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Inf,
10750 Cond: IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
10751 }
10752
10753 if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
10754 // TODO: Could handle ordered case, but it produces worse code for
10755 // x86. Maybe handle ordered if fabs is free?
10756
10757 ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10758 ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;
10759
10760 if (isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedOp : UnorderedOp,
10761 VT: OperandVT.getScalarType().getSimpleVT())) {
10762 // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal
10763
10764 // TODO: Maybe only makes sense if fabs is free. Integer test of
10765 // exponent bits seems better for x86.
10766 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10767 SDValue SmallestNormal = DAG.getConstantFP(
10768 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10769 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal,
10770 Cond: IsOrdered ? OrderedOp : UnorderedOp);
10771 }
10772 }
10773
10774 if (FPTestMask == fcNormal) {
10775 // TODO: Handle unordered
10776 ISD::CondCode IsFiniteOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10777 ISD::CondCode IsNormalOp = IsInvertedFP ? ISD::SETOLT : ISD::SETUGE;
10778
10779 if (isCondCodeLegalOrCustom(CC: IsFiniteOp,
10780 VT: OperandVT.getScalarType().getSimpleVT()) &&
10781 isCondCodeLegalOrCustom(CC: IsNormalOp,
10782 VT: OperandVT.getScalarType().getSimpleVT()) &&
10783 isFAbsFree(VT: OperandVT)) {
10784 // isnormal(x) --> fabs(x) < infinity && !(fabs(x) < smallest_normal)
10785 SDValue Inf =
10786 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10787 SDValue SmallestNormal = DAG.getConstantFP(
10788 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10789
10790 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10791 SDValue IsFinite = DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf, Cond: IsFiniteOp);
10792 SDValue IsNormal =
10793 DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal, Cond: IsNormalOp);
10794 unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
10795 return DAG.getNode(Opcode: LogicOp, DL, VT: ResultVT, N1: IsFinite, N2: IsNormal);
10796 }
10797 }
10798 }
10799
10800 // Some checks may be represented as inversion of simpler check, for example
10801 // "inf|normal|subnormal|zero" => !"nan".
10802 bool IsInverted = false;
10803
10804 if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, UseFCmp: false)) {
10805 Test = InvertedCheck;
10806 IsInverted = true;
10807 }
10808
10809 // In the general case use integer operations.
10810 unsigned BitSize = OperandVT.getScalarSizeInBits();
10811 EVT IntVT = OperandVT.changeElementType(
10812 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BitSize));
10813 SDValue OpAsInt = DAG.getBitcast(VT: IntVT, V: Op);
10814
10815 // Various masks.
10816 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10817 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10818 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10819 const unsigned ExplicitIntBitInF80 = 63;
10820 APInt ExpMask = Inf;
10821 if (IsF80)
10822 ExpMask.clearBit(BitPosition: ExplicitIntBitInF80);
10823 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10824 APInt QNaNBitMask =
10825 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10826 APInt InversionMask = APInt::getAllOnes(numBits: ResultVT.getScalarSizeInBits());
10827
10828 SDValue ValueMaskV = DAG.getConstant(Val: ValueMask, DL, VT: IntVT);
10829 SDValue SignBitV = DAG.getConstant(Val: SignBit, DL, VT: IntVT);
10830 SDValue ExpMaskV = DAG.getConstant(Val: ExpMask, DL, VT: IntVT);
10831 SDValue ZeroV = DAG.getConstant(Val: 0, DL, VT: IntVT);
10832 SDValue InfV = DAG.getConstant(Val: Inf, DL, VT: IntVT);
10833 SDValue ResultInversionMask = DAG.getConstant(Val: InversionMask, DL, VT: ResultVT);
10834
10835 SDValue Res;
10836 const auto appendResult = [&](SDValue PartialRes) {
10837 if (PartialRes) {
10838 if (Res)
10839 Res = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: Res, N2: PartialRes);
10840 else
10841 Res = PartialRes;
10842 }
10843 };
10844
10845 SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
10846 const auto getIntBitIsSet = [&]() -> SDValue {
10847 if (!IntBitIsSetV) {
10848 APInt IntBitMask(BitSize, 0);
10849 IntBitMask.setBit(ExplicitIntBitInF80);
10850 SDValue IntBitMaskV = DAG.getConstant(Val: IntBitMask, DL, VT: IntVT);
10851 SDValue IntBitV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: IntBitMaskV);
10852 IntBitIsSetV = DAG.getSetCC(DL, VT: ResultVT, LHS: IntBitV, RHS: ZeroV, Cond: ISD::SETNE);
10853 }
10854 return IntBitIsSetV;
10855 };
10856
10857 // Split the value into sign bit and absolute value.
10858 SDValue AbsV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ValueMaskV);
10859 SDValue SignV = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt,
10860 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETLT);
10861
10862 // Tests that involve more than one class should be processed first.
10863 SDValue PartialRes;
10864
10865 if (IsF80)
10866 ; // Detect finite numbers of f80 by checking individual classes because
10867 // they have different settings of the explicit integer bit.
10868 else if ((Test & fcFinite) == fcFinite) {
10869 // finite(V) ==> (a << 1) < (inf << 1)
10870 //
10871 // See https://github.com/llvm/llvm-project/issues/169270, this is slightly
10872 // shorter than the `finite(V) ==> abs(V) < exp_mask` formula used before.
10873
10874 assert(APFloat::isIEEELikeFP(OperandVT.getFltSemantics()) &&
10875 "finite check requires IEEE-like FP");
10876
10877 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: IntVT, DL);
10878 SDValue TwiceOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: OpAsInt, N2: One);
10879 SDValue TwiceInf = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: ExpMaskV, N2: One);
10880
10881 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: TwiceOp, RHS: TwiceInf, Cond: ISD::SETULT);
10882 Test &= ~fcFinite;
10883 } else if ((Test & fcFinite) == fcPosFinite) {
10884 // finite(V) && V > 0 ==> V < exp_mask
10885 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ExpMaskV, Cond: ISD::SETULT);
10886 Test &= ~fcPosFinite;
10887 } else if ((Test & fcFinite) == fcNegFinite) {
10888 // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
10889 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ExpMaskV, Cond: ISD::SETLT);
10890 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10891 Test &= ~fcNegFinite;
10892 }
10893 appendResult(PartialRes);
10894
10895 if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
10896 // fcZero | fcSubnormal => test all exponent bits are 0
10897 // TODO: Handle sign bit specific cases
10898 if (PartialCheck == (fcZero | fcSubnormal)) {
10899 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ExpMaskV);
10900 SDValue ExpIsZero =
10901 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
10902 appendResult(ExpIsZero);
10903 Test &= ~PartialCheck & fcAllFlags;
10904 }
10905 }
10906
10907 // Check for individual classes.
10908
10909 if (unsigned PartialCheck = Test & fcZero) {
10910 if (PartialCheck == fcPosZero)
10911 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ZeroV, Cond: ISD::SETEQ);
10912 else if (PartialCheck == fcZero)
10913 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ZeroV, Cond: ISD::SETEQ);
10914 else // ISD::fcNegZero
10915 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: SignBitV, Cond: ISD::SETEQ);
10916 appendResult(PartialRes);
10917 }
10918
10919 if (unsigned PartialCheck = Test & fcSubnormal) {
10920 // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
10921 // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
10922 SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
10923 SDValue MantissaV = DAG.getConstant(Val: AllOneMantissa, DL, VT: IntVT);
10924 SDValue VMinusOneV =
10925 DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: V, N2: DAG.getConstant(Val: 1, DL, VT: IntVT));
10926 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: VMinusOneV, RHS: MantissaV, Cond: ISD::SETULT);
10927 if (PartialCheck == fcNegSubnormal)
10928 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10929 appendResult(PartialRes);
10930 }
10931
10932 if (unsigned PartialCheck = Test & fcInf) {
10933 if (PartialCheck == fcPosInf)
10934 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: InfV, Cond: ISD::SETEQ);
10935 else if (PartialCheck == fcInf)
10936 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETEQ);
10937 else { // ISD::fcNegInf
10938 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10939 SDValue NegInfV = DAG.getConstant(Val: NegInf, DL, VT: IntVT);
10940 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: NegInfV, Cond: ISD::SETEQ);
10941 }
10942 appendResult(PartialRes);
10943 }
10944
10945 if (unsigned PartialCheck = Test & fcNan) {
10946 APInt InfWithQnanBit = Inf | QNaNBitMask;
10947 SDValue InfWithQnanBitV = DAG.getConstant(Val: InfWithQnanBit, DL, VT: IntVT);
10948 if (PartialCheck == fcNan) {
10949 // isnan(V) ==> abs(V) > int(inf)
10950 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
10951 if (IsF80) {
10952 // Recognize unsupported values as NaNs for compatibility with glibc.
10953 // In them (exp(V)==0) == int_bit.
10954 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: AbsV, N2: ExpMaskV);
10955 SDValue ExpIsZero =
10956 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
10957 SDValue IsPseudo =
10958 DAG.getSetCC(DL, VT: ResultVT, LHS: getIntBitIsSet(), RHS: ExpIsZero, Cond: ISD::SETEQ);
10959 PartialRes = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: PartialRes, N2: IsPseudo);
10960 }
10961 } else if (PartialCheck == fcQNan) {
10962 // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
10963 PartialRes =
10964 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETGE);
10965 } else { // ISD::fcSNan
10966 // issignaling(V) ==> abs(V) > unsigned(Inf) &&
10967 // abs(V) < (unsigned(Inf) | quiet_bit)
10968 SDValue IsNan = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
10969 SDValue IsNotQnan =
10970 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETLT);
10971 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: IsNan, N2: IsNotQnan);
10972 }
10973 appendResult(PartialRes);
10974 }
10975
10976 if (unsigned PartialCheck = Test & fcNormal) {
10977 // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
10978 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10979 SDValue ExpLSBV = DAG.getConstant(Val: ExpLSB, DL, VT: IntVT);
10980 SDValue ExpMinus1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: AbsV, N2: ExpLSBV);
10981 APInt ExpLimit = ExpMask - ExpLSB;
10982 SDValue ExpLimitV = DAG.getConstant(Val: ExpLimit, DL, VT: IntVT);
10983 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: ExpMinus1, RHS: ExpLimitV, Cond: ISD::SETULT);
10984 if (PartialCheck == fcNegNormal)
10985 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10986 else if (PartialCheck == fcPosNormal) {
10987 SDValue PosSignV =
10988 DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: SignV, N2: ResultInversionMask);
10989 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: PosSignV);
10990 }
10991 if (IsF80)
10992 PartialRes =
10993 DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: getIntBitIsSet());
10994 appendResult(PartialRes);
10995 }
10996
10997 if (!Res)
10998 return DAG.getConstant(Val: IsInverted, DL, VT: ResultVT);
10999 if (IsInverted)
11000 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: Res, N2: ResultInversionMask);
11001 return Res;
11002}
11003
11004// Only expand vector types if we have the appropriate vector bit operations.
11005static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
11006 assert(VT.isVector() && "Expected vector type");
11007 unsigned Len = VT.getScalarSizeInBits();
11008 return TLI.isOperationLegalOrCustom(Op: ISD::ADD, VT) &&
11009 TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT) &&
11010 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
11011 (Len == 8 || TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT)) &&
11012 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT);
11013}
11014
11015SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
11016 SDLoc dl(Node);
11017 EVT VT = Node->getValueType(ResNo: 0);
11018 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11019 SDValue Op = Node->getOperand(Num: 0);
11020 unsigned Len = VT.getScalarSizeInBits();
11021
11022 // Compute effective bit width from known bits, allowing us to shift the
11023 // active bits down if necessary to fit into smaller specialized expansions.
11024 KnownBits Known = DAG.computeKnownBits(Op);
11025 unsigned LZ = Known.countMinLeadingZeros();
11026 unsigned TZ = Known.countMinTrailingZeros();
11027 unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ);
11028
11029 // Round up to 8-bit boundary for byte-oriented SWAR algorithm
11030 unsigned EffectiveLen = Len;
11031 if (ShiftedActiveBits > 0 && ShiftedActiveBits < Len)
11032 EffectiveLen = std::min(a: alignTo(Value: ShiftedActiveBits, Align: 8), b: Len);
11033
11034 assert(VT.isInteger() && "CTPOP not implemented for this type.");
11035
11036 // TODO: Add support for irregular type lengths.
11037 if (!(Len <= 128 && Len % 8 == 0))
11038 return SDValue();
11039
11040 // Only expand vector types if we have the appropriate vector bit operations.
11041 if (VT.isVector() && !canExpandVectorCTPOP(TLI: *this, VT))
11042 return SDValue();
11043
11044 // If the active bits are not at the low end, shift them down
11045 if (EffectiveLen < Len && TZ > 0) {
11046 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11047 N2: DAG.getShiftAmountConstant(Val: TZ, VT, DL: dl));
11048 }
11049
11050 // This is the "best" algorithm from
11051 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
11052 SDValue Mask55 =
11053 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x55)), DL: dl, VT);
11054 SDValue Mask33 =
11055 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x33)), DL: dl, VT);
11056 SDValue Mask0F =
11057 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x0F)), DL: dl, VT);
11058
11059 // v = v - ((v >> 1) & 0x55555555...)
11060 Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op,
11061 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11062 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11063 N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT)),
11064 N2: Mask55));
11065 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
11066 Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: Mask33),
11067 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11068 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11069 N2: DAG.getConstant(Val: 2, DL: dl, VT: ShVT)),
11070 N2: Mask33));
11071 // v = (v + (v >> 4)) & 0x0F0F0F0F...
11072 Op = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11073 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
11074 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11075 N2: DAG.getConstant(Val: 4, DL: dl, VT: ShVT))),
11076 N2: Mask0F);
11077
11078 if (EffectiveLen <= 8)
11079 return Op;
11080
11081 // Avoid the multiply if we only have 2 bytes to add.
11082 // TODO: Only doing this for scalars because vectors weren't as obviously
11083 // improved.
11084 if (EffectiveLen == 16 && !VT.isVector()) {
11085 // v = (v + (v >> 8)) & 0x00FF;
11086 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
11087 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
11088 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
11089 N2: DAG.getConstant(Val: 8, DL: dl, VT: ShVT))),
11090 N2: DAG.getConstant(Val: 0xFF, DL: dl, VT));
11091 }
11092
11093 // v = (v * 0x01010101...) >> (Len - 8)
11094 SDValue V;
11095 if (isOperationLegalOrCustomOrPromote(
11096 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11097 SDValue Mask01 =
11098 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x01)), DL: dl, VT);
11099 V = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Op, N2: Mask01);
11100 } else {
11101 V = Op;
11102 for (unsigned Shift = 8; Shift < EffectiveLen; Shift *= 2) {
11103 SDValue ShiftC = DAG.getShiftAmountConstant(Val: Shift, VT, DL: dl);
11104 V = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: V,
11105 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: ShiftC));
11106 }
11107 }
11108 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: V, N2: DAG.getConstant(Val: Len - 8, DL: dl, VT: ShVT));
11109}
11110
11111SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
11112 SDLoc dl(Node);
11113 EVT VT = Node->getValueType(ResNo: 0);
11114 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11115 SDValue Op = Node->getOperand(Num: 0);
11116 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11117
11118 // If the non-ZERO_POISON version is supported we can use that instead.
11119 if (Node->getOpcode() == ISD::CTLZ_ZERO_POISON &&
11120 isOperationLegalOrCustom(Op: ISD::CTLZ, VT))
11121 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Op);
11122
11123 // If the ZERO_POISON version is supported use that and handle the zero case.
11124 if (isOperationLegalOrCustom(Op: ISD::CTLZ_ZERO_POISON, VT)) {
11125 EVT SetCCVT =
11126 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11127 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11128 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11129 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11130 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11131 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTLZ);
11132 }
11133
11134 // Only expand vector types if we have the appropriate vector bit operations.
11135 // This includes the operations needed to expand CTPOP if it isn't supported.
11136 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11137 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11138 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11139 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
11140 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
11141 return SDValue();
11142
11143 // for now, we do this:
11144 // x = x | (x >> 1);
11145 // x = x | (x >> 2);
11146 // ...
11147 // x = x | (x >>16);
11148 // x = x | (x >>32); // for 64-bit input
11149 // return popcount(~x);
11150 //
11151 // Ref: "Hacker's Delight" by Henry Warren
11152 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
11153 SDValue Tmp = DAG.getConstant(Val: 1ULL << i, DL: dl, VT: ShVT);
11154 Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op,
11155 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: Tmp));
11156 }
11157 Op = DAG.getNOT(DL: dl, Val: Op, VT);
11158 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
11159}
11160
11161SDValue TargetLowering::expandCTLS(SDNode *Node, SelectionDAG &DAG) const {
11162 SDLoc dl(Node);
11163 EVT VT = Node->getValueType(ResNo: 0);
11164 SDValue Op = DAG.getFreeze(V: Node->getOperand(Num: 0));
11165 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11166
11167 // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
11168 // This transforms the sign bits into leading zeros that can be counted.
11169 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: NumBitsPerElt - 1, VT, DL: dl);
11170 SDValue SignBit = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Op, N2: ShiftAmt);
11171 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: SignBit);
11172 SDValue Shl =
11173 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11174 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Shl, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11175 return DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Or);
11176}
11177
11178SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG,
11179 const SDLoc &DL, EVT VT, SDValue Op,
11180 unsigned BitWidth) const {
11181 if (BitWidth != 32 && BitWidth != 64)
11182 return SDValue();
11183
11184 const DataLayout &TD = DAG.getDataLayout();
11185 if (!isOperationCustom(Op: ISD::ConstantPool, VT: getPointerTy(DL: TD)))
11186 return SDValue();
11187
11188 APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
11189 : APInt(64, 0x0218A392CD3D5DBFULL);
11190 MachinePointerInfo PtrInfo =
11191 MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction());
11192 unsigned ShiftAmt = BitWidth - Log2_32(Value: BitWidth);
11193 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
11194 SDValue Lookup = DAG.getNode(
11195 Opcode: ISD::SRL, DL, VT,
11196 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op, N2: Neg),
11197 N2: DAG.getConstant(Val: DeBruijn, DL, VT)),
11198 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
11199 Lookup = DAG.getSExtOrTrunc(Op: Lookup, DL, VT: getPointerTy(DL: TD));
11200
11201 SmallVector<uint8_t> Table(BitWidth, 0);
11202 for (unsigned i = 0; i < BitWidth; i++) {
11203 APInt Shl = DeBruijn.shl(shiftAmt: i);
11204 APInt Lshr = Shl.lshr(shiftAmt: ShiftAmt);
11205 Table[Lshr.getZExtValue()] = i;
11206 }
11207
11208 // Create a ConstantArray in Constant Pool
11209 auto *CA = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: Table);
11210 SDValue CPIdx = DAG.getConstantPool(C: CA, VT: getPointerTy(DL: TD),
11211 Align: TD.getPrefTypeAlign(Ty: CA->getType()));
11212 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: DL, VT, Chain: DAG.getEntryNode(),
11213 Ptr: DAG.getMemBasePlusOffset(Base: CPIdx, Offset: Lookup, DL),
11214 PtrInfo, MemVT: MVT::i8);
11215 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
11216 return ExtLoad;
11217
11218 EVT SetCCVT =
11219 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11220 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
11221 SDValue SrcIsZero = DAG.getSetCC(DL, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11222 return DAG.getSelect(DL, VT, Cond: SrcIsZero,
11223 LHS: DAG.getConstant(Val: BitWidth, DL, VT), RHS: ExtLoad);
11224}
11225
11226SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
11227 SDLoc dl(Node);
11228 EVT VT = Node->getValueType(ResNo: 0);
11229 SDValue Op = Node->getOperand(Num: 0);
11230 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11231
11232 // If the non-ZERO_POISON version is supported we can use that instead.
11233 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON &&
11234 isOperationLegalOrCustom(Op: ISD::CTTZ, VT))
11235 return DAG.getNode(Opcode: ISD::CTTZ, DL: dl, VT, Operand: Op);
11236
11237 // If the ZERO_POISON version is supported use that and handle the zero case.
11238 if (isOperationLegalOrCustom(Op: ISD::CTTZ_ZERO_POISON, VT)) {
11239 EVT SetCCVT =
11240 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11241 SDValue CTTZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11242 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11243 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11244 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11245 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTTZ);
11246 }
11247
11248 // Only expand vector types if we have the appropriate vector bit operations.
11249 // This includes the operations needed to expand CTPOP if it isn't supported.
11250 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11251 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11252 !isOperationLegalOrCustom(Op: ISD::CTLZ, VT) &&
11253 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11254 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
11255 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) ||
11256 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11257 return SDValue();
11258
11259 // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
11260 // to be expanded or converted to a libcall.
11261 if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT) &&
11262 !isOperationLegal(Op: ISD::CTLZ, VT))
11263 if (SDValue V = CTTZTableLookup(Node, DAG, DL: dl, VT, Op, BitWidth: NumBitsPerElt))
11264 return V;
11265
11266 bool UseCTLZ =
11267 isOperationLegal(Op: ISD::CTLZ, VT) && !isOperationLegal(Op: ISD::CTPOP, VT);
11268
11269 // When only ctlz is available and the operand is nonzero we can use:
11270 // { return nlz(x & -x) ^ 31; }
11271 // which is more efficient than:
11272 // { return 32 - nlz(~x & (x - 1)); }.
11273 if (UseCTLZ && Node->getOpcode() == ISD::CTTZ_ZERO_POISON) {
11274 SDValue LowestBit =
11275 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: DAG.getNegative(Val: Op, DL: dl, VT));
11276 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT,
11277 N1: DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: LowestBit),
11278 N2: DAG.getConstant(Val: NumBitsPerElt - 1, DL: dl, VT));
11279 }
11280
11281 // If ctpop is available, we use:
11282 // { return popcount(~x & (x-1)); }
11283 // If the target has ctlz but not ctpop, we use:
11284 // { return 32 - nlz(~x & (x-1)); }
11285 // Ref: "Hacker's Delight" by Henry Warren
11286 SDValue Tmp = DAG.getNode(
11287 Opcode: ISD::AND, DL: dl, VT, N1: DAG.getNOT(DL: dl, Val: Op, VT),
11288 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 1, DL: dl, VT)));
11289
11290 if (UseCTLZ)
11291 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT),
11292 N2: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Tmp));
11293
11294 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Tmp);
11295}
11296
11297SDValue TargetLowering::expandVPCTTZElements(SDNode *N,
11298 SelectionDAG &DAG) const {
11299 // %cond = to_bool_vec %source
11300 // %splat = splat /*val=*/VL
11301 // %tz = step_vector
11302 // %v = select %cond, /*true=*/tz, /*false=*/%splat
11303 // %r = vp.reduce.umin %v
11304 SDLoc DL(N);
11305 SDValue Source = N->getOperand(Num: 0);
11306 SDValue Mask = N->getOperand(Num: 1);
11307 SDValue EVL = N->getOperand(Num: 2);
11308 EVT SrcVT = Source.getValueType();
11309 EVT ResVT = N->getValueType(ResNo: 0);
11310 EVT ResVecVT =
11311 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT, EC: SrcVT.getVectorElementCount());
11312
11313 // Convert to boolean vector.
11314 if (SrcVT.getScalarType() != MVT::i1) {
11315 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
11316 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
11317 EC: SrcVT.getVectorElementCount());
11318 Source = DAG.getSetCC(DL, VT: SrcVT, LHS: Source, RHS: AllZero, Cond: ISD::SETNE);
11319 }
11320
11321 SDValue ExtEVL = DAG.getZExtOrTrunc(Op: EVL, DL, VT: ResVT);
11322 SDValue Splat = DAG.getSplat(VT: ResVecVT, DL, Op: ExtEVL);
11323 SDValue StepVec = DAG.getStepVector(DL, ResVT: ResVecVT);
11324 SDValue Select = DAG.getSelect(DL, VT: ResVecVT, Cond: Source, LHS: StepVec, RHS: Splat);
11325 return DAG.getNode(Opcode: ISD::VP_REDUCE_UMIN, DL, VT: ResVT, N1: ExtEVL, N2: Select, N3: Mask, N4: EVL);
11326}
11327
11328/// Returns a type-legalized version of \p Mask as the first item in the
11329/// pair. The second item contains a type-legalized step vector that's
11330/// guaranteed to fit the number of elements in \p Mask.
11331/// If the stepvector would require splitting, returns an empty SDValue
11332/// as the second item to signal that the operation should be split instead.
11333static std::pair<SDValue, SDValue>
11334getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL,
11335 SelectionDAG &DAG) {
11336 EVT MaskVT = Mask.getValueType();
11337 EVT BoolVT = MaskVT.getScalarType();
11338
11339 // Find a suitable type for a stepvector.
11340 // If zero is poison, we can assume the upper limit of the result is VF-1.
11341 ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
11342 if (MaskVT.isScalableVector())
11343 VScaleRange = getVScaleRange(F: &DAG.getMachineFunction().getFunction(), BitWidth: 64);
11344 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11345 uint64_t EltWidth = TLI.getBitWidthForCttzElements(
11346 RetVT: EVT(TLI.getVectorIdxTy(DL: DAG.getDataLayout())),
11347 EC: MaskVT.getVectorElementCount(), ZeroIsPoison, VScaleRange: &VScaleRange);
11348 // If the step vector element type is smaller than the mask element type,
11349 // use the mask type directly to avoid widening issues.
11350 EltWidth = std::max(a: EltWidth, b: BoolVT.getFixedSizeInBits());
11351 EVT StepVT = MVT::getIntegerVT(BitWidth: EltWidth);
11352 EVT StepVecVT = MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: StepVT);
11353
11354 // If promotion or widening is required to make the type legal, do it here.
11355 // Promotion of integers within LegalizeVectorOps is looking for types of
11356 // the same size but with a smaller number of larger elements, not the usual
11357 // larger size with the same number of larger elements.
11358 TargetLowering::LegalizeTypeAction TypeAction =
11359 TLI.getTypeAction(Context&: *DAG.getContext(), VT: StepVecVT);
11360 SDValue StepVec;
11361 if (TypeAction == TargetLowering::TypePromoteInteger) {
11362 StepVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11363 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11364 } else if (TypeAction == TargetLowering::TypeWidenVector) {
11365 // For widening, the element count changes. Create a step vector with only
11366 // the original elements valid and zeros for padding. Also widen the mask.
11367 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11368 unsigned WideNumElts = WideVecVT.getVectorNumElements();
11369
11370 // Build widened step vector: <0, 1, ..., OrigNumElts-1, poison, poison, ..>
11371 SDValue OrigStepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11372 SDValue UndefStep = DAG.getPOISON(VT: WideVecVT);
11373 StepVec = DAG.getInsertSubvector(DL, Vec: UndefStep, SubVec: OrigStepVec, Idx: 0);
11374
11375 // Widen mask: pad with zeros.
11376 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: BoolVT, NumElements: WideNumElts);
11377 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
11378 Mask = DAG.getInsertSubvector(DL, Vec: ZeroMask, SubVec: Mask, Idx: 0);
11379 } else if (TypeAction == TargetLowering::TypeSplitVector) {
11380 // The stepvector type would require splitting. Signal to the caller
11381 // that the operation should be split instead of expanded.
11382 return {Mask, SDValue()};
11383 } else {
11384 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11385 }
11386
11387 return {Mask, StepVec};
11388}
11389
11390SDValue TargetLowering::expandVectorFindLastActive(SDNode *N,
11391 SelectionDAG &DAG) const {
11392 SDLoc DL(N);
11393 auto [Mask, StepVec] = getLegalMaskAndStepVector(
11394 Mask: N->getOperand(Num: 0), /*ZeroIsPoison=*/true, DL, DAG);
11395
11396 // If StepVec is empty, the stepvector would require splitting.
11397 // Split the operation instead and let it be recursively legalized.
11398 if (!StepVec) {
11399 EVT MaskVT = N->getOperand(Num: 0).getValueType();
11400 EVT ResVT = N->getValueType(ResNo: 0);
11401
11402 // Split the mask
11403 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: MaskVT);
11404 auto [MaskLo, MaskHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
11405
11406 // Create split VECTOR_FIND_LAST_ACTIVE operations
11407 SDValue LoResult =
11408 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskLo);
11409 SDValue HiResult =
11410 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskHi);
11411
11412 // Check if any lane is active in the high mask.
11413 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: MaskHi);
11414 SDValue Cond = DAG.getBoolExtOrTrunc(
11415 Op: AnyHiActive, SL: DL,
11416 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i1),
11417 OpVT: MVT::i1);
11418
11419 // Adjust HiResult by adding the number of elements in Lo
11420 SDValue LoNumElts =
11421 DAG.getElementCount(DL, VT: ResVT, EC: LoVT.getVectorElementCount());
11422 SDValue AdjustedHiResult =
11423 DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: HiResult, N2: LoNumElts);
11424
11425 // Return: AnyHiActive ? AdjustedHiResult : LoResult;
11426 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: ResVT, N1: Cond, N2: AdjustedHiResult,
11427 N3: LoResult);
11428 }
11429
11430 EVT StepVecVT = StepVec.getValueType();
11431 EVT StepVT = StepVec.getValueType().getVectorElementType();
11432
11433 // Zero out lanes with inactive elements, then find the highest remaining
11434 // value from the stepvector.
11435 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
11436 SDValue ActiveElts = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
11437 SDValue HighestIdx = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL, VT: StepVT, Operand: ActiveElts);
11438 return DAG.getZExtOrTrunc(Op: HighestIdx, DL, VT: N->getValueType(ResNo: 0));
11439}
11440
11441SDValue TargetLowering::expandLoopDependenceMask(SDNode *N,
11442 SelectionDAG &DAG) const {
11443 SDLoc DL(N);
11444 EVT VT = N->getValueType(ResNo: 0);
11445 SDValue SourceValue = N->getOperand(Num: 0);
11446 SDValue SinkValue = N->getOperand(Num: 1);
11447 SDValue EltSizeInBytes = N->getOperand(Num: 2);
11448
11449 // Note: The lane offset is scalable if the mask is scalable.
11450 ElementCount LaneOffsetEC =
11451 ElementCount::get(MinVal: N->getConstantOperandVal(Num: 3), Scalable: VT.isScalableVT());
11452
11453 EVT AddrVT = SourceValue->getValueType(ResNo: 0);
11454 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
11455
11456 EVT CmpVT =
11457 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: AddrVT);
11458
11459 // Unsigned compare: Source >= Sink.
11460 SDValue SourceAheadOfOrEqualToSink =
11461 DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETUGE);
11462
11463 // Take the difference between the pointers and divided by the element size,
11464 // to see how many lanes separate them.
11465 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: AddrVT, N1: SinkValue, N2: SourceValue);
11466
11467 // RAW_MASK: Diff = Source >= Sink ? (Source - Sink) : (Sink - Source)
11468 if (IsReadAfterWrite)
11469 Diff = DAG.getSelect(DL, VT: AddrVT, Cond: SourceAheadOfOrEqualToSink,
11470 LHS: DAG.getNegative(Val: Diff, DL, VT: AddrVT), RHS: Diff);
11471
11472 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: AddrVT, N1: Diff, N2: EltSizeInBytes);
11473
11474 // The pointers do not alias if:
11475 // - Source >= Sink (WAR_MASK)
11476 // - Source == Sink (RAW_MASK)
11477 SDValue NoAlias = SourceAheadOfOrEqualToSink;
11478 if (IsReadAfterWrite)
11479 NoAlias = DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETEQ);
11480
11481 // The pointers do not alias if:
11482 // Lane + LaneOffset < Diff (WAR/RAW_MASK)
11483 SDValue LaneOffset = DAG.getElementCount(DL, VT: AddrVT, EC: LaneOffsetEC);
11484 SDValue MaskN = DAG.getSelect(
11485 DL, VT: AddrVT, Cond: NoAlias,
11486 LHS: DAG.getConstant(Val: APInt::getMaxValue(numBits: AddrVT.getScalarSizeInBits()), DL,
11487 VT: AddrVT),
11488 RHS: Diff);
11489
11490 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT, N1: LaneOffset, N2: MaskN);
11491}
11492
11493SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
11494 bool IsNegative) const {
11495 SDLoc dl(N);
11496 EVT VT = N->getValueType(ResNo: 0);
11497 SDValue Op = N->getOperand(Num: 0);
11498
11499 // If expanding ABS_MIN_POISON, fall back to ABS if the target supports it.
11500 if (N->getOpcode() == ISD::ABS_MIN_POISON &&
11501 isOperationLegalOrCustom(Op: ISD::ABS, VT)) {
11502 SDValue AbsVal = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT, Operand: Op);
11503 if (IsNegative)
11504 return DAG.getNegative(Val: AbsVal, DL: dl, VT);
11505 return AbsVal;
11506 }
11507
11508 // abs(x) -> smax(x,sub(0,x))
11509 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11510 isOperationLegal(Op: ISD::SMAX, VT)) {
11511 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11512 Op = DAG.getFreeze(V: Op);
11513 return DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: Op,
11514 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11515 }
11516
11517 // abs(x) -> umin(x,sub(0,x))
11518 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11519 isOperationLegal(Op: ISD::UMIN, VT)) {
11520 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11521 Op = DAG.getFreeze(V: Op);
11522 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: Op,
11523 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11524 }
11525
11526 // 0 - abs(x) -> smin(x, sub(0,x))
11527 if (IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11528 isOperationLegal(Op: ISD::SMIN, VT)) {
11529 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11530 Op = DAG.getFreeze(V: Op);
11531 return DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: Op,
11532 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11533 }
11534
11535 // Only expand vector types if we have the appropriate vector operations.
11536 if (VT.isVector() &&
11537 (!isOperationLegalOrCustom(Op: ISD::SRA, VT) ||
11538 (!IsNegative && !isOperationLegalOrCustom(Op: ISD::ADD, VT)) ||
11539 (IsNegative && !isOperationLegalOrCustom(Op: ISD::SUB, VT)) ||
11540 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11541 return SDValue();
11542
11543 Op = DAG.getFreeze(V: Op);
11544 SDValue Shift = DAG.getNode(
11545 Opcode: ISD::SRA, DL: dl, VT, N1: Op,
11546 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11547 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Shift);
11548
11549 // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
11550 if (!IsNegative)
11551 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Shift);
11552
11553 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
11554 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Shift, N2: Xor);
11555}
11556
11557SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
11558 SDLoc dl(N);
11559 EVT VT = N->getValueType(ResNo: 0);
11560 SDValue LHS = N->getOperand(Num: 0);
11561 SDValue RHS = N->getOperand(Num: 1);
11562 bool IsSigned = N->getOpcode() == ISD::ABDS;
11563
11564 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
11565 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
11566 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
11567 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
11568 if (isOperationLegal(Op: MaxOpc, VT) && isOperationLegal(Op: MinOpc, VT)) {
11569 LHS = DAG.getFreeze(V: LHS);
11570 RHS = DAG.getFreeze(V: RHS);
11571 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
11572 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
11573 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
11574 }
11575
11576 // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
11577 if (!IsSigned && isOperationLegal(Op: ISD::USUBSAT, VT)) {
11578 LHS = DAG.getFreeze(V: LHS);
11579 RHS = DAG.getFreeze(V: RHS);
11580 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT,
11581 N1: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: LHS, N2: RHS),
11582 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: RHS, N2: LHS));
11583 }
11584
11585 // If the subtract doesn't overflow then just use abs(sub())
11586 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
11587
11588 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: LHS, N1: RHS))
11589 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11590 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS));
11591
11592 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: RHS, N1: LHS))
11593 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11594 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11595
11596 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11597 ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
11598 LHS = DAG.getFreeze(V: LHS);
11599 RHS = DAG.getFreeze(V: RHS);
11600 SDValue Cmp = DAG.getSetCC(DL: dl, VT: CCVT, LHS, RHS, Cond: CC);
11601
11602 // Branchless expansion iff cmp result is allbits:
11603 // abds(lhs, rhs) -> sub(sgt(lhs, rhs), xor(sgt(lhs, rhs), sub(lhs, rhs)))
11604 // abdu(lhs, rhs) -> sub(ugt(lhs, rhs), xor(ugt(lhs, rhs), sub(lhs, rhs)))
11605 if (CCVT == VT && getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
11606 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
11607 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Diff, N2: Cmp);
11608 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Cmp, N2: Xor);
11609 }
11610
11611 // Similar to the branchless expansion, if we don't prefer selects, use the
11612 // (sign-extended) usubo overflow flag if the (scalar) type is illegal as this
11613 // is more likely to legalize cleanly: abdu(lhs, rhs) -> sub(xor(sub(lhs,
11614 // rhs), uof(lhs, rhs)), uof(lhs, rhs))
11615 if (!IsSigned && VT.isScalarInteger() && !isTypeLegal(VT) &&
11616 !preferSelectsOverBooleanArithmetic(VT)) {
11617 SDValue USubO =
11618 DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {LHS, RHS});
11619 SDValue Cmp = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT, Operand: USubO.getValue(R: 1));
11620 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: USubO.getValue(R: 0), N2: Cmp);
11621 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Cmp);
11622 }
11623
11624 // FIXME: Should really try to split the vector in case it's legal on a
11625 // subvector.
11626 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
11627 return DAG.UnrollVectorOp(N);
11628
11629 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11630 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11631 return DAG.getSelect(DL: dl, VT, Cond: Cmp, LHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS),
11632 RHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11633}
11634
11635SDValue TargetLowering::expandAVG(SDNode *N, SelectionDAG &DAG) const {
11636 SDLoc dl(N);
11637 EVT VT = N->getValueType(ResNo: 0);
11638 SDValue LHS = N->getOperand(Num: 0);
11639 SDValue RHS = N->getOperand(Num: 1);
11640
11641 unsigned Opc = N->getOpcode();
11642 bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
11643 bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
11644 unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
11645 unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
11646 unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
11647 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
11648 assert((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS ||
11649 Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
11650 "Unknown AVG node");
11651
11652 // If the operands are already extended, we can add+shift.
11653 bool IsExt =
11654 (IsSigned && DAG.ComputeNumSignBits(Op: LHS) >= 2 &&
11655 DAG.ComputeNumSignBits(Op: RHS) >= 2) ||
11656 (!IsSigned && DAG.computeKnownBits(Op: LHS).countMinLeadingZeros() >= 1 &&
11657 DAG.computeKnownBits(Op: RHS).countMinLeadingZeros() >= 1);
11658 if (IsExt) {
11659 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
11660 if (!IsFloor)
11661 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Sum, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11662 return DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Sum,
11663 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11664 }
11665
11666 // For scalars, see if we can efficiently extend/truncate to use add+shift.
11667 if (VT.isScalarInteger()) {
11668 EVT ExtVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
11669 if (isTypeLegal(VT: ExtVT) && isTruncateFree(FromVT: ExtVT, ToVT: VT)) {
11670 LHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: LHS);
11671 RHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: RHS);
11672 SDValue Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: LHS, N2: RHS);
11673 if (!IsFloor)
11674 Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: Avg,
11675 N2: DAG.getConstant(Val: 1, DL: dl, VT: ExtVT));
11676 // Just use SRL as we will be truncating away the extended sign bits.
11677 Avg = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ExtVT, N1: Avg,
11678 N2: DAG.getShiftAmountConstant(Val: 1, VT: ExtVT, DL: dl));
11679 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Avg);
11680 }
11681 }
11682
11683 // avgflooru(lhs, rhs) -> or(lshr(add(lhs, rhs),1),shl(overflow, typesize-1))
11684 if (Opc == ISD::AVGFLOORU && VT.isScalarInteger() && !isTypeLegal(VT) &&
11685 isOperationLegalOrCustom(
11686 Op: ISD::UADDO, VT: getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11687 SDValue UAddWithOverflow =
11688 DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {RHS, LHS});
11689
11690 SDValue Sum = UAddWithOverflow.getValue(R: 0);
11691 SDValue Overflow = UAddWithOverflow.getValue(R: 1);
11692
11693 // Right shift the sum by 1
11694 SDValue LShrVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Sum,
11695 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11696
11697 SDValue ZeroExtOverflow = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Overflow);
11698 SDValue OverflowShl = DAG.getNode(
11699 Opcode: ISD::SHL, DL: dl, VT, N1: ZeroExtOverflow,
11700 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11701
11702 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: LShrVal, N2: OverflowShl);
11703 }
11704
11705 // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
11706 // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
11707 // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
11708 // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
11709 LHS = DAG.getFreeze(V: LHS);
11710 RHS = DAG.getFreeze(V: RHS);
11711 SDValue Sign = DAG.getNode(Opcode: SignOpc, DL: dl, VT, N1: LHS, N2: RHS);
11712 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
11713 SDValue Shift =
11714 DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11715 return DAG.getNode(Opcode: SumOpc, DL: dl, VT, N1: Sign, N2: Shift);
11716}
11717
11718SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
11719 SDLoc dl(N);
11720 EVT VT = N->getValueType(ResNo: 0);
11721 SDValue Op = N->getOperand(Num: 0);
11722
11723 if (!VT.isSimple())
11724 return SDValue();
11725
11726 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11727 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
11728 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
11729 default:
11730 return SDValue();
11731 case MVT::i16:
11732 // Use a rotate by 8. This can be further expanded if necessary.
11733 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11734 case MVT::i32:
11735 // This is meant for ARM specifically, which has ROTR but no ROTL.
11736 // t = x ^ rotr(x, 16)
11737 // t = bic(t, 0x00ff0000)
11738 // t = lshr(t, 8)
11739 // x = t ^ rotr(x, 8)
11740 if (isOperationLegalOrCustom(Op: ISD::ROTR, VT)) {
11741 SDValue Rotr16 =
11742 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 16, DL: dl, VT: SHVT));
11743 SDValue Tmp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Rotr16);
11744 Tmp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp,
11745 N2: DAG.getConstant(Val: 0xFF00FFFF, DL: dl, VT));
11746 Tmp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11747 SDValue Rotr8 =
11748 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11749 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Tmp, N2: Rotr8);
11750 }
11751 Tmp4 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11752 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11753 N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11754 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11755 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11756 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11757 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11758 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11759 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11760 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11761 case MVT::i64:
11762 Tmp8 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11763 Tmp7 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11764 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11765 Tmp7 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp7, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11766 Tmp6 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11767 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11768 Tmp6 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp6, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11769 Tmp5 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11770 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11771 Tmp5 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp5, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11772 Tmp4 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11773 Tmp4 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp4,
11774 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11775 Tmp3 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11776 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp3,
11777 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11778 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11779 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2,
11780 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11781 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11782 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp7);
11783 Tmp6 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp6, N2: Tmp5);
11784 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11785 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11786 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp6);
11787 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11788 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp4);
11789 }
11790}
11791
11792SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
11793 SDLoc dl(N);
11794 EVT VT = N->getValueType(ResNo: 0);
11795 SDValue Op = N->getOperand(Num: 0);
11796 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11797 unsigned Sz = VT.getScalarSizeInBits();
11798
11799 SDValue Tmp, Tmp2, Tmp3;
11800
11801 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
11802 // and finally the i1 pairs.
11803 // TODO: We can easily support i4/i2 legal types if any target ever does.
11804 if (Sz >= 8 && isPowerOf2_32(Value: Sz)) {
11805 // Create the masks - repeating the pattern every byte.
11806 APInt Mask4 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x0F));
11807 APInt Mask2 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x33));
11808 APInt Mask1 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x55));
11809
11810 // BSWAP if the type is wider than a single byte.
11811 Tmp = (Sz > 8 ? DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT, Operand: Op) : Op);
11812
11813 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
11814 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11815 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11816 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11817 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11818 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11819
11820 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
11821 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11822 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11823 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11824 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11825 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11826
11827 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
11828 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11829 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11830 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11831 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11832 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11833 return Tmp;
11834 }
11835
11836 Tmp = DAG.getConstant(Val: 0, DL: dl, VT);
11837 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
11838 if (I < J)
11839 Tmp2 =
11840 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: J - I, DL: dl, VT: SHVT));
11841 else
11842 Tmp2 =
11843 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: I - J, DL: dl, VT: SHVT));
11844
11845 APInt Shift = APInt::getOneBitSet(numBits: Sz, BitNo: J);
11846 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Shift, DL: dl, VT));
11847 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp, N2: Tmp2);
11848 }
11849
11850 return Tmp;
11851}
11852
11853std::pair<SDValue, SDValue>
11854TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
11855 SelectionDAG &DAG) const {
11856 SDLoc SL(LD);
11857 SDValue Chain = LD->getChain();
11858 SDValue BasePTR = LD->getBasePtr();
11859 EVT SrcVT = LD->getMemoryVT();
11860 EVT DstVT = LD->getValueType(ResNo: 0);
11861 ISD::LoadExtType ExtType = LD->getExtensionType();
11862
11863 if (SrcVT.isScalableVector())
11864 report_fatal_error(reason: "Cannot scalarize scalable vector loads");
11865
11866 unsigned NumElem = SrcVT.getVectorNumElements();
11867
11868 EVT SrcEltVT = SrcVT.getScalarType();
11869 EVT DstEltVT = DstVT.getScalarType();
11870
11871 // A vector must always be stored in memory as-is, i.e. without any padding
11872 // between the elements, since various code depend on it, e.g. in the
11873 // handling of a bitcast of a vector type to int, which may be done with a
11874 // vector store followed by an integer load. A vector that does not have
11875 // elements that are byte-sized must therefore be stored as an integer
11876 // built out of the extracted vector elements.
11877 if (!SrcEltVT.isByteSized()) {
11878 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
11879 EVT LoadVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumLoadBits);
11880
11881 unsigned NumSrcBits = SrcVT.getSizeInBits();
11882 EVT SrcIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumSrcBits);
11883
11884 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
11885 SDValue SrcEltBitMask = DAG.getConstant(
11886 Val: APInt::getLowBitsSet(numBits: NumLoadBits, loBitsSet: SrcEltBits), DL: SL, VT: LoadVT);
11887
11888 // Load the whole vector and avoid masking off the top bits as it makes
11889 // the codegen worse.
11890 SDValue Load = DAG.getExtLoad(
11891 ExtType: ISD::EXTLOAD, dl: SL, VT: LoadVT, Chain, Ptr: BasePTR, PtrInfo: LD->getPointerInfo(),
11892 MemVT: SrcIntVT, Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
11893 Metadata: LD->getMMOMetadataForSubAccess());
11894
11895 SmallVector<SDValue, 8> Vals;
11896 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11897 unsigned ShiftIntoIdx =
11898 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
11899 SDValue ShiftAmount = DAG.getShiftAmountConstant(
11900 Val: ShiftIntoIdx * SrcEltVT.getSizeInBits(), VT: LoadVT, DL: SL);
11901 SDValue ShiftedElt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: LoadVT, N1: Load, N2: ShiftAmount);
11902 SDValue Elt =
11903 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: LoadVT, N1: ShiftedElt, N2: SrcEltBitMask);
11904 SDValue Scalar = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: SrcEltVT, Operand: Elt);
11905
11906 if (ExtType != ISD::NON_EXTLOAD) {
11907 unsigned ExtendOp = ISD::getExtForLoadExtType(IsFP: false, ExtType);
11908 Scalar = DAG.getNode(Opcode: ExtendOp, DL: SL, VT: DstEltVT, Operand: Scalar);
11909 }
11910
11911 Vals.push_back(Elt: Scalar);
11912 }
11913
11914 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
11915 return std::make_pair(x&: Value, y: Load.getValue(R: 1));
11916 }
11917
11918 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
11919 assert(SrcEltVT.isByteSized());
11920
11921 SmallVector<SDValue, 8> Vals;
11922 SmallVector<SDValue, 8> LoadChains;
11923
11924 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11925 SDValue ScalarLoad = DAG.getExtLoad(
11926 ExtType, dl: SL, VT: DstEltVT, Chain, Ptr: BasePTR,
11927 PtrInfo: LD->getPointerInfo().getWithOffset(O: Idx * Stride), MemVT: SrcEltVT,
11928 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
11929 Metadata: LD->getMMOMetadataForSubAccess());
11930
11931 BasePTR = DAG.getObjectPtrOffset(SL, Ptr: BasePTR, Offset: TypeSize::getFixed(ExactSize: Stride));
11932
11933 Vals.push_back(Elt: ScalarLoad.getValue(R: 0));
11934 LoadChains.push_back(Elt: ScalarLoad.getValue(R: 1));
11935 }
11936
11937 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: LoadChains);
11938 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
11939
11940 return std::make_pair(x&: Value, y&: NewChain);
11941}
11942
11943SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
11944 SelectionDAG &DAG) const {
11945 SDLoc SL(ST);
11946
11947 SDValue Chain = ST->getChain();
11948 SDValue BasePtr = ST->getBasePtr();
11949 SDValue Value = ST->getValue();
11950 EVT StVT = ST->getMemoryVT();
11951
11952 if (StVT.isScalableVector())
11953 report_fatal_error(reason: "Cannot scalarize scalable vector stores");
11954
11955 // The type of the data we want to save
11956 EVT RegVT = Value.getValueType();
11957 EVT RegSclVT = RegVT.getScalarType();
11958
11959 // The type of data as saved in memory.
11960 EVT MemSclVT = StVT.getScalarType();
11961
11962 unsigned NumElem = StVT.getVectorNumElements();
11963
11964 // A vector must always be stored in memory as-is, i.e. without any padding
11965 // between the elements, since various code depend on it, e.g. in the
11966 // handling of a bitcast of a vector type to int, which may be done with a
11967 // vector store followed by an integer load. A vector that does not have
11968 // elements that are byte-sized must therefore be stored as an integer
11969 // built out of the extracted vector elements.
11970 if (!MemSclVT.isByteSized()) {
11971 unsigned NumBits = StVT.getSizeInBits();
11972 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
11973
11974 SDValue CurrVal = DAG.getConstant(Val: 0, DL: SL, VT: IntVT);
11975
11976 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11977 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
11978 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MemSclVT, Operand: Elt);
11979 SDValue ExtElt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: IntVT, Operand: Trunc);
11980 unsigned ShiftIntoIdx =
11981 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
11982 SDValue ShiftAmount =
11983 DAG.getConstant(Val: ShiftIntoIdx * MemSclVT.getSizeInBits(), DL: SL, VT: IntVT);
11984 SDValue ShiftedElt =
11985 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT, N1: ExtElt, N2: ShiftAmount);
11986 CurrVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: CurrVal, N2: ShiftedElt);
11987 }
11988
11989 return DAG.getStore(Chain, dl: SL, Val: CurrVal, Ptr: BasePtr, PtrInfo: ST->getPointerInfo(),
11990 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
11991 Metadata: ST->getMMOMetadataForSubAccess());
11992 }
11993
11994 // Store Stride in bytes
11995 unsigned Stride = MemSclVT.getSizeInBits() / 8;
11996 assert(Stride && "Zero stride!");
11997 // Extract each of the elements from the original vector and save them into
11998 // memory individually.
11999 SmallVector<SDValue, 8> Stores;
12000 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
12001 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
12002
12003 SDValue Ptr =
12004 DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Idx * Stride));
12005
12006 // This scalar TruncStore may be illegal, but we legalize it later.
12007 SDValue Store = DAG.getTruncStore(
12008 Chain, dl: SL, Val: Elt, Ptr, PtrInfo: ST->getPointerInfo().getWithOffset(O: Idx * Stride),
12009 SVT: MemSclVT, Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12010 Metadata: ST->getMMOMetadataForSubAccess());
12011
12012 Stores.push_back(Elt: Store);
12013 }
12014
12015 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: Stores);
12016}
12017
12018std::pair<SDValue, SDValue>
12019TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
12020 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
12021 "unaligned indexed loads not implemented!");
12022 SDValue Chain = LD->getChain();
12023 SDValue Ptr = LD->getBasePtr();
12024 EVT VT = LD->getValueType(ResNo: 0);
12025 EVT LoadedVT = LD->getMemoryVT();
12026 SDLoc dl(LD);
12027 auto &MF = DAG.getMachineFunction();
12028
12029 if (VT.isFloatingPoint() || VT.isVector()) {
12030 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadedVT.getSizeInBits());
12031 if (isTypeLegal(VT: intVT) && isTypeLegal(VT: LoadedVT)) {
12032 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: intVT) &&
12033 LoadedVT.isVector()) {
12034 // Scalarize the load and let the individual components be handled.
12035 return scalarizeVectorLoad(LD, DAG);
12036 }
12037
12038 // Expand to a (misaligned) integer load of the same size,
12039 // then bitconvert to floating point or vector.
12040 SDValue newLoad = DAG.getLoad(VT: intVT, dl, Chain, Ptr,
12041 MMO: LD->getMemOperand());
12042 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoadedVT, Operand: newLoad);
12043 if (LoadedVT != VT)
12044 Result = DAG.getNode(Opcode: VT.isFloatingPoint() ? ISD::FP_EXTEND :
12045 ISD::ANY_EXTEND, DL: dl, VT, Operand: Result);
12046
12047 return std::make_pair(x&: Result, y: newLoad.getValue(R: 1));
12048 }
12049
12050 // Copy the value to a (aligned) stack slot using (unaligned) integer
12051 // loads and stores, then do a (aligned) load from the stack slot.
12052 MVT RegVT = getRegisterType(Context&: *DAG.getContext(), VT: intVT);
12053 unsigned LoadedBytes = LoadedVT.getStoreSize();
12054 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12055 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
12056
12057 // Make sure the stack slot is also aligned for the register type.
12058 SDValue StackBase = DAG.CreateStackTemporary(VT1: LoadedVT, VT2: RegVT);
12059 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackBase.getNode())->getIndex();
12060 SmallVector<SDValue, 8> Stores;
12061 SDValue StackPtr = StackBase;
12062 unsigned Offset = 0;
12063
12064 EVT PtrVT = Ptr.getValueType();
12065 EVT StackPtrVT = StackPtr.getValueType();
12066
12067 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12068 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12069
12070 // Do all but one copies using the full register width.
12071 for (unsigned i = 1; i < NumRegs; i++) {
12072 // Load one integer register's worth from the original location.
12073 SDValue Load = DAG.getLoad(
12074 VT: RegVT, dl, Chain, Ptr, PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset),
12075 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(),
12076 Metadata: LD->getMMOMetadataForSubAccess());
12077 // Follow the load with a store to the stack slot. Remember the store.
12078 Stores.push_back(Elt: DAG.getStore(
12079 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12080 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset)));
12081 // Increment the pointers.
12082 Offset += RegBytes;
12083
12084 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12085 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12086 }
12087
12088 // The last copy may be partial. Do an extending load.
12089 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12090 BitWidth: 8 * (LoadedBytes - Offset));
12091 SDValue Load = DAG.getExtLoad(
12092 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain, Ptr,
12093 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT, Alignment: LD->getBaseAlign(),
12094 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getMMOMetadataForSubAccess());
12095 // Follow the load with a store to the stack slot. Remember the store.
12096 // On big-endian machines this requires a truncating store to ensure
12097 // that the bits end up in the right place.
12098 Stores.push_back(Elt: DAG.getTruncStore(
12099 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12100 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), SVT: MemVT));
12101
12102 // The order of the stores doesn't matter - say it with a TokenFactor.
12103 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12104
12105 // Finally, perform the original load only redirected to the stack slot.
12106 Load = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT, Chain: TF, Ptr: StackBase,
12107 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0),
12108 MemVT: LoadedVT);
12109
12110 // Callers expect a MERGE_VALUES node.
12111 return std::make_pair(x&: Load, y&: TF);
12112 }
12113
12114 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
12115 "Unaligned load of unsupported type.");
12116
12117 // Compute the new VT that is half the size of the old one. This is an
12118 // integer MVT.
12119 unsigned NumBits = LoadedVT.getSizeInBits();
12120 EVT NewLoadedVT;
12121 NewLoadedVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits/2);
12122 NumBits >>= 1;
12123
12124 Align Alignment = LD->getBaseAlign();
12125 unsigned IncrementSize = NumBits / 8;
12126 ISD::LoadExtType HiExtType = LD->getExtensionType();
12127
12128 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
12129 if (HiExtType == ISD::NON_EXTLOAD)
12130 HiExtType = ISD::ZEXTLOAD;
12131
12132 // Load the value in two parts
12133 SDValue Lo, Hi;
12134 if (DAG.getDataLayout().isLittleEndian()) {
12135 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12136 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12137 Metadata: LD->getMMOMetadataForSubAccess());
12138
12139 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12140 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr,
12141 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12142 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12143 Metadata: LD->getMMOMetadataForSubAccess());
12144 } else {
12145 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12146 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12147 Metadata: LD->getMMOMetadataForSubAccess());
12148
12149 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12150 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
12151 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12152 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12153 Metadata: LD->getMMOMetadataForSubAccess());
12154 }
12155
12156 // aggregate the two parts
12157 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: NumBits, VT, DL: dl);
12158 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: ShiftAmount);
12159 Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Result, N2: Lo);
12160
12161 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
12162 N2: Hi.getValue(R: 1));
12163
12164 return std::make_pair(x&: Result, y&: TF);
12165}
12166
12167SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
12168 SelectionDAG &DAG) const {
12169 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
12170 "unaligned indexed stores not implemented!");
12171 SDValue Chain = ST->getChain();
12172 SDValue Ptr = ST->getBasePtr();
12173 SDValue Val = ST->getValue();
12174 EVT VT = Val.getValueType();
12175 Align Alignment = ST->getBaseAlign();
12176 auto &MF = DAG.getMachineFunction();
12177 EVT StoreMemVT = ST->getMemoryVT();
12178
12179 SDLoc dl(ST);
12180 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
12181 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
12182 if (isTypeLegal(VT: intVT)) {
12183 if (!isOperationLegalOrCustom(Op: ISD::STORE, VT: intVT) &&
12184 StoreMemVT.isVector()) {
12185 // Scalarize the store and let the individual components be handled.
12186 SDValue Result = scalarizeVectorStore(ST, DAG);
12187 return Result;
12188 }
12189 // Expand to a bitconvert of the value to the integer type of the
12190 // same size, then a (misaligned) int store.
12191 // FIXME: Does not handle truncating floating point stores!
12192 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: intVT, Operand: Val);
12193 Result = DAG.getStore(Chain, dl, Val: Result, Ptr, PtrInfo: ST->getPointerInfo(),
12194 Alignment, MMOFlags: ST->getMemOperand()->getFlags(),
12195 Metadata: ST->getMMOMetadataForSubAccess());
12196 return Result;
12197 }
12198 // Do a (aligned) store to a stack slot, then copy from the stack slot
12199 // to the final destination using (unaligned) integer loads and stores.
12200 MVT RegVT = getRegisterType(
12201 Context&: *DAG.getContext(),
12202 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreMemVT.getSizeInBits()));
12203 EVT PtrVT = Ptr.getValueType();
12204 unsigned StoredBytes = StoreMemVT.getStoreSize();
12205 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12206 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
12207
12208 // Make sure the stack slot is also aligned for the register type.
12209 SDValue StackPtr = DAG.CreateStackTemporary(VT1: StoreMemVT, VT2: RegVT);
12210 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
12211
12212 // Perform the original store, only redirected to the stack slot.
12213 SDValue Store = DAG.getTruncStore(
12214 Chain, dl, Val, Ptr: StackPtr,
12215 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0), SVT: StoreMemVT);
12216
12217 EVT StackPtrVT = StackPtr.getValueType();
12218
12219 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12220 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12221 SmallVector<SDValue, 8> Stores;
12222 unsigned Offset = 0;
12223
12224 // Do all but one copies using the full register width.
12225 for (unsigned i = 1; i < NumRegs; i++) {
12226 // Load one integer register's worth from the stack slot.
12227 SDValue Load = DAG.getLoad(
12228 VT: RegVT, dl, Chain: Store, Ptr: StackPtr,
12229 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset));
12230 // Store it to the final location. Remember the store.
12231 Stores.push_back(Elt: DAG.getStore(
12232 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12233 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), Alignment: ST->getBaseAlign(),
12234 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess()));
12235 // Increment the pointers.
12236 Offset += RegBytes;
12237 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12238 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12239 }
12240
12241 // The last store may be partial. Do a truncating store. On big-endian
12242 // machines this requires an extending load from the stack slot to ensure
12243 // that the bits are in the right place.
12244 EVT LoadMemVT =
12245 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: 8 * (StoredBytes - Offset));
12246
12247 // Load from the stack slot.
12248 SDValue Load = DAG.getExtLoad(
12249 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain: Store, Ptr: StackPtr,
12250 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), MemVT: LoadMemVT);
12251
12252 Stores.push_back(
12253 Elt: DAG.getTruncStore(Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12254 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), SVT: LoadMemVT,
12255 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
12256 Metadata: ST->getMMOMetadataForSubAccess()));
12257 // The order of the stores doesn't matter - say it with a TokenFactor.
12258 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12259 return Result;
12260 }
12261
12262 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
12263 "Unaligned store of unknown type.");
12264 // Get the half-size VT
12265 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
12266 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
12267 unsigned IncrementSize = NumBits / 8;
12268
12269 // Divide the stored value in two parts.
12270 SDValue ShiftAmount =
12271 DAG.getShiftAmountConstant(Val: NumBits, VT: Val.getValueType(), DL: dl);
12272 SDValue Lo = Val;
12273 // If Val is a constant, replace the upper bits with 0. The SRL will constant
12274 // fold and not use the upper bits. A smaller constant may be easier to
12275 // materialize.
12276 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Lo); C && !C->isOpaque())
12277 Lo = DAG.getNode(
12278 Opcode: ISD::AND, DL: dl, VT, N1: Lo,
12279 N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: NumBits), DL: dl,
12280 VT));
12281 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Val, N2: ShiftAmount);
12282
12283 // Store the two parts
12284 SDValue Store1, Store2;
12285 Store1 = DAG.getTruncStore(
12286 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Lo : Hi, Ptr,
12287 PtrInfo: ST->getPointerInfo(), SVT: NewStoredVT, Alignment,
12288 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess());
12289
12290 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12291 Store2 = DAG.getTruncStore(
12292 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
12293 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize), SVT: NewStoredVT, Alignment,
12294 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getMMOMetadataForSubAccess());
12295
12296 SDValue Result =
12297 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
12298 return Result;
12299}
12300
12301SDValue
12302TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
12303 const SDLoc &DL, EVT DataVT,
12304 SelectionDAG &DAG,
12305 bool IsCompressedMemory) const {
12306 SDValue Increment;
12307 EVT AddrVT = Addr.getValueType();
12308 EVT MaskVT = Mask.getValueType();
12309 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
12310 "Incompatible types of Data and Mask");
12311 if (IsCompressedMemory) {
12312 // Incrementing the pointer according to number of '1's in the mask.
12313 if (DataVT.isScalableVector()) {
12314 EVT MaskExtVT = MaskVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
12315 SDValue MaskExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MaskExtVT, Operand: Mask);
12316 Increment = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: MaskExt);
12317 } else {
12318 EVT MaskIntVT =
12319 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MaskVT.getSizeInBits());
12320 SDValue MaskInIntReg = DAG.getBitcast(VT: MaskIntVT, V: Mask);
12321 if (MaskIntVT.getSizeInBits() < 32) {
12322 MaskInIntReg =
12323 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: MaskInIntReg);
12324 MaskIntVT = MVT::i32;
12325 }
12326 Increment = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MaskIntVT, Operand: MaskInIntReg);
12327 }
12328 // Scale is an element size in bytes.
12329 SDValue Scale = DAG.getConstant(Val: DataVT.getScalarSizeInBits() / 8, DL,
12330 VT: AddrVT);
12331 Increment = DAG.getZExtOrTrunc(Op: Increment, DL, VT: AddrVT);
12332 Increment = DAG.getNode(Opcode: ISD::MUL, DL, VT: AddrVT, N1: Increment, N2: Scale);
12333 } else
12334 Increment = DAG.getTypeSize(DL, VT: AddrVT, TS: DataVT.getStoreSize());
12335
12336 return DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: Addr, N2: Increment);
12337}
12338
12339static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
12340 EVT VecVT, const SDLoc &dl,
12341 ElementCount SubEC) {
12342 assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
12343 "Cannot index a scalable vector within a fixed-width vector");
12344
12345 unsigned NElts = VecVT.getVectorMinNumElements();
12346 unsigned NumSubElts = SubEC.getKnownMinValue();
12347 EVT IdxVT = Idx.getValueType();
12348
12349 if (VecVT.isScalableVector() && !SubEC.isScalable()) {
12350 // If this is a constant index and we know the value plus the number of the
12351 // elements in the subvector minus one is less than the minimum number of
12352 // elements then it's safe to return Idx.
12353 if (auto *IdxCst = dyn_cast<ConstantSDNode>(Val&: Idx))
12354 if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
12355 return Idx;
12356 SDValue VS =
12357 DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getFixedSizeInBits(), NElts));
12358 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
12359 SDValue Sub = DAG.getNode(Opcode: SubOpcode, DL: dl, VT: IdxVT, N1: VS,
12360 N2: DAG.getConstant(Val: NumSubElts, DL: dl, VT: IdxVT));
12361 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx, N2: Sub);
12362 }
12363 if (isPowerOf2_32(Value: NElts) && NumSubElts == 1) {
12364 APInt Imm = APInt::getLowBitsSet(numBits: IdxVT.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
12365 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IdxVT, N1: Idx,
12366 N2: DAG.getConstant(Val: Imm, DL: dl, VT: IdxVT));
12367 }
12368 unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
12369 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx,
12370 N2: DAG.getConstant(Val: MaxIndex, DL: dl, VT: IdxVT));
12371}
12372
12373SDValue
12374TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr,
12375 EVT VecVT, SDValue Index,
12376 const SDNodeFlags PtrArithFlags) const {
12377 return getVectorSubVecPointer(
12378 DAG, VecPtr, VecVT,
12379 SubVecVT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(), NumElements: 1),
12380 Index, PtrArithFlags);
12381}
12382
12383SDValue
12384TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr,
12385 EVT VecVT, EVT SubVecVT, SDValue Index,
12386 const SDNodeFlags PtrArithFlags) const {
12387 SDLoc dl(Index);
12388 // Make sure the index type is big enough to compute in.
12389 Index = DAG.getZExtOrTrunc(Op: Index, DL: dl, VT: VecPtr.getValueType());
12390
12391 EVT EltVT = VecVT.getVectorElementType();
12392
12393 // Calculate the element offset and add it to the pointer.
12394 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
12395 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
12396 "Converting bits to bytes lost precision");
12397 assert(SubVecVT.getVectorElementType() == EltVT &&
12398 "Sub-vector must be a vector with matching element type");
12399
12400 // An out-of-range index only makes the vector operation return poison, but
12401 // a load/store through the pointer computed below would be immediate UB, so
12402 // freeze the index before clamping it into range.
12403 if (!DAG.isGuaranteedNotToBePoison(Op: Index))
12404 Index = DAG.getFreeze(V: Index);
12405
12406 Index = clampDynamicVectorIndex(DAG, Idx: Index, VecVT, dl,
12407 SubEC: SubVecVT.getVectorElementCount());
12408
12409 EVT IdxVT = Index.getValueType();
12410 if (SubVecVT.isScalableVector())
12411 Index =
12412 DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12413 N2: DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getSizeInBits(), 1)));
12414
12415 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12416 N2: DAG.getConstant(Val: EltSize, DL: dl, VT: IdxVT));
12417 return DAG.getMemBasePlusOffset(Base: VecPtr, Offset: Index, DL: dl, Flags: PtrArithFlags);
12418}
12419
12420//===----------------------------------------------------------------------===//
12421// Implementation of Emulated TLS Model
12422//===----------------------------------------------------------------------===//
12423
12424SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
12425 SelectionDAG &DAG) const {
12426 // Access to address of TLS varialbe xyz is lowered to a function call:
12427 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
12428 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12429 PointerType *VoidPtrType = PointerType::get(C&: *DAG.getContext(), AddressSpace: 0);
12430 SDLoc dl(GA);
12431
12432 ArgListTy Args;
12433 const GlobalValue *GV =
12434 cast<GlobalValue>(Val: GA->getGlobal()->stripPointerCastsAndAliases());
12435 SmallString<32> NameString("__emutls_v.");
12436 NameString += GV->getName();
12437 StringRef EmuTlsVarName(NameString);
12438 const GlobalVariable *EmuTlsVar =
12439 GV->getParent()->getNamedGlobal(Name: EmuTlsVarName);
12440 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
12441 Args.emplace_back(args: DAG.getGlobalAddress(GV: EmuTlsVar, DL: dl, VT: PtrVT), args&: VoidPtrType);
12442
12443 SDValue EmuTlsGetAddr = DAG.getExternalSymbol(Sym: "__emutls_get_address", VT: PtrVT);
12444
12445 TargetLowering::CallLoweringInfo CLI(DAG);
12446 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
12447 CLI.setLibCallee(CC: CallingConv::C, ResultType: VoidPtrType, Target: EmuTlsGetAddr, ArgsList: std::move(Args));
12448 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
12449
12450 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
12451 // At last for X86 targets, maybe good for other targets too?
12452 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
12453 MFI.setAdjustsStack(true); // Is this only for X86 target?
12454 MFI.setHasCalls(true);
12455
12456 assert((GA->getOffset() == 0) &&
12457 "Emulated TLS must have zero offset in GlobalAddressSDNode");
12458 return CallResult.first;
12459}
12460
12461SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
12462 SelectionDAG &DAG) const {
12463 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
12464 if (!isCtlzFast())
12465 return SDValue();
12466 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
12467 SDLoc dl(Op);
12468 if (isNullConstant(V: Op.getOperand(i: 1)) && CC == ISD::SETEQ) {
12469 EVT VT = Op.getOperand(i: 0).getValueType();
12470 SDValue Zext = Op.getOperand(i: 0);
12471 if (VT.bitsLT(VT: MVT::i32)) {
12472 VT = MVT::i32;
12473 Zext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Op.getOperand(i: 0));
12474 }
12475 unsigned Log2b = Log2_32(Value: VT.getSizeInBits());
12476 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Zext);
12477 SDValue Scc = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Clz,
12478 N2: DAG.getConstant(Val: Log2b, DL: dl, VT: MVT::i32));
12479 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Scc);
12480 }
12481 return SDValue();
12482}
12483
12484SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
12485 SDValue Op0 = Node->getOperand(Num: 0);
12486 SDValue Op1 = Node->getOperand(Num: 1);
12487 EVT VT = Op0.getValueType();
12488 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12489 unsigned Opcode = Node->getOpcode();
12490 SDLoc DL(Node);
12491
12492 // If both sign bits are zero, flip UMIN/UMAX <-> SMIN/SMAX if legal.
12493 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(MinMaxOpc: Opcode);
12494 if (isOperationLegal(Op: AltOpcode, VT) && DAG.SignBitIsZero(Op: Op0) &&
12495 DAG.SignBitIsZero(Op: Op1))
12496 return DAG.getNode(Opcode: AltOpcode, DL, VT, N1: Op0, N2: Op1);
12497
12498 // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
12499 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(V: Op1, AllowUndefs: true) && BoolVT == VT &&
12500 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12501 Op0 = DAG.getFreeze(V: Op0);
12502 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
12503 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12504 N2: DAG.getSetCC(DL, VT, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ));
12505 }
12506
12507 // umin(x,y) -> sub(x,usubsat(x,y))
12508 // TODO: Missing freeze(Op0)?
12509 if (Opcode == ISD::UMIN && isOperationLegal(Op: ISD::SUB, VT) &&
12510 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12511 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12512 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op0, N2: Op1));
12513 }
12514
12515 // umax(x,y) -> add(x,usubsat(y,x))
12516 // TODO: Missing freeze(Op0)?
12517 if (Opcode == ISD::UMAX && isOperationLegal(Op: ISD::ADD, VT) &&
12518 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12519 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0,
12520 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op1, N2: Op0));
12521 }
12522
12523 // FIXME: Should really try to split the vector in case it's legal on a
12524 // subvector.
12525 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12526 return DAG.UnrollVectorOp(N: Node);
12527
12528 // Attempt to find an existing SETCC node that we can reuse.
12529 // TODO: Do we need a generic doesSETCCNodeExist?
12530 // TODO: Missing freeze(Op0)/freeze(Op1)?
12531 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
12532 ISD::CondCode PrefCommuteCC,
12533 ISD::CondCode AltCommuteCC) {
12534 SDVTList BoolVTList = DAG.getVTList(VT: BoolVT);
12535 for (ISD::CondCode CC : {PrefCC, AltCC}) {
12536 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12537 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12538 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12539 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12540 }
12541 }
12542 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
12543 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12544 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12545 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12546 return DAG.getSelect(DL, VT, Cond, LHS: Op1, RHS: Op0);
12547 }
12548 }
12549 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: PrefCC);
12550 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12551 };
12552
12553 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
12554 // -> Y = (A < B) ? B : A
12555 // -> Y = (A >= B) ? A : B
12556 // -> Y = (A <= B) ? B : A
12557 switch (Opcode) {
12558 case ISD::SMAX:
12559 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
12560 case ISD::SMIN:
12561 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
12562 case ISD::UMAX:
12563 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
12564 case ISD::UMIN:
12565 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
12566 }
12567
12568 llvm_unreachable("How did we get here?");
12569}
12570
12571SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
12572 unsigned Opcode = Node->getOpcode();
12573 SDValue LHS = Node->getOperand(Num: 0);
12574 SDValue RHS = Node->getOperand(Num: 1);
12575 EVT VT = LHS.getValueType();
12576 SDLoc dl(Node);
12577
12578 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
12579 assert(VT.isInteger() && "Expected operands to be integers");
12580
12581 // usub.sat(a, b) -> umax(a, b) - b
12582 if (Opcode == ISD::USUBSAT && isOperationLegal(Op: ISD::UMAX, VT)) {
12583 SDValue Max = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT, N1: LHS, N2: RHS);
12584 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: RHS);
12585 }
12586
12587 // usub.sat(a, 1) -> sub(a, zext(a != 0))
12588 // Prefer this on targets without legal/cost-effective overflow-carry nodes.
12589 if (Opcode == ISD::USUBSAT && isOneOrOneSplat(V: RHS) &&
12590 !isOperationLegalOrCustom(Op: ISD::USUBO_CARRY, VT)) {
12591 LHS = DAG.getFreeze(V: LHS);
12592 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12593 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12594 SDValue IsNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETNE);
12595 SDValue Subtrahend = DAG.getBoolExtOrTrunc(Op: IsNonZero, SL: dl, VT, OpVT: BoolVT);
12596 Subtrahend =
12597 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Subtrahend, N2: DAG.getConstant(Val: 1, DL: dl, VT));
12598 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: Subtrahend);
12599 }
12600
12601 // uadd.sat(a, b) -> umin(a, ~b) + b
12602 if (Opcode == ISD::UADDSAT && isOperationLegal(Op: ISD::UMIN, VT)) {
12603 SDValue InvRHS = DAG.getNOT(DL: dl, Val: RHS, VT);
12604 SDValue Min = DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: LHS, N2: InvRHS);
12605 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Min, N2: RHS);
12606 }
12607
12608 unsigned OverflowOp;
12609 switch (Opcode) {
12610 case ISD::SADDSAT:
12611 OverflowOp = ISD::SADDO;
12612 break;
12613 case ISD::UADDSAT:
12614 OverflowOp = ISD::UADDO;
12615 break;
12616 case ISD::SSUBSAT:
12617 OverflowOp = ISD::SSUBO;
12618 break;
12619 case ISD::USUBSAT:
12620 OverflowOp = ISD::USUBO;
12621 break;
12622 default:
12623 llvm_unreachable("Expected method to receive signed or unsigned saturation "
12624 "addition or subtraction node.");
12625 }
12626
12627 // FIXME: Should really try to split the vector in case it's legal on a
12628 // subvector.
12629 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12630 return DAG.UnrollVectorOp(N: Node);
12631
12632 unsigned BitWidth = LHS.getScalarValueSizeInBits();
12633 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12634 SDValue Result = DAG.getNode(Opcode: OverflowOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12635 SDValue SumDiff = Result.getValue(R: 0);
12636 SDValue Overflow = Result.getValue(R: 1);
12637 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12638 SDValue AllOnes = DAG.getAllOnesConstant(DL: dl, VT);
12639
12640 if (Opcode == ISD::UADDSAT) {
12641 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12642 // (LHS + RHS) | OverflowMask
12643 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12644 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SumDiff, N2: OverflowMask);
12645 }
12646 // Overflow ? 0xffff.... : (LHS + RHS)
12647 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: AllOnes, RHS: SumDiff);
12648 }
12649
12650 if (Opcode == ISD::USUBSAT) {
12651 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12652 // (LHS - RHS) & ~OverflowMask
12653 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12654 SDValue Not = DAG.getNOT(DL: dl, Val: OverflowMask, VT);
12655 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SumDiff, N2: Not);
12656 }
12657 // Overflow ? 0 : (LHS - RHS)
12658 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Zero, RHS: SumDiff);
12659 }
12660
12661 assert((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
12662 "Expected signed saturating add/sub opcode");
12663
12664 const APInt MinVal = APInt::getSignedMinValue(numBits: BitWidth);
12665 const APInt MaxVal = APInt::getSignedMaxValue(numBits: BitWidth);
12666
12667 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
12668 KnownBits KnownRHS = DAG.computeKnownBits(Op: RHS);
12669
12670 // If either of the operand signs are known, then they are guaranteed to
12671 // only saturate in one direction. If non-negative they will saturate
12672 // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
12673 //
12674 // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
12675 // sign of 'y' has to be flipped.
12676
12677 bool LHSIsNonNegative = KnownLHS.isNonNegative();
12678 bool RHSIsNonNegative =
12679 Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() : KnownRHS.isNegative();
12680 if (LHSIsNonNegative || RHSIsNonNegative) {
12681 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12682 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: SumDiff);
12683 }
12684
12685 bool LHSIsNegative = KnownLHS.isNegative();
12686 bool RHSIsNegative =
12687 Opcode == ISD::SADDSAT ? KnownRHS.isNegative() : KnownRHS.isNonNegative();
12688 if (LHSIsNegative || RHSIsNegative) {
12689 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12690 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMin, RHS: SumDiff);
12691 }
12692
12693 // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
12694 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12695 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SumDiff,
12696 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT));
12697 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Shift, N2: SatMin);
12698 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: SumDiff);
12699}
12700
12701SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
12702 unsigned Opcode = Node->getOpcode();
12703 SDValue LHS = Node->getOperand(Num: 0);
12704 SDValue RHS = Node->getOperand(Num: 1);
12705 EVT VT = LHS.getValueType();
12706 EVT ResVT = Node->getValueType(ResNo: 0);
12707 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12708 SDLoc dl(Node);
12709
12710 auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT);
12711 auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT);
12712 SDValue IsLT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: LTPredicate);
12713 SDValue IsGT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: GTPredicate);
12714
12715 // We can't perform arithmetic on i1 values. Extending them would
12716 // probably result in worse codegen, so let's just use two selects instead.
12717 // Some targets are also just better off using selects rather than subtraction
12718 // because one of the conditions can be merged with one of the selects.
12719 // And finally, if we don't know the contents of high bits of a boolean value
12720 // we can't perform any arithmetic either.
12721 if (preferSelectsOverBooleanArithmetic(VT) ||
12722 BoolVT.getScalarSizeInBits() == 1 ||
12723 getBooleanContents(Type: BoolVT) == UndefinedBooleanContent) {
12724 SDValue SelectZeroOrOne =
12725 DAG.getSelect(DL: dl, VT: ResVT, Cond: IsGT, LHS: DAG.getConstant(Val: 1, DL: dl, VT: ResVT),
12726 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ResVT));
12727 return DAG.getSelect(DL: dl, VT: ResVT, Cond: IsLT, LHS: DAG.getAllOnesConstant(DL: dl, VT: ResVT),
12728 RHS: SelectZeroOrOne);
12729 }
12730
12731 if (getBooleanContents(Type: BoolVT) == ZeroOrNegativeOneBooleanContent)
12732 std::swap(a&: IsGT, b&: IsLT);
12733 return DAG.getSExtOrTrunc(Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: BoolVT, N1: IsGT, N2: IsLT), DL: dl,
12734 VT: ResVT);
12735}
12736
12737SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
12738 unsigned Opcode = Node->getOpcode();
12739 bool IsSigned = Opcode == ISD::SSHLSAT;
12740 SDValue LHS = Node->getOperand(Num: 0);
12741 SDValue RHS = Node->getOperand(Num: 1);
12742 EVT VT = LHS.getValueType();
12743 SDLoc dl(Node);
12744
12745 assert((Node->getOpcode() == ISD::SSHLSAT ||
12746 Node->getOpcode() == ISD::USHLSAT) &&
12747 "Expected a SHLSAT opcode");
12748 assert(VT.isInteger() && "Expected operands to be integers");
12749
12750 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12751 return DAG.UnrollVectorOp(N: Node);
12752
12753 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
12754
12755 unsigned BW = VT.getScalarSizeInBits();
12756 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12757 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: RHS);
12758 SDValue Orig =
12759 DAG.getNode(Opcode: IsSigned ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: Result, N2: RHS);
12760
12761 SDValue SatVal;
12762 if (IsSigned) {
12763 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: BW), DL: dl, VT);
12764 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: BW), DL: dl, VT);
12765 SDValue Cond =
12766 DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT);
12767 SatVal = DAG.getSelect(DL: dl, VT, Cond, LHS: SatMin, RHS: SatMax);
12768 } else {
12769 SatVal = DAG.getConstant(Val: APInt::getMaxValue(numBits: BW), DL: dl, VT);
12770 }
12771 SDValue Cond = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Orig, Cond: ISD::SETNE);
12772 return DAG.getSelect(DL: dl, VT, Cond, LHS: SatVal, RHS: Result);
12773}
12774
12775void TargetLowering::forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl,
12776 bool Signed, SDValue &Lo, SDValue &Hi,
12777 SDValue LHS, SDValue RHS,
12778 SDValue HiLHS, SDValue HiRHS) const {
12779 EVT VT = LHS.getValueType();
12780 assert(RHS.getValueType() == VT && "Mismatching operand types");
12781
12782 assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
12783 assert((!Signed || !HiLHS) &&
12784 "Signed flag should only be set when HiLHS and RiRHS are null");
12785
12786 // We'll expand the multiplication by brute force because we have no other
12787 // options. This is a trivially-generalized version of the code from
12788 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
12789 // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
12790 // sign bits while calculating the Hi half.
12791 unsigned Bits = VT.getScalarSizeInBits();
12792 unsigned HalfBits = Bits / 2;
12793 SDValue Mask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: Bits, loBitsSet: HalfBits), DL: dl, VT);
12794 SDValue LL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Mask);
12795 SDValue RL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: RHS, N2: Mask);
12796
12797 SDValue T = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RL);
12798 SDValue TL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: Mask);
12799
12800 SDValue Shift = DAG.getShiftAmountConstant(Val: HalfBits, VT, DL: dl);
12801 // This is always an unsigned shift.
12802 SDValue TH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: T, N2: Shift);
12803
12804 unsigned ShiftOpc = Signed ? ISD::SRA : ISD::SRL;
12805 SDValue LH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: LHS, N2: Shift);
12806 SDValue RH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: RHS, N2: Shift);
12807
12808 SDValue U =
12809 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RL), N2: TH);
12810 SDValue UL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: U, N2: Mask);
12811 SDValue UH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: U, N2: Shift);
12812
12813 SDValue V =
12814 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RH), N2: UL);
12815 SDValue VH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: V, N2: Shift);
12816
12817 Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: TL,
12818 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: Shift));
12819
12820 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RH),
12821 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: UH, N2: VH));
12822
12823 // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
12824 // the products to Hi.
12825 if (HiLHS) {
12826 SDValue RHLL = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: HiRHS, N2: LHS);
12827 SDValue RLLH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: RHS, N2: HiLHS);
12828 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Hi,
12829 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: RHLL, N2: RLLH));
12830 }
12831}
12832
12833void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
12834 bool Signed, const SDValue LHS,
12835 const SDValue RHS, SDValue &Lo,
12836 SDValue &Hi) const {
12837 EVT VT = LHS.getValueType();
12838 assert(RHS.getValueType() == VT && "Mismatching operand types");
12839 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
12840 // We can fall back to a libcall with an illegal type for the MUL if we
12841 // have a libcall big enough.
12842 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
12843 if (WideVT == MVT::i16)
12844 LC = RTLIB::MUL_I16;
12845 else if (WideVT == MVT::i32)
12846 LC = RTLIB::MUL_I32;
12847 else if (WideVT == MVT::i64)
12848 LC = RTLIB::MUL_I64;
12849 else if (WideVT == MVT::i128)
12850 LC = RTLIB::MUL_I128;
12851
12852 RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(Call: LC);
12853 if (LibcallImpl == RTLIB::Unsupported) {
12854 forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
12855 return;
12856 }
12857
12858 SDValue HiLHS, HiRHS;
12859 if (Signed) {
12860 // The high part is obtained by SRA'ing all but one of the bits of low
12861 // part.
12862 unsigned LoSize = VT.getFixedSizeInBits();
12863 SDValue Shift = DAG.getShiftAmountConstant(Val: LoSize - 1, VT, DL: dl);
12864 HiLHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS, N2: Shift);
12865 HiRHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: RHS, N2: Shift);
12866 } else {
12867 HiLHS = DAG.getConstant(Val: 0, DL: dl, VT);
12868 HiRHS = DAG.getConstant(Val: 0, DL: dl, VT);
12869 }
12870
12871 // Attempt a libcall.
12872 SDValue Ret;
12873 TargetLowering::MakeLibCallOptions CallOptions;
12874 CallOptions.setIsSigned(Signed);
12875 CallOptions.setIsPostTypeLegalization(true);
12876 if (shouldSplitFunctionArgumentsAsLittleEndian(DL: DAG.getDataLayout())) {
12877 // Halves of WideVT are packed into registers in different order
12878 // depending on platform endianness. This is usually handled by
12879 // the C calling convention, but we can't defer to it in
12880 // the legalizer.
12881 SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
12882 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12883 } else {
12884 SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
12885 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12886 }
12887 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
12888 "Ret value is a collection of constituent nodes holding result.");
12889 if (DAG.getDataLayout().isLittleEndian()) {
12890 // Same as above.
12891 Lo = Ret.getOperand(i: 0);
12892 Hi = Ret.getOperand(i: 1);
12893 } else {
12894 Lo = Ret.getOperand(i: 1);
12895 Hi = Ret.getOperand(i: 0);
12896 }
12897}
12898
12899SDValue
12900TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
12901 assert((Node->getOpcode() == ISD::SMULFIX ||
12902 Node->getOpcode() == ISD::UMULFIX ||
12903 Node->getOpcode() == ISD::SMULFIXSAT ||
12904 Node->getOpcode() == ISD::UMULFIXSAT) &&
12905 "Expected a fixed point multiplication opcode");
12906
12907 SDLoc dl(Node);
12908 SDValue LHS = Node->getOperand(Num: 0);
12909 SDValue RHS = Node->getOperand(Num: 1);
12910 EVT VT = LHS.getValueType();
12911 unsigned Scale = Node->getConstantOperandVal(Num: 2);
12912 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
12913 Node->getOpcode() == ISD::UMULFIXSAT);
12914 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
12915 Node->getOpcode() == ISD::SMULFIXSAT);
12916 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12917 unsigned VTSize = VT.getScalarSizeInBits();
12918
12919 if (!Scale) {
12920 // [us]mul.fix(a, b, 0) -> mul(a, b)
12921 if (!Saturating) {
12922 if (isOperationLegalOrCustom(Op: ISD::MUL, VT))
12923 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
12924 } else if (Signed && isOperationLegalOrCustom(Op: ISD::SMULO, VT)) {
12925 SDValue Result =
12926 DAG.getNode(Opcode: ISD::SMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12927 SDValue Product = Result.getValue(R: 0);
12928 SDValue Overflow = Result.getValue(R: 1);
12929 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12930
12931 APInt MinVal = APInt::getSignedMinValue(numBits: VTSize);
12932 APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize);
12933 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12934 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12935 // Xor the inputs, if resulting sign bit is 0 the product will be
12936 // positive, else negative.
12937 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
12938 SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT);
12939 Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax);
12940 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product);
12941 } else if (!Signed && isOperationLegalOrCustom(Op: ISD::UMULO, VT)) {
12942 SDValue Result =
12943 DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12944 SDValue Product = Result.getValue(R: 0);
12945 SDValue Overflow = Result.getValue(R: 1);
12946
12947 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
12948 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12949 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product);
12950 }
12951 }
12952
12953 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
12954 "Expected scale to be less than the number of bits if signed or at "
12955 "most the number of bits if unsigned.");
12956 assert(LHS.getValueType() == RHS.getValueType() &&
12957 "Expected both operands to be the same type");
12958
12959 // Select the saturated value when Cond0 <CC> Cond1, keeping it vectorized:
12960 // SELECT_CC is scalarized for vector types, so build SETCC + VSELECT there.
12961 auto getSaturatingSelect = [&](SDValue Cond0, SDValue Cond1, SDValue Sat,
12962 SDValue Val, ISD::CondCode CC) {
12963 if (VT.isVector())
12964 return DAG.getSelect(DL: dl, VT, Cond: DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Cond0, RHS: Cond1, Cond: CC),
12965 LHS: Sat, RHS: Val);
12966 return DAG.getSelectCC(DL: dl, LHS: Cond0, RHS: Cond1, True: Sat, False: Val, Cond: CC);
12967 };
12968
12969 // Get the upper and lower bits of the result.
12970 SDValue Lo, Hi;
12971 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
12972 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
12973 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
12974 if (isOperationLegalOrCustom(Op: LoHiOp, VT)) {
12975 SDValue Result = DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
12976 Lo = Result.getValue(R: 0);
12977 Hi = Result.getValue(R: 1);
12978 } else if (isOperationLegalOrCustom(Op: HiOp, VT)) {
12979 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
12980 Hi = DAG.getNode(Opcode: HiOp, DL: dl, VT, N1: LHS, N2: RHS);
12981 } else if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
12982 // Try for a multiplication using a wider type.
12983 unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
12984 SDValue LHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: LHS);
12985 SDValue RHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: RHS);
12986 SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHSExt, N2: RHSExt);
12987 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Res);
12988 SDValue Shifted =
12989 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: WideVT, N1: Res,
12990 N2: DAG.getShiftAmountConstant(Val: VTSize, VT: WideVT, DL: dl));
12991 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Shifted);
12992 } else if (VT.isVector()) {
12993 return SDValue();
12994 } else {
12995 forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
12996 }
12997
12998 if (Scale == VTSize)
12999 // Result is just the top half since we'd be shifting by the width of the
13000 // operand. Overflow impossible so this works for both UMULFIX and
13001 // UMULFIXSAT.
13002 return Hi;
13003
13004 // The result will need to be shifted right by the scale since both operands
13005 // are scaled. The result is given to us in 2 halves, so we only want part of
13006 // both in the result.
13007 SDValue Result = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: Hi, N2: Lo,
13008 N3: DAG.getShiftAmountConstant(Val: Scale, VT, DL: dl));
13009 if (!Saturating)
13010 return Result;
13011
13012 if (!Signed) {
13013 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
13014 // widened multiplication) aren't all zeroes.
13015
13016 // Saturate to max if ((Hi >> Scale) != 0),
13017 // which is the same as if (Hi > ((1 << Scale) - 1))
13018 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
13019 SDValue LowMask =
13020 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale), DL: dl, VT);
13021 return getSaturatingSelect(Hi, LowMask, DAG.getConstant(Val: MaxVal, DL: dl, VT),
13022 Result, ISD::SETUGT);
13023 }
13024
13025 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
13026 // widened multiplication) aren't all ones or all zeroes.
13027
13028 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: VTSize), DL: dl, VT);
13029 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: VTSize), DL: dl, VT);
13030
13031 if (Scale == 0) {
13032 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Lo,
13033 N2: DAG.getShiftAmountConstant(Val: VTSize - 1, VT, DL: dl));
13034 SDValue Overflow = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Hi, RHS: Sign, Cond: ISD::SETNE);
13035 // Saturated to SatMin if wide product is negative, and SatMax if wide
13036 // product is positive ...
13037 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13038 SDValue ResultIfOverflow =
13039 getSaturatingSelect(Hi, Zero, SatMin, SatMax, ISD::SETLT);
13040 // ... but only if we overflowed.
13041 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: ResultIfOverflow, RHS: Result);
13042 }
13043
13044 // We handled Scale==0 above so all the bits to examine is in Hi.
13045
13046 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
13047 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
13048 SDValue LowMask =
13049 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale - 1), DL: dl, VT);
13050 // Saturate to min if (Hi >> (Scale - 1)) < -1),
13051 // which is the same as if (HI < (-1 << (Scale - 1))
13052 SDValue HighMask = DAG.getConstant(
13053 Val: APInt::getHighBitsSet(numBits: VTSize, hiBitsSet: VTSize - Scale + 1), DL: dl, VT);
13054 Result = getSaturatingSelect(Hi, LowMask, SatMax, Result, ISD::SETGT);
13055 Result = getSaturatingSelect(Hi, HighMask, SatMin, Result, ISD::SETLT);
13056 return Result;
13057}
13058
13059SDValue
13060TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
13061 SDValue LHS, SDValue RHS,
13062 unsigned Scale, SelectionDAG &DAG) const {
13063 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
13064 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
13065 "Expected a fixed point division opcode");
13066
13067 EVT VT = LHS.getValueType();
13068 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
13069 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
13070 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13071
13072 // If there is enough room in the type to upscale the LHS or downscale the
13073 // RHS before the division, we can perform it in this type without having to
13074 // resize. For signed operations, the LHS headroom is the number of
13075 // redundant sign bits, and for unsigned ones it is the number of zeroes.
13076 // The headroom for the RHS is the number of trailing zeroes.
13077 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(Op: LHS) - 1
13078 : DAG.computeKnownBits(Op: LHS).countMinLeadingZeros();
13079 unsigned RHSTrail = DAG.computeKnownBits(Op: RHS).countMinTrailingZeros();
13080
13081 // For signed saturating operations, we need to be able to detect true integer
13082 // division overflow; that is, when you have MIN / -EPS. However, this
13083 // is undefined behavior and if we emit divisions that could take such
13084 // values it may cause undesired behavior (arithmetic exceptions on x86, for
13085 // example).
13086 // Avoid this by requiring an extra bit so that we never get this case.
13087 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
13088 // signed saturating division, we need to emit a whopping 32-bit division.
13089 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
13090 return SDValue();
13091
13092 unsigned LHSShift = std::min(a: LHSLead, b: Scale);
13093 unsigned RHSShift = Scale - LHSShift;
13094
13095 // At this point, we know that if we shift the LHS up by LHSShift and the
13096 // RHS down by RHSShift, we can emit a regular division with a final scaling
13097 // factor of Scale.
13098
13099 if (LHSShift)
13100 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS,
13101 N2: DAG.getShiftAmountConstant(Val: LHSShift, VT, DL: dl));
13102 if (RHSShift)
13103 RHS = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: RHS,
13104 N2: DAG.getShiftAmountConstant(Val: RHSShift, VT, DL: dl));
13105
13106 SDValue Quot;
13107 if (Signed) {
13108 // For signed operations, if the resulting quotient is negative and the
13109 // remainder is nonzero, subtract 1 from the quotient to round towards
13110 // negative infinity.
13111 SDValue Rem;
13112 // FIXME: Ideally we would always produce an SDIVREM here, but if the
13113 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
13114 // we couldn't just form a libcall, but the type legalizer doesn't do it.
13115 if (isTypeLegal(VT) &&
13116 isOperationLegalOrCustom(Op: ISD::SDIVREM, VT)) {
13117 Quot = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl,
13118 VTList: DAG.getVTList(VT1: VT, VT2: VT),
13119 N1: LHS, N2: RHS);
13120 Rem = Quot.getValue(R: 1);
13121 Quot = Quot.getValue(R: 0);
13122 } else {
13123 Quot = DAG.getNode(Opcode: ISD::SDIV, DL: dl, VT,
13124 N1: LHS, N2: RHS);
13125 Rem = DAG.getNode(Opcode: ISD::SREM, DL: dl, VT,
13126 N1: LHS, N2: RHS);
13127 }
13128 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13129 SDValue RemNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Rem, RHS: Zero, Cond: ISD::SETNE);
13130 SDValue LHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETLT);
13131 SDValue RHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13132 SDValue QuotNeg = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: BoolVT, N1: LHSNeg, N2: RHSNeg);
13133 SDValue Sub1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Quot,
13134 N2: DAG.getConstant(Val: 1, DL: dl, VT));
13135 Quot = DAG.getSelect(DL: dl, VT,
13136 Cond: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolVT, N1: RemNonZero, N2: QuotNeg),
13137 LHS: Sub1, RHS: Quot);
13138 } else
13139 Quot = DAG.getNode(Opcode: ISD::UDIV, DL: dl, VT,
13140 N1: LHS, N2: RHS);
13141
13142 return Quot;
13143}
13144
13145void TargetLowering::expandUADDSUBO(
13146 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13147 SDLoc dl(Node);
13148 SDValue LHS = Node->getOperand(Num: 0);
13149 SDValue RHS = Node->getOperand(Num: 1);
13150 bool IsAdd = Node->getOpcode() == ISD::UADDO;
13151
13152 // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
13153 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
13154 if (isOperationLegalOrCustom(Op: OpcCarry, VT: Node->getValueType(ResNo: 0))) {
13155 SDValue CarryIn = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 1));
13156 SDValue NodeCarry = DAG.getNode(Opcode: OpcCarry, DL: dl, VTList: Node->getVTList(),
13157 Ops: { LHS, RHS, CarryIn });
13158 Result = SDValue(NodeCarry.getNode(), 0);
13159 Overflow = SDValue(NodeCarry.getNode(), 1);
13160 return;
13161 }
13162
13163 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13164 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13165
13166 EVT ResultType = Node->getValueType(ResNo: 1);
13167 EVT SetCCType = getSetCCResultType(
13168 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13169 SDValue SetCC;
13170 if (IsAdd && isOneConstant(V: RHS)) {
13171 // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
13172 // the live range of X. We assume comparing with 0 is cheap.
13173 // The general case (X + C) < C is not necessarily beneficial. Although we
13174 // reduce the live range of X, we may introduce the materialization of
13175 // constant C.
13176 SetCC =
13177 DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result,
13178 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETEQ);
13179 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
13180 // Special case: uaddo X, -1 overflows if X != 0.
13181 SetCC =
13182 DAG.getSetCC(DL: dl, VT: SetCCType, LHS,
13183 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETNE);
13184 } else {
13185 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
13186 SetCC = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result, RHS: LHS, Cond: CC);
13187 }
13188 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13189}
13190
13191void TargetLowering::expandSADDSUBO(
13192 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13193 SDLoc dl(Node);
13194 SDValue LHS = Node->getOperand(Num: 0);
13195 SDValue RHS = Node->getOperand(Num: 1);
13196 bool IsAdd = Node->getOpcode() == ISD::SADDO;
13197
13198 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13199 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13200
13201 EVT ResultType = Node->getValueType(ResNo: 1);
13202 EVT OType = getSetCCResultType(
13203 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13204
13205 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
13206 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
13207 if (isOperationLegal(Op: OpcSat, VT: LHS.getValueType())) {
13208 SDValue Sat = DAG.getNode(Opcode: OpcSat, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS);
13209 SDValue SetCC = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Sat, Cond: ISD::SETNE);
13210 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13211 return;
13212 }
13213
13214 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
13215
13216 if (IsAdd) {
13217 // For an addition, the result should be less than one of the operands (LHS)
13218 // if and only if the other operand (RHS) is negative, otherwise there will
13219 // be overflow.
13220 SDValue ResultLowerThanLHS =
13221 DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: LHS, Cond: ISD::SETLT);
13222 SDValue RHSNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13223 Overflow = DAG.getBoolExtOrTrunc(
13224 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: RHSNegative, N2: ResultLowerThanLHS), SL: dl,
13225 VT: ResultType, OpVT: ResultType);
13226 } else {
13227 // For subtraction, overflow occurs when the signed comparison of operands
13228 // doesn't match the sign of the result.
13229 SDValue LHSLessThanRHS = DAG.getSetCC(DL: dl, VT: OType, LHS, RHS, Cond: ISD::SETLT);
13230 SDValue ResultNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Zero, Cond: ISD::SETLT);
13231 Overflow = DAG.getBoolExtOrTrunc(
13232 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: LHSLessThanRHS, N2: ResultNegative), SL: dl,
13233 VT: ResultType, OpVT: ResultType);
13234 }
13235}
13236
13237bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
13238 SDValue &Overflow, SelectionDAG &DAG) const {
13239 SDLoc dl(Node);
13240 EVT VT = Node->getValueType(ResNo: 0);
13241 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13242 SDValue LHS = Node->getOperand(Num: 0);
13243 SDValue RHS = Node->getOperand(Num: 1);
13244 bool isSigned = Node->getOpcode() == ISD::SMULO;
13245
13246 // For power-of-two multiplications we can use a simpler shift expansion.
13247 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
13248 const APInt &C = RHSC->getAPIntValue();
13249 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
13250 if (C.isPowerOf2()) {
13251 // smulo(x, signed_min) is same as umulo(x, signed_min).
13252 bool UseArithShift = isSigned && !C.isMinSignedValue();
13253 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: C.logBase2(), VT, DL: dl);
13254 Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: ShiftAmt);
13255 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT,
13256 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL,
13257 DL: dl, VT, N1: Result, N2: ShiftAmt),
13258 RHS: LHS, Cond: ISD::SETNE);
13259 return true;
13260 }
13261 }
13262
13263 SDValue BottomHalf;
13264 SDValue TopHalf;
13265 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13266
13267 static const unsigned Ops[2][3] =
13268 { { ISD::UMUL_LOHI, ISD::MULHU, ISD::ZERO_EXTEND },
13269 { ISD::SMUL_LOHI, ISD::MULHS, ISD::SIGN_EXTEND }};
13270 if (isOperationLegalOrCustom(Op: Ops[isSigned][0], VT)) {
13271 BottomHalf = DAG.getNode(Opcode: Ops[isSigned][0], DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS,
13272 N2: RHS);
13273 TopHalf = BottomHalf.getValue(R: 1);
13274 } else if (isOperationLegalOrCustom(Op: Ops[isSigned][1], VT)) {
13275 BottomHalf = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13276 TopHalf = DAG.getNode(Opcode: Ops[isSigned][1], DL: dl, VT, N1: LHS, N2: RHS);
13277 } else if (isTypeLegal(VT: WideVT)) {
13278 LHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: LHS);
13279 RHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: RHS);
13280 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS);
13281 BottomHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Mul);
13282 SDValue ShiftAmt =
13283 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits(), VT: WideVT, DL: dl);
13284 TopHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13285 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT, N1: Mul, N2: ShiftAmt));
13286 } else {
13287 if (VT.isVector())
13288 return false;
13289
13290 forceExpandWideMUL(DAG, dl, Signed: isSigned, LHS, RHS, Lo&: BottomHalf, Hi&: TopHalf);
13291 }
13292
13293 Result = BottomHalf;
13294 if (isSigned) {
13295 SDValue ShiftAmt = DAG.getShiftAmountConstant(
13296 Val: VT.getScalarSizeInBits() - 1, VT: BottomHalf.getValueType(), DL: dl);
13297 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: BottomHalf, N2: ShiftAmt);
13298 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf, RHS: Sign, Cond: ISD::SETNE);
13299 } else {
13300 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf,
13301 RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
13302 }
13303
13304 // Truncate the result if SetCC returns a larger type than needed.
13305 EVT RType = Node->getValueType(ResNo: 1);
13306 if (RType.bitsLT(VT: Overflow.getValueType()))
13307 Overflow = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RType, Operand: Overflow);
13308
13309 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
13310 "Unexpected result type for S/UMULO legalization");
13311 return true;
13312}
13313
13314SDValue TargetLowering::expandMULH(SDNode *Node, SelectionDAG &DAG) const {
13315 SDLoc dl(Node);
13316 EVT VT = Node->getValueType(ResNo: 0);
13317 SDValue LHS = Node->getOperand(Num: 0);
13318 SDValue RHS = Node->getOperand(Num: 1);
13319 bool IsSigned = Node->getOpcode() == ISD::MULHS;
13320
13321 // Use MUL_LOHI if legal/custom for the original type.
13322 unsigned LoHiOp = IsSigned ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13323 if (isOperationLegalOrCustom(Op: LoHiOp, VT))
13324 return DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS).getValue(R: 1);
13325
13326 // Use a wide multiply if available.
13327 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13328 if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
13329 unsigned BW = VT.getScalarSizeInBits();
13330 LHS = DAG.getExtOrTrunc(IsSigned, Op: LHS, DL: dl, VT: WideVT);
13331 RHS = DAG.getExtOrTrunc(IsSigned, Op: RHS, DL: dl, VT: WideVT);
13332 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13333 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT,
13334 N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS),
13335 N2: DAG.getShiftAmountConstant(Val: BW, VT: WideVT, DL: dl)));
13336 }
13337
13338 // Let fixed-length vectors be scalarised by the caller.
13339 // Expand everything else with a wide multiply.
13340 if (!VT.isFixedLengthVector()) {
13341 SDValue Lo, Hi;
13342 forceExpandWideMUL(DAG, dl, Signed: IsSigned, LHS, RHS, Lo, Hi);
13343 return Hi;
13344 }
13345
13346 return SDValue();
13347}
13348
13349SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
13350 SDLoc dl(Node);
13351 ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13352 SDValue Op = Node->getOperand(Num: 0);
13353 SDNodeFlags Flags = Node->getFlags();
13354 EVT VT = Op.getValueType();
13355
13356 // Try to use a shuffle reduction for power of two vectors.
13357 if (VT.isPow2VectorType()) {
13358 // See if the reduction opcode is safe to use with widened types.
13359 bool WidenSrc = false;
13360 switch (Node->getOpcode()) {
13361 case ISD::VECREDUCE_FADD:
13362 case ISD::VECREDUCE_FMUL:
13363 case ISD::VECREDUCE_ADD:
13364 case ISD::VECREDUCE_MUL:
13365 case ISD::VECREDUCE_AND:
13366 case ISD::VECREDUCE_OR:
13367 case ISD::VECREDUCE_XOR:
13368 case ISD::VECREDUCE_SMAX:
13369 case ISD::VECREDUCE_SMIN:
13370 case ISD::VECREDUCE_UMAX:
13371 case ISD::VECREDUCE_UMIN:
13372 WidenSrc = VT.isFixedLengthVector();
13373 break;
13374 }
13375
13376 while (VT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
13377 EVT HalfVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
13378 if (!isOperationLegalOrCustom(Op: BaseOpcode, VT: HalfVT)) {
13379 if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
13380 // Attempt to widen the source vectors to a legal op.
13381 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: HalfVT);
13382 if (WideVT.isVector() &&
13383 WideVT.getScalarType() == HalfVT.getScalarType() &&
13384 WideVT.getVectorNumElements() >= HalfVT.getVectorNumElements() &&
13385 isOperationLegalOrCustom(Op: BaseOpcode, VT: WideVT)) {
13386 SDValue Lo, Hi;
13387 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13388 Lo = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Lo, Idx: 0);
13389 Hi = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Hi, Idx: 0);
13390 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: WideVT, N1: Lo, N2: Hi, Flags);
13391 Op = DAG.getExtractSubvector(DL: dl, VT: HalfVT, Vec: Op, Idx: 0);
13392 VT = HalfVT;
13393 continue;
13394 }
13395 }
13396 break;
13397 }
13398
13399 SDValue Lo, Hi;
13400 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13401 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: HalfVT, N1: Lo, N2: Hi, Flags);
13402 VT = HalfVT;
13403
13404 // Stop if splitting is enough to make the reduction legal.
13405 if (isOperationLegalOrCustom(Op: Node->getOpcode(), VT: HalfVT))
13406 return DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op,
13407 Flags);
13408 }
13409 }
13410
13411 if (VT.isScalableVector())
13412 reportFatalInternalError(
13413 reason: "Expanding reductions for scalable vectors is undefined.");
13414
13415 EVT EltVT = VT.getVectorElementType();
13416 unsigned NumElts = VT.getVectorNumElements();
13417
13418 SmallVector<SDValue, 8> Ops;
13419 DAG.ExtractVectorElements(Op, Args&: Ops, Start: 0, Count: NumElts);
13420
13421 SDValue Res = Ops[0];
13422 for (unsigned i = 1; i < NumElts; i++)
13423 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13424
13425 // Result type may be wider than element type.
13426 if (EltVT != Node->getValueType(ResNo: 0))
13427 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res);
13428 return Res;
13429}
13430
13431SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
13432 SDLoc dl(Node);
13433 SDValue AccOp = Node->getOperand(Num: 0);
13434 SDValue VecOp = Node->getOperand(Num: 1);
13435 SDNodeFlags Flags = Node->getFlags();
13436
13437 EVT VT = VecOp.getValueType();
13438 EVT EltVT = VT.getVectorElementType();
13439
13440 if (VT.isScalableVector())
13441 report_fatal_error(
13442 reason: "Expanding reductions for scalable vectors is undefined.");
13443
13444 unsigned NumElts = VT.getVectorNumElements();
13445
13446 SmallVector<SDValue, 8> Ops;
13447 DAG.ExtractVectorElements(Op: VecOp, Args&: Ops, Start: 0, Count: NumElts);
13448
13449 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13450
13451 SDValue Res = AccOp;
13452 for (unsigned i = 0; i < NumElts; i++)
13453 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13454
13455 return Res;
13456}
13457
13458bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
13459 SelectionDAG &DAG) const {
13460 EVT VT = Node->getValueType(ResNo: 0);
13461 SDLoc dl(Node);
13462 bool isSigned = Node->getOpcode() == ISD::SREM;
13463 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
13464 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
13465 SDValue Dividend = Node->getOperand(Num: 0);
13466 SDValue Divisor = Node->getOperand(Num: 1);
13467 if (isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
13468 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
13469 Result = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Dividend, N2: Divisor).getValue(R: 1);
13470 return true;
13471 }
13472 if (isOperationLegalOrCustom(Op: DivOpc, VT)) {
13473 // X % Y -> X-X/Y*Y
13474 SDValue Divide = DAG.getNode(Opcode: DivOpc, DL: dl, VT, N1: Dividend, N2: Divisor);
13475 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Divide, N2: Divisor);
13476 Result = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
13477 return true;
13478 }
13479 return false;
13480}
13481
13482SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
13483 SelectionDAG &DAG) const {
13484 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
13485 SDLoc dl(SDValue(Node, 0));
13486 SDValue Src = Node->getOperand(Num: 0);
13487
13488 // DstVT is the result type, while SatVT is the size to which we saturate
13489 EVT SrcVT = Src.getValueType();
13490 EVT DstVT = Node->getValueType(ResNo: 0);
13491
13492 EVT SatVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
13493 unsigned SatWidth = SatVT.getScalarSizeInBits();
13494 unsigned DstWidth = DstVT.getScalarSizeInBits();
13495 assert(SatWidth <= DstWidth &&
13496 "Expected saturation width smaller than result width");
13497
13498 // Determine minimum and maximum integer values and their corresponding
13499 // floating-point values.
13500 APInt MinInt, MaxInt;
13501 if (IsSigned) {
13502 MinInt = APInt::getSignedMinValue(numBits: SatWidth).sext(width: DstWidth);
13503 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth).sext(width: DstWidth);
13504 } else {
13505 MinInt = APInt::getMinValue(numBits: SatWidth).zext(width: DstWidth);
13506 MaxInt = APInt::getMaxValue(numBits: SatWidth).zext(width: DstWidth);
13507 }
13508
13509 // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
13510 // libcall emission cannot handle this. Large result types will fail.
13511 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
13512 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f32, Operand: Src);
13513 SrcVT = Src.getValueType();
13514 }
13515
13516 const fltSemantics &Sem = SrcVT.getFltSemantics();
13517 APFloat MinFloat(Sem);
13518 APFloat MaxFloat(Sem);
13519
13520 APFloat::opStatus MinStatus =
13521 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
13522 APFloat::opStatus MaxStatus =
13523 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
13524 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
13525 !(MaxStatus & APFloat::opStatus::opInexact);
13526
13527 SDValue MinFloatNode = DAG.getConstantFP(Val: MinFloat, DL: dl, VT: SrcVT);
13528 SDValue MaxFloatNode = DAG.getConstantFP(Val: MaxFloat, DL: dl, VT: SrcVT);
13529
13530 // If the integer bounds are exactly representable as floats and min/max are
13531 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
13532 // of comparisons and selects.
13533 auto EmitMinMax = [&](unsigned MinOpcode, unsigned MaxOpcode,
13534 bool MayPropagateNaN) {
13535 bool MinMaxLegal = isOperationLegalOrCustom(Op: MinOpcode, VT: SrcVT) &&
13536 isOperationLegalOrCustom(Op: MaxOpcode, VT: SrcVT);
13537 if (!MinMaxLegal)
13538 return SDValue();
13539
13540 SDValue Clamped = Src;
13541
13542 // Clamp Src by MinFloat from below. If !MayPropagateNaN and Src is NaN
13543 // then the result is MinFloat.
13544 Clamped = DAG.getNode(Opcode: MaxOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MinFloatNode);
13545 // Clamp by MaxFloat from above. If !MayPropagateNaN then NaN cannot occur.
13546 Clamped = DAG.getNode(Opcode: MinOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MaxFloatNode);
13547 // Convert clamped value to integer.
13548 SDValue FpToInt = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
13549 DL: dl, VT: DstVT, Operand: Clamped);
13550
13551 // If !MayPropagateNan and the conversion is unsigned case we're done,
13552 // because we mapped NaN to MinFloat, which will cast to zero.
13553 if (!MayPropagateNaN && !IsSigned)
13554 return FpToInt;
13555
13556 // Otherwise, select 0 if Src is NaN.
13557 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13558 EVT SetCCVT =
13559 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13560 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13561 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: FpToInt);
13562 };
13563 if (AreExactFloatBounds) {
13564 if (SDValue Res = EmitMinMax(ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
13565 /*MayPropagateNaN=*/false))
13566 return Res;
13567 // These may propagate NaN for sNaN operands.
13568 if (SDValue Res =
13569 EmitMinMax(ISD::FMINNUM, ISD::FMAXNUM, /*MayPropagateNaN=*/true))
13570 return Res;
13571 // These always propagate NaN.
13572 if (SDValue Res =
13573 EmitMinMax(ISD::FMINIMUM, ISD::FMAXIMUM, /*MayPropagateNaN=*/true))
13574 return Res;
13575 }
13576
13577 SDValue MinIntNode = DAG.getConstant(Val: MinInt, DL: dl, VT: DstVT);
13578 SDValue MaxIntNode = DAG.getConstant(Val: MaxInt, DL: dl, VT: DstVT);
13579
13580 // Result of direct conversion. The assumption here is that the operation is
13581 // non-trapping and it's fine to apply it to an out-of-range value if we
13582 // select it away later.
13583 SDValue FpToInt =
13584 DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL: dl, VT: DstVT, Operand: Src);
13585
13586 SDValue Select = FpToInt;
13587
13588 EVT SetCCVT =
13589 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13590
13591 // If Src ULT MinFloat, select MinInt. In particular, this also selects
13592 // MinInt if Src is NaN.
13593 SDValue ULT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MinFloatNode, Cond: ISD::SETULT);
13594 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: ULT, LHS: MinIntNode, RHS: Select);
13595 // If Src OGT MaxFloat, select MaxInt.
13596 SDValue OGT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MaxFloatNode, Cond: ISD::SETOGT);
13597 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: OGT, LHS: MaxIntNode, RHS: Select);
13598
13599 // In the unsigned case we are done, because we mapped NaN to MinInt, which
13600 // is already zero.
13601 if (!IsSigned)
13602 return Select;
13603
13604 // Otherwise, select 0 if Src is NaN.
13605 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13606 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13607 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: Select);
13608}
13609
13610SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op,
13611 const SDLoc &dl,
13612 SelectionDAG &DAG) const {
13613 EVT OperandVT = Op.getValueType();
13614 if (OperandVT.getScalarType() == ResultVT.getScalarType())
13615 return Op;
13616 EVT ResultIntVT = ResultVT.changeTypeToInteger();
13617 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13618 // can induce double-rounding which may alter the results. We can
13619 // correct for this using a trick explained in: Boldo, Sylvie, and
13620 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13621 // World Congress. 2005.
13622 SDValue Narrow = DAG.getFPExtendOrRound(Op, DL: dl, VT: ResultVT);
13623 SDValue NarrowAsWide = DAG.getFPExtendOrRound(Op: Narrow, DL: dl, VT: OperandVT);
13624
13625 // We can keep the narrow value as-is if narrowing was exact (no
13626 // rounding error), the wide value was NaN (the narrow value is also
13627 // NaN and should be preserved) or if we rounded to the odd value.
13628 SDValue NarrowBits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultIntVT, Operand: Narrow);
13629 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResultIntVT);
13630 SDValue NegativeOne = DAG.getAllOnesConstant(DL: dl, VT: ResultIntVT);
13631 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: One);
13632 EVT ResultIntVTCCVT = getSetCCResultType(
13633 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: And.getValueType());
13634 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ResultIntVT);
13635 // The result is already odd so we don't need to do anything.
13636 SDValue AlreadyOdd = DAG.getSetCC(DL: dl, VT: ResultIntVTCCVT, LHS: And, RHS: Zero, Cond: ISD::SETNE);
13637
13638 EVT WideSetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
13639 VT: Op.getValueType());
13640 // We keep results which are exact, odd or NaN.
13641 SDValue KeepNarrow =
13642 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: Op, RHS: NarrowAsWide, Cond: ISD::SETUEQ);
13643 KeepNarrow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: WideSetCCVT, N1: KeepNarrow, N2: AlreadyOdd);
13644 // We morally performed a round-down if AbsNarrow is smaller than
13645 // AbsWide.
13646 SDValue AbsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: Op);
13647 SDValue AbsNarrowAsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: NarrowAsWide);
13648 SDValue NarrowIsRd =
13649 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: AbsWide, RHS: AbsNarrowAsWide, Cond: ISD::SETOGT);
13650 // If the narrow value is odd or exact, pick it.
13651 // Otherwise, narrow is even and corresponds to either the rounded-up
13652 // or rounded-down value. If narrow is the rounded-down value, we want
13653 // the rounded-up value as it will be odd.
13654 SDValue Adjust = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: NarrowIsRd, LHS: One, RHS: NegativeOne);
13655 SDValue Adjusted = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: Adjust);
13656 Op = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: KeepNarrow, LHS: NarrowBits, RHS: Adjusted);
13657 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultVT, Operand: Op);
13658}
13659
13660SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const {
13661 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
13662 SDValue Op = Node->getOperand(Num: 0);
13663 EVT VT = Node->getValueType(ResNo: 0);
13664 SDLoc dl(Node);
13665 if (VT.getScalarType() == MVT::bf16) {
13666 if (Node->getConstantOperandVal(Num: 1) == 1) {
13667 return DAG.getNode(Opcode: ISD::FP_TO_BF16, DL: dl, VT, Operand: Node->getOperand(Num: 0));
13668 }
13669 EVT OperandVT = Op.getValueType();
13670 SDValue IsNaN = DAG.getSetCC(
13671 DL: dl,
13672 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT),
13673 LHS: Op, RHS: Op, Cond: ISD::SETUO);
13674
13675 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13676 // can induce double-rounding which may alter the results. We can
13677 // correct for this using a trick explained in: Boldo, Sylvie, and
13678 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13679 // World Congress. 2005.
13680 EVT F32 = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
13681 EVT I32 = F32.changeTypeToInteger();
13682 Op = expandRoundInexactToOdd(ResultVT: F32, Op, dl, DAG);
13683 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: I32, Operand: Op);
13684
13685 // Conversions should set NaN's quiet bit. This also prevents NaNs from
13686 // turning into infinities.
13687 SDValue NaN =
13688 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: I32, N1: Op, N2: DAG.getConstant(Val: 0x400000, DL: dl, VT: I32));
13689
13690 // Factor in the contribution of the low 16 bits.
13691 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: I32);
13692 SDValue Lsb = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13693 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13694 Lsb = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: I32, N1: Lsb, N2: One);
13695 SDValue RoundingBias =
13696 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Lsb, N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: I32));
13697 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Op, N2: RoundingBias);
13698
13699 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
13700 // 0x80000000.
13701 Op = DAG.getSelect(DL: dl, VT: I32, Cond: IsNaN, LHS: NaN, RHS: Add);
13702
13703 // Now that we have rounded, shift the bits into position.
13704 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13705 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13706 EVT I16 = I32.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i16);
13707 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: I16, Operand: Op);
13708 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
13709 }
13710 return SDValue();
13711}
13712
13713SDValue TargetLowering::expandVectorSplice(SDNode *Node,
13714 SelectionDAG &DAG) const {
13715 assert((Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT ||
13716 Node->getOpcode() == ISD::VECTOR_SPLICE_RIGHT) &&
13717 "Unexpected opcode!");
13718 assert((Node->getValueType(0).isScalableVector() ||
13719 !isa<ConstantSDNode>(Node->getOperand(2))) &&
13720 "Fixed length vector types with constant offsets expected to use "
13721 "SHUFFLE_VECTOR!");
13722
13723 EVT VT = Node->getValueType(ResNo: 0);
13724 SDValue V1 = Node->getOperand(Num: 0);
13725 SDValue V2 = Node->getOperand(Num: 1);
13726 SDValue Offset = Node->getOperand(Num: 2);
13727 SDLoc DL(Node);
13728
13729 // Expand through memory thusly:
13730 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
13731 // Store V1, Ptr
13732 // Store V2, Ptr + sizeof(V1)
13733 // if (VECTOR_SPLICE_LEFT)
13734 // Ptr = Ptr + (Offset * sizeof(VT.Elt))
13735 // else
13736 // Ptr = Ptr + sizeof(V1) - (Offset * size(VT.Elt))
13737 // Res = Load Ptr
13738
13739 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
13740
13741 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
13742 EC: VT.getVectorElementCount() * 2);
13743 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
13744 EVT PtrVT = StackPtr.getValueType();
13745 auto &MF = DAG.getMachineFunction();
13746 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13747 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13748
13749 // Store the lo part of CONCAT_VECTORS(V1, V2)
13750 SDValue StoreV1 =
13751 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, PtrInfo, Alignment);
13752 // Store the hi part of CONCAT_VECTORS(V1, V2)
13753 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
13754 SDValue StackPtr2 = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: VTBytes);
13755 SDValue StoreV2 =
13756 DAG.getStore(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, PtrInfo, Alignment);
13757
13758 // NOTE: TrailingBytes must be clamped so as not to read outside of V1:V2.
13759 SDValue EltByteSize =
13760 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
13761 Offset = DAG.getZExtOrTrunc(Op: Offset, DL, VT: PtrVT);
13762 SDValue TrailingBytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: Offset, N2: EltByteSize);
13763
13764 TrailingBytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: VTBytes);
13765
13766 if (Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT)
13767 StackPtr = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TrailingBytes, DL);
13768 else
13769 StackPtr = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
13770
13771 // Load the spliced result
13772 return DAG.getLoad(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr,
13773 PtrInfo: MachinePointerInfo::getUnknownStack(MF), Alignment);
13774}
13775
13776SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
13777 SelectionDAG &DAG) const {
13778 SDLoc DL(Node);
13779 SDValue Vec = Node->getOperand(Num: 0);
13780 SDValue Mask = Node->getOperand(Num: 1);
13781 SDValue Passthru = Node->getOperand(Num: 2);
13782
13783 EVT VecVT = Vec.getValueType();
13784 EVT ScalarVT = VecVT.getScalarType();
13785 EVT MaskVT = Mask.getValueType();
13786 EVT MaskScalarVT = MaskVT.getScalarType();
13787
13788 // Needs to be handled by targets that have scalable vector types.
13789 if (VecVT.isScalableVector())
13790 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
13791
13792 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
13793 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment);
13794 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13795 MachinePointerInfo PtrInfo =
13796 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
13797
13798 MVT PositionVT = getVectorIdxTy(DL: DAG.getDataLayout());
13799 SDValue Chain = DAG.getEntryNode();
13800 SDValue OutPos = DAG.getConstant(Val: 0, DL, VT: PositionVT);
13801
13802 bool HasPassthru = !Passthru.isUndef();
13803
13804 // If we have a passthru vector, store it on the stack, overwrite the matching
13805 // positions and then re-write the last element that was potentially
13806 // overwritten even though mask[i] = false.
13807 if (HasPassthru)
13808 Chain = DAG.getStore(Chain, dl: DL, Val: Passthru, Ptr: StackPtr, PtrInfo, Alignment);
13809
13810 SDValue LastWriteVal;
13811 APInt PassthruSplatVal;
13812 bool IsSplatPassthru =
13813 ISD::isConstantSplatVector(N: Passthru.getNode(), SplatValue&: PassthruSplatVal);
13814
13815 if (IsSplatPassthru) {
13816 // As we do not know which position we wrote to last, we cannot simply
13817 // access that index from the passthru vector. So we first check if passthru
13818 // is a splat vector, to use any element ...
13819 LastWriteVal = DAG.getConstant(Val: PassthruSplatVal, DL, VT: ScalarVT);
13820 } else if (HasPassthru) {
13821 // ... if it is not a splat vector, we need to get the passthru value at
13822 // position = popcount(mask) and re-load it from the stack before it is
13823 // overwritten in the loop below.
13824 EVT PopcountVT = ScalarVT.changeTypeToInteger();
13825 SDValue Popcount = DAG.getNode(
13826 Opcode: ISD::TRUNCATE, DL,
13827 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1), Operand: Mask);
13828 Popcount = DAG.getNode(
13829 Opcode: ISD::ZERO_EXTEND, DL,
13830 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: PopcountVT),
13831 Operand: Popcount);
13832 Popcount = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: PopcountVT, Operand: Popcount);
13833 SDValue LastElmtPtr =
13834 getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Popcount);
13835 LastWriteVal = DAG.getLoad(
13836 VT: ScalarVT, dl: DL, Chain, Ptr: LastElmtPtr,
13837 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13838 Chain = LastWriteVal.getValue(R: 1);
13839 }
13840
13841 unsigned NumElms = VecVT.getVectorNumElements();
13842 for (unsigned I = 0; I < NumElms; I++) {
13843 SDValue ValI = DAG.getExtractVectorElt(DL, VT: ScalarVT, Vec, Idx: I);
13844 SDValue OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
13845 Chain = DAG.getStore(
13846 Chain, dl: DL, Val: ValI, Ptr: OutPtr,
13847 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13848
13849 // Get the mask value and add it to the current output position. This
13850 // either increments by 1 if MaskI is true or adds 0 otherwise.
13851 // Freeze in case we have poison/undef mask entries.
13852 SDValue MaskI = DAG.getExtractVectorElt(DL, VT: MaskScalarVT, Vec: Mask, Idx: I);
13853 MaskI = DAG.getFreeze(V: MaskI);
13854 MaskI = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: MaskI);
13855 MaskI = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PositionVT, Operand: MaskI);
13856 OutPos = DAG.getNode(Opcode: ISD::ADD, DL, VT: PositionVT, N1: OutPos, N2: MaskI);
13857
13858 if (HasPassthru && I == NumElms - 1) {
13859 SDValue EndOfVector =
13860 DAG.getConstant(Val: VecVT.getVectorNumElements() - 1, DL, VT: PositionVT);
13861 SDValue AllLanesSelected =
13862 DAG.getSetCC(DL, VT: MVT::i1, LHS: OutPos, RHS: EndOfVector, Cond: ISD::CondCode::SETUGT);
13863 OutPos = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PositionVT, N1: OutPos, N2: EndOfVector);
13864 OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
13865
13866 // Re-write the last ValI if all lanes were selected. Otherwise,
13867 // overwrite the last write it with the passthru value.
13868 LastWriteVal = DAG.getSelect(DL, VT: ScalarVT, Cond: AllLanesSelected, LHS: ValI,
13869 RHS: LastWriteVal, Flags: SDNodeFlags::Unpredictable);
13870 Chain = DAG.getStore(
13871 Chain, dl: DL, Val: LastWriteVal, Ptr: OutPtr,
13872 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13873 }
13874 }
13875
13876 return DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo, Alignment);
13877}
13878
13879SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
13880 SDLoc DL(Node);
13881 EVT VT = Node->getValueType(ResNo: 0);
13882 SDValue Op = Node->getOperand(Num: 0);
13883 ElementCount EC = Op.getValueType().getVectorElementCount();
13884
13885 bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
13886 auto [Mask, StepVec] = getLegalMaskAndStepVector(Mask: Op, ZeroIsPoison, DL, DAG);
13887
13888 // No legal step vector: split mask in half and recombine results.
13889 // LoNumElts uses the non-poison CTTZ_ELTS so its result is well-defined
13890 // (== LoNumElts when no active lane), allowing the SETNE comparison.
13891 // Result: (ResLo != LoNumElts) ? ResLo : (LoNumElts + ResHi)
13892 if (!StepVec) {
13893 EVT ResVT = Node->getValueType(ResNo: 0);
13894 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Op, DL);
13895 SDValue LoNumElts = DAG.getElementCount(
13896 DL, VT: ResVT, EC: MaskLo.getValueType().getVectorElementCount());
13897 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: MaskLo);
13898 SDValue ResHi = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Operand: MaskHi);
13899 SDValue ResLoNotNumElts = DAG.getSetCC(
13900 DL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ResVT),
13901 LHS: ResLo, RHS: LoNumElts, Cond: ISD::SETNE);
13902 // Per LangRef, ResVT must be wide enough to hold the total element count,
13903 // so the sum cannot wrap as an unsigned add. NSW is not guaranteed since
13904 // the count is only required to fit unsigned.
13905 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: LoNumElts, N2: ResHi,
13906 Flags: SDNodeFlags::NoUnsignedWrap);
13907 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotNumElts, LHS: ResLo, RHS: Sum);
13908 }
13909
13910 EVT StepVecVT = StepVec.getValueType();
13911 EVT StepVT = StepVecVT.getVectorElementType();
13912
13913 // Promote the scalar result type early to avoid redundant zexts.
13914 if (getTypeAction(VT: StepVT.getSimpleVT()) == TypePromoteInteger)
13915 StepVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVT);
13916
13917 SDValue VL = DAG.getElementCount(DL, VT: StepVT, EC);
13918 SDValue SplatVL = DAG.getSplat(VT: StepVecVT, DL, Op: VL);
13919 StepVec = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVecVT, N1: SplatVL, N2: StepVec);
13920 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
13921 SDValue Select = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
13922 SDValue Max = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL,
13923 VT: StepVecVT.getVectorElementType(), Operand: Select);
13924 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVT, N1: VL,
13925 N2: DAG.getZExtOrTrunc(Op: Max, DL, VT: StepVT));
13926
13927 return DAG.getZExtOrTrunc(Op: Sub, DL, VT);
13928}
13929
13930SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
13931 SDLoc DL(N);
13932 SDValue Source = N->getOperand(Num: 0);
13933 SDValue Needle = N->getOperand(Num: 1);
13934 SDValue Mask = N->getOperand(Num: 2);
13935 EVT SourceVT = Source.getValueType();
13936 EVT NeedleVT = Needle.getValueType();
13937 EVT ResVT = N->getValueType(ResNo: 0);
13938 EVT CmpVT =
13939 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SourceVT);
13940
13941 assert(NeedleVT.isFixedLengthVector() && "Needle must be a fixed vector");
13942
13943 SDValue Ret = DAG.getConstant(Val: 0, DL, VT: CmpVT);
13944 EVT NeedleEltVT = NeedleVT.getVectorElementType();
13945 for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
13946 SDValue Splat;
13947 if (NeedleVT == SourceVT) {
13948 // Prefer a shuffle over scalar extracts + splat for fixed vectors.
13949 Splat = DAG.getVectorShuffle(
13950 VT: SourceVT, dl: DL, N1: Needle, N2: DAG.getUNDEF(VT: SourceVT),
13951 Mask: SmallVector<int>(NeedleVT.getVectorNumElements(), I));
13952 } else {
13953 SDValue NeedleElt = DAG.getExtractVectorElt(DL, VT: NeedleEltVT, Vec: Needle, Idx: I);
13954 Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: SourceVT, Operand: NeedleElt);
13955 }
13956 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Source, RHS: Splat, Cond: ISD::SETEQ);
13957 Ret = DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Ret, N2: Cmp);
13958 }
13959
13960 EVT UseVT = ResVT;
13961 // If the result is immediately truncated, only extend to that type (to avoid
13962 // unnecessary sign/zero extends).
13963 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TRUNCATE)
13964 UseVT = N->user_begin()->getValueType(ResNo: 0);
13965
13966 Mask = DAG.getBoolExtOrTrunc(Op: Mask, SL: DL, VT: UseVT, OpVT: Mask.getValueType());
13967 Ret = DAG.getBoolExtOrTrunc(Op: Ret, SL: DL, VT: UseVT, OpVT: Ret.getValueType());
13968
13969 Ret = DAG.getNode(Opcode: ISD::AND, DL, VT: UseVT, N1: Ret, N2: Mask);
13970 if (UseVT != ResVT)
13971 Ret = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ResVT, Operand: Ret);
13972 return Ret;
13973}
13974
13975SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
13976 SelectionDAG &DAG) const {
13977 SDLoc DL(N);
13978 SDValue Acc = N->getOperand(Num: 0);
13979 SDValue MulLHS = N->getOperand(Num: 1);
13980 SDValue MulRHS = N->getOperand(Num: 2);
13981 EVT AccVT = Acc.getValueType();
13982 EVT MulOpVT = MulLHS.getValueType();
13983
13984 EVT ExtMulOpVT =
13985 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccVT.getVectorElementType(),
13986 EC: MulOpVT.getVectorElementCount());
13987
13988 unsigned ExtOpcLHS, ExtOpcRHS;
13989 switch (N->getOpcode()) {
13990 default:
13991 llvm_unreachable("Unexpected opcode");
13992 case ISD::PARTIAL_REDUCE_UMLA:
13993 ExtOpcLHS = ExtOpcRHS = ISD::ZERO_EXTEND;
13994 break;
13995 case ISD::PARTIAL_REDUCE_SMLA:
13996 ExtOpcLHS = ExtOpcRHS = ISD::SIGN_EXTEND;
13997 break;
13998 case ISD::PARTIAL_REDUCE_SUMLA:
13999 ExtOpcLHS = ISD::SIGN_EXTEND;
14000 ExtOpcRHS = ISD::ZERO_EXTEND;
14001 break;
14002 case ISD::PARTIAL_REDUCE_FMLA:
14003 ExtOpcLHS = ExtOpcRHS = ISD::FP_EXTEND;
14004 break;
14005 }
14006
14007 // A wide partial reduction is built from a ladder of narrower ones, a rung
14008 // at a time, each halving the element count and doubling the width.
14009 unsigned Opc = N->getOpcode();
14010 ElementCount MulEC = MulOpVT.getVectorElementCount();
14011 ElementCount AccEC = AccVT.getVectorElementCount();
14012 unsigned CountRatio =
14013 MulEC.hasKnownScalarFactor(RHS: AccEC) ? MulEC.getKnownScalarFactor(RHS: AccEC) : 0;
14014 unsigned WidthRatio =
14015 AccVT.getScalarSizeInBits() / MulOpVT.getScalarSizeInBits();
14016 if (Opc != ISD::PARTIAL_REDUCE_FMLA && CountRatio > 2 && WidthRatio >= 2) {
14017 LLVMContext &Ctx = *DAG.getContext();
14018 EVT ProdVT = MulOpVT.widenIntegerVectorElementType(Context&: Ctx);
14019
14020 // A pure reduction peels one rung and re-enters.
14021 if (llvm::isOneOrOneSplat(V: MulRHS)) {
14022 EVT RungVT = ProdVT.getHalfNumVectorElementsVT(Context&: Ctx);
14023 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc,
14024 N2: DAG.getNode(Opcode: Opc, DL, VT: RungVT,
14025 N1: DAG.getConstant(Val: 0, DL, VT: RungVT), N2: MulLHS,
14026 N3: MulRHS),
14027 N3: DAG.getConstant(Val: 1, DL, VT: RungVT));
14028 }
14029
14030 // A multiply widens the products by one rung, which legalizes back into a
14031 // widening multiply per half, and the ladder re-enters as a plain sum.
14032 SDValue Prod = DAG.getNode(Opcode: ISD::MUL, DL, VT: ProdVT,
14033 N1: DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ProdVT, Operand: MulLHS),
14034 N2: DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ProdVT, Operand: MulRHS));
14035 auto [Lo, Hi] = DAG.SplitVector(N: Prod, DL);
14036 SDValue One = DAG.getConstant(Val: 1, DL, VT: Lo.getValueType());
14037
14038 // The halves meet at the narrowest rung, so the accumulator is added once.
14039 EVT MidVT = Lo.getValueType()
14040 .widenIntegerVectorElementType(Context&: Ctx)
14041 .getHalfNumVectorElementsVT(Context&: Ctx);
14042 if (ElementCount::isKnownLE(LHS: MidVT.getVectorElementCount(), RHS: AccEC))
14043 return DAG.getNode(Opcode: Opc, DL, VT: AccVT,
14044 N1: DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Lo, N3: One), N2: Hi, N3: One);
14045 SDValue Mid =
14046 DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: DAG.getConstant(Val: 0, DL, VT: MidVT), N2: Lo, N3: One);
14047 Mid = DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: Mid, N2: Hi, N3: One);
14048 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Mid, N3: DAG.getConstant(Val: 1, DL, VT: MidVT));
14049 }
14050
14051 if (ExtMulOpVT != MulOpVT) {
14052 MulLHS = DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ExtMulOpVT, Operand: MulLHS);
14053 MulRHS = DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ExtMulOpVT, Operand: MulRHS);
14054 }
14055 SDValue Input = MulLHS;
14056 if (N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA) {
14057 if (!llvm::isOneOrOneSplatFP(V: MulRHS))
14058 Input = DAG.getNode(Opcode: ISD::FMUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
14059 } else if (!llvm::isOneOrOneSplat(V: MulRHS)) {
14060 Input = DAG.getNode(Opcode: ISD::MUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
14061 }
14062
14063 unsigned Stride = AccVT.getVectorMinNumElements();
14064 unsigned ScaleFactor = MulOpVT.getVectorMinNumElements() / Stride;
14065
14066 // Collect all of the subvectors
14067 std::deque<SDValue> Subvectors = {Acc};
14068 for (unsigned I = 0; I < ScaleFactor; I++)
14069 Subvectors.push_back(x: DAG.getExtractSubvector(DL, VT: AccVT, Vec: Input, Idx: I * Stride));
14070
14071 unsigned FlatNode =
14072 N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA ? ISD::FADD : ISD::ADD;
14073
14074 // Flatten the subvector tree
14075 while (Subvectors.size() > 1) {
14076 Subvectors.push_back(
14077 x: DAG.getNode(Opcode: FlatNode, DL, VT: AccVT, Ops: {Subvectors[0], Subvectors[1]}));
14078 Subvectors.pop_front();
14079 Subvectors.pop_front();
14080 }
14081
14082 assert(Subvectors.size() == 1 &&
14083 "There should only be one subvector after tree flattening");
14084
14085 return Subvectors[0];
14086}
14087
14088/// Given a store node \p StoreNode, return true if it is safe to fold that node
14089/// into \p FPNode, which expands to a library call with output pointers.
14090static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
14091 SDNode *FPNode) {
14092 SmallVector<const SDNode *, 8> Worklist;
14093 SmallVector<const SDNode *, 8> DeferredNodes;
14094 SmallPtrSet<const SDNode *, 16> Visited;
14095
14096 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
14097 for (SDValue Op : StoreNode->ops())
14098 if (Op.getNode() != FPNode)
14099 Worklist.push_back(Elt: Op.getNode());
14100
14101 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
14102 while (!Worklist.empty()) {
14103 const SDNode *Node = Worklist.pop_back_val();
14104 auto [_, Inserted] = Visited.insert(Ptr: Node);
14105 if (!Inserted)
14106 continue;
14107
14108 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
14109 return false;
14110
14111 // Reached the FPNode (would result in a cycle).
14112 // OR Reached CALLSEQ_START (would result in nested call sequences).
14113 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
14114 return false;
14115
14116 if (Node->getOpcode() == ISD::CALLSEQ_END) {
14117 // Defer looking into call sequences (so we can check we're outside one).
14118 // We still need to look through these for the predecessor check.
14119 DeferredNodes.push_back(Elt: Node);
14120 continue;
14121 }
14122
14123 for (SDValue Op : Node->ops())
14124 Worklist.push_back(Elt: Op.getNode());
14125 }
14126
14127 // True if we're outside a call sequence and don't have the FPNode as a
14128 // predecessor. No cycles or nested call sequences possible.
14129 return !SDNode::hasPredecessorHelper(N: FPNode, Visited, Worklist&: DeferredNodes,
14130 MaxSteps);
14131}
14132
14133bool TargetLowering::expandMultipleResultFPLibCall(
14134 SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
14135 SmallVectorImpl<SDValue> &Results,
14136 std::optional<unsigned> CallRetResNo) const {
14137 if (LC == RTLIB::UNKNOWN_LIBCALL)
14138 return false;
14139
14140 RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(Call: LC);
14141 if (LibcallImpl == RTLIB::Unsupported)
14142 return false;
14143
14144 LLVMContext &Ctx = *DAG.getContext();
14145 EVT VT = Node->getValueType(ResNo: 0);
14146 unsigned NumResults = Node->getNumValues();
14147
14148 // Find users of the node that store the results (and share input chains). The
14149 // destination pointers can be used instead of creating stack allocations.
14150 SDValue StoresInChain;
14151 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
14152 for (SDNode *User : Node->users()) {
14153 if (!ISD::isNormalStore(N: User))
14154 continue;
14155 auto *ST = cast<StoreSDNode>(Val: User);
14156 SDValue StoreValue = ST->getValue();
14157 unsigned ResNo = StoreValue.getResNo();
14158 // Ensure the store corresponds to an output pointer.
14159 if (CallRetResNo == ResNo)
14160 continue;
14161 // Ensure the store to the default address space and not atomic or volatile.
14162 if (!ST->isSimple() || ST->getAddressSpace() != 0)
14163 continue;
14164 // Ensure all store chains are the same (so they don't alias).
14165 if (StoresInChain && ST->getChain() != StoresInChain)
14166 continue;
14167 // Ensure the store is properly aligned.
14168 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Context&: Ctx);
14169 if (ST->getAlign() <
14170 DAG.getDataLayout().getABITypeAlign(Ty: StoreType->getScalarType()))
14171 continue;
14172 // Avoid:
14173 // 1. Creating cyclic dependencies.
14174 // 2. Expanding the node to a call within a call sequence.
14175 if (!canFoldStoreIntoLibCallOutputPointers(StoreNode: ST, FPNode: Node))
14176 continue;
14177 ResultStores[ResNo] = ST;
14178 StoresInChain = ST->getChain();
14179 }
14180
14181 ArgListTy Args;
14182
14183 // Pass the arguments.
14184 for (const SDValue &Op : Node->op_values()) {
14185 EVT ArgVT = Op.getValueType();
14186 Type *ArgTy = ArgVT.getTypeForEVT(Context&: Ctx);
14187 Args.emplace_back(args: Op, args&: ArgTy);
14188 }
14189
14190 // Pass the output pointers.
14191 SmallVector<SDValue, 2> ResultPtrs(NumResults);
14192 Type *PointerTy = PointerType::getUnqual(C&: Ctx);
14193 for (auto [ResNo, ST] : llvm::enumerate(First&: ResultStores)) {
14194 if (ResNo == CallRetResNo)
14195 continue;
14196 EVT ResVT = Node->getValueType(ResNo);
14197 SDValue ResultPtr = ST ? ST->getBasePtr() : DAG.CreateStackTemporary(VT: ResVT);
14198 ResultPtrs[ResNo] = ResultPtr;
14199 Args.emplace_back(args&: ResultPtr, args&: PointerTy);
14200 }
14201
14202 SDLoc DL(Node);
14203
14204 if (RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(Impl: LibcallImpl)) {
14205 // Pass the vector mask (if required).
14206 EVT MaskVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
14207 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT);
14208 Args.emplace_back(args&: Mask, args: MaskVT.getTypeForEVT(Context&: Ctx));
14209 }
14210
14211 Type *RetType = CallRetResNo.has_value()
14212 ? Node->getValueType(ResNo: *CallRetResNo).getTypeForEVT(Context&: Ctx)
14213 : Type::getVoidTy(C&: Ctx);
14214 SDValue InChain = StoresInChain ? StoresInChain : DAG.getEntryNode();
14215 SDValue Callee =
14216 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
14217 TargetLowering::CallLoweringInfo CLI(DAG);
14218 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
14219 CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetType, Target: Callee, ArgsList: std::move(Args));
14220
14221 auto [Call, CallChain] = LowerCallTo(CLI);
14222
14223 for (auto [ResNo, ResultPtr] : llvm::enumerate(First&: ResultPtrs)) {
14224 if (ResNo == CallRetResNo) {
14225 Results.push_back(Elt: Call);
14226 continue;
14227 }
14228 MachinePointerInfo PtrInfo;
14229 SDValue LoadResult = DAG.getLoad(VT: Node->getValueType(ResNo), dl: DL, Chain: CallChain,
14230 Ptr: ResultPtr, PtrInfo);
14231 SDValue OutChain = LoadResult.getValue(R: 1);
14232
14233 if (StoreSDNode *ST = ResultStores[ResNo]) {
14234 // Replace store with the library call.
14235 DAG.ReplaceAllUsesOfValueWith(From: SDValue(ST, 0), To: OutChain);
14236 PtrInfo = ST->getPointerInfo();
14237 } else {
14238 PtrInfo = MachinePointerInfo::getFixedStack(
14239 MF&: DAG.getMachineFunction(),
14240 FI: cast<FrameIndexSDNode>(Val&: ResultPtr)->getIndex());
14241 }
14242
14243 Results.push_back(Elt: LoadResult);
14244 }
14245
14246 return true;
14247}
14248
14249bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
14250 SDValue &LHS, SDValue &RHS,
14251 SDValue &CC, bool &NeedInvert,
14252 const SDLoc &dl, SDValue &Chain,
14253 bool IsSignaling) const {
14254 MVT OpVT = LHS.getSimpleValueType();
14255 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
14256 NeedInvert = false;
14257 switch (getCondCodeAction(CC: CCCode, VT: OpVT)) {
14258 default:
14259 llvm_unreachable("Unknown condition code action!");
14260 case TargetLowering::Legal:
14261 // Nothing to do.
14262 break;
14263 case TargetLowering::Expand: {
14264 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(Operation: CCCode);
14265 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14266 std::swap(a&: LHS, b&: RHS);
14267 CC = DAG.getCondCode(Cond: InvCC);
14268 return true;
14269 }
14270 // Swapping operands didn't work. Try inverting the condition.
14271 bool NeedSwap = false;
14272 InvCC = getSetCCInverse(Operation: CCCode, Type: OpVT);
14273 if (!isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14274 // If inverting the condition is not enough, try swapping operands
14275 // on top of it.
14276 InvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
14277 NeedSwap = true;
14278 }
14279 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14280 CC = DAG.getCondCode(Cond: InvCC);
14281 NeedInvert = true;
14282 if (NeedSwap)
14283 std::swap(a&: LHS, b&: RHS);
14284 return true;
14285 }
14286
14287 // Special case: expand i1 comparisons using logical operations.
14288 if (OpVT == MVT::i1) {
14289 SDValue Ret;
14290 switch (CCCode) {
14291 default:
14292 llvm_unreachable("Unknown integer setcc!");
14293 case ISD::SETEQ: // X == Y --> ~(X ^ Y)
14294 Ret = DAG.getNOT(DL: dl, Val: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS),
14295 VT: MVT::i1);
14296 break;
14297 case ISD::SETNE: // X != Y --> (X ^ Y)
14298 Ret = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS);
14299 break;
14300 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
14301 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
14302 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: RHS,
14303 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14304 break;
14305 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
14306 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
14307 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: LHS,
14308 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14309 break;
14310 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
14311 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
14312 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: RHS,
14313 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14314 break;
14315 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
14316 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
14317 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: LHS,
14318 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14319 break;
14320 }
14321
14322 LHS = DAG.getZExtOrTrunc(Op: Ret, DL: dl, VT);
14323 RHS = SDValue();
14324 CC = SDValue();
14325 return true;
14326 }
14327
14328 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
14329 unsigned Opc = 0;
14330 switch (CCCode) {
14331 default:
14332 llvm_unreachable("Don't know how to expand this condition!");
14333 case ISD::SETUO:
14334 if (isCondCodeLegal(CC: ISD::SETUNE, VT: OpVT)) {
14335 CC1 = ISD::SETUNE;
14336 CC2 = ISD::SETUNE;
14337 Opc = ISD::OR;
14338 break;
14339 }
14340 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14341 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
14342 NeedInvert = true;
14343 [[fallthrough]];
14344 case ISD::SETO:
14345 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14346 "If SETO is expanded, SETOEQ must be legal!");
14347 CC1 = ISD::SETOEQ;
14348 CC2 = ISD::SETOEQ;
14349 Opc = ISD::AND;
14350 break;
14351 case ISD::SETONE:
14352 case ISD::SETUEQ:
14353 // If the SETUO or SETO CC isn't legal, we might be able to use
14354 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
14355 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
14356 // the operands.
14357 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14358 if (!isCondCodeLegal(CC: CC2, VT: OpVT) && (isCondCodeLegal(CC: ISD::SETOGT, VT: OpVT) ||
14359 isCondCodeLegal(CC: ISD::SETOLT, VT: OpVT))) {
14360 CC1 = ISD::SETOGT;
14361 CC2 = ISD::SETOLT;
14362 Opc = ISD::OR;
14363 NeedInvert = ((unsigned)CCCode & 0x8U);
14364 break;
14365 }
14366 [[fallthrough]];
14367 case ISD::SETOEQ:
14368 case ISD::SETOGT:
14369 case ISD::SETOGE:
14370 case ISD::SETOLT:
14371 case ISD::SETOLE:
14372 case ISD::SETUNE:
14373 case ISD::SETUGT:
14374 case ISD::SETUGE:
14375 case ISD::SETULT:
14376 case ISD::SETULE:
14377 // If we are floating point, assign and break, otherwise fall through.
14378 if (!OpVT.isInteger()) {
14379 // We can use the 4th bit to tell if we are the unordered
14380 // or ordered version of the opcode.
14381 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14382 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
14383 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
14384 break;
14385 }
14386 // Fallthrough if we are unsigned integer.
14387 [[fallthrough]];
14388 case ISD::SETLE:
14389 case ISD::SETGT:
14390 case ISD::SETGE:
14391 case ISD::SETLT:
14392 case ISD::SETNE:
14393 case ISD::SETEQ:
14394 // If all combinations of inverting the condition and swapping operands
14395 // didn't work then we have no means to expand the condition.
14396 llvm_unreachable("Don't know how to expand this condition!");
14397 }
14398
14399 SDValue SetCC1, SetCC2;
14400 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
14401 // If we aren't the ordered or unorder operation,
14402 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
14403 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC1, Chain, IsSignaling);
14404 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC2, Chain, IsSignaling);
14405 } else {
14406 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
14407 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS: LHS, Cond: CC1, Chain, IsSignaling);
14408 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS: RHS, RHS, Cond: CC2, Chain, IsSignaling);
14409 }
14410 if (Chain)
14411 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: SetCC1.getValue(R: 1),
14412 N2: SetCC2.getValue(R: 1));
14413 LHS = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SetCC1, N2: SetCC2);
14414 RHS = SDValue();
14415 CC = SDValue();
14416 return true;
14417 }
14418 }
14419 return false;
14420}
14421
14422SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
14423 SelectionDAG &DAG) const {
14424 EVT VT = Node->getValueType(ResNo: 0);
14425 // Despite its documentation, GetSplitDestVTs will assert if VT cannot be
14426 // split into two equal parts.
14427 if (!VT.isVector() || !VT.getVectorElementCount().isKnownMultipleOf(RHS: 2))
14428 return SDValue();
14429
14430 // Restrict expansion to cases where both parts can be concatenated.
14431 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
14432 if (LoVT != HiVT || !isTypeLegal(VT: LoVT))
14433 return SDValue();
14434
14435 SDLoc DL(Node);
14436 unsigned Opcode = Node->getOpcode();
14437
14438 // Don't expand if the result is likely to be unrolled anyway.
14439 if (!isOperationLegalOrCustomOrPromote(Op: Opcode, VT: LoVT))
14440 return SDValue();
14441
14442 SmallVector<SDValue, 4> LoOps, HiOps;
14443 for (const SDValue &V : Node->op_values()) {
14444 if (!V.getValueType().isVector()) {
14445 // Scalar operands pass through to both halves unchanged.
14446 LoOps.push_back(Elt: V);
14447 HiOps.push_back(Elt: V);
14448 continue;
14449 }
14450 auto [Lo, Hi] = DAG.SplitVector(N: V, DL, LoVT, HiVT);
14451 LoOps.push_back(Elt: Lo);
14452 HiOps.push_back(Elt: Hi);
14453 }
14454
14455 SDValue SplitOpLo = DAG.getNode(Opcode, DL, VT: LoVT, Ops: LoOps, Flags: Node->getFlags());
14456 SDValue SplitOpHi = DAG.getNode(Opcode, DL, VT: HiVT, Ops: HiOps, Flags: Node->getFlags());
14457 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: SplitOpLo, N2: SplitOpHi);
14458}
14459
14460SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
14461 const SDLoc &DL,
14462 EVT InVecVT, SDValue EltNo,
14463 LoadSDNode *OriginalLoad,
14464 SelectionDAG &DAG) const {
14465 assert(OriginalLoad->isSimple());
14466
14467 EVT VecEltVT = InVecVT.getVectorElementType();
14468
14469 // If the vector element type is not a multiple of a byte then we are unable
14470 // to correctly compute an address to load only the extracted element as a
14471 // scalar.
14472 if (!VecEltVT.isByteSized())
14473 return SDValue();
14474
14475 ISD::LoadExtType ExtTy =
14476 ResultVT.bitsGT(VT: VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
14477 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: VecEltVT))
14478 return SDValue();
14479
14480 std::optional<unsigned> ByteOffset;
14481 Align Alignment = OriginalLoad->getAlign();
14482 MachinePointerInfo MPI;
14483 if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo)) {
14484 int Elt = ConstEltNo->getZExtValue();
14485 ByteOffset = VecEltVT.getSizeInBits() * Elt / 8;
14486 MPI = OriginalLoad->getPointerInfo().getWithOffset(O: *ByteOffset);
14487 Alignment = commonAlignment(A: Alignment, Offset: *ByteOffset);
14488 } else {
14489 // Discard the pointer info except the address space because the memory
14490 // operand can't represent this new access since the offset is variable.
14491 MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
14492 Alignment = commonAlignment(A: Alignment, Offset: VecEltVT.getSizeInBits() / 8);
14493 }
14494
14495 if (!shouldReduceLoadWidth(Load: OriginalLoad, ExtTy, NewVT: VecEltVT, ByteOffset))
14496 return SDValue();
14497
14498 unsigned IsFast = 0;
14499 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: VecEltVT,
14500 AddrSpace: OriginalLoad->getAddressSpace(), Alignment,
14501 Flags: OriginalLoad->getMemOperand()->getFlags(), Fast: &IsFast) ||
14502 !IsFast)
14503 return SDValue();
14504
14505 // The original DAG loaded the entire vector from memory, so arithmetic
14506 // within it must be inbounds.
14507 SDValue NewPtr = getInboundsVectorElementPointer(
14508 DAG, VecPtr: OriginalLoad->getBasePtr(), VecVT: InVecVT, Index: EltNo);
14509
14510 // We are replacing a vector load with a scalar load. The new load must have
14511 // identical memory op ordering to the original.
14512 SDValue Load;
14513 if (ResultVT.bitsGT(VT: VecEltVT)) {
14514 // If the result type of vextract is wider than the load, then issue an
14515 // extending load instead.
14516 ISD::LoadExtType ExtType =
14517 isLoadLegal(ValVT: ResultVT, MemVT: VecEltVT, Alignment,
14518 AddrSpace: OriginalLoad->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false)
14519 ? ISD::ZEXTLOAD
14520 : ISD::EXTLOAD;
14521 Load = DAG.getExtLoad(ExtType, dl: DL, VT: ResultVT, Chain: OriginalLoad->getChain(),
14522 Ptr: NewPtr, PtrInfo: MPI, MemVT: VecEltVT, Alignment,
14523 MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14524 Metadata: OriginalLoad->getMMOMetadataForSubAccess());
14525 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14526 } else {
14527 // The result type is narrower or the same width as the vector element
14528 Load = DAG.getLoad(VT: VecEltVT, dl: DL, Chain: OriginalLoad->getChain(), Ptr: NewPtr, PtrInfo: MPI,
14529 Alignment, MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14530 Metadata: OriginalLoad->getMMOMetadataForSubAccess());
14531 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14532 if (ResultVT.bitsLT(VT: VecEltVT))
14533 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: Load);
14534 else
14535 Load = DAG.getBitcast(VT: ResultVT, V: Load);
14536 }
14537
14538 return Load;
14539}
14540
14541// Set type id for call site info and metadata 'call_target'.
14542// We are filtering for:
14543// a) The call-graph-section use case that wants to know about indirect
14544// calls, or
14545// b) We want to annotate indirect calls.
14546void TargetLowering::setTypeIdForCallsiteInfo(
14547 const CallBase *CB, MachineFunction &MF,
14548 MachineFunction::CallSiteInfo &CSInfo) const {
14549 if (CB && CB->isIndirectCall() &&
14550 (MF.getTarget().Options.EmitCallGraphSection ||
14551 MF.getTarget().Options.EmitCallSiteInfo))
14552 CSInfo = MachineFunction::CallSiteInfo(*CB);
14553}
14554