1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/Analysis/VectorUtils.h"
17#include "llvm/CodeGen/Analysis.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/CodeGenCommonISel.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/SDPatternMatch.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/GlobalVariable.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCExpr.h"
33#include "llvm/Support/DivisionByConstantInfo.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Target/TargetMachine.h"
38#include <cctype>
39#include <deque>
40using namespace llvm;
41using namespace llvm::SDPatternMatch;
42
43/// NOTE: The TargetMachine owns TLOF.
44TargetLowering::TargetLowering(const TargetMachine &tm,
45 const TargetSubtargetInfo &STI)
46 : TargetLoweringBase(tm, STI) {}
47
48// Define the virtual destructor out-of-line for build efficiency.
49TargetLowering::~TargetLowering() = default;
50
51const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
52 return nullptr;
53}
54
55bool TargetLowering::isPositionIndependent() const {
56 return getTargetMachine().isPositionIndependent();
57}
58
59/// Check whether a given call node is in tail position within its function. If
60/// so, it sets Chain to the input chain of the tail call.
61bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
62 SDValue &Chain) const {
63 const Function &F = DAG.getMachineFunction().getFunction();
64
65 // First, check if tail calls have been disabled in this function.
66 if (F.getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
67 return false;
68
69 // Conservatively require the attributes of the call to match those of
70 // the return. Ignore following attributes because they don't affect the
71 // call sequence.
72 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
73 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
74 Attribute::DereferenceableOrNull, Attribute::NoAlias,
75 Attribute::NonNull, Attribute::NoUndef,
76 Attribute::Range, Attribute::NoFPClass})
77 CallerAttrs.removeAttribute(Val: Attr);
78
79 if (CallerAttrs.hasAttributes())
80 return false;
81
82 // It's not safe to eliminate the sign / zero extension of the return value.
83 if (CallerAttrs.contains(A: Attribute::ZExt) ||
84 CallerAttrs.contains(A: Attribute::SExt))
85 return false;
86
87 // Check if the only use is a function return node.
88 return isUsedByReturnOnly(Node, Chain);
89}
90
91bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
92 const uint32_t *CallerPreservedMask,
93 const SmallVectorImpl<CCValAssign> &ArgLocs,
94 const SmallVectorImpl<SDValue> &OutVals) const {
95 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
96 const CCValAssign &ArgLoc = ArgLocs[I];
97 if (!ArgLoc.isRegLoc())
98 continue;
99 MCRegister Reg = ArgLoc.getLocReg();
100 // Only look at callee saved registers.
101 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg: Reg))
102 continue;
103 // Check that we pass the value used for the caller.
104 // (We look for a CopyFromReg reading a virtual register that is used
105 // for the function live-in value of register Reg)
106 SDValue Value = OutVals[I];
107 if (Value->getOpcode() == ISD::AssertZext)
108 Value = Value.getOperand(i: 0);
109 if (Value->getOpcode() != ISD::CopyFromReg)
110 return false;
111 Register ArgReg = cast<RegisterSDNode>(Val: Value->getOperand(Num: 1))->getReg();
112 if (MRI.getLiveInPhysReg(VReg: ArgReg) != Reg)
113 return false;
114 }
115 return true;
116}
117
118/// Set CallLoweringInfo attribute flags based on a call instruction
119/// and called function attributes.
120void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
121 unsigned ArgIdx) {
122 IsSExt = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SExt);
123 IsZExt = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::ZExt);
124 IsNoExt = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::NoExt);
125 IsInReg = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::InReg);
126 IsSRet = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::StructRet);
127 IsNest = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::Nest);
128 IsByVal = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::ByVal);
129 IsPreallocated = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::Preallocated);
130 IsInAlloca = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::InAlloca);
131 IsReturned = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::Returned);
132 IsSwiftSelf = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SwiftSelf);
133 IsSwiftAsync = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SwiftAsync);
134 IsSwiftError = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SwiftError);
135 Alignment = Call->getParamStackAlign(ArgNo: ArgIdx);
136 IndirectType = nullptr;
137 assert(IsByVal + IsPreallocated + IsInAlloca + IsSRet <= 1 &&
138 "multiple ABI attributes?");
139 if (IsByVal) {
140 IndirectType = Call->getParamByValType(ArgNo: ArgIdx);
141 if (!Alignment)
142 Alignment = Call->getParamAlign(ArgNo: ArgIdx);
143 }
144 if (IsPreallocated)
145 IndirectType = Call->getParamPreallocatedType(ArgNo: ArgIdx);
146 if (IsInAlloca)
147 IndirectType = Call->getParamInAllocaType(ArgNo: ArgIdx);
148 if (IsSRet)
149 IndirectType = Call->getParamStructRetType(ArgNo: ArgIdx);
150}
151
152/// Generate a libcall taking the given operands as arguments and returning a
153/// result of type RetVT.
154std::pair<SDValue, SDValue>
155TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl,
156 EVT RetVT, ArrayRef<SDValue> Ops,
157 MakeLibCallOptions CallOptions, const SDLoc &dl,
158 SDValue InChain) const {
159 if (LibcallImpl == RTLIB::Unsupported)
160 reportFatalInternalError(reason: "unsupported library call operation");
161
162 if (!InChain)
163 InChain = DAG.getEntryNode();
164
165 TargetLowering::ArgListTy Args;
166 Args.reserve(n: Ops.size());
167
168 ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
169 for (unsigned i = 0; i < Ops.size(); ++i) {
170 SDValue NewOp = Ops[i];
171 Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
172 ? OpsTypeOverrides[i]
173 : NewOp.getValueType().getTypeForEVT(Context&: *DAG.getContext());
174 TargetLowering::ArgListEntry Entry(NewOp, Ty);
175 if (CallOptions.IsSoften)
176 Entry.OrigTy =
177 CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(Context&: *DAG.getContext());
178
179 Entry.IsSExt =
180 shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned: CallOptions.IsSigned);
181 Entry.IsZExt = !Entry.IsSExt;
182
183 if (CallOptions.IsSoften &&
184 !shouldExtendTypeInLibCall(Type: CallOptions.OpsVTBeforeSoften[i])) {
185 Entry.IsSExt = Entry.IsZExt = false;
186 }
187 Args.push_back(x: Entry);
188 }
189
190 SDValue Callee =
191 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
192
193 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
194 Type *OrigRetTy = RetTy;
195 TargetLowering::CallLoweringInfo CLI(DAG);
196 bool signExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: CallOptions.IsSigned);
197 bool zeroExtend = !signExtend;
198
199 if (CallOptions.IsSoften) {
200 OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(Context&: *DAG.getContext());
201 if (!shouldExtendTypeInLibCall(Type: CallOptions.RetVTBeforeSoften))
202 signExtend = zeroExtend = false;
203 }
204
205 CLI.setDebugLoc(dl)
206 .setChain(InChain)
207 .setLibCallee(CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetTy, OrigResultType: OrigRetTy,
208 Target: Callee, ArgsList: std::move(Args))
209 .setNoReturn(CallOptions.DoesNotReturn)
210 .setDiscardResult(!CallOptions.IsReturnValueUsed)
211 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
212 .setSExtResult(signExtend)
213 .setZExtResult(zeroExtend);
214 return LowerCallTo(CLI);
215}
216
217bool TargetLowering::findOptimalMemOpLowering(
218 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
219 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
220 const AttributeList &FuncAttributes, EVT *LargestVT) const {
221 EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);
222
223 if (VT == MVT::Other) {
224 // Use the largest integer type whose alignment constraints are satisfied.
225 VT = MVT::LAST_INTEGER_VALUETYPE;
226 if (Op.isFixedDstAlign()) {
227 bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
228 while (VT != MVT::i8) {
229 unsigned VTSize = VT.getSizeInBits() / 8;
230 bool DstOk =
231 Op.getDstAlign() >= VTSize ||
232 allowsMisalignedMemoryAccesses(VT, AddrSpace: DstAS, Alignment: Op.getDstAlign());
233 bool SrcOk =
234 !LoadsFromSrc || Op.getSrcAlign() >= VTSize ||
235 allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcAS, Alignment: Op.getSrcAlign());
236 if (DstOk && SrcOk)
237 break;
238 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
239 }
240 }
241 assert(VT.isInteger());
242
243 // Find the largest legal integer type.
244 MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
245 while (!isTypeLegal(VT: LVT))
246 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
247 assert(LVT.isInteger());
248
249 // If the type we've chosen is larger than the largest legal integer type
250 // then use the largest legal type.
251 if (VT.bitsGT(VT: LVT))
252 VT = LVT;
253 }
254
255 unsigned NumMemOps = 0;
256 uint64_t Size = Op.size();
257 while (Size) {
258 unsigned VTSize = VT.getSizeInBits() / 8;
259 while (VTSize > Size) {
260 // For now, only use non-vector load / store's for the left-over pieces.
261 EVT NewVT = VT;
262 unsigned NewVTSize;
263
264 bool Found = false;
265 if (VT.isVector() || VT.isFloatingPoint()) {
266 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
267 if (isOperationLegalOrCustom(Op: ISD::STORE, VT: NewVT) &&
268 isSafeMemOpType(NewVT.getSimpleVT()))
269 Found = true;
270 else if (NewVT == MVT::i64 &&
271 isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::f64) &&
272 isSafeMemOpType(MVT::f64)) {
273 // i64 is usually not legal on 32-bit targets, but f64 may be.
274 NewVT = MVT::f64;
275 Found = true;
276 }
277 }
278
279 if (!Found) {
280 do {
281 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
282 if (NewVT == MVT::i8)
283 break;
284 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
285 }
286 NewVTSize = NewVT.getSizeInBits() / 8;
287
288 // If the new VT cannot cover all of the remaining bits, then consider
289 // issuing a (or a pair of) unaligned and overlapping load / store.
290 unsigned Fast;
291 if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
292 allowsMisalignedMemoryAccesses(
293 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
294 Flags: MachineMemOperand::MONone, &Fast) &&
295 Fast)
296 VTSize = Size;
297 else {
298 VT = NewVT;
299 VTSize = NewVTSize;
300 }
301 }
302
303 if (++NumMemOps > Limit)
304 return false;
305
306 MemOps.push_back(x: VT);
307 Size -= VTSize;
308 }
309
310 return true;
311}
312
313/// Soften the operands of a comparison. This code is shared among BR_CC,
314/// SELECT_CC, and SETCC handlers.
315void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
316 SDValue &NewLHS, SDValue &NewRHS,
317 ISD::CondCode &CCCode,
318 const SDLoc &dl, const SDValue OldLHS,
319 const SDValue OldRHS) const {
320 SDValue Chain;
321 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: dl, OldLHS,
322 OldRHS, Chain);
323}
324
325/// Select the libcall and the condition code to test its result against 0 for
326/// an ordered floating-point compare. \p BoolLC is the boolean helper (result
327/// is 0/1). \p TriStateLC is the per-predicate three-way helper and \p
328/// GenericLC the generic single-symbol three-way helper (both return -1/0/1,
329/// tested against 0 with \p TriStateCC). The boolean form is preferred, then
330/// the per-predicate three-way, then the generic three-way.
331static std::pair<RTLIB::Libcall, ISD::CondCode>
332selectFPCmpLibcall(const LibcallLoweringInfo &Libcalls, RTLIB::Libcall BoolLC,
333 RTLIB::Libcall TriStateLC, RTLIB::Libcall GenericLC,
334 ISD::CondCode TriStateCC) {
335 if (Libcalls.getLibcallImpl(Call: BoolLC) != RTLIB::Unsupported)
336 return {BoolLC, ISD::SETNE};
337 if (Libcalls.getLibcallImpl(Call: TriStateLC) != RTLIB::Unsupported)
338 return {TriStateLC, TriStateCC};
339 return {GenericLC, TriStateCC};
340}
341
342void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
343 SDValue &NewLHS, SDValue &NewRHS,
344 ISD::CondCode &CCCode,
345 const SDLoc &dl, const SDValue OldLHS,
346 const SDValue OldRHS,
347 SDValue &Chain,
348 bool IsSignaling) const {
349 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
350 // not supporting it. We can update this code when libgcc provides such
351 // functions.
352
353 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
354 && "Unsupported setcc type!");
355
356 // Expand into one or more soft-fp libcall(s).
357 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
358 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
359 bool ShouldInvertCC = false;
360
361 // Expand a compare libcall family name (e.g. OEQ, FCMP3_PRED_OEQ) to the
362 // RTLIB::Libcall for VT.
363#define FP_CMP_LIBCALL(BASE) \
364 RTLIB::getFPLibCall(VT, RTLIB::BASE##_F32, RTLIB::BASE##_F64, \
365 RTLIB::UNKNOWN_LIBCALL, RTLIB::BASE##_F128, \
366 RTLIB::BASE##_PPCF128)
367
368 switch (CCCode) {
369 case ISD::SETEQ:
370 case ISD::SETOEQ:
371 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
372 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
373 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
374 break;
375 case ISD::SETNE:
376 case ISD::SETUNE:
377 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
378 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(UNE), FP_CMP_LIBCALL(FCMP3_PRED_UNE),
379 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETNE);
380 // Some ABIs (e.g. AEABI) provide neither a not-equal nor a three-way
381 // compare; obtain not-equal (UNE = !OEQ) by inverting ordered-equal.
382 if (DAG.getLibcalls().getLibcallImpl(Call: LC1) == RTLIB::Unsupported) {
383 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
384 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ),
385 FP_CMP_LIBCALL(FCMP3_PRED_OEQ), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
386 ShouldInvertCC = true;
387 }
388 break;
389 case ISD::SETGE:
390 case ISD::SETOGE:
391 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
392 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE), FP_CMP_LIBCALL(FCMP3_PRED_OGE),
393 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
394 break;
395 case ISD::SETLT:
396 case ISD::SETOLT:
397 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
398 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT), FP_CMP_LIBCALL(FCMP3_PRED_OLT),
399 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
400 break;
401 case ISD::SETLE:
402 case ISD::SETOLE:
403 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
404 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE), FP_CMP_LIBCALL(FCMP3_PRED_OLE),
405 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
406 break;
407 case ISD::SETGT:
408 case ISD::SETOGT:
409 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
410 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT), FP_CMP_LIBCALL(FCMP3_PRED_OGT),
411 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
412 break;
413 case ISD::SETO:
414 ShouldInvertCC = true;
415 [[fallthrough]];
416 case ISD::SETUO:
417 // Unordered is a boolean everywhere (__unordXf2 returns 0/1).
418 LC1 = FP_CMP_LIBCALL(UO);
419 CC1 = ISD::SETNE;
420 break;
421 case ISD::SETONE:
422 // SETONE = O && UNE
423 ShouldInvertCC = true;
424 [[fallthrough]];
425 case ISD::SETUEQ:
426 LC1 = FP_CMP_LIBCALL(UO);
427 CC1 = ISD::SETNE;
428 std::tie(args&: LC2, args&: CC2) = selectFPCmpLibcall(
429 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OEQ), FP_CMP_LIBCALL(FCMP3_PRED_OEQ),
430 FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETEQ);
431 break;
432 default:
433 // Invert CC for unordered comparisons, handled by the ordered inverse.
434 ShouldInvertCC = true;
435 switch (CCCode) {
436 case ISD::SETULT:
437 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
438 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGE),
439 FP_CMP_LIBCALL(FCMP3_PRED_OGE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGE);
440 break;
441 case ISD::SETULE:
442 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
443 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OGT),
444 FP_CMP_LIBCALL(FCMP3_PRED_OGT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETGT);
445 break;
446 case ISD::SETUGT:
447 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
448 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLE),
449 FP_CMP_LIBCALL(FCMP3_PRED_OLE), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLE);
450 break;
451 case ISD::SETUGE:
452 std::tie(args&: LC1, args&: CC1) = selectFPCmpLibcall(
453 Libcalls: DAG.getLibcalls(), FP_CMP_LIBCALL(OLT),
454 FP_CMP_LIBCALL(FCMP3_PRED_OLT), FP_CMP_LIBCALL(FCMP3), TriStateCC: ISD::SETLT);
455 break;
456 default:
457 llvm_unreachable("Do not know how to soften this setcc!");
458 }
459 }
460
461#undef FP_CMP_LIBCALL
462
463 // Use the target specific return value for comparison lib calls.
464 EVT RetVT = getCmpLibcallReturnType();
465 SDValue Ops[2] = {NewLHS, NewRHS};
466 TargetLowering::MakeLibCallOptions CallOptions;
467 EVT OpsVT[2] = { OldLHS.getValueType(),
468 OldRHS.getValueType() };
469 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
470 auto Call = makeLibCall(DAG, LC: LC1, RetVT, Ops, CallOptions, dl, Chain);
471 NewLHS = Call.first;
472 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
473
474 if (DAG.getLibcalls().getLibcallImpl(Call: LC1) == RTLIB::Unsupported) {
475 reportFatalUsageError(
476 reason: "no libcall available to soften floating-point compare");
477 }
478
479 CCCode = CC1;
480 if (ShouldInvertCC) {
481 assert(RetVT.isInteger());
482 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
483 }
484
485 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
486 // Update Chain.
487 Chain = Call.second;
488 } else {
489 if (DAG.getLibcalls().getLibcallImpl(Call: LC2) == RTLIB::Unsupported) {
490 reportFatalUsageError(
491 reason: "no libcall available to soften floating-point compare");
492 }
493
494 assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
495 "unordered call should be simple boolean");
496
497 EVT SetCCVT =
498 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: RetVT);
499 if (getBooleanContents(Type: RetVT) == ZeroOrOneBooleanContent) {
500 NewLHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: RetVT, N1: Call.first,
501 N2: DAG.getValueType(MVT::i1));
502 }
503
504 SDValue Tmp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NewLHS, RHS: NewRHS, Cond: CCCode);
505 auto Call2 = makeLibCall(DAG, LC: LC2, RetVT, Ops, CallOptions, dl, Chain);
506 CCCode = CC2;
507 if (ShouldInvertCC)
508 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
509 NewLHS = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Call2.first, RHS: NewRHS, Cond: CCCode);
510 if (Chain)
511 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Call.second,
512 N2: Call2.second);
513 NewLHS = DAG.getNode(Opcode: ShouldInvertCC ? ISD::AND : ISD::OR, DL: dl,
514 VT: Tmp.getValueType(), N1: Tmp, N2: NewLHS);
515 NewRHS = SDValue();
516 }
517}
518
519/// Return the entry encoding for a jump table in the current function. The
520/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
521unsigned TargetLowering::getJumpTableEncoding() const {
522 // In non-pic modes, just use the address of a block.
523 if (!isPositionIndependent())
524 return MachineJumpTableInfo::EK_BlockAddress;
525
526 // Otherwise, use a label difference.
527 return MachineJumpTableInfo::EK_LabelDifference32;
528}
529
530SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
531 SelectionDAG &DAG) const {
532 return Table;
533}
534
535/// This returns the relocation base for the given PIC jumptable, the same as
536/// getPICJumpTableRelocBase, but as an MCExpr.
537const MCExpr *
538TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
539 unsigned JTI,MCContext &Ctx) const{
540 // The normal PIC reloc base is the label at the start of the jump table.
541 return MCSymbolRefExpr::create(Symbol: MF->getJTISymbol(JTI, Ctx), Ctx);
542}
543
544SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
545 SDValue Addr, int JTI,
546 SelectionDAG &DAG) const {
547 SDValue Chain = Value;
548 // Jump table debug info is only needed if CodeView is enabled.
549 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF()) {
550 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
551 }
552 return DAG.getNode(Opcode: ISD::BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
553}
554
555bool
556TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
557 const TargetMachine &TM = getTargetMachine();
558 const GlobalValue *GV = GA->getGlobal();
559
560 // If the address is not even local to this DSO we will have to load it from
561 // a got and then add the offset.
562 if (!TM.shouldAssumeDSOLocal(GV))
563 return false;
564
565 // If the code is position independent we will have to add a base register.
566 if (isPositionIndependent())
567 return false;
568
569 // Otherwise we can do it.
570 return true;
571}
572
573//===----------------------------------------------------------------------===//
574// Optimization Methods
575//===----------------------------------------------------------------------===//
576
577/// If the specified instruction has a constant integer operand and there are
578/// bits set in that constant that are not demanded, then clear those bits and
579/// return true.
580bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
581 const APInt &DemandedBits,
582 const APInt &DemandedElts,
583 TargetLoweringOpt &TLO) const {
584 SDLoc DL(Op);
585 unsigned Opcode = Op.getOpcode();
586
587 // Early-out if we've ended up calling an undemanded node, leave this to
588 // constant folding.
589 if (DemandedBits.isZero() || DemandedElts.isZero())
590 return false;
591
592 // Do target-specific constant optimization.
593 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
594 return TLO.New.getNode();
595
596 // FIXME: ISD::SELECT, ISD::SELECT_CC
597 switch (Opcode) {
598 default:
599 break;
600 case ISD::XOR:
601 case ISD::AND:
602 case ISD::OR: {
603 auto *Op1C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
604 if (!Op1C || Op1C->isOpaque())
605 return false;
606
607 // If this is a 'not' op, don't touch it because that's a canonical form.
608 const APInt &C = Op1C->getAPIntValue();
609 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(RHS: C))
610 return false;
611
612 if (!C.isSubsetOf(RHS: DemandedBits)) {
613 EVT VT = Op.getValueType();
614 SDValue NewC = TLO.DAG.getConstant(Val: DemandedBits & C, DL, VT);
615 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 0), N2: NewC,
616 Flags: Op->getFlags());
617 return TLO.CombineTo(O: Op, N: NewOp);
618 }
619
620 break;
621 }
622 }
623
624 return false;
625}
626
627bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
628 const APInt &DemandedBits,
629 TargetLoweringOpt &TLO) const {
630 EVT VT = Op.getValueType();
631 APInt DemandedElts = VT.isVector()
632 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
633 : APInt(1, 1);
634 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
635}
636
637/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
638/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
639/// but it could be generalized for targets with other types of implicit
640/// widening casts.
641bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
642 const APInt &DemandedBits,
643 TargetLoweringOpt &TLO) const {
644 assert(Op.getNumOperands() == 2 &&
645 "ShrinkDemandedOp only supports binary operators!");
646 assert(Op.getNode()->getNumValues() == 1 &&
647 "ShrinkDemandedOp only supports nodes with one result!");
648
649 EVT VT = Op.getValueType();
650 SelectionDAG &DAG = TLO.DAG;
651 SDLoc dl(Op);
652
653 // Early return, as this function cannot handle vector types.
654 if (VT.isVector())
655 return false;
656
657 assert(Op.getOperand(0).getValueType().getScalarSizeInBits() == BitWidth &&
658 Op.getOperand(1).getValueType().getScalarSizeInBits() == BitWidth &&
659 "ShrinkDemandedOp only supports operands that have the same size!");
660
661 // Don't do this if the node has another user, which may require the
662 // full value.
663 if (!Op.getNode()->hasOneUse())
664 return false;
665
666 // Search for the smallest integer type with free casts to and from
667 // Op's type. For expedience, just check power-of-2 integer types.
668 unsigned DemandedSize = DemandedBits.getActiveBits();
669 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
670 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
671 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SmallVTBits);
672 if (isTruncateFree(Val: Op, VT2: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT)) {
673 // We found a type with free casts.
674
675 // If the operation has the 'disjoint' flag, then the
676 // operands on the new node are also disjoint.
677 SDNodeFlags Flags(Op->getFlags().hasDisjoint() ? SDNodeFlags::Disjoint
678 : SDNodeFlags::None);
679 unsigned Opcode = Op.getOpcode();
680 if (Opcode == ISD::PTRADD) {
681 // It isn't a ptradd anymore if it doesn't operate on the entire
682 // pointer.
683 Opcode = ISD::ADD;
684 }
685 SDValue X = DAG.getNode(
686 Opcode, DL: dl, VT: SmallVT,
687 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
688 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 1)), Flags);
689 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
690 SDValue Z = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: X);
691 return TLO.CombineTo(O: Op, N: Z);
692 }
693 }
694 return false;
695}
696
697bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
698 DAGCombinerInfo &DCI) const {
699 SelectionDAG &DAG = DCI.DAG;
700 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
701 !DCI.isBeforeLegalizeOps());
702 KnownBits Known;
703
704 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
705 if (Simplified) {
706 DCI.AddToWorklist(N: Op.getNode());
707 DCI.CommitTargetLoweringOpt(TLO);
708 }
709 return Simplified;
710}
711
712bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
713 const APInt &DemandedElts,
714 DAGCombinerInfo &DCI) const {
715 SelectionDAG &DAG = DCI.DAG;
716 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
717 !DCI.isBeforeLegalizeOps());
718 KnownBits Known;
719
720 bool Simplified =
721 SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
722 if (Simplified) {
723 DCI.AddToWorklist(N: Op.getNode());
724 DCI.CommitTargetLoweringOpt(TLO);
725 }
726 return Simplified;
727}
728
729bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
730 KnownBits &Known,
731 TargetLoweringOpt &TLO,
732 unsigned Depth,
733 bool AssumeSingleUse) const {
734 EVT VT = Op.getValueType();
735
736 // Since the number of lanes in a scalable vector is unknown at compile time,
737 // we track one bit which is implicitly broadcast to all lanes. This means
738 // that all lanes in a scalable vector are considered demanded.
739 APInt DemandedElts = VT.isFixedLengthVector()
740 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
741 : APInt(1, 1);
742 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
743 AssumeSingleUse);
744}
745
746// TODO: Under what circumstances can we create nodes? Constant folding?
747SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
748 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
749 SelectionDAG &DAG, unsigned Depth) const {
750 EVT VT = Op.getValueType();
751
752 // Limit search depth.
753 if (Depth >= SelectionDAG::MaxRecursionDepth)
754 return SDValue();
755
756 // Ignore UNDEFs.
757 if (Op.isUndef())
758 return SDValue();
759
760 // Not demanding any bits/elts from Op.
761 if (DemandedBits == 0 || DemandedElts == 0)
762 return DAG.getUNDEF(VT);
763
764 bool IsLE = DAG.getDataLayout().isLittleEndian();
765 unsigned NumElts = DemandedElts.getBitWidth();
766 unsigned BitWidth = DemandedBits.getBitWidth();
767 KnownBits LHSKnown, RHSKnown;
768 switch (Op.getOpcode()) {
769 case ISD::BITCAST: {
770 if (VT.isScalableVector())
771 return SDValue();
772
773 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
774 EVT SrcVT = Src.getValueType();
775 EVT DstVT = Op.getValueType();
776 if (SrcVT == DstVT)
777 return Src;
778
779 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
780 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
781 if (NumSrcEltBits == NumDstEltBits)
782 if (SDValue V = SimplifyMultipleUseDemandedBits(
783 Op: Src, DemandedBits, DemandedElts, DAG, Depth: Depth + 1))
784 return DAG.getBitcast(VT: DstVT, V);
785
786 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
787 unsigned Scale = NumDstEltBits / NumSrcEltBits;
788 unsigned NumSrcElts = SrcVT.getVectorNumElements();
789 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
790 for (unsigned i = 0; i != Scale; ++i) {
791 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
792 unsigned BitOffset = EltOffset * NumSrcEltBits;
793 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
794 }
795 // Recursive calls below may turn not demanded elements into poison, so we
796 // need to demand all smaller source elements that maps to a demanded
797 // destination element.
798 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
799
800 if (SDValue V = SimplifyMultipleUseDemandedBits(
801 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
802 return DAG.getBitcast(VT: DstVT, V);
803 }
804
805 // TODO - bigendian once we have test coverage.
806 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
807 unsigned Scale = NumSrcEltBits / NumDstEltBits;
808 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
809 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
810 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
811 for (unsigned i = 0; i != NumElts; ++i)
812 if (DemandedElts[i]) {
813 unsigned Offset = (i % Scale) * NumDstEltBits;
814 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
815 DemandedSrcElts.setBit(i / Scale);
816 }
817
818 if (SDValue V = SimplifyMultipleUseDemandedBits(
819 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
820 return DAG.getBitcast(VT: DstVT, V);
821 }
822
823 break;
824 }
825 case ISD::AND: {
826 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
827 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
828
829 // If all of the demanded bits are known 1 on one side, return the other.
830 // These bits cannot contribute to the result of the 'and' in this
831 // context.
832 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero | RHSKnown.One))
833 return Op.getOperand(i: 0);
834 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.One))
835 return Op.getOperand(i: 1);
836 break;
837 }
838 case ISD::OR: {
839 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
840 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
841
842 // If all of the demanded bits are known zero on one side, return the
843 // other. These bits cannot contribute to the result of the 'or' in this
844 // context.
845 if (DemandedBits.isSubsetOf(RHS: LHSKnown.One | RHSKnown.Zero))
846 return Op.getOperand(i: 0);
847 if (DemandedBits.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
848 return Op.getOperand(i: 1);
849 break;
850 }
851 case ISD::XOR: {
852 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
853 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
854
855 // If all of the demanded bits are known zero on one side, return the
856 // other.
857 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero))
858 return Op.getOperand(i: 0);
859 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero))
860 return Op.getOperand(i: 1);
861 break;
862 }
863 case ISD::ADD:
864 case ISD::MUL:
865 case ISD::SMIN:
866 case ISD::SMAX:
867 case ISD::UMIN:
868 case ISD::UMAX: {
869 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 1),
870 DemandedElts, OperandNo: 1, Depth: Depth + 1))
871 return Op.getOperand(i: 0);
872
873 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 0),
874 DemandedElts, OperandNo: 0, Depth: Depth + 1))
875 return Op.getOperand(i: 1);
876 break;
877 }
878 case ISD::SHL: {
879 // If we are only demanding sign bits then we can use the shift source
880 // directly.
881 if (std::optional<unsigned> MaxSA =
882 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
883 SDValue Op0 = Op.getOperand(i: 0);
884 unsigned ShAmt = *MaxSA;
885 unsigned NumSignBits =
886 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
887 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
888 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
889 return Op0;
890 }
891 break;
892 }
893 case ISD::SRL: {
894 // If we are only demanding sign bits then we can use the shift source
895 // directly.
896 if (std::optional<unsigned> MaxSA =
897 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
898 SDValue Op0 = Op.getOperand(i: 0);
899 unsigned ShAmt = *MaxSA;
900 // Must already be signbits in DemandedBits bounds, and can't demand any
901 // shifted in zeroes.
902 if (DemandedBits.countl_zero() >= ShAmt) {
903 unsigned NumSignBits =
904 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
905 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
906 return Op0;
907 }
908 }
909 break;
910 }
911 case ISD::SETCC: {
912 SDValue Op0 = Op.getOperand(i: 0);
913 SDValue Op1 = Op.getOperand(i: 1);
914 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
915 // If (1) we only need the sign-bit, (2) the setcc operands are the same
916 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
917 // -1, we may be able to bypass the setcc.
918 if (DemandedBits.isSignMask() &&
919 Op0.getScalarValueSizeInBits() == BitWidth &&
920 getBooleanContents(Type: Op0.getValueType()) ==
921 BooleanContent::ZeroOrNegativeOneBooleanContent) {
922 // If we're testing X < 0, then this compare isn't needed - just use X!
923 // FIXME: We're limiting to integer types here, but this should also work
924 // if we don't care about FP signed-zero. The use of SETLT with FP means
925 // that we don't care about NaNs.
926 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
927 (isNullConstant(V: Op1) || ISD::isBuildVectorAllZeros(N: Op1.getNode())))
928 return Op0;
929 }
930 break;
931 }
932 case ISD::SIGN_EXTEND_INREG: {
933 // If none of the extended bits are demanded, eliminate the sextinreg.
934 SDValue Op0 = Op.getOperand(i: 0);
935 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
936 unsigned ExBits = ExVT.getScalarSizeInBits();
937 if (DemandedBits.getActiveBits() <= ExBits &&
938 shouldRemoveRedundantExtend(Op))
939 return Op0;
940 // If the input is already sign extended, just drop the extension.
941 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
942 if (NumSignBits >= (BitWidth - ExBits + 1))
943 return Op0;
944 break;
945 }
946 case ISD::ANY_EXTEND_VECTOR_INREG:
947 case ISD::SIGN_EXTEND_VECTOR_INREG:
948 case ISD::ZERO_EXTEND_VECTOR_INREG: {
949 if (VT.isScalableVector())
950 return SDValue();
951
952 // If we only want the lowest element and none of extended bits, then we can
953 // return the bitcasted source vector.
954 SDValue Src = Op.getOperand(i: 0);
955 EVT SrcVT = Src.getValueType();
956 EVT DstVT = Op.getValueType();
957 if (IsLE && DemandedElts == 1 &&
958 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
959 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
960 return DAG.getBitcast(VT: DstVT, V: Src);
961 }
962 break;
963 }
964 case ISD::INSERT_VECTOR_ELT: {
965 if (VT.isScalableVector())
966 return SDValue();
967
968 // If we don't demand the inserted element, return the base vector.
969 SDValue Vec = Op.getOperand(i: 0);
970 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
971 EVT VecVT = Vec.getValueType();
972 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements()) &&
973 !DemandedElts[CIdx->getZExtValue()])
974 return Vec;
975 break;
976 }
977 case ISD::INSERT_SUBVECTOR: {
978 if (VT.isScalableVector())
979 return SDValue();
980
981 SDValue Vec = Op.getOperand(i: 0);
982 SDValue Sub = Op.getOperand(i: 1);
983 uint64_t Idx = Op.getConstantOperandVal(i: 2);
984 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
985 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
986 // If we don't demand the inserted subvector, return the base vector.
987 if (DemandedSubElts == 0)
988 return Vec;
989 break;
990 }
991 case ISD::VECTOR_SHUFFLE: {
992 assert(!VT.isScalableVector());
993 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
994
995 // If all the demanded elts are from one operand and are inline,
996 // then we can use the operand directly.
997 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
998 for (unsigned i = 0; i != NumElts; ++i) {
999 int M = ShuffleMask[i];
1000 if (M < 0 || !DemandedElts[i])
1001 continue;
1002 AllUndef = false;
1003 IdentityLHS &= (M == (int)i);
1004 IdentityRHS &= ((M - NumElts) == i);
1005 }
1006
1007 if (AllUndef)
1008 return DAG.getUNDEF(VT: Op.getValueType());
1009 if (IdentityLHS)
1010 return Op.getOperand(i: 0);
1011 if (IdentityRHS)
1012 return Op.getOperand(i: 1);
1013 break;
1014 }
1015 default:
1016 // TODO: Probably okay to remove after audit; here to reduce change size
1017 // in initial enablement patch for scalable vectors
1018 if (VT.isScalableVector())
1019 return SDValue();
1020
1021 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
1022 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
1023 Op, DemandedBits, DemandedElts, DAG, Depth))
1024 return V;
1025 break;
1026 }
1027 return SDValue();
1028}
1029
1030SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
1031 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
1032 unsigned Depth) const {
1033 EVT VT = Op.getValueType();
1034 // Since the number of lanes in a scalable vector is unknown at compile time,
1035 // we track one bit which is implicitly broadcast to all lanes. This means
1036 // that all lanes in a scalable vector are considered demanded.
1037 APInt DemandedElts = VT.isFixedLengthVector()
1038 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1039 : APInt(1, 1);
1040 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1041 Depth);
1042}
1043
1044SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
1045 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
1046 unsigned Depth) const {
1047 APInt DemandedBits = APInt::getAllOnes(numBits: Op.getScalarValueSizeInBits());
1048 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1049 Depth);
1050}
1051
1052// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
1053// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
1054static SDValue combineShiftToAVG(SDValue Op,
1055 TargetLowering::TargetLoweringOpt &TLO,
1056 const TargetLowering &TLI,
1057 const APInt &DemandedBits,
1058 const APInt &DemandedElts, unsigned Depth) {
1059 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1060 "SRL or SRA node is required here!");
1061 // Is the right shift using an immediate value of 1?
1062 ConstantSDNode *N1C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
1063 if (!N1C || !N1C->isOne())
1064 return SDValue();
1065
1066 // We are looking for an avgfloor
1067 // add(ext, ext)
1068 // or one of these as a avgceil
1069 // add(add(ext, ext), 1)
1070 // add(add(ext, 1), ext)
1071 // add(ext, add(ext, 1))
1072 SDValue Add = Op.getOperand(i: 0);
1073 if (Add.getOpcode() != ISD::ADD)
1074 return SDValue();
1075
1076 SDValue ExtOpA = Add.getOperand(i: 0);
1077 SDValue ExtOpB = Add.getOperand(i: 1);
1078 SDValue Add2;
1079 auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
1080 ConstantSDNode *ConstOp;
1081 if ((ConstOp = isConstOrConstSplat(N: Op2, DemandedElts)) &&
1082 ConstOp->isOne()) {
1083 ExtOpA = Op1;
1084 ExtOpB = Op3;
1085 Add2 = A;
1086 return true;
1087 }
1088 if ((ConstOp = isConstOrConstSplat(N: Op3, DemandedElts)) &&
1089 ConstOp->isOne()) {
1090 ExtOpA = Op1;
1091 ExtOpB = Op2;
1092 Add2 = A;
1093 return true;
1094 }
1095 return false;
1096 };
1097 bool IsCeil =
1098 (ExtOpA.getOpcode() == ISD::ADD &&
1099 MatchOperands(ExtOpA.getOperand(i: 0), ExtOpA.getOperand(i: 1), ExtOpB, ExtOpA)) ||
1100 (ExtOpB.getOpcode() == ISD::ADD &&
1101 MatchOperands(ExtOpB.getOperand(i: 0), ExtOpB.getOperand(i: 1), ExtOpA, ExtOpB));
1102
1103 // If the shift is signed (sra):
1104 // - Needs >= 2 sign bit for both operands.
1105 // - Needs >= 2 zero bits.
1106 // If the shift is unsigned (srl):
1107 // - Needs >= 1 zero bit for both operands.
1108 // - Needs 1 demanded bit zero and >= 2 sign bits.
1109 SelectionDAG &DAG = TLO.DAG;
1110 unsigned ShiftOpc = Op.getOpcode();
1111 bool IsSigned = false;
1112 unsigned KnownBits;
1113 unsigned NumSignedA = DAG.ComputeNumSignBits(Op: ExtOpA, DemandedElts, Depth);
1114 unsigned NumSignedB = DAG.ComputeNumSignBits(Op: ExtOpB, DemandedElts, Depth);
1115 unsigned NumSigned = std::min(a: NumSignedA, b: NumSignedB) - 1;
1116 unsigned NumZeroA =
1117 DAG.computeKnownBits(Op: ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
1118 unsigned NumZeroB =
1119 DAG.computeKnownBits(Op: ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
1120 unsigned NumZero = std::min(a: NumZeroA, b: NumZeroB);
1121
1122 switch (ShiftOpc) {
1123 default:
1124 llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
1125 case ISD::SRA: {
1126 if (NumZero >= 2 && NumSigned < NumZero) {
1127 IsSigned = false;
1128 KnownBits = NumZero;
1129 break;
1130 }
1131 if (NumSigned >= 1) {
1132 IsSigned = true;
1133 KnownBits = NumSigned;
1134 break;
1135 }
1136 return SDValue();
1137 }
1138 case ISD::SRL: {
1139 if (NumZero >= 1 && NumSigned < NumZero) {
1140 IsSigned = false;
1141 KnownBits = NumZero;
1142 break;
1143 }
1144 if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
1145 IsSigned = true;
1146 KnownBits = NumSigned;
1147 break;
1148 }
1149 return SDValue();
1150 }
1151 }
1152
1153 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
1154 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);
1155
1156 // Find the smallest power-2 type that is legal for this vector size and
1157 // operation, given the original type size and the number of known sign/zero
1158 // bits.
1159 EVT VT = Op.getValueType();
1160 unsigned MinWidth =
1161 std::max<unsigned>(a: VT.getScalarSizeInBits() - KnownBits, b: 8);
1162 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_ceil(Value: MinWidth));
1163 if (NVT.getScalarSizeInBits() > VT.getScalarSizeInBits())
1164 return SDValue();
1165 if (VT.isVector())
1166 NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT, EC: VT.getVectorElementCount());
1167 if (TLO.LegalTypes() && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT)) {
1168 // If we could not transform, and (both) adds are nuw/nsw, we can use the
1169 // larger type size to do the transform.
1170 if (TLO.LegalOperations() && !TLI.isOperationLegal(Op: AVGOpc, VT))
1171 return SDValue();
1172 if (DAG.willNotOverflowAdd(IsSigned, N0: Add.getOperand(i: 0),
1173 N1: Add.getOperand(i: 1)) &&
1174 (!Add2 || DAG.willNotOverflowAdd(IsSigned, N0: Add2.getOperand(i: 0),
1175 N1: Add2.getOperand(i: 1))))
1176 NVT = VT;
1177 else
1178 return SDValue();
1179 }
1180
1181 // Don't create a AVGFLOOR node with a scalar constant unless its legal as
1182 // this is likely to stop other folds (reassociation, value tracking etc.)
1183 if (!IsCeil && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT) &&
1184 (isa<ConstantSDNode>(Val: ExtOpA) || isa<ConstantSDNode>(Val: ExtOpB)))
1185 return SDValue();
1186
1187 SDLoc DL(Op);
1188 SDValue ResultAVG =
1189 DAG.getNode(Opcode: AVGOpc, DL, VT: NVT, N1: DAG.getExtOrTrunc(IsSigned, Op: ExtOpA, DL, VT: NVT),
1190 N2: DAG.getExtOrTrunc(IsSigned, Op: ExtOpB, DL, VT: NVT));
1191 return DAG.getExtOrTrunc(IsSigned, Op: ResultAVG, DL, VT);
1192}
1193
1194/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
1195/// result of Op are ever used downstream. If we can use this information to
1196/// simplify Op, create a new simplified DAG node and return true, returning the
1197/// original and new nodes in Old and New. Otherwise, analyze the expression and
1198/// return a mask of Known bits for the expression (used to simplify the
1199/// caller). The Known bits may only be accurate for those bits in the
1200/// OriginalDemandedBits and OriginalDemandedElts.
1201bool TargetLowering::SimplifyDemandedBits(
1202 SDValue Op, const APInt &OriginalDemandedBits,
1203 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
1204 unsigned Depth, bool AssumeSingleUse) const {
1205 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
1206 assert(Op.getScalarValueSizeInBits() == BitWidth &&
1207 "Mask size mismatches value type size!");
1208
1209 // Don't know anything.
1210 Known = KnownBits(BitWidth);
1211
1212 EVT VT = Op.getValueType();
1213 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
1214 unsigned NumElts = OriginalDemandedElts.getBitWidth();
1215 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
1216 "Unexpected vector size");
1217
1218 APInt DemandedBits = OriginalDemandedBits;
1219 APInt DemandedElts = OriginalDemandedElts;
1220 SDLoc dl(Op);
1221
1222 // Undef operand.
1223 if (Op.isUndef())
1224 return false;
1225
1226 // We can't simplify target constants.
1227 if (Op.getOpcode() == ISD::TargetConstant)
1228 return false;
1229
1230 if (Op.getOpcode() == ISD::Constant) {
1231 // We know all of the bits for a constant!
1232 Known = KnownBits::makeConstant(C: Op->getAsAPIntVal());
1233 return false;
1234 }
1235
1236 if (Op.getOpcode() == ISD::ConstantFP) {
1237 // We know all of the bits for a floating point constant!
1238 Known = KnownBits::makeConstant(
1239 C: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF().bitcastToAPInt());
1240 return false;
1241 }
1242
1243 // Other users may use these bits.
1244 bool HasMultiUse = false;
1245 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1246 if (Depth >= SelectionDAG::MaxRecursionDepth) {
1247 // Limit search depth.
1248 return false;
1249 }
1250 // Allow multiple uses, just set the DemandedBits/Elts to all bits.
1251 DemandedBits = APInt::getAllOnes(numBits: BitWidth);
1252 DemandedElts = APInt::getAllOnes(numBits: NumElts);
1253 HasMultiUse = true;
1254 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1255 // Not demanding any bits/elts from Op.
1256 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
1257 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
1258 // Limit search depth.
1259 return false;
1260 }
1261
1262 KnownBits Known2;
1263 switch (Op.getOpcode()) {
1264 case ISD::SCALAR_TO_VECTOR: {
1265 if (VT.isScalableVector())
1266 return false;
1267 if (!DemandedElts[0])
1268 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
1269
1270 KnownBits SrcKnown;
1271 SDValue Src = Op.getOperand(i: 0);
1272 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1273 APInt SrcDemandedBits = DemandedBits.zext(width: SrcBitWidth);
1274 if (SimplifyDemandedBits(Op: Src, DemandedBits: SrcDemandedBits, Known&: SrcKnown, TLO, Depth: Depth + 1))
1275 return true;
1276
1277 // Upper elements are undef, so only get the knownbits if we just demand
1278 // the bottom element.
1279 if (DemandedElts == 1)
1280 Known = SrcKnown.anyextOrTrunc(BitWidth);
1281 break;
1282 }
1283 case ISD::BUILD_VECTOR:
1284 // Collect the known bits that are shared by every demanded element.
1285 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1286 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1287 return false; // Don't fall through, will infinitely loop.
1288 case ISD::SPLAT_VECTOR: {
1289 SDValue Scl = Op.getOperand(i: 0);
1290 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: Scl.getValueSizeInBits());
1291 KnownBits KnownScl;
1292 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1293 return true;
1294
1295 // Implicitly truncate the bits to match the official semantics of
1296 // SPLAT_VECTOR.
1297 Known = KnownScl.trunc(BitWidth);
1298 break;
1299 }
1300 case ISD::FREEZE: {
1301 SDValue N0 = Op.getOperand(i: 0);
1302 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
1303 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
1304 return TLO.CombineTo(O: Op, N: N0);
1305 break;
1306 }
1307 case ISD::LOAD: {
1308 auto *LD = cast<LoadSDNode>(Val&: Op);
1309 if (getTargetConstantFromLoad(LD)) {
1310 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1311 return false; // Don't fall through, will infinitely loop.
1312 }
1313 if (ISD::isZEXTLoad(N: Op.getNode()) && Op.getResNo() == 0) {
1314 // If this is a ZEXTLoad and we are looking at the loaded value.
1315 EVT MemVT = LD->getMemoryVT();
1316 unsigned MemBits = MemVT.getScalarSizeInBits();
1317 Known.Zero.setBitsFrom(MemBits);
1318 return false; // Don't fall through, will infinitely loop.
1319 }
1320 break;
1321 }
1322 case ISD::INSERT_VECTOR_ELT: {
1323 if (VT.isScalableVector())
1324 return false;
1325 SDValue Vec = Op.getOperand(i: 0);
1326 SDValue Scl = Op.getOperand(i: 1);
1327 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1328 EVT VecVT = Vec.getValueType();
1329
1330 // If index isn't constant, assume we need all vector elements AND the
1331 // inserted element.
1332 APInt DemandedVecElts(DemandedElts);
1333 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements())) {
1334 unsigned Idx = CIdx->getZExtValue();
1335 DemandedVecElts.clearBit(BitPosition: Idx);
1336
1337 // Inserted element is not required.
1338 if (!DemandedElts[Idx])
1339 return TLO.CombineTo(O: Op, N: Vec);
1340 }
1341
1342 KnownBits KnownScl;
1343 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1344 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: NumSclBits);
1345 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1346 return true;
1347
1348 Known = KnownScl.anyextOrTrunc(BitWidth);
1349
1350 KnownBits KnownVec;
1351 if (SimplifyDemandedBits(Op: Vec, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedVecElts, Known&: KnownVec, TLO,
1352 Depth: Depth + 1))
1353 return true;
1354
1355 if (!!DemandedVecElts)
1356 Known = Known.intersectWith(RHS: KnownVec);
1357
1358 return false;
1359 }
1360 case ISD::INSERT_SUBVECTOR: {
1361 if (VT.isScalableVector())
1362 return false;
1363 // Demand any elements from the subvector and the remainder from the src its
1364 // inserted into.
1365 SDValue Src = Op.getOperand(i: 0);
1366 SDValue Sub = Op.getOperand(i: 1);
1367 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1368 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1369 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1370 APInt DemandedSrcElts = DemandedElts;
1371 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
1372
1373 KnownBits KnownSub, KnownSrc;
1374 if (SimplifyDemandedBits(Op: Sub, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts, Known&: KnownSub, TLO,
1375 Depth: Depth + 1))
1376 return true;
1377 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known&: KnownSrc, TLO,
1378 Depth: Depth + 1))
1379 return true;
1380
1381 Known.setAllConflict();
1382 if (!!DemandedSubElts)
1383 Known = Known.intersectWith(RHS: KnownSub);
1384 if (!!DemandedSrcElts)
1385 Known = Known.intersectWith(RHS: KnownSrc);
1386
1387 // Attempt to avoid multi-use src if we don't need anything from it.
1388 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1389 !DemandedSrcElts.isAllOnes()) {
1390 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1391 Op: Sub, DemandedBits, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
1392 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1393 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1394 if (NewSub || NewSrc) {
1395 NewSub = NewSub ? NewSub : Sub;
1396 NewSrc = NewSrc ? NewSrc : Src;
1397 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: NewSrc, N2: NewSub,
1398 N3: Op.getOperand(i: 2));
1399 return TLO.CombineTo(O: Op, N: NewOp);
1400 }
1401 }
1402 break;
1403 }
1404 case ISD::EXTRACT_SUBVECTOR: {
1405 if (VT.isScalableVector())
1406 return false;
1407 // Offset the demanded elts by the subvector index.
1408 SDValue Src = Op.getOperand(i: 0);
1409 if (Src.getValueType().isScalableVector())
1410 break;
1411 uint64_t Idx = Op.getConstantOperandVal(i: 1);
1412 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1413 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
1414
1415 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known, TLO,
1416 Depth: Depth + 1))
1417 return true;
1418
1419 // Attempt to avoid multi-use src if we don't need anything from it.
1420 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1421 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1422 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1423 if (DemandedSrc) {
1424 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc,
1425 N2: Op.getOperand(i: 1));
1426 return TLO.CombineTo(O: Op, N: NewOp);
1427 }
1428 }
1429 break;
1430 }
1431 case ISD::CONCAT_VECTORS: {
1432 if (VT.isScalableVector())
1433 return false;
1434 Known.setAllConflict();
1435 EVT SubVT = Op.getOperand(i: 0).getValueType();
1436 unsigned NumSubVecs = Op.getNumOperands();
1437 unsigned NumSubElts = SubVT.getVectorNumElements();
1438 for (unsigned i = 0; i != NumSubVecs; ++i) {
1439 APInt DemandedSubElts =
1440 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
1441 if (SimplifyDemandedBits(Op: Op.getOperand(i), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts,
1442 Known&: Known2, TLO, Depth: Depth + 1))
1443 return true;
1444 // Known bits are shared by every demanded subvector element.
1445 if (!!DemandedSubElts)
1446 Known = Known.intersectWith(RHS: Known2);
1447 }
1448 break;
1449 }
1450 case ISD::VECTOR_SHUFFLE: {
1451 assert(!VT.isScalableVector());
1452 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1453
1454 // Collect demanded elements from shuffle operands..
1455 APInt DemandedLHS, DemandedRHS;
1456 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: ShuffleMask, DemandedElts, DemandedLHS,
1457 DemandedRHS))
1458 break;
1459
1460 if (!!DemandedLHS || !!DemandedRHS) {
1461 SDValue Op0 = Op.getOperand(i: 0);
1462 SDValue Op1 = Op.getOperand(i: 1);
1463
1464 Known.setAllConflict();
1465 if (!!DemandedLHS) {
1466 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedLHS, Known&: Known2, TLO,
1467 Depth: Depth + 1))
1468 return true;
1469 Known = Known.intersectWith(RHS: Known2);
1470 }
1471 if (!!DemandedRHS) {
1472 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedRHS, Known&: Known2, TLO,
1473 Depth: Depth + 1))
1474 return true;
1475 Known = Known.intersectWith(RHS: Known2);
1476 }
1477
1478 // Attempt to avoid multi-use ops if we don't need anything from them.
1479 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1480 Op: Op0, DemandedBits, DemandedElts: DemandedLHS, DAG&: TLO.DAG, Depth: Depth + 1);
1481 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1482 Op: Op1, DemandedBits, DemandedElts: DemandedRHS, DAG&: TLO.DAG, Depth: Depth + 1);
1483 if (DemandedOp0 || DemandedOp1) {
1484 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1485 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1486 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, N1: Op0, N2: Op1, Mask: ShuffleMask);
1487 return TLO.CombineTo(O: Op, N: NewOp);
1488 }
1489 }
1490 break;
1491 }
1492 case ISD::AND: {
1493 SDValue Op0 = Op.getOperand(i: 0);
1494 SDValue Op1 = Op.getOperand(i: 1);
1495
1496 // If the RHS is a constant, check to see if the LHS would be zero without
1497 // using the bits from the RHS. Below, we use knowledge about the RHS to
1498 // simplify the LHS, here we're using information from the LHS to simplify
1499 // the RHS.
1500 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: Op1, DemandedElts)) {
1501 // Do not increment Depth here; that can cause an infinite loop.
1502 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth);
1503 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1504 if ((LHSKnown.Zero & DemandedBits) ==
1505 (~RHSC->getAPIntValue() & DemandedBits))
1506 return TLO.CombineTo(O: Op, N: Op0);
1507
1508 // If any of the set bits in the RHS are known zero on the LHS, shrink
1509 // the constant.
1510 if (ShrinkDemandedConstant(Op, DemandedBits: ~LHSKnown.Zero & DemandedBits,
1511 DemandedElts, TLO))
1512 return true;
1513
1514 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1515 // constant, but if this 'and' is only clearing bits that were just set by
1516 // the xor, then this 'and' can be eliminated by shrinking the mask of
1517 // the xor. For example, for a 32-bit X:
1518 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1519 if (isBitwiseNot(V: Op0) && Op0.hasOneUse() &&
1520 LHSKnown.One == ~RHSC->getAPIntValue()) {
1521 SDValue Xor = TLO.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1522 return TLO.CombineTo(O: Op, N: Xor);
1523 }
1524 }
1525
1526 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2 (or zero).
1527 SDValue X, Y;
1528 if (sd_match(N: Op,
1529 P: m_And(L: m_Value(N&: Y),
1530 R: m_OneUse(P: m_AnyOf(preds: m_Add(L: m_Value(N&: X), R: m_Deferred(V&: Y)),
1531 preds: m_Sub(L: m_Value(N&: X), R: m_Deferred(V&: Y)))))) &&
1532 TLO.DAG.isKnownToBeAPowerOfTwo(Val: Y, DemandedElts, /*OrZero=*/true)) {
1533 return TLO.CombineTo(
1534 O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TLO.DAG.getNOT(DL: dl, Val: X, VT), N2: Y));
1535 }
1536
1537 // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
1538 // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
1539 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
1540 (Op0.getOperand(i: 0).isUndef() ||
1541 ISD::isBuildVectorOfConstantSDNodes(N: Op0.getOperand(i: 0).getNode())) &&
1542 Op0->hasOneUse()) {
1543 unsigned NumSubElts =
1544 Op0.getOperand(i: 1).getValueType().getVectorNumElements();
1545 unsigned SubIdx = Op0.getConstantOperandVal(i: 2);
1546 APInt DemandedSub =
1547 APInt::getBitsSet(numBits: NumElts, loBit: SubIdx, hiBit: SubIdx + NumSubElts);
1548 KnownBits KnownSubMask =
1549 TLO.DAG.computeKnownBits(Op: Op1, DemandedElts: DemandedSub & DemandedElts, Depth: Depth + 1);
1550 if (DemandedBits.isSubsetOf(RHS: KnownSubMask.One)) {
1551 SDValue NewAnd =
1552 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1553 SDValue NewInsert =
1554 TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: NewAnd,
1555 N2: Op0.getOperand(i: 1), N3: Op0.getOperand(i: 2));
1556 return TLO.CombineTo(O: Op, N: NewInsert);
1557 }
1558 }
1559
1560 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1561 Depth: Depth + 1))
1562 return true;
1563 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.Zero & DemandedBits, OriginalDemandedElts: DemandedElts,
1564 Known&: Known2, TLO, Depth: Depth + 1))
1565 return true;
1566
1567 // If all of the demanded bits are known one on one side, return the other.
1568 // These bits cannot contribute to the result of the 'and'.
1569 if (DemandedBits.isSubsetOf(RHS: Known2.Zero | Known.One))
1570 return TLO.CombineTo(O: Op, N: Op0);
1571 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.One))
1572 return TLO.CombineTo(O: Op, N: Op1);
1573 // If all of the demanded bits in the inputs are known zeros, return zero.
1574 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1575 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: dl, VT));
1576 // If the RHS is a constant, see if we can simplify it.
1577 if (ShrinkDemandedConstant(Op, DemandedBits: ~Known2.Zero & DemandedBits, DemandedElts,
1578 TLO))
1579 return true;
1580 // If the operation can be done in a smaller type, do so.
1581 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1582 return true;
1583
1584 // Attempt to avoid multi-use ops if we don't need anything from them.
1585 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1586 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1587 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1588 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1589 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1590 if (DemandedOp0 || DemandedOp1) {
1591 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1592 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1593 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1594 return TLO.CombineTo(O: Op, N: NewOp);
1595 }
1596 }
1597
1598 Known &= Known2;
1599 break;
1600 }
1601 case ISD::OR: {
1602 SDValue Op0 = Op.getOperand(i: 0);
1603 SDValue Op1 = Op.getOperand(i: 1);
1604 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1605 Depth: Depth + 1)) {
1606 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1607 return true;
1608 }
1609
1610 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.One & DemandedBits, OriginalDemandedElts: DemandedElts,
1611 Known&: Known2, TLO, Depth: Depth + 1)) {
1612 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1613 return true;
1614 }
1615
1616 // If all of the demanded bits are known zero on one side, return the other.
1617 // These bits cannot contribute to the result of the 'or'.
1618 if (DemandedBits.isSubsetOf(RHS: Known2.One | Known.Zero))
1619 return TLO.CombineTo(O: Op, N: Op0);
1620 if (DemandedBits.isSubsetOf(RHS: Known.One | Known2.Zero))
1621 return TLO.CombineTo(O: Op, N: Op1);
1622 // If the RHS is a constant, see if we can simplify it.
1623 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1624 return true;
1625 // If the operation can be done in a smaller type, do so.
1626 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1627 return true;
1628
1629 // Attempt to avoid multi-use ops if we don't need anything from them.
1630 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1631 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1632 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1633 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1634 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1635 if (DemandedOp0 || DemandedOp1) {
1636 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1637 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1638 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1639 return TLO.CombineTo(O: Op, N: NewOp);
1640 }
1641 }
1642
1643 // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
1644 // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
1645 SDValue X, Y, C1, C2;
1646 if (sd_match(N: Op, P: m_Or(L: m_OneUse(P: m_And(L: m_Value(N&: X), R: m_Value(N&: C1))),
1647 R: m_OneUse(P: m_And(L: m_Or(L: m_Deferred(V&: X), R: m_Value(N&: Y)),
1648 R: m_Value(N&: C2)))))) {
1649 if (SDValue C12 =
1650 TLO.DAG.FoldConstantArithmetic(Opcode: ISD::OR, DL: dl, VT, Ops: {C1, C2})) {
1651 SDValue MaskX = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: C12);
1652 SDValue MaskY = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Y, N2: C2);
1653 return TLO.CombineTo(O: Op,
1654 N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: MaskX, N2: MaskY));
1655 }
1656 }
1657
1658 Known |= Known2;
1659 break;
1660 }
1661 case ISD::XOR: {
1662 SDValue Op0 = Op.getOperand(i: 0);
1663 SDValue Op1 = Op.getOperand(i: 1);
1664
1665 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1666 Depth: Depth + 1))
1667 return true;
1668 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
1669 Depth: Depth + 1))
1670 return true;
1671
1672 // If all of the demanded bits are known zero on one side, return the other.
1673 // These bits cannot contribute to the result of the 'xor'.
1674 if (DemandedBits.isSubsetOf(RHS: Known.Zero))
1675 return TLO.CombineTo(O: Op, N: Op0);
1676 if (DemandedBits.isSubsetOf(RHS: Known2.Zero))
1677 return TLO.CombineTo(O: Op, N: Op1);
1678 // If the operation can be done in a smaller type, do so.
1679 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1680 return true;
1681
1682 // If all of the unknown bits are known to be zero on one side or the other
1683 // turn this into an *inclusive* or.
1684 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1685 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1686 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op0, N2: Op1));
1687
1688 ConstantSDNode *C = isConstOrConstSplat(N: Op1, DemandedElts);
1689 if (C) {
1690 // If one side is a constant, and all of the set bits in the constant are
1691 // also known set on the other side, turn this into an AND, as we know
1692 // the bits will be cleared.
1693 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1694 // NB: it is okay if more bits are known than are requested
1695 if (C->getAPIntValue() == Known2.One) {
1696 SDValue ANDC =
1697 TLO.DAG.getConstant(Val: ~C->getAPIntValue() & DemandedBits, DL: dl, VT);
1698 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0, N2: ANDC));
1699 }
1700
1701 // If the RHS is a constant, see if we can change it. Don't alter a -1
1702 // constant because that's a 'not' op, and that is better for combining
1703 // and codegen.
1704 if (!C->isAllOnes() && DemandedBits.isSubsetOf(RHS: C->getAPIntValue())) {
1705 // We're flipping all demanded bits. Flip the undemanded bits too.
1706 SDValue New = TLO.DAG.getNOT(DL: dl, Val: Op0, VT);
1707 return TLO.CombineTo(O: Op, N: New);
1708 }
1709
1710 unsigned Op0Opcode = Op0.getOpcode();
1711 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
1712 if (ConstantSDNode *ShiftC =
1713 isConstOrConstSplat(N: Op0.getOperand(i: 1), DemandedElts)) {
1714 // Don't crash on an oversized shift. We can not guarantee that a
1715 // bogus shift has been simplified to undef.
1716 if (ShiftC->getAPIntValue().ult(RHS: BitWidth)) {
1717 uint64_t ShiftAmt = ShiftC->getZExtValue();
1718 APInt Ones = APInt::getAllOnes(numBits: BitWidth);
1719 Ones = Op0Opcode == ISD::SHL ? Ones.shl(shiftAmt: ShiftAmt)
1720 : Ones.lshr(shiftAmt: ShiftAmt);
1721 if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
1722 isDesirableToCommuteXorWithShift(N: Op.getNode())) {
1723 // If the xor constant is a demanded mask, do a 'not' before the
1724 // shift:
1725 // xor (X << ShiftC), XorC --> (not X) << ShiftC
1726 // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
1727 SDValue Not = TLO.DAG.getNOT(DL: dl, Val: Op0.getOperand(i: 0), VT);
1728 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op0Opcode, DL: dl, VT, N1: Not,
1729 N2: Op0.getOperand(i: 1)));
1730 }
1731 }
1732 }
1733 }
1734 }
1735
1736 // If we can't turn this into a 'not', try to shrink the constant.
1737 if (!C || !C->isAllOnes())
1738 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1739 return true;
1740
1741 // Attempt to avoid multi-use ops if we don't need anything from them.
1742 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1743 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1744 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1745 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1746 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1747 if (DemandedOp0 || DemandedOp1) {
1748 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1749 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1750 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1751 return TLO.CombineTo(O: Op, N: NewOp);
1752 }
1753 }
1754
1755 Known ^= Known2;
1756 break;
1757 }
1758 case ISD::SELECT:
1759 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1760 Known, TLO, Depth: Depth + 1))
1761 return true;
1762 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1763 Known&: Known2, TLO, Depth: Depth + 1))
1764 return true;
1765
1766 // If the operands are constants, see if we can simplify them.
1767 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1768 return true;
1769
1770 // Only known if known in both the LHS and RHS.
1771 Known = Known.intersectWith(RHS: Known2);
1772 break;
1773 case ISD::VSELECT:
1774 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1775 Known, TLO, Depth: Depth + 1))
1776 return true;
1777 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1778 Known&: Known2, TLO, Depth: Depth + 1))
1779 return true;
1780
1781 // Only known if known in both the LHS and RHS.
1782 Known = Known.intersectWith(RHS: Known2);
1783 break;
1784 case ISD::SELECT_CC:
1785 if (SimplifyDemandedBits(Op: Op.getOperand(i: 3), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1786 Known, TLO, Depth: Depth + 1))
1787 return true;
1788 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1789 Known&: Known2, TLO, Depth: Depth + 1))
1790 return true;
1791
1792 // If the operands are constants, see if we can simplify them.
1793 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1794 return true;
1795
1796 // Only known if known in both the LHS and RHS.
1797 Known = Known.intersectWith(RHS: Known2);
1798 break;
1799 case ISD::SETCC: {
1800 SDValue Op0 = Op.getOperand(i: 0);
1801 SDValue Op1 = Op.getOperand(i: 1);
1802 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1803 // If we're testing X < 0, X >= 0, X <= -1 or X > -1
1804 // (X is of integer type) then we only need the sign mask of the previous
1805 // result
1806 if (Op1.getValueType().isInteger() &&
1807 (((CC == ISD::SETLT || CC == ISD::SETGE) && isNullOrNullSplat(V: Op1)) ||
1808 ((CC == ISD::SETLE || CC == ISD::SETGT) &&
1809 isAllOnesOrAllOnesSplat(V: Op1)))) {
1810 KnownBits KnownOp0;
1811 if (SimplifyDemandedBits(
1812 Op: Op0, OriginalDemandedBits: APInt::getSignMask(BitWidth: Op0.getScalarValueSizeInBits()),
1813 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1))
1814 return true;
1815 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1816 // width as the setcc result, and (3) the result of a setcc conforms to 0
1817 // or -1, we may be able to bypass the setcc.
1818 if (DemandedBits.isSignMask() &&
1819 Op0.getScalarValueSizeInBits() == BitWidth &&
1820 getBooleanContents(Type: Op0.getValueType()) ==
1821 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1822 // If we remove a >= 0 or > -1 (for integers), we need to introduce a
1823 // NOT Operation
1824 if (CC == ISD::SETGE || CC == ISD::SETGT) {
1825 SDLoc DL(Op);
1826 EVT VT = Op0.getValueType();
1827 SDValue NotOp0 = TLO.DAG.getNOT(DL, Val: Op0, VT);
1828 return TLO.CombineTo(O: Op, N: NotOp0);
1829 }
1830 return TLO.CombineTo(O: Op, N: Op0);
1831 }
1832 }
1833 if (getBooleanContents(Type: Op0.getValueType()) ==
1834 TargetLowering::ZeroOrOneBooleanContent &&
1835 BitWidth > 1)
1836 Known.Zero.setBitsFrom(1);
1837 break;
1838 }
1839 case ISD::SHL: {
1840 SDValue Op0 = Op.getOperand(i: 0);
1841 SDValue Op1 = Op.getOperand(i: 1);
1842 EVT ShiftVT = Op1.getValueType();
1843
1844 if (std::optional<unsigned> KnownSA =
1845 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
1846 unsigned ShAmt = *KnownSA;
1847 if (ShAmt == 0)
1848 return TLO.CombineTo(O: Op, N: Op0);
1849
1850 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1851 // single shift. We can do this if the bottom bits (which are shifted
1852 // out) are never demanded.
1853 // TODO - support non-uniform vector amounts.
1854 if (Op0.getOpcode() == ISD::SRL) {
1855 if (!DemandedBits.intersects(RHS: APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ShAmt))) {
1856 if (std::optional<unsigned> InnerSA =
1857 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
1858 unsigned C1 = *InnerSA;
1859 unsigned Opc = ISD::SHL;
1860 int Diff = ShAmt - C1;
1861 if (Diff < 0) {
1862 Diff = -Diff;
1863 Opc = ISD::SRL;
1864 }
1865 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
1866 return TLO.CombineTo(
1867 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
1868 }
1869 }
1870 }
1871
1872 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1873 // are not demanded. This will likely allow the anyext to be folded away.
1874 // TODO - support non-uniform vector amounts.
1875 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1876 SDValue InnerOp = Op0.getOperand(i: 0);
1877 EVT InnerVT = InnerOp.getValueType();
1878 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1879 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1880 isTypeDesirableForOp(ISD::SHL, VT: InnerVT)) {
1881 SDValue NarrowShl = TLO.DAG.getNode(
1882 Opcode: ISD::SHL, DL: dl, VT: InnerVT, N1: InnerOp,
1883 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: InnerVT, DL: dl));
1884 return TLO.CombineTo(
1885 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
1886 }
1887
1888 // Repeat the SHL optimization above in cases where an extension
1889 // intervenes: (shl (anyext (shr x, c1)), c2) to
1890 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1891 // aren't demanded (as above) and that the shifted upper c1 bits of
1892 // x aren't demanded.
1893 // TODO - support non-uniform vector amounts.
1894 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
1895 InnerOp.hasOneUse()) {
1896 if (std::optional<unsigned> SA2 = TLO.DAG.getValidShiftAmount(
1897 V: InnerOp, DemandedElts, Depth: Depth + 2)) {
1898 unsigned InnerShAmt = *SA2;
1899 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1900 DemandedBits.getActiveBits() <=
1901 (InnerBits - InnerShAmt + ShAmt) &&
1902 DemandedBits.countr_zero() >= ShAmt) {
1903 SDValue NewSA =
1904 TLO.DAG.getConstant(Val: ShAmt - InnerShAmt, DL: dl, VT: ShiftVT);
1905 SDValue NewExt = TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT,
1906 Operand: InnerOp.getOperand(i: 0));
1907 return TLO.CombineTo(
1908 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: NewExt, N2: NewSA));
1909 }
1910 }
1911 }
1912 }
1913
1914 APInt InDemandedMask = DemandedBits.lshr(shiftAmt: ShAmt);
1915 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
1916 Depth: Depth + 1)) {
1917 // Disable the nsw and nuw flags. We can no longer guarantee that we
1918 // won't wrap after simplification.
1919 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
1920 return true;
1921 }
1922 Known <<= ShAmt;
1923 // low bits known zero.
1924 Known.Zero.setLowBits(ShAmt);
1925
1926 // Attempt to avoid multi-use ops if we don't need anything from them.
1927 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1928 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1929 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1930 if (DemandedOp0) {
1931 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
1932 return TLO.CombineTo(O: Op, N: NewOp);
1933 }
1934 }
1935
1936 // TODO: Can we merge this fold with the one below?
1937 // Try shrinking the operation as long as the shift amount will still be
1938 // in range.
1939 if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() &&
1940 Op.getNode()->hasOneUse()) {
1941 // Search for the smallest integer type with free casts to and from
1942 // Op's type. For expedience, just check power-of-2 integer types.
1943 unsigned DemandedSize = DemandedBits.getActiveBits();
1944 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
1945 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
1946 EVT SmallVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: SmallVTBits);
1947 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: SmallVT) &&
1948 isTypeDesirableForOp(ISD::SHL, VT: SmallVT) &&
1949 isTruncateFree(FromVT: VT, ToVT: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT) &&
1950 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: SmallVT))) {
1951 assert(DemandedSize <= SmallVTBits &&
1952 "Narrowed below demanded bits?");
1953 // We found a type with free casts.
1954 SDValue NarrowShl = TLO.DAG.getNode(
1955 Opcode: ISD::SHL, DL: dl, VT: SmallVT,
1956 N1: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
1957 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: SmallVT, DL: dl));
1958 return TLO.CombineTo(
1959 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
1960 }
1961 }
1962 }
1963
1964 // Narrow shift to lower half - similar to ShrinkDemandedOp.
1965 // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
1966 // Only do this if we demand the upper half so the knownbits are correct.
1967 unsigned HalfWidth = BitWidth / 2;
1968 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
1969 DemandedBits.countLeadingOnes() >= HalfWidth) {
1970 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: HalfWidth);
1971 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
1972 isTypeDesirableForOp(ISD::SHL, VT: HalfVT) &&
1973 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
1974 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: HalfVT))) {
1975 // If we're demanding the upper bits at all, we must ensure
1976 // that the upper bits of the shift result are known to be zero,
1977 // which is equivalent to the narrow shift being NUW.
1978 if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
1979 bool IsNSW = Known.countMinSignBits() > HalfWidth;
1980 SDNodeFlags Flags;
1981 Flags.setNoSignedWrap(IsNSW);
1982 Flags.setNoUnsignedWrap(IsNUW);
1983 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
1984 SDValue NewShiftAmt =
1985 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
1986 SDValue NewShift = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HalfVT, N1: NewOp,
1987 N2: NewShiftAmt, Flags);
1988 SDValue NewExt =
1989 TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift);
1990 return TLO.CombineTo(O: Op, N: NewExt);
1991 }
1992 }
1993 }
1994 } else {
1995 // This is a variable shift, so we can't shift the demand mask by a known
1996 // amount. But if we are not demanding high bits, then we are not
1997 // demanding those bits from the pre-shifted operand either.
1998 if (unsigned CTLZ = DemandedBits.countl_zero()) {
1999 APInt DemandedFromOp(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - CTLZ));
2000 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedFromOp, OriginalDemandedElts: DemandedElts, Known, TLO,
2001 Depth: Depth + 1)) {
2002 // Disable the nsw and nuw flags. We can no longer guarantee that we
2003 // won't wrap after simplification.
2004 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2005 return true;
2006 }
2007 Known.resetAll();
2008 }
2009 }
2010
2011 // If we are only demanding sign bits then we can use the shift source
2012 // directly.
2013 if (std::optional<unsigned> MaxSA =
2014 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2015 unsigned ShAmt = *MaxSA;
2016 unsigned NumSignBits =
2017 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2018 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
2019 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
2020 return TLO.CombineTo(O: Op, N: Op0);
2021 }
2022 break;
2023 }
2024 case ISD::SRL: {
2025 SDValue Op0 = Op.getOperand(i: 0);
2026 SDValue Op1 = Op.getOperand(i: 1);
2027 EVT ShiftVT = Op1.getValueType();
2028
2029 if (std::optional<unsigned> KnownSA =
2030 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2031 unsigned ShAmt = *KnownSA;
2032 if (ShAmt == 0)
2033 return TLO.CombineTo(O: Op, N: Op0);
2034
2035 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
2036 // single shift. We can do this if the top bits (which are shifted out)
2037 // are never demanded.
2038 // TODO - support non-uniform vector amounts.
2039 if (Op0.getOpcode() == ISD::SHL) {
2040 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2041 if (std::optional<unsigned> InnerSA =
2042 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2043 unsigned C1 = *InnerSA;
2044 unsigned Opc = ISD::SRL;
2045 int Diff = ShAmt - C1;
2046 if (Diff < 0) {
2047 Diff = -Diff;
2048 Opc = ISD::SHL;
2049 }
2050 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
2051 return TLO.CombineTo(
2052 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
2053 }
2054 }
2055 }
2056
2057 // If this is (srl (sra X, C1), ShAmt), see if we can combine this into a
2058 // single sra. We can do this if the top bits are never demanded.
2059 if (Op0.getOpcode() == ISD::SRA && Op0.hasOneUse()) {
2060 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2061 if (std::optional<unsigned> InnerSA =
2062 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2063 unsigned C1 = *InnerSA;
2064 // Clamp the combined shift amount if it exceeds the bit width.
2065 unsigned Combined = std::min(a: C1 + ShAmt, b: BitWidth - 1);
2066 SDValue NewSA = TLO.DAG.getConstant(Val: Combined, DL: dl, VT: ShiftVT);
2067 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT,
2068 N1: Op0.getOperand(i: 0), N2: NewSA));
2069 }
2070 }
2071 }
2072
2073 APInt InDemandedMask = (DemandedBits << ShAmt);
2074
2075 // If the shift is exact, then it does demand the low bits (and knows that
2076 // they are zero).
2077 if (Op->getFlags().hasExact())
2078 InDemandedMask.setLowBits(ShAmt);
2079
2080 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2081 // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
2082 if ((BitWidth % 2) == 0 && !VT.isVector()) {
2083 APInt HiBits = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth / 2);
2084 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: BitWidth / 2);
2085 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2086 isTypeDesirableForOp(ISD::SRL, VT: HalfVT) &&
2087 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2088 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT: HalfVT)) &&
2089 ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
2090 TLO.DAG.MaskedValueIsZero(Op: Op0, Mask: HiBits))) {
2091 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2092 SDValue NewShiftAmt =
2093 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2094 SDValue NewShift =
2095 TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HalfVT, N1: NewOp, N2: NewShiftAmt);
2096 return TLO.CombineTo(
2097 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift));
2098 }
2099 }
2100
2101 // Compute the new bits that are at the top now.
2102 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2103 Depth: Depth + 1))
2104 return true;
2105 Known >>= ShAmt;
2106 // High bits known zero.
2107 Known.Zero.setHighBits(ShAmt);
2108
2109 // Attempt to avoid multi-use ops if we don't need anything from them.
2110 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2111 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2112 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2113 if (DemandedOp0) {
2114 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2115 return TLO.CombineTo(O: Op, N: NewOp);
2116 }
2117 }
2118 } else {
2119 // Use generic knownbits computation as it has support for non-uniform
2120 // shift amounts.
2121 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2122 }
2123
2124 // If we are only demanding sign bits then we can use the shift source
2125 // directly.
2126 if (std::optional<unsigned> MaxSA =
2127 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2128 unsigned ShAmt = *MaxSA;
2129 // Must already be signbits in DemandedBits bounds, and can't demand any
2130 // shifted in zeroes.
2131 if (DemandedBits.countl_zero() >= ShAmt) {
2132 unsigned NumSignBits =
2133 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2134 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
2135 return TLO.CombineTo(O: Op, N: Op0);
2136 }
2137 }
2138
2139 // Try to match AVG patterns (after shift simplification).
2140 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2141 DemandedElts, Depth: Depth + 1))
2142 return TLO.CombineTo(O: Op, N: AVG);
2143
2144 break;
2145 }
2146 case ISD::SRA: {
2147 SDValue Op0 = Op.getOperand(i: 0);
2148 SDValue Op1 = Op.getOperand(i: 1);
2149 EVT ShiftVT = Op1.getValueType();
2150
2151 // If we only want bits that already match the signbit then we don't need
2152 // to shift.
2153 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
2154 if (TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1) >=
2155 NumHiDemandedBits)
2156 return TLO.CombineTo(O: Op, N: Op0);
2157
2158 // If this is an arithmetic shift right and only the low-bit is set, we can
2159 // always convert this into a logical shr, even if the shift amount is
2160 // variable. The low bit of the shift cannot be an input sign bit unless
2161 // the shift amount is >= the size of the datatype, which is undefined.
2162 if (DemandedBits.isOne())
2163 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2164
2165 if (std::optional<unsigned> KnownSA =
2166 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2167 unsigned ShAmt = *KnownSA;
2168 if (ShAmt == 0)
2169 return TLO.CombineTo(O: Op, N: Op0);
2170
2171 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
2172 // supports sext_inreg.
2173 if (Op0.getOpcode() == ISD::SHL) {
2174 if (std::optional<unsigned> InnerSA =
2175 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2176 unsigned LowBits = BitWidth - ShAmt;
2177 EVT ExtVT = VT.changeElementType(
2178 Context&: *TLO.DAG.getContext(),
2179 EltVT: EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: LowBits));
2180
2181 if (*InnerSA == ShAmt) {
2182 if (!TLO.LegalOperations() ||
2183 getOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: ExtVT) == Legal)
2184 return TLO.CombineTo(
2185 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT,
2186 N1: Op0.getOperand(i: 0),
2187 N2: TLO.DAG.getValueType(ExtVT)));
2188
2189 // Even if we can't convert to sext_inreg, we might be able to
2190 // remove this shift pair if the input is already sign extended.
2191 unsigned NumSignBits =
2192 TLO.DAG.ComputeNumSignBits(Op: Op0.getOperand(i: 0), DemandedElts);
2193 if (NumSignBits > ShAmt)
2194 return TLO.CombineTo(O: Op, N: Op0.getOperand(i: 0));
2195 }
2196 }
2197 }
2198
2199 APInt InDemandedMask = (DemandedBits << ShAmt);
2200
2201 // If the shift is exact, then it does demand the low bits (and knows that
2202 // they are zero).
2203 if (Op->getFlags().hasExact())
2204 InDemandedMask.setLowBits(ShAmt);
2205
2206 // If any of the demanded bits are produced by the sign extension, we also
2207 // demand the input sign bit.
2208 if (DemandedBits.countl_zero() < ShAmt)
2209 InDemandedMask.setSignBit();
2210
2211 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2212 Depth: Depth + 1))
2213 return true;
2214 Known >>= ShAmt;
2215
2216 // If the input sign bit is known to be zero, or if none of the top bits
2217 // are demanded, turn this into an unsigned shift right.
2218 if (Known.Zero[BitWidth - ShAmt - 1] ||
2219 DemandedBits.countl_zero() >= ShAmt) {
2220 SDNodeFlags Flags;
2221 Flags.setExact(Op->getFlags().hasExact());
2222 return TLO.CombineTo(
2223 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1, Flags));
2224 }
2225
2226 int Log2 = DemandedBits.exactLogBase2();
2227 if (Log2 >= 0) {
2228 // The bit must come from the sign.
2229 SDValue NewSA = TLO.DAG.getConstant(Val: BitWidth - 1 - Log2, DL: dl, VT: ShiftVT);
2230 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: NewSA));
2231 }
2232
2233 if (Known.One[BitWidth - ShAmt - 1])
2234 // New bits are known one.
2235 Known.One.setHighBits(ShAmt);
2236
2237 // Attempt to avoid multi-use ops if we don't need anything from them.
2238 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2239 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2240 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2241 if (DemandedOp0) {
2242 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2243 return TLO.CombineTo(O: Op, N: NewOp);
2244 }
2245 }
2246 }
2247
2248 // Try to match AVG patterns (after shift simplification).
2249 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2250 DemandedElts, Depth: Depth + 1))
2251 return TLO.CombineTo(O: Op, N: AVG);
2252
2253 break;
2254 }
2255 case ISD::FSHL:
2256 case ISD::FSHR: {
2257 SDValue Op0 = Op.getOperand(i: 0);
2258 SDValue Op1 = Op.getOperand(i: 1);
2259 SDValue Op2 = Op.getOperand(i: 2);
2260 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
2261
2262 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op2, DemandedElts)) {
2263 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2264
2265 // For fshl, 0-shift returns the 1st arg.
2266 // For fshr, 0-shift returns the 2nd arg.
2267 if (Amt == 0) {
2268 if (SimplifyDemandedBits(Op: IsFSHL ? Op0 : Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2269 Known, TLO, Depth: Depth + 1))
2270 return true;
2271 break;
2272 }
2273
2274 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
2275 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
2276 APInt Demanded0 = DemandedBits.lshr(shiftAmt: IsFSHL ? Amt : (BitWidth - Amt));
2277 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
2278 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2279 Depth: Depth + 1))
2280 return true;
2281 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: Demanded1, OriginalDemandedElts: DemandedElts, Known, TLO,
2282 Depth: Depth + 1))
2283 return true;
2284
2285 Known2 <<= (IsFSHL ? Amt : (BitWidth - Amt));
2286 Known >>= (IsFSHL ? (BitWidth - Amt) : Amt);
2287 Known = Known.unionWith(RHS: Known2);
2288
2289 // Attempt to avoid multi-use ops if we don't need anything from them.
2290 if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
2291 !DemandedElts.isAllOnes()) {
2292 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2293 Op: Op0, DemandedBits: Demanded0, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2294 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2295 Op: Op1, DemandedBits: Demanded1, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2296 if (DemandedOp0 || DemandedOp1) {
2297 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2298 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2299 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedOp0,
2300 N2: DemandedOp1, N3: Op2);
2301 return TLO.CombineTo(O: Op, N: NewOp);
2302 }
2303 }
2304 }
2305
2306 if (isPowerOf2_32(Value: BitWidth)) {
2307 // Fold FSHR(Op0,Op1,Op2) -> SRL(Op1,Op2)
2308 // iff we're guaranteed not to use Op0.
2309 // TODO: Add FSHL equivalent?
2310 if (!IsFSHL && !DemandedBits.isAllOnes() &&
2311 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT))) {
2312 KnownBits KnownAmt =
2313 TLO.DAG.computeKnownBits(Op: Op2, DemandedElts, Depth: Depth + 1);
2314 unsigned MaxShiftAmt =
2315 KnownAmt.getMaxValue().getLimitedValue(Limit: BitWidth - 1);
2316 // Check we don't demand any shifted bits outside Op1.
2317 if (DemandedBits.countl_zero() >= MaxShiftAmt) {
2318 EVT AmtVT = Op2.getValueType();
2319 SDValue NewAmt =
2320 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Op2,
2321 N2: TLO.DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
2322 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op1, N2: NewAmt);
2323 return TLO.CombineTo(O: Op, N: NewOp);
2324 }
2325 }
2326
2327 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2328 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
2329 if (SimplifyDemandedBits(Op: Op2, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2330 Depth: Depth + 1))
2331 return true;
2332 }
2333 break;
2334 }
2335 case ISD::ROTL:
2336 case ISD::ROTR: {
2337 SDValue Op0 = Op.getOperand(i: 0);
2338 SDValue Op1 = Op.getOperand(i: 1);
2339 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
2340
2341 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
2342 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1))
2343 return TLO.CombineTo(O: Op, N: Op0);
2344
2345 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op1, DemandedElts)) {
2346 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2347 unsigned RevAmt = BitWidth - Amt;
2348
2349 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
2350 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
2351 APInt Demanded0 = DemandedBits.rotr(rotateAmt: IsROTL ? Amt : RevAmt);
2352 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2353 Depth: Depth + 1))
2354 return true;
2355
2356 // rot*(x, 0) --> x
2357 if (Amt == 0)
2358 return TLO.CombineTo(O: Op, N: Op0);
2359
2360 // See if we don't demand either half of the rotated bits.
2361 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT)) &&
2362 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2363 Op1 = TLO.DAG.getConstant(Val: IsROTL ? Amt : RevAmt, DL: dl, VT: Op1.getValueType());
2364 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: Op1));
2365 }
2366 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT)) &&
2367 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2368 Op1 = TLO.DAG.getConstant(Val: IsROTL ? RevAmt : Amt, DL: dl, VT: Op1.getValueType());
2369 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2370 }
2371 }
2372
2373 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2374 if (isPowerOf2_32(Value: BitWidth)) {
2375 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
2376 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2377 Depth: Depth + 1))
2378 return true;
2379 }
2380 break;
2381 }
2382 case ISD::SMIN:
2383 case ISD::SMAX:
2384 case ISD::UMIN:
2385 case ISD::UMAX: {
2386 unsigned Opc = Op.getOpcode();
2387 SDValue Op0 = Op.getOperand(i: 0);
2388 SDValue Op1 = Op.getOperand(i: 1);
2389
2390 // If we're only demanding signbits, then we can simplify to OR/AND node.
2391 unsigned BitOp =
2392 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
2393 unsigned NumSignBits =
2394 std::min(a: TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1),
2395 b: TLO.DAG.ComputeNumSignBits(Op: Op1, DemandedElts, Depth: Depth + 1));
2396 unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
2397 if (NumSignBits >= NumDemandedUpperBits)
2398 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: BitOp, DL: SDLoc(Op), VT, N1: Op0, N2: Op1));
2399
2400 // Check if one arg is always less/greater than (or equal) to the other arg.
2401 KnownBits Known0 = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2402 KnownBits Known1 = TLO.DAG.computeKnownBits(Op: Op1, DemandedElts, Depth: Depth + 1);
2403 switch (Opc) {
2404 case ISD::SMIN:
2405 if (std::optional<bool> IsSLE = KnownBits::sle(LHS: Known0, RHS: Known1))
2406 return TLO.CombineTo(O: Op, N: *IsSLE ? Op0 : Op1);
2407 if (std::optional<bool> IsSLT = KnownBits::slt(LHS: Known0, RHS: Known1))
2408 return TLO.CombineTo(O: Op, N: *IsSLT ? Op0 : Op1);
2409 Known = KnownBits::smin(LHS: Known0, RHS: Known1);
2410 break;
2411 case ISD::SMAX:
2412 if (std::optional<bool> IsSGE = KnownBits::sge(LHS: Known0, RHS: Known1))
2413 return TLO.CombineTo(O: Op, N: *IsSGE ? Op0 : Op1);
2414 if (std::optional<bool> IsSGT = KnownBits::sgt(LHS: Known0, RHS: Known1))
2415 return TLO.CombineTo(O: Op, N: *IsSGT ? Op0 : Op1);
2416 Known = KnownBits::smax(LHS: Known0, RHS: Known1);
2417 break;
2418 case ISD::UMIN:
2419 if (std::optional<bool> IsULE = KnownBits::ule(LHS: Known0, RHS: Known1))
2420 return TLO.CombineTo(O: Op, N: *IsULE ? Op0 : Op1);
2421 if (std::optional<bool> IsULT = KnownBits::ult(LHS: Known0, RHS: Known1))
2422 return TLO.CombineTo(O: Op, N: *IsULT ? Op0 : Op1);
2423 Known = KnownBits::umin(LHS: Known0, RHS: Known1);
2424 break;
2425 case ISD::UMAX:
2426 if (std::optional<bool> IsUGE = KnownBits::uge(LHS: Known0, RHS: Known1))
2427 return TLO.CombineTo(O: Op, N: *IsUGE ? Op0 : Op1);
2428 if (std::optional<bool> IsUGT = KnownBits::ugt(LHS: Known0, RHS: Known1))
2429 return TLO.CombineTo(O: Op, N: *IsUGT ? Op0 : Op1);
2430 Known = KnownBits::umax(LHS: Known0, RHS: Known1);
2431 break;
2432 }
2433 break;
2434 }
2435 case ISD::BITREVERSE: {
2436 SDValue Src = Op.getOperand(i: 0);
2437 APInt DemandedSrcBits = DemandedBits.reverseBits();
2438 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2439 Depth: Depth + 1))
2440 return true;
2441 Known = Known2.reverseBits();
2442 break;
2443 }
2444 case ISD::BSWAP: {
2445 SDValue Src = Op.getOperand(i: 0);
2446
2447 // If the only bits demanded come from one byte of the bswap result,
2448 // just shift the input byte into position to eliminate the bswap.
2449 unsigned NLZ = DemandedBits.countl_zero();
2450 unsigned NTZ = DemandedBits.countr_zero();
2451
2452 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
2453 // we need all the bits down to bit 8. Likewise, round NLZ. If we
2454 // have 14 leading zeros, round to 8.
2455 NLZ = alignDown(Value: NLZ, Align: 8);
2456 NTZ = alignDown(Value: NTZ, Align: 8);
2457 // If we need exactly one byte, we can do this transformation.
2458 if (BitWidth - NLZ - NTZ == 8) {
2459 // Replace this with either a left or right shift to get the byte into
2460 // the right place.
2461 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
2462 if (!TLO.LegalOperations() || isOperationLegal(Op: ShiftOpcode, VT)) {
2463 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2464 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
2465 SDValue NewOp = TLO.DAG.getNode(Opcode: ShiftOpcode, DL: dl, VT, N1: Src, N2: ShAmt);
2466 return TLO.CombineTo(O: Op, N: NewOp);
2467 }
2468 }
2469
2470 APInt DemandedSrcBits = DemandedBits.byteSwap();
2471 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2472 Depth: Depth + 1))
2473 return true;
2474 Known = Known2.byteSwap();
2475 break;
2476 }
2477 case ISD::CTPOP: {
2478 // If only 1 bit is demanded, replace with PARITY as long as we're before
2479 // op legalization.
2480 // FIXME: Limit to scalars for now.
2481 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
2482 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT,
2483 Operand: Op.getOperand(i: 0)));
2484
2485 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2486 break;
2487 }
2488 case ISD::PDEP: {
2489 SDValue Op0 = Op.getOperand(i: 0);
2490 SDValue Op1 = Op.getOperand(i: 1);
2491
2492 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2493 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
2494
2495 // If the demanded bits has leading zeroes, we don't demand those from the
2496 // mask.
2497 if (SimplifyDemandedBits(Op: Op1, DemandedBits: LoMask, Known, TLO, Depth: Depth + 1))
2498 return true;
2499
2500 // The number of possible 1s in the mask determines the number of LSBs of
2501 // operand 0 used. Undemanded bits from the mask don't matter so filter
2502 // them before counting.
2503 KnownBits Known2;
2504 uint64_t Count = (~Known.Zero & LoMask).popcount();
2505 APInt DemandedMask(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: Count));
2506 if (SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, Known&: Known2, TLO, Depth: Depth + 1))
2507 return true;
2508
2509 // Zeroes are retained from the mask, but not ones.
2510 Known.One.clearAllBits();
2511 // The result will have at least as many trailing zeros as the non-mask
2512 // operand since bits can only map to the same or higher bit position.
2513 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
2514 break;
2515 }
2516 case ISD::SIGN_EXTEND_INREG: {
2517 SDValue Op0 = Op.getOperand(i: 0);
2518 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2519 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2520
2521 // If we only care about the highest bit, don't bother shifting right.
2522 if (DemandedBits.isSignMask()) {
2523 unsigned MinSignedBits =
2524 TLO.DAG.ComputeMaxSignificantBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2525 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2526 // However if the input is already sign extended we expect the sign
2527 // extension to be dropped altogether later and do not simplify.
2528 if (!AlreadySignExtended) {
2529 // Compute the correct shift amount type, which must be getShiftAmountTy
2530 // for scalar types after legalization.
2531 SDValue ShiftAmt =
2532 TLO.DAG.getShiftAmountConstant(Val: BitWidth - ExVTBits, VT, DL: dl);
2533 return TLO.CombineTo(O: Op,
2534 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: ShiftAmt));
2535 }
2536 }
2537
2538 // If none of the extended bits are demanded, eliminate the sextinreg.
2539 if (DemandedBits.getActiveBits() <= ExVTBits)
2540 return TLO.CombineTo(O: Op, N: Op0);
2541
2542 APInt InputDemandedBits = DemandedBits.getLoBits(numBits: ExVTBits);
2543
2544 // Since the sign extended bits are demanded, we know that the sign
2545 // bit is demanded.
2546 InputDemandedBits.setBit(ExVTBits - 1);
2547
2548 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InputDemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
2549 Depth: Depth + 1))
2550 return true;
2551
2552 // If the sign bit of the input is known set or clear, then we know the
2553 // top bits of the result.
2554
2555 // If the input sign bit is known zero, convert this into a zero extension.
2556 if (Known.Zero[ExVTBits - 1])
2557 return TLO.CombineTo(O: Op, N: TLO.DAG.getZeroExtendInReg(Op: Op0, DL: dl, VT: ExVT));
2558
2559 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ExVTBits);
2560 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
2561 Known.One.setBitsFrom(ExVTBits);
2562 Known.Zero &= Mask;
2563 } else { // Input sign bit unknown
2564 Known.Zero &= Mask;
2565 Known.One &= Mask;
2566 }
2567 break;
2568 }
2569 case ISD::BUILD_PAIR: {
2570 EVT HalfVT = Op.getOperand(i: 0).getValueType();
2571 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
2572
2573 APInt MaskLo = DemandedBits.getLoBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2574 APInt MaskHi = DemandedBits.getHiBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2575
2576 KnownBits KnownLo, KnownHi;
2577
2578 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: MaskLo, Known&: KnownLo, TLO, Depth: Depth + 1))
2579 return true;
2580
2581 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: MaskHi, Known&: KnownHi, TLO, Depth: Depth + 1))
2582 return true;
2583
2584 Known = KnownHi.concat(Lo: KnownLo);
2585 break;
2586 }
2587 case ISD::ZERO_EXTEND_VECTOR_INREG:
2588 if (VT.isScalableVector())
2589 return false;
2590 [[fallthrough]];
2591 case ISD::ZERO_EXTEND: {
2592 SDValue Src = Op.getOperand(i: 0);
2593 EVT SrcVT = Src.getValueType();
2594 unsigned InBits = SrcVT.getScalarSizeInBits();
2595 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2596 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
2597
2598 // If none of the top bits are demanded, convert this into an any_extend.
2599 if (DemandedBits.getActiveBits() <= InBits) {
2600 // If we only need the non-extended bits of the bottom element
2601 // then we can just bitcast to the result.
2602 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2603 VT.getSizeInBits() == SrcVT.getSizeInBits())
2604 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2605
2606 unsigned Opc =
2607 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2608 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2609 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2610 }
2611
2612 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2613 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2614 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2615 Depth: Depth + 1)) {
2616 Op->dropFlags(Mask: SDNodeFlags::NonNeg);
2617 return true;
2618 }
2619 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2620 Known = Known.zext(BitWidth);
2621
2622 // Attempt to avoid multi-use ops if we don't need anything from them.
2623 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2624 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2625 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2626 break;
2627 }
2628 case ISD::SIGN_EXTEND_VECTOR_INREG:
2629 if (VT.isScalableVector())
2630 return false;
2631 [[fallthrough]];
2632 case ISD::SIGN_EXTEND: {
2633 SDValue Src = Op.getOperand(i: 0);
2634 EVT SrcVT = Src.getValueType();
2635 unsigned InBits = SrcVT.getScalarSizeInBits();
2636 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2637 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
2638
2639 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2640 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2641
2642 // Since some of the sign extended bits are demanded, we know that the sign
2643 // bit is demanded.
2644 InDemandedBits.setBit(InBits - 1);
2645
2646 // If none of the top bits are demanded, convert this into an any_extend.
2647 if (DemandedBits.getActiveBits() <= InBits) {
2648 // If we only need the non-extended bits of the bottom element
2649 // then we can just bitcast to the result.
2650 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2651 VT.getSizeInBits() == SrcVT.getSizeInBits())
2652 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2653
2654 // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
2655 if (getBooleanContents(Type: VT) != ZeroOrNegativeOneBooleanContent ||
2656 TLO.DAG.ComputeNumSignBits(Op: Src, DemandedElts: InDemandedElts, Depth: Depth + 1) !=
2657 InBits) {
2658 unsigned Opc =
2659 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2660 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2661 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2662 }
2663 }
2664
2665 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2666 Depth: Depth + 1))
2667 return true;
2668 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2669
2670 // If the sign bit is known one, the top bits match.
2671 Known = Known.sext(BitWidth);
2672
2673 // If the sign bit is known zero, convert this to a zero extend.
2674 if (Known.isNonNegative()) {
2675 unsigned Opc =
2676 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
2677 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT)) {
2678 SDNodeFlags Flags;
2679 if (!IsVecInReg)
2680 Flags |= SDNodeFlags::NonNeg;
2681 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src, Flags));
2682 }
2683 }
2684
2685 // Attempt to avoid multi-use ops if we don't need anything from them.
2686 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2687 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2688 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2689 break;
2690 }
2691 case ISD::ANY_EXTEND_VECTOR_INREG:
2692 if (VT.isScalableVector())
2693 return false;
2694 [[fallthrough]];
2695 case ISD::ANY_EXTEND: {
2696 SDValue Src = Op.getOperand(i: 0);
2697 EVT SrcVT = Src.getValueType();
2698 unsigned InBits = SrcVT.getScalarSizeInBits();
2699 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2700 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2701
2702 // If we only need the bottom element then we can just bitcast.
2703 // TODO: Handle ANY_EXTEND?
2704 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2705 VT.getSizeInBits() == SrcVT.getSizeInBits())
2706 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2707
2708 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2709 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2710 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2711 Depth: Depth + 1))
2712 return true;
2713 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2714 Known = Known.anyext(BitWidth);
2715
2716 // Attempt to avoid multi-use ops if we don't need anything from them.
2717 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2718 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2719 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2720 break;
2721 }
2722 case ISD::TRUNCATE: {
2723 SDValue Src = Op.getOperand(i: 0);
2724
2725 // Simplify the input, using demanded bit information, and compute the known
2726 // zero/one bits live out.
2727 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2728 APInt TruncMask = DemandedBits.zext(width: OperandBitWidth);
2729 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: TruncMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2730 Depth: Depth + 1)) {
2731 // Disable the nsw and nuw flags. We can no longer guarantee that we
2732 // won't wrap after simplification.
2733 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2734 return true;
2735 }
2736 Known = Known.trunc(BitWidth);
2737
2738 // Attempt to avoid multi-use ops if we don't need anything from them.
2739 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2740 Op: Src, DemandedBits: TruncMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2741 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: NewSrc));
2742
2743 // If the input is only used by this truncate, see if we can shrink it based
2744 // on the known demanded bits.
2745 switch (Src.getOpcode()) {
2746 default:
2747 break;
2748 case ISD::SRL:
2749 // Shrink SRL by a constant if none of the high bits shifted in are
2750 // demanded.
2751 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2752 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2753 // undesirable.
2754 break;
2755
2756 if (Src.getNode()->hasOneUse()) {
2757 if (isTruncateFree(Val: Src, VT2: VT) &&
2758 !isTruncateFree(FromVT: Src.getValueType(), ToVT: VT)) {
2759 // If truncate is only free at trunc(srl), do not turn it into
2760 // srl(trunc). The check is done by first check the truncate is free
2761 // at Src's opcode(srl), then check the truncate is not done by
2762 // referencing sub-register. In test, if both trunc(srl) and
2763 // srl(trunc)'s trunc are free, srl(trunc) performs better. If only
2764 // trunc(srl)'s trunc is free, trunc(srl) is better.
2765 break;
2766 }
2767
2768 std::optional<unsigned> ShAmtC =
2769 TLO.DAG.getValidShiftAmount(V: Src, DemandedElts, Depth: Depth + 2);
2770 if (!ShAmtC || *ShAmtC >= BitWidth)
2771 break;
2772 unsigned ShVal = *ShAmtC;
2773
2774 APInt HighBits =
2775 APInt::getHighBitsSet(numBits: OperandBitWidth, hiBitsSet: OperandBitWidth - BitWidth);
2776 HighBits.lshrInPlace(ShiftAmt: ShVal);
2777 HighBits = HighBits.trunc(width: BitWidth);
2778 if (!(HighBits & DemandedBits)) {
2779 // None of the shifted in bits are needed. Add a truncate of the
2780 // shift input, then shift it.
2781 SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
2782 SDValue NewTrunc =
2783 TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Src.getOperand(i: 0));
2784 return TLO.CombineTo(
2785 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NewTrunc, N2: NewShAmt));
2786 }
2787 }
2788 break;
2789 }
2790
2791 break;
2792 }
2793 case ISD::AssertZext: {
2794 // AssertZext demands all of the high bits, plus any of the low bits
2795 // demanded by its users.
2796 EVT ZVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2797 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ZVT.getSizeInBits());
2798 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: ~InMask | DemandedBits, Known,
2799 TLO, Depth: Depth + 1))
2800 return true;
2801
2802 Known.Zero |= ~InMask;
2803 Known.One &= (~Known.Zero);
2804 break;
2805 }
2806 case ISD::EXTRACT_VECTOR_ELT: {
2807 SDValue Src = Op.getOperand(i: 0);
2808 SDValue Idx = Op.getOperand(i: 1);
2809 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2810 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2811
2812 if (SrcEltCnt.isScalable())
2813 return false;
2814
2815 // Demand the bits from every vector element without a constant index.
2816 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2817 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
2818 if (auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx))
2819 if (CIdx->getAPIntValue().ult(RHS: NumSrcElts))
2820 DemandedSrcElts = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: CIdx->getZExtValue());
2821
2822 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2823 // anything about the extended bits.
2824 APInt DemandedSrcBits = DemandedBits;
2825 if (BitWidth > EltBitWidth)
2826 DemandedSrcBits = DemandedSrcBits.trunc(width: EltBitWidth);
2827
2828 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts, Known&: Known2, TLO,
2829 Depth: Depth + 1))
2830 return true;
2831
2832 // Attempt to avoid multi-use ops if we don't need anything from them.
2833 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2834 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2835 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
2836 SDValue NewOp =
2837 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc, N2: Idx);
2838 return TLO.CombineTo(O: Op, N: NewOp);
2839 }
2840 }
2841
2842 Known = Known2;
2843 if (BitWidth > EltBitWidth)
2844 Known = Known.anyext(BitWidth);
2845 break;
2846 }
2847 case ISD::BITCAST: {
2848 if (VT.isScalableVector())
2849 return false;
2850 SDValue Src = Op.getOperand(i: 0);
2851 EVT SrcVT = Src.getValueType();
2852 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2853
2854 // If this is an FP->Int bitcast and if the sign bit is the only
2855 // thing demanded, turn this into a FGETSIGN.
2856 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2857 DemandedBits == APInt::getSignMask(BitWidth: Op.getValueSizeInBits()) &&
2858 SrcVT.isFloatingPoint()) {
2859 if (isOperationLegalOrCustom(Op: ISD::FGETSIGN, VT)) {
2860 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2861 // place. We expect the SHL to be eliminated by other optimizations.
2862 SDValue Sign = TLO.DAG.getNode(Opcode: ISD::FGETSIGN, DL: dl, VT, Operand: Src);
2863 unsigned ShVal = Op.getValueSizeInBits() - 1;
2864 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
2865 return TLO.CombineTo(O: Op,
2866 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Sign, N2: ShAmt));
2867 }
2868 }
2869
2870 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2871 // Demand the elt/bit if any of the original elts/bits are demanded.
2872 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
2873 unsigned Scale = BitWidth / NumSrcEltBits;
2874 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2875 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
2876 for (unsigned i = 0; i != Scale; ++i) {
2877 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2878 unsigned BitOffset = EltOffset * NumSrcEltBits;
2879 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
2880 }
2881 // Recursive calls below may turn not demanded elements into poison, so we
2882 // need to demand all smaller source elements that maps to a demanded
2883 // destination element.
2884 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
2885
2886 APInt KnownSrcUndef, KnownSrcZero;
2887 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
2888 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
2889 return true;
2890
2891 KnownBits KnownSrcBits;
2892 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
2893 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
2894 return true;
2895 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
2896 // TODO - bigendian once we have test coverage.
2897 unsigned Scale = NumSrcEltBits / BitWidth;
2898 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2899 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
2900 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
2901 for (unsigned i = 0; i != NumElts; ++i)
2902 if (DemandedElts[i]) {
2903 unsigned Offset = (i % Scale) * BitWidth;
2904 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
2905 DemandedSrcElts.setBit(i / Scale);
2906 }
2907
2908 if (SrcVT.isVector()) {
2909 APInt KnownSrcUndef, KnownSrcZero;
2910 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
2911 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
2912 return true;
2913 }
2914
2915 KnownBits KnownSrcBits;
2916 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
2917 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
2918 return true;
2919
2920 // Attempt to avoid multi-use ops if we don't need anything from them.
2921 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2922 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2923 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
2924 SDValue NewOp = TLO.DAG.getBitcast(VT, V: DemandedSrc);
2925 return TLO.CombineTo(O: Op, N: NewOp);
2926 }
2927 }
2928 }
2929
2930 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2931 // recursive call where Known may be useful to the caller.
2932 if (Depth > 0) {
2933 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2934 return false;
2935 }
2936 break;
2937 }
2938 case ISD::MUL:
2939 if (DemandedBits.isPowerOf2()) {
2940 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
2941 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
2942 // odd (has LSB set), then the left-shifted low bit of X is the answer.
2943 unsigned CTZ = DemandedBits.countr_zero();
2944 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
2945 if (C && C->getAPIntValue().countr_zero() == CTZ) {
2946 SDValue AmtC = TLO.DAG.getShiftAmountConstant(Val: CTZ, VT, DL: dl);
2947 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op.getOperand(i: 0), N2: AmtC);
2948 return TLO.CombineTo(O: Op, N: Shl);
2949 }
2950 }
2951 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
2952 // X * X is odd iff X is odd.
2953 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
2954 if (Op.getOperand(i: 0) == Op.getOperand(i: 1) && DemandedBits.ult(RHS: 4)) {
2955 SDValue One = TLO.DAG.getConstant(Val: 1, DL: dl, VT);
2956 SDValue And1 = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op.getOperand(i: 0), N2: One);
2957 return TLO.CombineTo(O: Op, N: And1);
2958 }
2959 [[fallthrough]];
2960 case ISD::PTRADD:
2961 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
2962 break;
2963 // PTRADD behaves like ADD if pointers are represented as integers.
2964 [[fallthrough]];
2965 case ISD::ADD:
2966 case ISD::SUB: {
2967 // Add, Sub, and Mul don't demand any bits in positions beyond that
2968 // of the highest bit demanded of them.
2969 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
2970 SDNodeFlags Flags = Op.getNode()->getFlags();
2971 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2972 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
2973 KnownBits KnownOp0, KnownOp1;
2974 auto GetDemandedBitsLHSMask = [&](APInt Demanded,
2975 const KnownBits &KnownRHS) {
2976 if (Op.getOpcode() == ISD::MUL)
2977 Demanded.clearHighBits(hiBits: KnownRHS.countMinTrailingZeros());
2978 return Demanded;
2979 };
2980 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: LoMask, OriginalDemandedElts: DemandedElts, Known&: KnownOp1, TLO,
2981 Depth: Depth + 1) ||
2982 SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: GetDemandedBitsLHSMask(LoMask, KnownOp1),
2983 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1) ||
2984 // See if the operation should be performed at a smaller bit width.
2985 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2986 // Disable the nsw and nuw flags. We can no longer guarantee that we
2987 // won't wrap after simplification.
2988 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2989 return true;
2990 }
2991
2992 // neg x with only low bit demanded is simply x.
2993 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
2994 isNullConstant(V: Op0))
2995 return TLO.CombineTo(O: Op, N: Op1);
2996
2997 // Attempt to avoid multi-use ops if we don't need anything from them.
2998 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2999 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
3000 Op: Op0, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3001 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
3002 Op: Op1, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
3003 if (DemandedOp0 || DemandedOp1) {
3004 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
3005 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
3006 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1,
3007 Flags: Flags & ~SDNodeFlags::NoWrap);
3008 return TLO.CombineTo(O: Op, N: NewOp);
3009 }
3010 }
3011
3012 // If we have a constant operand, we may be able to turn it into -1 if we
3013 // do not demand the high bits. This can make the constant smaller to
3014 // encode, allow more general folding, or match specialized instruction
3015 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
3016 // is probably not useful (and could be detrimental).
3017 ConstantSDNode *C = isConstOrConstSplat(N: Op1);
3018 APInt HighMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: DemandedBitsLZ);
3019 if (C && !C->isAllOnes() && !C->isOne() &&
3020 (C->getAPIntValue() | HighMask).isAllOnes()) {
3021 SDValue Neg1 = TLO.DAG.getAllOnesConstant(DL: dl, VT);
3022 // Disable the nsw and nuw flags. We can no longer guarantee that we
3023 // won't wrap after simplification.
3024 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Neg1,
3025 Flags: Flags & ~SDNodeFlags::NoWrap);
3026 return TLO.CombineTo(O: Op, N: NewOp);
3027 }
3028
3029 // Match a multiply with a disguised negated-power-of-2 and convert to a
3030 // an equivalent shift-left amount.
3031 // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3032 auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
3033 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
3034 return 0;
3035
3036 // Don't touch opaque constants. Also, ignore zero and power-of-2
3037 // multiplies. Those will get folded later.
3038 ConstantSDNode *MulC = isConstOrConstSplat(N: Mul.getOperand(i: 1));
3039 if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
3040 !MulC->getAPIntValue().isPowerOf2()) {
3041 APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
3042 if (UnmaskedC.isNegatedPowerOf2())
3043 return (-UnmaskedC).logBase2();
3044 }
3045 return 0;
3046 };
3047
3048 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
3049 unsigned ShlAmt) {
3050 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(Val: ShlAmt, VT, DL: dl);
3051 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: X, N2: ShlAmtC);
3052 SDValue Res = TLO.DAG.getNode(Opcode: NT, DL: dl, VT, N1: Y, N2: Shl);
3053 return TLO.CombineTo(O: Op, N: Res);
3054 };
3055
3056 if (isOperationLegalOrCustom(Op: ISD::SHL, VT)) {
3057 if (Op.getOpcode() == ISD::ADD) {
3058 // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3059 if (unsigned ShAmt = getShiftLeftAmt(Op0))
3060 return foldMul(ISD::SUB, Op0.getOperand(i: 0), Op1, ShAmt);
3061 // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
3062 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3063 return foldMul(ISD::SUB, Op1.getOperand(i: 0), Op0, ShAmt);
3064 }
3065 if (Op.getOpcode() == ISD::SUB) {
3066 // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
3067 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3068 return foldMul(ISD::ADD, Op1.getOperand(i: 0), Op0, ShAmt);
3069 }
3070 }
3071
3072 if (Op.getOpcode() == ISD::MUL) {
3073 Known = KnownBits::mul(LHS: KnownOp0, RHS: KnownOp1);
3074 } else { // Op.getOpcode() is either ISD::ADD, ISD::PTRADD, or ISD::SUB.
3075 Known = KnownBits::computeForAddSub(
3076 Add: Op.getOpcode() != ISD::SUB, NSW: Flags.hasNoSignedWrap(),
3077 NUW: Flags.hasNoUnsignedWrap(), LHS: KnownOp0, RHS: KnownOp1);
3078 }
3079 break;
3080 }
3081 case ISD::FABS: {
3082 SDValue Op0 = Op.getOperand(i: 0);
3083 APInt SignMask = APInt::getSignMask(BitWidth);
3084
3085 if (!DemandedBits.intersects(RHS: SignMask))
3086 return TLO.CombineTo(O: Op, N: Op0);
3087
3088 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3089 Depth: Depth + 1))
3090 return true;
3091
3092 if (Known.isNonNegative())
3093 return TLO.CombineTo(O: Op, N: Op0);
3094 if (Known.isNegative())
3095 return TLO.CombineTo(
3096 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3097
3098 Known.Zero |= SignMask;
3099 Known.One &= ~SignMask;
3100
3101 break;
3102 }
3103 case ISD::FCOPYSIGN: {
3104 SDValue Op0 = Op.getOperand(i: 0);
3105 SDValue Op1 = Op.getOperand(i: 1);
3106
3107 unsigned BitWidth0 = Op0.getScalarValueSizeInBits();
3108 unsigned BitWidth1 = Op1.getScalarValueSizeInBits();
3109 APInt SignMask0 = APInt::getSignMask(BitWidth: BitWidth0);
3110 APInt SignMask1 = APInt::getSignMask(BitWidth: BitWidth1);
3111
3112 if (!DemandedBits.intersects(RHS: SignMask0))
3113 return TLO.CombineTo(O: Op, N: Op0);
3114
3115 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~SignMask0 & DemandedBits, OriginalDemandedElts: DemandedElts,
3116 Known, TLO, Depth: Depth + 1) ||
3117 SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: SignMask1, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
3118 Depth: Depth + 1))
3119 return true;
3120
3121 if (Known2.isNonNegative())
3122 return TLO.CombineTo(
3123 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3124
3125 if (Known2.isNegative())
3126 return TLO.CombineTo(
3127 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT,
3128 Operand: TLO.DAG.getNode(Opcode: ISD::FABS, DL: SDLoc(Op0), VT, Operand: Op0)));
3129
3130 Known.Zero &= ~SignMask0;
3131 Known.One &= ~SignMask0;
3132 break;
3133 }
3134 case ISD::FNEG: {
3135 SDValue Op0 = Op.getOperand(i: 0);
3136 APInt SignMask = APInt::getSignMask(BitWidth);
3137
3138 if (!DemandedBits.intersects(RHS: SignMask))
3139 return TLO.CombineTo(O: Op, N: Op0);
3140
3141 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3142 Depth: Depth + 1))
3143 return true;
3144
3145 if (!Known.isSignUnknown()) {
3146 Known.Zero ^= SignMask;
3147 Known.One ^= SignMask;
3148 }
3149
3150 break;
3151 }
3152 default:
3153 // We also ask the target about intrinsics (which could be specific to it).
3154 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3155 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
3156 // TODO: Probably okay to remove after audit; here to reduce change size
3157 // in initial enablement patch for scalable vectors
3158 if (Op.getValueType().isScalableVector())
3159 break;
3160 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
3161 Known, TLO, Depth))
3162 return true;
3163 break;
3164 }
3165
3166 // Just use computeKnownBits to compute output bits.
3167 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3168 break;
3169 }
3170
3171 // If we know the value of all of the demanded bits, return this as a
3172 // constant.
3173 if (!isTargetCanonicalConstantNode(Op) &&
3174 DemandedBits.isSubsetOf(RHS: Known.Zero | Known.One)) {
3175 // Avoid folding to a constant if any OpaqueConstant is involved.
3176 if (llvm::any_of(Range: Op->ops(), P: [](SDValue V) {
3177 auto *C = dyn_cast<ConstantSDNode>(Val&: V);
3178 return C && C->isOpaque();
3179 }))
3180 return false;
3181 if (VT.isInteger())
3182 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: Known.One, DL: dl, VT));
3183 if (VT.isFloatingPoint())
3184 return TLO.CombineTo(
3185 O: Op, N: TLO.DAG.getConstantFP(Val: APFloat(VT.getFltSemantics(), Known.One),
3186 DL: dl, VT));
3187 }
3188
3189 // A multi use 'all demanded elts' simplify failed to find any knownbits.
3190 // Try again just for the original demanded elts.
3191 // Ensure we do this AFTER constant folding above.
3192 if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
3193 Known = TLO.DAG.computeKnownBits(Op, DemandedElts: OriginalDemandedElts, Depth);
3194
3195 return false;
3196}
3197
3198bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
3199 const APInt &DemandedElts,
3200 DAGCombinerInfo &DCI) const {
3201 SelectionDAG &DAG = DCI.DAG;
3202 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
3203 !DCI.isBeforeLegalizeOps());
3204
3205 APInt KnownUndef, KnownZero;
3206 bool Simplified =
3207 SimplifyDemandedVectorElts(Op, DemandedEltMask: DemandedElts, KnownUndef, KnownZero, TLO);
3208 if (Simplified) {
3209 DCI.AddToWorklist(N: Op.getNode());
3210 DCI.CommitTargetLoweringOpt(TLO);
3211 }
3212
3213 return Simplified;
3214}
3215
3216/// Given a vector binary operation and known undefined elements for each input
3217/// operand, compute whether each element of the output is undefined.
3218static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
3219 const APInt &UndefOp0,
3220 const APInt &UndefOp1) {
3221 EVT VT = BO.getValueType();
3222 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
3223 "Vector binop only");
3224
3225 EVT EltVT = VT.getVectorElementType();
3226 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
3227 assert(UndefOp0.getBitWidth() == NumElts &&
3228 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
3229
3230 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
3231 const APInt &UndefVals) {
3232 if (UndefVals[Index])
3233 return DAG.getUNDEF(VT: EltVT);
3234
3235 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
3236 // Try hard to make sure that the getNode() call is not creating temporary
3237 // nodes. Ignore opaque integers because they do not constant fold.
3238 SDValue Elt = BV->getOperand(Num: Index);
3239 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
3240 if (isa<ConstantFPSDNode>(Val: Elt) || Elt.isUndef() || (C && !C->isOpaque()))
3241 return Elt;
3242 }
3243
3244 return SDValue();
3245 };
3246
3247 APInt KnownUndef = APInt::getZero(numBits: NumElts);
3248 for (unsigned i = 0; i != NumElts; ++i) {
3249 // If both inputs for this element are either constant or undef and match
3250 // the element type, compute the constant/undef result for this element of
3251 // the vector.
3252 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
3253 // not handle FP constants. The code within getNode() should be refactored
3254 // to avoid the danger of creating a bogus temporary node here.
3255 SDValue C0 = getUndefOrConstantElt(BO.getOperand(i: 0), i, UndefOp0);
3256 SDValue C1 = getUndefOrConstantElt(BO.getOperand(i: 1), i, UndefOp1);
3257 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
3258 if (DAG.getNode(Opcode: BO.getOpcode(), DL: SDLoc(BO), VT: EltVT, N1: C0, N2: C1).isUndef())
3259 KnownUndef.setBit(i);
3260 }
3261 return KnownUndef;
3262}
3263
3264bool TargetLowering::SimplifyDemandedVectorElts(
3265 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
3266 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
3267 bool AssumeSingleUse) const {
3268 EVT VT = Op.getValueType();
3269 unsigned Opcode = Op.getOpcode();
3270 APInt DemandedElts = OriginalDemandedElts;
3271 unsigned NumElts = DemandedElts.getBitWidth();
3272 assert(VT.isVector() && "Expected vector op");
3273
3274 KnownUndef = KnownZero = APInt::getZero(numBits: NumElts);
3275
3276 if (!shouldSimplifyDemandedVectorElts(Op, TLO))
3277 return false;
3278
3279 // TODO: For now we assume we know nothing about scalable vectors.
3280 if (VT.isScalableVector())
3281 return false;
3282
3283 assert(VT.getVectorNumElements() == NumElts &&
3284 "Mask size mismatches value type element count!");
3285
3286 // Undef operand.
3287 if (Op.isUndef()) {
3288 KnownUndef.setAllBits();
3289 return false;
3290 }
3291
3292 // If Op has other users, assume that all elements are needed.
3293 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
3294 DemandedElts.setAllBits();
3295
3296 // Not demanding any elements from Op.
3297 if (DemandedElts == 0) {
3298 KnownUndef.setAllBits();
3299 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3300 }
3301
3302 // Limit search depth.
3303 if (Depth >= SelectionDAG::MaxRecursionDepth)
3304 return false;
3305
3306 SDLoc DL(Op);
3307 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3308 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
3309
3310 auto TryShrinkBinOp = [&](SDValue Op0, SDValue Op1) {
3311 unsigned ShrunkSize = getPreferredShrunkVectorSizeInBits(Op, DemandedElts);
3312 if (!ShrunkSize)
3313 return false;
3314
3315 assert(ShrunkSize % EltSizeInBits == 0 &&
3316 "Shrunk size not a multiple of element size");
3317 assert(ShrunkSize < VT.getSizeInBits() &&
3318 "Shrunk size must be < original vector size");
3319 assert(ShrunkSize >= EltSizeInBits * DemandedElts.getActiveBits() &&
3320 "Shrunk size must be >= demanded size");
3321
3322 EVT ShrunkVT = VT.changeVectorElementCount(
3323 Context&: *TLO.DAG.getContext(),
3324 EC: ElementCount::getFixed(MinVal: ShrunkSize / EltSizeInBits));
3325 Op0 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op0, Idx: 0);
3326 Op1 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op1, Idx: 0);
3327 SDValue NewOp =
3328 TLO.DAG.getNode(Opcode, DL, VT: ShrunkVT, N1: Op0, N2: Op1, Flags: Op->getFlags());
3329 return TLO.CombineTo(
3330 O: Op, N: TLO.DAG.getInsertSubvector(DL, Vec: TLO.DAG.getUNDEF(VT), SubVec: NewOp, Idx: 0));
3331 };
3332
3333 // Helper for demanding the specified elements and all the bits of both binary
3334 // operands.
3335 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
3336 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op: Op0, DemandedElts,
3337 DAG&: TLO.DAG, Depth: Depth + 1);
3338 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op: Op1, DemandedElts,
3339 DAG&: TLO.DAG, Depth: Depth + 1);
3340 if (NewOp0 || NewOp1) {
3341 SDValue NewOp =
3342 TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, N1: NewOp0 ? NewOp0 : Op0,
3343 N2: NewOp1 ? NewOp1 : Op1, Flags: Op->getFlags());
3344 return TLO.CombineTo(O: Op, N: NewOp);
3345 }
3346
3347 if (TryShrinkBinOp(Op0, Op1))
3348 return true;
3349
3350 return false;
3351 };
3352
3353 switch (Opcode) {
3354 case ISD::SCALAR_TO_VECTOR: {
3355 if (!DemandedElts[0]) {
3356 KnownUndef.setAllBits();
3357 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3358 }
3359 KnownUndef.setHighBits(NumElts - 1);
3360 break;
3361 }
3362 case ISD::BITCAST: {
3363 SDValue Src = Op.getOperand(i: 0);
3364 EVT SrcVT = Src.getValueType();
3365
3366 if (!SrcVT.isVector()) {
3367 // TODO - bigendian once we have test coverage.
3368 if (IsLE) {
3369 APInt DemandedSrcBits = APInt::getZero(numBits: SrcVT.getSizeInBits());
3370 unsigned EltSize = VT.getScalarSizeInBits();
3371 for (unsigned I = 0; I != NumElts; ++I) {
3372 if (DemandedElts[I]) {
3373 unsigned Offset = I * EltSize;
3374 DemandedSrcBits.setBits(loBit: Offset, hiBit: Offset + EltSize);
3375 }
3376 }
3377 KnownBits Known;
3378 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, Known, TLO, Depth: Depth + 1))
3379 return true;
3380 }
3381 break;
3382 }
3383
3384 // Fast handling of 'identity' bitcasts.
3385 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3386 if (NumSrcElts == NumElts)
3387 return SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedElts, KnownUndef,
3388 KnownZero, TLO, Depth: Depth + 1);
3389
3390 APInt SrcDemandedElts, SrcZero, SrcUndef;
3391
3392 // Bitcast from 'large element' src vector to 'small element' vector, we
3393 // must demand a source element if any DemandedElt maps to it.
3394 if ((NumElts % NumSrcElts) == 0) {
3395 unsigned Scale = NumElts / NumSrcElts;
3396 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3397 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3398 TLO, Depth: Depth + 1))
3399 return true;
3400
3401 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
3402 // of the large element.
3403 // TODO - bigendian once we have test coverage.
3404 if (IsLE) {
3405 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
3406 APInt SrcDemandedBits = APInt::getZero(numBits: SrcEltSizeInBits);
3407 for (unsigned i = 0; i != NumElts; ++i)
3408 if (DemandedElts[i]) {
3409 unsigned Ofs = (i % Scale) * EltSizeInBits;
3410 SrcDemandedBits.setBits(loBit: Ofs, hiBit: Ofs + EltSizeInBits);
3411 }
3412
3413 KnownBits Known;
3414 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: SrcDemandedBits, OriginalDemandedElts: SrcDemandedElts, Known,
3415 TLO, Depth: Depth + 1))
3416 return true;
3417
3418 // The bitcast has split each wide element into a number of
3419 // narrow subelements. We have just computed the Known bits
3420 // for wide elements. See if element splitting results in
3421 // some subelements being zero. Only for demanded elements!
3422 for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3423 if (!Known.Zero.extractBits(numBits: EltSizeInBits, bitPosition: SubElt * EltSizeInBits)
3424 .isAllOnes())
3425 continue;
3426 for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3427 unsigned Elt = Scale * SrcElt + SubElt;
3428 // A wholly-undef source lane is reported as undef below; don't also
3429 // flag it as zero, keeping the undef and zero sets disjoint.
3430 if (DemandedElts[Elt] && !SrcUndef[SrcElt])
3431 KnownZero.setBit(Elt);
3432 }
3433 }
3434 }
3435
3436 // If the src element is zero/undef then all the output elements will be -
3437 // only demanded elements are guaranteed to be correct.
3438 for (unsigned i = 0; i != NumSrcElts; ++i) {
3439 if (SrcDemandedElts[i]) {
3440 if (SrcZero[i])
3441 KnownZero.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3442 if (SrcUndef[i])
3443 KnownUndef.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3444 }
3445 }
3446 }
3447
3448 // Bitcast from 'small element' src vector to 'large element' vector, we
3449 // demand all smaller source elements covered by the larger demanded element
3450 // of this vector.
3451 if ((NumSrcElts % NumElts) == 0) {
3452 unsigned Scale = NumSrcElts / NumElts;
3453 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3454 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3455 TLO, Depth: Depth + 1))
3456 return true;
3457
3458 // If all the src elements covering an output element are zero/undef, then
3459 // the output element will be as well, assuming it was demanded.
3460 for (unsigned i = 0; i != NumElts; ++i) {
3461 if (DemandedElts[i]) {
3462 if (SrcZero.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3463 KnownZero.setBit(i);
3464 if (SrcUndef.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3465 KnownUndef.setBit(i);
3466 }
3467 }
3468 }
3469 break;
3470 }
3471 case ISD::FREEZE: {
3472 SDValue N0 = Op.getOperand(i: 0);
3473 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
3474 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
3475 return TLO.CombineTo(O: Op, N: N0);
3476
3477 // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE
3478 // freeze(op(x, ...)) -> op(freeze(x), ...).
3479 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1)
3480 return TLO.CombineTo(
3481 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT,
3482 Operand: TLO.DAG.getFreeze(V: N0.getOperand(i: 0))));
3483 break;
3484 }
3485 case ISD::BUILD_VECTOR: {
3486 // Check all elements and simplify any unused elements with UNDEF.
3487 if (!DemandedElts.isAllOnes()) {
3488 // Don't simplify BROADCASTS.
3489 if (llvm::any_of(Range: Op->op_values(),
3490 P: [&](SDValue Elt) { return Op.getOperand(i: 0) != Elt; })) {
3491 SmallVector<SDValue, 32> Ops(Op->ops());
3492 bool Updated = false;
3493 for (unsigned i = 0; i != NumElts; ++i) {
3494 if (!DemandedElts[i] && !Ops[i].isUndef()) {
3495 Ops[i] = TLO.DAG.getUNDEF(VT: Ops[0].getValueType());
3496 KnownUndef.setBit(i);
3497 Updated = true;
3498 }
3499 }
3500 if (Updated)
3501 return TLO.CombineTo(O: Op, N: TLO.DAG.getBuildVector(VT, DL, Ops));
3502 }
3503 }
3504 for (unsigned i = 0; i != NumElts; ++i) {
3505 SDValue SrcOp = Op.getOperand(i);
3506 if (SrcOp.isUndef()) {
3507 KnownUndef.setBit(i);
3508 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
3509 (isNullConstant(V: SrcOp) || isNullFPConstant(V: SrcOp))) {
3510 KnownZero.setBit(i);
3511 }
3512 }
3513 break;
3514 }
3515 case ISD::CONCAT_VECTORS: {
3516 EVT SubVT = Op.getOperand(i: 0).getValueType();
3517 unsigned NumSubVecs = Op.getNumOperands();
3518 unsigned NumSubElts = SubVT.getVectorNumElements();
3519 for (unsigned i = 0; i != NumSubVecs; ++i) {
3520 SDValue SubOp = Op.getOperand(i);
3521 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3522 APInt SubUndef, SubZero;
3523 if (SimplifyDemandedVectorElts(Op: SubOp, OriginalDemandedElts: SubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3524 Depth: Depth + 1))
3525 return true;
3526 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: i * NumSubElts);
3527 KnownZero.insertBits(SubBits: SubZero, bitPosition: i * NumSubElts);
3528 }
3529
3530 // Attempt to avoid multi-use ops if we don't need anything from them.
3531 if (!DemandedElts.isAllOnes()) {
3532 bool FoundNewSub = false;
3533 SmallVector<SDValue, 2> DemandedSubOps;
3534 for (unsigned i = 0; i != NumSubVecs; ++i) {
3535 SDValue SubOp = Op.getOperand(i);
3536 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3537 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3538 Op: SubOp, DemandedElts: SubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3539 DemandedSubOps.push_back(Elt: NewSubOp ? NewSubOp : SubOp);
3540 FoundNewSub = NewSubOp ? true : FoundNewSub;
3541 }
3542 if (FoundNewSub) {
3543 SDValue NewOp =
3544 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, Ops: DemandedSubOps);
3545 return TLO.CombineTo(O: Op, N: NewOp);
3546 }
3547 }
3548 break;
3549 }
3550 case ISD::INSERT_SUBVECTOR: {
3551 // Demand any elements from the subvector and the remainder from the src it
3552 // is inserted into.
3553 SDValue Src = Op.getOperand(i: 0);
3554 SDValue Sub = Op.getOperand(i: 1);
3555 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3556 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3557 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3558 APInt DemandedSrcElts = DemandedElts;
3559 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3560
3561 // If none of the sub operand elements are demanded, bypass the insert.
3562 if (!DemandedSubElts)
3563 return TLO.CombineTo(O: Op, N: Src);
3564
3565 APInt SubUndef, SubZero;
3566 if (SimplifyDemandedVectorElts(Op: Sub, OriginalDemandedElts: DemandedSubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3567 Depth: Depth + 1))
3568 return true;
3569
3570 // If none of the src operand elements are demanded, replace it with undef.
3571 if (!DemandedSrcElts && !Src.isUndef())
3572 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT,
3573 N1: TLO.DAG.getUNDEF(VT), N2: Sub,
3574 N3: Op.getOperand(i: 2)));
3575
3576 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef, KnownZero,
3577 TLO, Depth: Depth + 1))
3578 return true;
3579 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: Idx);
3580 KnownZero.insertBits(SubBits: SubZero, bitPosition: Idx);
3581
3582 // Attempt to avoid multi-use ops if we don't need anything from them.
3583 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
3584 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3585 Op: Src, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
3586 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3587 Op: Sub, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3588 if (NewSrc || NewSub) {
3589 NewSrc = NewSrc ? NewSrc : Src;
3590 NewSub = NewSub ? NewSub : Sub;
3591 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3592 N2: NewSub, N3: Op.getOperand(i: 2));
3593 return TLO.CombineTo(O: Op, N: NewOp);
3594 }
3595 }
3596 break;
3597 }
3598 case ISD::EXTRACT_SUBVECTOR: {
3599 // Offset the demanded elts by the subvector index.
3600 SDValue Src = Op.getOperand(i: 0);
3601 if (Src.getValueType().isScalableVector())
3602 break;
3603 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3604 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3605 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3606
3607 APInt SrcUndef, SrcZero;
3608 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3609 Depth: Depth + 1))
3610 return true;
3611 KnownUndef = SrcUndef.extractBits(numBits: NumElts, bitPosition: Idx);
3612 KnownZero = SrcZero.extractBits(numBits: NumElts, bitPosition: Idx);
3613
3614 // Attempt to avoid multi-use ops if we don't need anything from them.
3615 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(Op: Src, DemandedElts: DemandedSrcElts,
3616 DAG&: TLO.DAG, Depth: Depth + 1);
3617 if (NewSrc) {
3618 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3619 N2: Op.getOperand(i: 1));
3620 return TLO.CombineTo(O: Op, N: NewOp);
3621 }
3622 break;
3623 }
3624 case ISD::INSERT_VECTOR_ELT: {
3625 SDValue Vec = Op.getOperand(i: 0);
3626 SDValue Scl = Op.getOperand(i: 1);
3627 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
3628
3629 // For a legal, constant insertion index, if we don't need this insertion
3630 // then strip it, else remove it from the demanded elts.
3631 if (CIdx && CIdx->getAPIntValue().ult(RHS: NumElts)) {
3632 unsigned Idx = CIdx->getZExtValue();
3633 if (!DemandedElts[Idx])
3634 return TLO.CombineTo(O: Op, N: Vec);
3635
3636 APInt DemandedVecElts(DemandedElts);
3637 DemandedVecElts.clearBit(BitPosition: Idx);
3638 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedVecElts, KnownUndef,
3639 KnownZero, TLO, Depth: Depth + 1))
3640 return true;
3641
3642 KnownUndef.setBitVal(BitPosition: Idx, BitValue: Scl.isUndef());
3643
3644 KnownZero.setBitVal(BitPosition: Idx, BitValue: isNullConstant(V: Scl) || isNullFPConstant(V: Scl));
3645 break;
3646 }
3647
3648 APInt VecUndef, VecZero;
3649 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedElts, KnownUndef&: VecUndef, KnownZero&: VecZero, TLO,
3650 Depth: Depth + 1))
3651 return true;
3652 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
3653 break;
3654 }
3655 case ISD::VSELECT: {
3656 SDValue Sel = Op.getOperand(i: 0);
3657 SDValue LHS = Op.getOperand(i: 1);
3658 SDValue RHS = Op.getOperand(i: 2);
3659
3660 // Try to transform the select condition based on the current demanded
3661 // elements.
3662 APInt UndefSel, ZeroSel;
3663 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3664 Depth: Depth + 1))
3665 return true;
3666
3667 // See if we can simplify either vselect operand.
3668 APInt DemandedLHS(DemandedElts);
3669 APInt DemandedRHS(DemandedElts);
3670 APInt UndefLHS, ZeroLHS;
3671 APInt UndefRHS, ZeroRHS;
3672 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3673 Depth: Depth + 1))
3674 return true;
3675 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3676 Depth: Depth + 1))
3677 return true;
3678
3679 KnownUndef = UndefLHS & UndefRHS;
3680 KnownZero = ZeroLHS & ZeroRHS;
3681
3682 // If we know that the selected element is always zero, we don't need the
3683 // select value element.
3684 APInt DemandedSel = DemandedElts & ~KnownZero;
3685 if (DemandedSel != DemandedElts)
3686 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedSel, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3687 Depth: Depth + 1))
3688 return true;
3689
3690 break;
3691 }
3692 case ISD::VECTOR_SHUFFLE: {
3693 SDValue LHS = Op.getOperand(i: 0);
3694 SDValue RHS = Op.getOperand(i: 1);
3695 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
3696
3697 // Collect demanded elements from shuffle operands..
3698 APInt DemandedLHS(NumElts, 0);
3699 APInt DemandedRHS(NumElts, 0);
3700 for (unsigned i = 0; i != NumElts; ++i) {
3701 int M = ShuffleMask[i];
3702 if (M < 0 || !DemandedElts[i])
3703 continue;
3704 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
3705 if (M < (int)NumElts)
3706 DemandedLHS.setBit(M);
3707 else
3708 DemandedRHS.setBit(M - NumElts);
3709 }
3710
3711 // If either side isn't demanded, replace it by UNDEF. We handle this
3712 // explicitly here to also simplify in case of multiple uses (on the
3713 // contrary to the SimplifyDemandedVectorElts calls below).
3714 bool FoldLHS = !DemandedLHS && !LHS.isUndef();
3715 bool FoldRHS = !DemandedRHS && !RHS.isUndef();
3716 if (FoldLHS || FoldRHS) {
3717 LHS = FoldLHS ? TLO.DAG.getUNDEF(VT: LHS.getValueType()) : LHS;
3718 RHS = FoldRHS ? TLO.DAG.getUNDEF(VT: RHS.getValueType()) : RHS;
3719 SDValue NewOp =
3720 TLO.DAG.getVectorShuffle(VT, dl: SDLoc(Op), N1: LHS, N2: RHS, Mask: ShuffleMask);
3721 return TLO.CombineTo(O: Op, N: NewOp);
3722 }
3723
3724 // See if we can simplify either shuffle operand.
3725 APInt UndefLHS, ZeroLHS;
3726 APInt UndefRHS, ZeroRHS;
3727 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3728 Depth: Depth + 1))
3729 return true;
3730 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3731 Depth: Depth + 1))
3732 return true;
3733
3734 // Simplify mask using undef elements from LHS/RHS.
3735 bool Updated = false;
3736 bool IdentityLHS = true, IdentityRHS = true;
3737 SmallVector<int, 32> NewMask(ShuffleMask);
3738 for (unsigned i = 0; i != NumElts; ++i) {
3739 int &M = NewMask[i];
3740 if (M < 0)
3741 continue;
3742 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
3743 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
3744 Updated = true;
3745 M = -1;
3746 }
3747 IdentityLHS &= (M < 0) || (M == (int)i);
3748 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3749 }
3750
3751 // Update legal shuffle masks based on demanded elements if it won't reduce
3752 // to Identity which can cause premature removal of the shuffle mask.
3753 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
3754 SDValue LegalShuffle =
3755 buildLegalVectorShuffle(VT, DL, N0: LHS, N1: RHS, Mask: NewMask, DAG&: TLO.DAG);
3756 if (LegalShuffle)
3757 return TLO.CombineTo(O: Op, N: LegalShuffle);
3758 }
3759
3760 // Propagate undef/zero elements from LHS/RHS.
3761 for (unsigned i = 0; i != NumElts; ++i) {
3762 int M = ShuffleMask[i];
3763 if (M < 0) {
3764 KnownUndef.setBit(i);
3765 } else if (M < (int)NumElts) {
3766 if (UndefLHS[M])
3767 KnownUndef.setBit(i);
3768 if (ZeroLHS[M])
3769 KnownZero.setBit(i);
3770 } else {
3771 if (UndefRHS[M - NumElts])
3772 KnownUndef.setBit(i);
3773 if (ZeroRHS[M - NumElts])
3774 KnownZero.setBit(i);
3775 }
3776 }
3777 break;
3778 }
3779 case ISD::ANY_EXTEND_VECTOR_INREG:
3780 case ISD::SIGN_EXTEND_VECTOR_INREG:
3781 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3782 APInt SrcUndef, SrcZero;
3783 SDValue Src = Op.getOperand(i: 0);
3784 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3785 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3786 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3787 Depth: Depth + 1))
3788 return true;
3789 KnownZero = SrcZero.zextOrTrunc(width: NumElts);
3790 KnownUndef = SrcUndef.zextOrTrunc(width: NumElts);
3791
3792 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
3793 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3794 DemandedSrcElts == 1) {
3795 // aext - if we just need the bottom element then we can bitcast.
3796 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
3797 }
3798
3799 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
3800 // zext(undef) upper bits are guaranteed to be zero.
3801 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
3802 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3803 KnownUndef.clearAllBits();
3804
3805 // zext - if we just need the bottom element then we can mask:
3806 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
3807 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
3808 Op->isOnlyUserOf(N: Src.getNode()) &&
3809 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3810 SDLoc DL(Op);
3811 EVT SrcVT = Src.getValueType();
3812 EVT SrcSVT = SrcVT.getScalarType();
3813
3814 // If we're after type legalization and SrcSVT is not legal, use the
3815 // promoted type for creating constants to avoid creating nodes with
3816 // illegal types.
3817 if (TLO.LegalTypes())
3818 SrcSVT = getLegalTypeToTransformTo(Context&: *TLO.DAG.getContext(), VT: SrcSVT);
3819
3820 SmallVector<SDValue> MaskElts;
3821 MaskElts.push_back(Elt: TLO.DAG.getAllOnesConstant(DL, VT: SrcSVT));
3822 MaskElts.append(NumInputs: NumSrcElts - 1, Elt: TLO.DAG.getConstant(Val: 0, DL, VT: SrcSVT));
3823 SDValue Mask = TLO.DAG.getBuildVector(VT: SrcVT, DL, Ops: MaskElts);
3824 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
3825 Opcode: ISD::AND, DL, VT: SrcVT, Ops: {Src.getOperand(i: 1), Mask})) {
3826 Fold = TLO.DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src.getOperand(i: 0), N2: Fold);
3827 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Fold));
3828 }
3829 }
3830 }
3831 break;
3832 }
3833
3834 // TODO: There are more binop opcodes that could be handled here - MIN,
3835 // MAX, saturated math, etc.
3836 case ISD::ADD: {
3837 SDValue Op0 = Op.getOperand(i: 0);
3838 SDValue Op1 = Op.getOperand(i: 1);
3839 if (Op0 == Op1 && Op->isOnlyUserOf(N: Op0.getNode())) {
3840 APInt UndefLHS, ZeroLHS;
3841 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3842 Depth: Depth + 1, /*AssumeSingleUse*/ true))
3843 return true;
3844 }
3845 [[fallthrough]];
3846 }
3847 case ISD::AVGCEILS:
3848 case ISD::AVGCEILU:
3849 case ISD::AVGFLOORS:
3850 case ISD::AVGFLOORU:
3851 case ISD::OR:
3852 case ISD::XOR:
3853 case ISD::SUB:
3854 case ISD::FADD:
3855 case ISD::FSUB:
3856 case ISD::FMUL:
3857 case ISD::FDIV:
3858 case ISD::FREM:
3859 case ISD::PSEUDO_FMIN:
3860 case ISD::PSEUDO_FMAX: {
3861 SDValue Op0 = Op.getOperand(i: 0);
3862 SDValue Op1 = Op.getOperand(i: 1);
3863
3864 APInt UndefRHS, ZeroRHS;
3865 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3866 Depth: Depth + 1))
3867 return true;
3868 APInt UndefLHS, ZeroLHS;
3869 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3870 Depth: Depth + 1))
3871 return true;
3872
3873 KnownZero = ZeroLHS & ZeroRHS;
3874 KnownUndef = getKnownUndefForVectorBinop(BO: Op, DAG&: TLO.DAG, UndefOp0: UndefLHS, UndefOp1: UndefRHS);
3875
3876 // Attempt to avoid multi-use ops if we don't need anything from them.
3877 // TODO - use KnownUndef to relax the demandedelts?
3878 if (!DemandedElts.isAllOnes())
3879 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3880 return true;
3881 break;
3882 }
3883 case ISD::SHL:
3884 case ISD::SRL:
3885 case ISD::SRA:
3886 case ISD::ROTL:
3887 case ISD::ROTR: {
3888 SDValue Op0 = Op.getOperand(i: 0);
3889 SDValue Op1 = Op.getOperand(i: 1);
3890
3891 APInt UndefRHS, ZeroRHS;
3892 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3893 Depth: Depth + 1))
3894 return true;
3895 APInt UndefLHS, ZeroLHS;
3896 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3897 Depth: Depth + 1))
3898 return true;
3899
3900 KnownZero = ZeroLHS;
3901 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
3902
3903 // Attempt to avoid multi-use ops if we don't need anything from them.
3904 // TODO - use KnownUndef to relax the demandedelts?
3905 if (!DemandedElts.isAllOnes())
3906 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3907 return true;
3908 break;
3909 }
3910 case ISD::MUL:
3911 case ISD::MULHU:
3912 case ISD::MULHS:
3913 case ISD::AND: {
3914 SDValue Op0 = Op.getOperand(i: 0);
3915 SDValue Op1 = Op.getOperand(i: 1);
3916
3917 APInt SrcUndef, SrcZero;
3918 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3919 Depth: Depth + 1))
3920 return true;
3921 // FIXME: If we know that a demanded element was zero in Op1 we don't need
3922 // to demand it in Op0 - its guaranteed to be zero. There is however a
3923 // restriction, as we must not make any of the originally demanded elements
3924 // more poisonous. We could reduce amount of elements demanded, but then we
3925 // also need a to inform SimplifyDemandedVectorElts that some elements must
3926 // not be made more poisonous.
3927 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef, KnownZero,
3928 TLO, Depth: Depth + 1))
3929 return true;
3930
3931 KnownUndef &= DemandedElts;
3932 KnownZero &= DemandedElts;
3933
3934 // If every element pair has a zero/undef/poison then just fold to zero.
3935 // fold (and x, undef/poison) -> 0 / (and x, 0) -> 0
3936 // fold (mul x, undef/poison) -> 0 / (mul x, 0) -> 0
3937 if (DemandedElts.isSubsetOf(RHS: SrcZero | KnownZero | SrcUndef | KnownUndef))
3938 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3939
3940 // If either side has a zero element, then the result element is zero, even
3941 // if the other is an UNDEF.
3942 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
3943 // and then handle 'and' nodes with the rest of the binop opcodes.
3944 KnownZero |= SrcZero;
3945 KnownUndef &= SrcUndef;
3946 KnownUndef &= ~KnownZero;
3947
3948 // Attempt to avoid multi-use ops if we don't need anything from them.
3949 if (!DemandedElts.isAllOnes())
3950 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3951 return true;
3952 break;
3953 }
3954 case ISD::TRUNCATE:
3955 case ISD::SIGN_EXTEND:
3956 case ISD::ZERO_EXTEND:
3957 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
3958 KnownZero, TLO, Depth: Depth + 1))
3959 return true;
3960
3961 if (!DemandedElts.isAllOnes())
3962 if (SDValue NewOp = SimplifyMultipleUseDemandedVectorElts(
3963 Op: Op.getOperand(i: 0), DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
3964 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, Operand: NewOp));
3965
3966 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
3967 // zext(undef) upper bits are guaranteed to be zero.
3968 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
3969 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3970 KnownUndef.clearAllBits();
3971 }
3972 break;
3973 case ISD::SINT_TO_FP:
3974 case ISD::UINT_TO_FP:
3975 case ISD::FP_TO_SINT:
3976 case ISD::FP_TO_UINT:
3977 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
3978 KnownZero, TLO, Depth: Depth + 1))
3979 return true;
3980 // Don't fall through to generic undef -> undef handling.
3981 return false;
3982 default: {
3983 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
3984 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3985 KnownZero, TLO, Depth))
3986 return true;
3987 } else {
3988 KnownBits Known;
3989 APInt DemandedBits = APInt::getAllOnes(numBits: EltSizeInBits);
3990 if (SimplifyDemandedBits(Op, OriginalDemandedBits: DemandedBits, OriginalDemandedElts, Known,
3991 TLO, Depth, AssumeSingleUse))
3992 return true;
3993 }
3994 break;
3995 }
3996 }
3997
3998 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
3999
4000 // Constant fold all undef cases.
4001 // TODO: Handle zero cases as well.
4002 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
4003 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
4004
4005 return false;
4006}
4007
4008/// Determine which of the bits specified in Mask are known to be either zero or
4009/// one and return them in the Known.
4010void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
4011 KnownBits &Known,
4012 const APInt &DemandedElts,
4013 const SelectionDAG &DAG,
4014 unsigned Depth) const {
4015 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4016 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4017 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4018 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4019 "Should use MaskedValueIsZero if you don't know whether Op"
4020 " is a target node!");
4021 Known.resetAll();
4022}
4023
4024void TargetLowering::computeKnownBitsForTargetInstr(
4025 GISelValueTracking &Analysis, Register R, KnownBits &Known,
4026 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4027 unsigned Depth) const {
4028 Known.resetAll();
4029}
4030
4031void TargetLowering::computeKnownFPClassForTargetInstr(
4032 GISelValueTracking &Analysis, Register R, KnownFPClass &Known,
4033 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4034 unsigned Depth) const {
4035 Known.resetAll();
4036}
4037
4038void TargetLowering::computeKnownBitsForStackObjectPointer(
4039 KnownBits &Known, const MachineFunction &, Align Alignment) const {
4040 // The low bits are known zero if the pointer is aligned.
4041 Known.Zero.setLowBits(Log2(A: Alignment));
4042}
4043
4044SDValue TargetLowering::annotateStackObjectPointer(SDValue Ptr,
4045 SelectionDAG &DAG,
4046 const SDLoc &DL,
4047 Align Alignment) const {
4048 // Materialize leading-zero stack object pointer facts as AssertZext.
4049 // Alignment-derived low zero bits are not represented on the returned DAG
4050 // value here.
4051 EVT PtrVT = Ptr.getValueType();
4052
4053 unsigned RegSize = PtrVT.getScalarSizeInBits();
4054 KnownBits Known(RegSize);
4055 computeKnownBitsForStackObjectPointer(Known, DAG.getMachineFunction(),
4056 Alignment);
4057
4058 unsigned NumZeroBits = Known.countMinLeadingZeros();
4059 if (!NumZeroBits)
4060 return Ptr;
4061
4062 EVT FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
4063 return DAG.getNode(Opcode: ISD::AssertZext, DL, VT: PtrVT, N1: Ptr, N2: DAG.getValueType(FromVT));
4064}
4065
4066Align TargetLowering::computeKnownAlignForTargetInstr(
4067 GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI,
4068 unsigned Depth) const {
4069 return Align(1);
4070}
4071
4072/// This method can be implemented by targets that want to expose additional
4073/// information about sign bits to the DAG Combiner.
4074unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
4075 const APInt &,
4076 const SelectionDAG &,
4077 unsigned Depth) const {
4078 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4079 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4080 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4081 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4082 "Should use ComputeNumSignBits if you don't know whether Op"
4083 " is a target node!");
4084 return 1;
4085}
4086
4087unsigned TargetLowering::computeNumSignBitsForTargetInstr(
4088 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4089 const MachineRegisterInfo &MRI, unsigned Depth) const {
4090 return 1;
4091}
4092
4093bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
4094 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
4095 TargetLoweringOpt &TLO, unsigned Depth) const {
4096 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4097 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4098 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4099 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4100 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
4101 " is a target node!");
4102 return false;
4103}
4104
4105bool TargetLowering::SimplifyDemandedBitsForTargetNode(
4106 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4107 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
4108 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4109 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4110 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4111 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4112 "Should use SimplifyDemandedBits if you don't know whether Op"
4113 " is a target node!");
4114 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
4115 return false;
4116}
4117
4118SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
4119 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4120 SelectionDAG &DAG, unsigned Depth) const {
4121 assert(
4122 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4123 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4124 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4125 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4126 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
4127 " is a target node!");
4128 return SDValue();
4129}
4130
4131SDValue
4132TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
4133 SDValue N1, MutableArrayRef<int> Mask,
4134 SelectionDAG &DAG) const {
4135 bool LegalMask = isShuffleMaskLegal(Mask, VT);
4136 if (!LegalMask) {
4137 std::swap(a&: N0, b&: N1);
4138 ShuffleVectorSDNode::commuteMask(Mask);
4139 LegalMask = isShuffleMaskLegal(Mask, VT);
4140 }
4141
4142 if (!LegalMask)
4143 return SDValue();
4144
4145 return DAG.getVectorShuffle(VT, dl: DL, N1: N0, N2: N1, Mask);
4146}
4147
4148const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
4149 return nullptr;
4150}
4151
4152bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4153 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4154 UndefPoisonKind Kind, unsigned Depth) const {
4155 assert(
4156 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4157 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4158 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4159 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4160 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
4161 " is a target node!");
4162
4163 // If Op can't create undef/poison and none of its operands are undef/poison
4164 // then Op is never undef/poison.
4165 return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, Kind,
4166 /*ConsiderFlags*/ true, Depth) &&
4167 all_of(Range: Op->ops(), P: [&](SDValue V) {
4168 return DAG.isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
4169 });
4170}
4171
4172bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
4173 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4174 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
4175 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4176 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4177 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4178 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4179 "Should use canCreateUndefOrPoison if you don't know whether Op"
4180 " is a target node!");
4181 // Be conservative and return true.
4182 return true;
4183}
4184
4185void TargetLowering::computeKnownFPClassForTargetNode(const SDValue Op,
4186 KnownFPClass &Known,
4187 const APInt &DemandedElts,
4188 const SelectionDAG &DAG,
4189 unsigned Depth) const {
4190 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4191 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4192 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4193 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4194 "Should use computeKnownFPClass if you don't know whether Op"
4195 " is a target node!");
4196}
4197
4198bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
4199 const APInt &DemandedElts,
4200 const SelectionDAG &DAG,
4201 bool SNaN,
4202 unsigned Depth) const {
4203 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4204 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4205 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4206 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4207 "Should use isKnownNeverNaN if you don't know whether Op"
4208 " is a target node!");
4209 return false;
4210}
4211
4212bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
4213 const APInt &DemandedElts,
4214 APInt &UndefElts,
4215 const SelectionDAG &DAG,
4216 unsigned Depth) const {
4217 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4218 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4219 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4220 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4221 "Should use isSplatValue if you don't know whether Op"
4222 " is a target node!");
4223 return false;
4224}
4225
4226// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
4227// work with truncating build vectors and vectors with elements of less than
4228// 8 bits.
4229bool TargetLowering::isConstTrueVal(SDValue N) const {
4230 if (!N)
4231 return false;
4232
4233 unsigned EltWidth;
4234 APInt CVal;
4235 if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
4236 /*AllowTruncation=*/true)) {
4237 CVal = CN->getAPIntValue();
4238 EltWidth = N.getValueType().getScalarSizeInBits();
4239 } else
4240 return false;
4241
4242 // If this is a truncating splat, truncate the splat value.
4243 // Otherwise, we may fail to match the expected values below.
4244 if (EltWidth < CVal.getBitWidth())
4245 CVal = CVal.trunc(width: EltWidth);
4246
4247 switch (getBooleanContents(Type: N.getValueType())) {
4248 case UndefinedBooleanContent:
4249 return CVal[0];
4250 case ZeroOrOneBooleanContent:
4251 return CVal.isOne();
4252 case ZeroOrNegativeOneBooleanContent:
4253 return CVal.isAllOnes();
4254 }
4255
4256 llvm_unreachable("Invalid boolean contents");
4257}
4258
4259bool TargetLowering::isConstFalseVal(SDValue N) const {
4260 if (!N)
4261 return false;
4262
4263 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N);
4264 if (!CN) {
4265 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
4266 if (!BV)
4267 return false;
4268
4269 // Only interested in constant splats, we don't care about undef
4270 // elements in identifying boolean constants and getConstantSplatNode
4271 // returns NULL if all ops are undef;
4272 CN = BV->getConstantSplatNode();
4273 if (!CN)
4274 return false;
4275 }
4276
4277 if (getBooleanContents(Type: N->getValueType(ResNo: 0)) == UndefinedBooleanContent)
4278 return !CN->getAPIntValue()[0];
4279
4280 return CN->isZero();
4281}
4282
4283bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
4284 bool SExt) const {
4285 if (VT == MVT::i1)
4286 return N->isOne();
4287
4288 TargetLowering::BooleanContent Cnt = getBooleanContents(Type: VT);
4289 switch (Cnt) {
4290 case TargetLowering::ZeroOrOneBooleanContent:
4291 // An extended value of 1 is always true, unless its original type is i1,
4292 // in which case it will be sign extended to -1.
4293 return (N->isOne() && !SExt) || (SExt && (N->getValueType(ResNo: 0) != MVT::i1));
4294 case TargetLowering::UndefinedBooleanContent:
4295 case TargetLowering::ZeroOrNegativeOneBooleanContent:
4296 return N->isAllOnes() && SExt;
4297 }
4298 llvm_unreachable("Unexpected enumeration.");
4299}
4300
4301/// This helper function of SimplifySetCC tries to optimize the comparison when
4302/// either operand of the SetCC node is a bitwise-and instruction.
4303SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
4304 ISD::CondCode Cond, const SDLoc &DL,
4305 DAGCombinerInfo &DCI) const {
4306 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
4307 std::swap(a&: N0, b&: N1);
4308
4309 SelectionDAG &DAG = DCI.DAG;
4310 EVT OpVT = N0.getValueType();
4311 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
4312 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4313 return SDValue();
4314
4315 // (X & Y) != 0 --> zextOrTrunc(X & Y)
4316 // iff everything but LSB is known zero:
4317 if (Cond == ISD::SETNE && isNullConstant(V: N1) &&
4318 (getBooleanContents(Type: OpVT) == TargetLowering::UndefinedBooleanContent ||
4319 getBooleanContents(Type: OpVT) == TargetLowering::ZeroOrOneBooleanContent)) {
4320 unsigned NumEltBits = OpVT.getScalarSizeInBits();
4321 APInt UpperBits = APInt::getHighBitsSet(numBits: NumEltBits, hiBitsSet: NumEltBits - 1);
4322 if (DAG.MaskedValueIsZero(Op: N0, Mask: UpperBits))
4323 return DAG.getBoolExtOrTrunc(Op: N0, SL: DL, VT, OpVT);
4324 }
4325
4326 // Try to eliminate a power-of-2 mask constant by converting to a signbit
4327 // test in a narrow type that we can truncate to with no cost. Examples:
4328 // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
4329 // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
4330 // TODO: This conservatively checks for type legality on the source and
4331 // destination types. That may inhibit optimizations, but it also
4332 // allows setcc->shift transforms that may be more beneficial.
4333 auto *AndC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
4334 if (AndC && isNullConstant(V: N1) && AndC->getAPIntValue().isPowerOf2() &&
4335 isTypeLegal(VT: OpVT) && N0.hasOneUse()) {
4336 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
4337 BitWidth: AndC->getAPIntValue().getActiveBits());
4338 if (isTruncateFree(FromVT: OpVT, ToVT: NarrowVT) && isTypeLegal(VT: NarrowVT)) {
4339 SDValue Trunc = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 0), DL, VT: NarrowVT);
4340 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: NarrowVT);
4341 return DAG.getSetCC(DL, VT, LHS: Trunc, RHS: Zero,
4342 Cond: Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT);
4343 }
4344 }
4345
4346 // Match these patterns in any of their permutations:
4347 // (X & Y) == Y
4348 // (X & Y) != Y
4349 SDValue X, Y;
4350 if (N0.getOperand(i: 0) == N1) {
4351 X = N0.getOperand(i: 1);
4352 Y = N0.getOperand(i: 0);
4353 } else if (N0.getOperand(i: 1) == N1) {
4354 X = N0.getOperand(i: 0);
4355 Y = N0.getOperand(i: 1);
4356 } else {
4357 return SDValue();
4358 }
4359
4360 // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
4361 // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
4362 // its liable to create and infinite loop.
4363 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: OpVT);
4364 if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
4365 DAG.isKnownToBeAPowerOfTwo(Val: Y)) {
4366 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
4367 // Note that where Y is variable and is known to have at most one bit set
4368 // (for example, if it is Z & 1) we cannot do this; the expressions are not
4369 // equivalent when Y == 0.
4370 assert(OpVT.isInteger());
4371 Cond = ISD::getSetCCInverse(Operation: Cond, Type: OpVT);
4372 if (DCI.isBeforeLegalizeOps() ||
4373 isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType()))
4374 return DAG.getSetCC(DL, VT, LHS: N0, RHS: Zero, Cond);
4375 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
4376 // If the target supports an 'and-not' or 'and-complement' logic operation,
4377 // try to use that to make a comparison operation more efficient.
4378 // But don't do this transform if the mask is a single bit because there are
4379 // more efficient ways to deal with that case (for example, 'bt' on x86 or
4380 // 'rlwinm' on PPC).
4381
4382 // Bail out if the compare operand that we want to turn into a zero is
4383 // already a zero (otherwise, infinite loop).
4384 if (isNullConstant(V: Y))
4385 return SDValue();
4386
4387 // Transform this into: ~X & Y == 0.
4388 SDValue NotX = DAG.getNOT(DL: SDLoc(X), Val: X, VT: OpVT);
4389 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: NotX, N2: Y);
4390 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: Zero, Cond);
4391 }
4392
4393 return SDValue();
4394}
4395
4396/// This helper function of SimplifySetCC tries to optimize the comparison when
4397/// either operand of the SetCC node is a bitwise-or instruction.
4398/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
4399SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
4400 ISD::CondCode Cond, const SDLoc &DL,
4401 DAGCombinerInfo &DCI) const {
4402 if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
4403 std::swap(a&: N0, b&: N1);
4404
4405 SelectionDAG &DAG = DCI.DAG;
4406 EVT OpVT = N0.getValueType();
4407 if (!N0.hasOneUse() || !OpVT.isInteger() ||
4408 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4409 return SDValue();
4410
4411 // (X | Y) == Y
4412 // (X | Y) != Y
4413 SDValue X;
4414 if (sd_match(N: N0, P: m_Or(L: m_Value(N&: X), R: m_Specific(N: N1))) && hasAndNotCompare(Y: X)) {
4415 // If the target supports an 'and-not' or 'and-complement' logic operation,
4416 // try to use that to make a comparison operation more efficient.
4417
4418 // Bail out if the compare operand that we want to turn into a zero is
4419 // already a zero (otherwise, infinite loop).
4420 if (isNullConstant(V: N1))
4421 return SDValue();
4422
4423 // Transform this into: X & ~Y ==/!= 0.
4424 SDValue NotY = DAG.getNOT(DL: SDLoc(N1), Val: N1, VT: OpVT);
4425 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: X, N2: NotY);
4426 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4427 }
4428
4429 return SDValue();
4430}
4431
4432/// There are multiple IR patterns that could be checking whether certain
4433/// truncation of a signed number would be lossy or not. The pattern which is
4434/// best at IR level, may not lower optimally. Thus, we want to unfold it.
4435/// We are looking for the following pattern: (KeptBits is a constant)
4436/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
4437/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
4438/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
4439/// We will unfold it into the natural trunc+sext pattern:
4440/// ((%x << C) a>> C) dstcond %x
4441/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
4442SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4443 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
4444 const SDLoc &DL) const {
4445 // We must be comparing with a constant.
4446 ConstantSDNode *C1;
4447 if (!(C1 = dyn_cast<ConstantSDNode>(Val&: N1)))
4448 return SDValue();
4449
4450 // N0 should be: add %x, (1 << (KeptBits-1))
4451 if (N0->getOpcode() != ISD::ADD)
4452 return SDValue();
4453
4454 // And we must be 'add'ing a constant.
4455 ConstantSDNode *C01;
4456 if (!(C01 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1))))
4457 return SDValue();
4458
4459 SDValue X = N0->getOperand(Num: 0);
4460 EVT XVT = X.getValueType();
4461
4462 // Validate constants ...
4463
4464 APInt I1 = C1->getAPIntValue();
4465
4466 ISD::CondCode NewCond;
4467 if (Cond == ISD::CondCode::SETULT) {
4468 NewCond = ISD::CondCode::SETEQ;
4469 } else if (Cond == ISD::CondCode::SETULE) {
4470 NewCond = ISD::CondCode::SETEQ;
4471 // But need to 'canonicalize' the constant.
4472 I1 += 1;
4473 } else if (Cond == ISD::CondCode::SETUGT) {
4474 NewCond = ISD::CondCode::SETNE;
4475 // But need to 'canonicalize' the constant.
4476 I1 += 1;
4477 } else if (Cond == ISD::CondCode::SETUGE) {
4478 NewCond = ISD::CondCode::SETNE;
4479 } else
4480 return SDValue();
4481
4482 APInt I01 = C01->getAPIntValue();
4483
4484 auto checkConstants = [&I1, &I01]() -> bool {
4485 // Both of them must be power-of-two, and the constant from setcc is bigger.
4486 return I1.ugt(RHS: I01) && I1.isPowerOf2() && I01.isPowerOf2();
4487 };
4488
4489 if (checkConstants()) {
4490 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
4491 } else {
4492 // What if we invert constants? (and the target predicate)
4493 I1.negate();
4494 I01.negate();
4495 assert(XVT.isInteger());
4496 NewCond = getSetCCInverse(Operation: NewCond, Type: XVT);
4497 if (!checkConstants())
4498 return SDValue();
4499 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
4500 }
4501
4502 // They are power-of-two, so which bit is set?
4503 const unsigned KeptBits = I1.logBase2();
4504 const unsigned KeptBitsMinusOne = I01.logBase2();
4505
4506 // Magic!
4507 if (KeptBits != (KeptBitsMinusOne + 1))
4508 return SDValue();
4509 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
4510
4511 // We don't want to do this in every single case.
4512 SelectionDAG &DAG = DCI.DAG;
4513 if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
4514 return SDValue();
4515
4516 // Unfold into: sext_inreg(%x) cond %x
4517 // Where 'cond' will be either 'eq' or 'ne'.
4518 SDValue SExtInReg = DAG.getNode(
4519 Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: XVT, N1: X,
4520 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: KeptBits)));
4521 return DAG.getSetCC(DL, VT: SCCVT, LHS: SExtInReg, RHS: X, Cond: NewCond);
4522}
4523
4524// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4525SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4526 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4527 DAGCombinerInfo &DCI, const SDLoc &DL) const {
4528 assert(isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->isZero() &&
4529 "Should be a comparison with 0.");
4530 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4531 "Valid only for [in]equality comparisons.");
4532
4533 unsigned NewShiftOpcode;
4534 SDValue X, C, Y;
4535
4536 SelectionDAG &DAG = DCI.DAG;
4537
4538 // Look for '(C l>>/<< Y)'.
4539 auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
4540 // The shift should be one-use.
4541 if (!V.hasOneUse())
4542 return false;
4543 unsigned OldShiftOpcode = V.getOpcode();
4544 switch (OldShiftOpcode) {
4545 case ISD::SHL:
4546 NewShiftOpcode = ISD::SRL;
4547 break;
4548 case ISD::SRL:
4549 NewShiftOpcode = ISD::SHL;
4550 break;
4551 default:
4552 return false; // must be a logical shift.
4553 }
4554 // We should be shifting a constant.
4555 // FIXME: best to use isConstantOrConstantVector().
4556 C = V.getOperand(i: 0);
4557 ConstantSDNode *CC =
4558 isConstOrConstSplat(N: C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4559 if (!CC)
4560 return false;
4561 Y = V.getOperand(i: 1);
4562
4563 ConstantSDNode *XC =
4564 isConstOrConstSplat(N: X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4565 return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
4566 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
4567 };
4568
4569 // LHS of comparison should be an one-use 'and'.
4570 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
4571 return SDValue();
4572
4573 X = N0.getOperand(i: 0);
4574 SDValue Mask = N0.getOperand(i: 1);
4575
4576 // 'and' is commutative!
4577 if (!Match(Mask)) {
4578 std::swap(a&: X, b&: Mask);
4579 if (!Match(Mask))
4580 return SDValue();
4581 }
4582
4583 EVT VT = X.getValueType();
4584
4585 // Produce:
4586 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
4587 SDValue T0 = DAG.getNode(Opcode: NewShiftOpcode, DL, VT, N1: X, N2: Y);
4588 SDValue T1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T0, N2: C);
4589 SDValue T2 = DAG.getSetCC(DL, VT: SCCVT, LHS: T1, RHS: N1C, Cond);
4590 return T2;
4591}
4592
4593/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
4594/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
4595/// handle the commuted versions of these patterns.
4596SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
4597 ISD::CondCode Cond, const SDLoc &DL,
4598 DAGCombinerInfo &DCI) const {
4599 unsigned BOpcode = N0.getOpcode();
4600 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
4601 "Unexpected binop");
4602 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
4603
4604 // (X + Y) == X --> Y == 0
4605 // (X - Y) == X --> Y == 0
4606 // (X ^ Y) == X --> Y == 0
4607 SelectionDAG &DAG = DCI.DAG;
4608 EVT OpVT = N0.getValueType();
4609 SDValue X = N0.getOperand(i: 0);
4610 SDValue Y = N0.getOperand(i: 1);
4611 if (X == N1)
4612 return DAG.getSetCC(DL, VT, LHS: Y, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4613
4614 if (Y != N1)
4615 return SDValue();
4616
4617 // (X + Y) == Y --> X == 0
4618 // (X ^ Y) == Y --> X == 0
4619 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
4620 return DAG.getSetCC(DL, VT, LHS: X, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4621
4622 // The shift would not be valid if the operands are boolean (i1).
4623 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
4624 return SDValue();
4625
4626 // (X - Y) == Y --> X == Y << 1
4627 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: OpVT, DL);
4628 SDValue YShl1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: N1.getValueType(), N1: Y, N2: One);
4629 if (!DCI.isCalledByLegalizer())
4630 DCI.AddToWorklist(N: YShl1.getNode());
4631 return DAG.getSetCC(DL, VT, LHS: X, RHS: YShl1, Cond);
4632}
4633
4634static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
4635 SDValue N0, const APInt &C1,
4636 ISD::CondCode Cond, const SDLoc &dl,
4637 SelectionDAG &DAG) {
4638 // Look through truncs that don't change the value of a ctpop.
4639 // FIXME: Add vector support? Need to be careful with setcc result type below.
4640 SDValue CTPOP = N0;
4641 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
4642 N0.getScalarValueSizeInBits() > Log2_32(Value: N0.getOperand(i: 0).getScalarValueSizeInBits()))
4643 CTPOP = N0.getOperand(i: 0);
4644
4645 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
4646 return SDValue();
4647
4648 EVT CTVT = CTPOP.getValueType();
4649 SDValue CTOp = CTPOP.getOperand(i: 0);
4650
4651 // Expand a power-of-2-or-zero comparison based on ctpop:
4652 // (ctpop x) u< 2 -> (x & x-1) == 0
4653 // (ctpop x) u> 1 -> (x & x-1) != 0
4654 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
4655 // Keep the CTPOP if it is a cheap vector op.
4656 if (CTVT.isVector() && TLI.isCtpopFast(VT: CTVT))
4657 return SDValue();
4658
4659 unsigned CostLimit = TLI.getCustomCtpopCost(VT: CTVT, Cond);
4660 if (C1.ugt(RHS: CostLimit + (Cond == ISD::SETULT)))
4661 return SDValue();
4662 if (C1 == 0 && (Cond == ISD::SETULT))
4663 return SDValue(); // This is handled elsewhere.
4664
4665 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
4666
4667 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4668 SDValue Result = CTOp;
4669 for (unsigned i = 0; i < Passes; i++) {
4670 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: Result, N2: NegOne);
4671 Result = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: Result, N2: Add);
4672 }
4673 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
4674 return DAG.getSetCC(DL: dl, VT, LHS: Result, RHS: DAG.getConstant(Val: 0, DL: dl, VT: CTVT), Cond: CC);
4675 }
4676
4677 // Expand a power-of-2 comparison based on ctpop
4678 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
4679 // Keep the CTPOP if it is cheap.
4680 if (TLI.isCtpopFast(VT: CTVT))
4681 return SDValue();
4682
4683 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: CTVT);
4684 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4685 assert(CTVT.isInteger());
4686 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: CTOp, N2: NegOne);
4687
4688 // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
4689 // check before emitting a potentially unnecessary op.
4690 if (DAG.isKnownNeverZero(Op: CTOp)) {
4691 // (ctpop x) == 1 --> (x & x-1) == 0
4692 // (ctpop x) != 1 --> (x & x-1) != 0
4693 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4694 SDValue RHS = DAG.getSetCC(DL: dl, VT, LHS: And, RHS: Zero, Cond);
4695 return RHS;
4696 }
4697
4698 // (ctpop x) == 1 --> (x ^ x-1) > x-1
4699 // (ctpop x) != 1 --> (x ^ x-1) <= x-1
4700 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4701 ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE;
4702 return DAG.getSetCC(DL: dl, VT, LHS: Xor, RHS: Add, Cond: CmpCond);
4703 }
4704
4705 return SDValue();
4706}
4707
4708static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1,
4709 ISD::CondCode Cond, const SDLoc &dl,
4710 SelectionDAG &DAG) {
4711 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4712 return SDValue();
4713
4714 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4715 if (!C1 || !(C1->isZero() || C1->isAllOnes()))
4716 return SDValue();
4717
4718 auto getRotateSource = [](SDValue X) {
4719 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
4720 return X.getOperand(i: 0);
4721 return SDValue();
4722 };
4723
4724 // Peek through a rotated value compared against 0 or -1:
4725 // (rot X, Y) == 0/-1 --> X == 0/-1
4726 // (rot X, Y) != 0/-1 --> X != 0/-1
4727 if (SDValue R = getRotateSource(N0))
4728 return DAG.getSetCC(DL: dl, VT, LHS: R, RHS: N1, Cond);
4729
4730 // Peek through an 'or' of a rotated value compared against 0:
4731 // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
4732 // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
4733 //
4734 // TODO: Add the 'and' with -1 sibling.
4735 // TODO: Recurse through a series of 'or' ops to find the rotate.
4736 EVT OpVT = N0.getValueType();
4737 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
4738 if (SDValue R = getRotateSource(N0.getOperand(i: 0))) {
4739 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 1));
4740 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4741 }
4742 if (SDValue R = getRotateSource(N0.getOperand(i: 1))) {
4743 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 0));
4744 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4745 }
4746 }
4747
4748 return SDValue();
4749}
4750
4751static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1,
4752 ISD::CondCode Cond, const SDLoc &dl,
4753 SelectionDAG &DAG) {
4754 // If we are testing for all-bits-clear, we might be able to do that with
4755 // less shifting since bit-order does not matter.
4756 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4757 return SDValue();
4758
4759 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4760 if (!C1 || !C1->isZero())
4761 return SDValue();
4762
4763 if (!N0.hasOneUse() ||
4764 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
4765 return SDValue();
4766
4767 unsigned BitWidth = N0.getScalarValueSizeInBits();
4768 auto *ShAmtC = isConstOrConstSplat(N: N0.getOperand(i: 2));
4769 if (!ShAmtC)
4770 return SDValue();
4771
4772 uint64_t ShAmt = ShAmtC->getAPIntValue().urem(RHS: BitWidth);
4773 if (ShAmt == 0)
4774 return SDValue();
4775
4776 // Canonicalize fshr as fshl to reduce pattern-matching.
4777 if (N0.getOpcode() == ISD::FSHR)
4778 ShAmt = BitWidth - ShAmt;
4779
4780 // Match an 'or' with a specific operand 'Other' in either commuted variant.
4781 SDValue X, Y;
4782 auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
4783 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
4784 return false;
4785 if (Or.getOperand(i: 0) == Other) {
4786 X = Or.getOperand(i: 0);
4787 Y = Or.getOperand(i: 1);
4788 return true;
4789 }
4790 if (Or.getOperand(i: 1) == Other) {
4791 X = Or.getOperand(i: 1);
4792 Y = Or.getOperand(i: 0);
4793 return true;
4794 }
4795 return false;
4796 };
4797
4798 EVT OpVT = N0.getValueType();
4799 EVT ShAmtVT = N0.getOperand(i: 2).getValueType();
4800 SDValue F0 = N0.getOperand(i: 0);
4801 SDValue F1 = N0.getOperand(i: 1);
4802 if (matchOr(F0, F1)) {
4803 // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
4804 SDValue NewShAmt = DAG.getConstant(Val: ShAmt, DL: dl, VT: ShAmtVT);
4805 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4806 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4807 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4808 }
4809 if (matchOr(F1, F0)) {
4810 // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
4811 SDValue NewShAmt = DAG.getConstant(Val: BitWidth - ShAmt, DL: dl, VT: ShAmtVT);
4812 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4813 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4814 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4815 }
4816
4817 return SDValue();
4818}
4819
4820/// Try to simplify a setcc built with the specified operands and cc. If it is
4821/// unable to simplify it, return a null SDValue.
4822SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
4823 ISD::CondCode Cond, bool foldBooleans,
4824 DAGCombinerInfo &DCI,
4825 const SDLoc &dl) const {
4826 SelectionDAG &DAG = DCI.DAG;
4827 const DataLayout &Layout = DAG.getDataLayout();
4828 EVT OpVT = N0.getValueType();
4829 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4830
4831 // Constant fold or commute setcc.
4832 if (SDValue Fold = DAG.FoldSetCC(VT, N1: N0, N2: N1, Cond, dl))
4833 return Fold;
4834
4835 bool N0ConstOrSplat =
4836 isConstOrConstSplat(N: N0, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
4837 bool N1ConstOrSplat =
4838 isConstOrConstSplat(N: N1, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
4839
4840 // Canonicalize toward having the constant on the RHS.
4841 // TODO: Handle non-splat vector constants. All undef causes trouble.
4842 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
4843 // infinite loop here when we encounter one.
4844 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Operation: Cond);
4845 if (N0ConstOrSplat && !N1ConstOrSplat &&
4846 (DCI.isBeforeLegalizeOps() ||
4847 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())))
4848 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
4849
4850 // If we have a subtract with the same 2 non-constant operands as this setcc
4851 // -- but in reverse order -- then try to commute the operands of this setcc
4852 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
4853 // instruction on some targets.
4854 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4855 (DCI.isBeforeLegalizeOps() ||
4856 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())) &&
4857 DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N1, N0}) &&
4858 !DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N0, N1}))
4859 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
4860
4861 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
4862 return V;
4863
4864 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
4865 return V;
4866
4867 if (auto *N1C = isConstOrConstSplat(N: N1)) {
4868 const APInt &C1 = N1C->getAPIntValue();
4869
4870 // Optimize some CTPOP cases.
4871 if (SDValue V = simplifySetCCWithCTPOP(TLI: *this, VT, N0, C1, Cond, dl, DAG))
4872 return V;
4873
4874 // For equality to 0 of a no-wrap multiply, decompose and test each op:
4875 // X * Y == 0 --> (X == 0) || (Y == 0)
4876 // X * Y != 0 --> (X != 0) && (Y != 0)
4877 // TODO: This bails out if minsize is set, but if the target doesn't have a
4878 // single instruction multiply for this type, it would likely be
4879 // smaller to decompose.
4880 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4881 N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
4882 (N0->getFlags().hasNoUnsignedWrap() ||
4883 N0->getFlags().hasNoSignedWrap()) &&
4884 !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
4885 SDValue IsXZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
4886 SDValue IsYZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1, Cond);
4887 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
4888 return DAG.getNode(Opcode: LogicOp, DL: dl, VT, N1: IsXZero, N2: IsYZero);
4889 }
4890
4891 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
4892 // equality comparison, then we're just comparing whether X itself is
4893 // zero.
4894 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
4895 N0.getOperand(i: 0).getOpcode() == ISD::CTLZ &&
4896 llvm::has_single_bit<uint32_t>(Value: N0.getScalarValueSizeInBits())) {
4897 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
4898 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4899 ShAmt->getAPIntValue() == Log2_32(Value: N0.getScalarValueSizeInBits())) {
4900 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
4901 // (srl (ctlz x), 5) == 0 -> X != 0
4902 // (srl (ctlz x), 5) != 1 -> X != 0
4903 Cond = ISD::SETNE;
4904 } else {
4905 // (srl (ctlz x), 5) != 0 -> X == 0
4906 // (srl (ctlz x), 5) == 1 -> X == 0
4907 Cond = ISD::SETEQ;
4908 }
4909 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: N0.getValueType());
4910 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0).getOperand(i: 0), RHS: Zero,
4911 Cond);
4912 }
4913 }
4914 }
4915 }
4916
4917 // setcc X, 0, setlt --> X (when X is all sign bits)
4918 // setcc X, 0, setne --> X (when X is all sign bits)
4919 //
4920 // When we know that X has 0 or -1 in each element (or scalar), this
4921 // comparison will produce X. This is only true when boolean contents are
4922 // represented via 0s and -1s.
4923 if (VT == OpVT &&
4924 // Check that the result of setcc is 0 and -1.
4925 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent &&
4926 // Match only for checks X < 0 and X != 0
4927 (Cond == ISD::SETLT || Cond == ISD::SETNE) && isNullOrNullSplat(V: N1) &&
4928 // The identity holds iff we know all sign bits for all lanes.
4929 DAG.ComputeNumSignBits(Op: N0) == N0.getScalarValueSizeInBits())
4930 return N0;
4931
4932 // FIXME: Support vectors.
4933 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
4934 const APInt &C1 = N1C->getAPIntValue();
4935
4936 // (zext x) == C --> x == (trunc C)
4937 // (sext x) == C --> x == (trunc C)
4938 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4939 DCI.isBeforeLegalize() && N0->hasOneUse()) {
4940 unsigned MinBits = N0.getValueSizeInBits();
4941 SDValue PreExt;
4942 bool Signed = false;
4943 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
4944 // ZExt
4945 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
4946 PreExt = N0->getOperand(Num: 0);
4947 } else if (N0->getOpcode() == ISD::AND) {
4948 // DAGCombine turns costly ZExts into ANDs
4949 if (auto *C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
4950 if ((C->getAPIntValue()+1).isPowerOf2()) {
4951 MinBits = C->getAPIntValue().countr_one();
4952 PreExt = N0->getOperand(Num: 0);
4953 }
4954 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
4955 // SExt
4956 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
4957 PreExt = N0->getOperand(Num: 0);
4958 Signed = true;
4959 } else if (auto *LN0 = dyn_cast<LoadSDNode>(Val&: N0)) {
4960 // ZEXTLOAD / SEXTLOAD
4961 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
4962 MinBits = LN0->getMemoryVT().getSizeInBits();
4963 PreExt = N0;
4964 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
4965 Signed = true;
4966 MinBits = LN0->getMemoryVT().getSizeInBits();
4967 PreExt = N0;
4968 }
4969 }
4970
4971 // Figure out how many bits we need to preserve this constant.
4972 unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();
4973
4974 // Make sure we're not losing bits from the constant.
4975 if (MinBits > 0 &&
4976 MinBits < C1.getBitWidth() &&
4977 MinBits >= ReqdBits) {
4978 EVT MinVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MinBits);
4979 if (isTypeDesirableForOp(ISD::SETCC, VT: MinVT)) {
4980 // Will get folded away.
4981 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MinVT, Operand: PreExt);
4982 if (MinBits == 1 && C1 == 1)
4983 // Invert the condition.
4984 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i1),
4985 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
4986 SDValue C = DAG.getConstant(Val: C1.trunc(width: MinBits), DL: dl, VT: MinVT);
4987 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: C, Cond);
4988 }
4989
4990 // If truncating the setcc operands is not desirable, we can still
4991 // simplify the expression in some cases:
4992 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
4993 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
4994 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
4995 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
4996 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
4997 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
4998 SDValue TopSetCC = N0->getOperand(Num: 0);
4999 unsigned N0Opc = N0->getOpcode();
5000 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
5001 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
5002 TopSetCC.getOpcode() == ISD::SETCC &&
5003 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
5004 (isConstFalseVal(N: N1) ||
5005 isExtendedTrueVal(N: N1C, VT: N0->getValueType(ResNo: 0), SExt))) {
5006
5007 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
5008 (!N1C->isZero() && Cond == ISD::SETNE);
5009
5010 if (!Inverse)
5011 return TopSetCC;
5012
5013 ISD::CondCode InvCond = ISD::getSetCCInverse(
5014 Operation: cast<CondCodeSDNode>(Val: TopSetCC.getOperand(i: 2))->get(),
5015 Type: TopSetCC.getOperand(i: 0).getValueType());
5016 return DAG.getSetCC(DL: dl, VT, LHS: TopSetCC.getOperand(i: 0),
5017 RHS: TopSetCC.getOperand(i: 1),
5018 Cond: InvCond);
5019 }
5020 }
5021 }
5022
5023 // If the LHS is '(and load, const)', the RHS is 0, the test is for
5024 // equality or unsigned, and all 1 bits of the const are in the same
5025 // partial word, see if we can shorten the load.
5026 if (DCI.isBeforeLegalize() &&
5027 !ISD::isSignedIntSetCC(Code: Cond) &&
5028 N0.getOpcode() == ISD::AND && C1 == 0 &&
5029 N0.getNode()->hasOneUse() &&
5030 isa<LoadSDNode>(Val: N0.getOperand(i: 0)) &&
5031 N0.getOperand(i: 0).getNode()->hasOneUse() &&
5032 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5033 auto *Lod = cast<LoadSDNode>(Val: N0.getOperand(i: 0));
5034 APInt bestMask;
5035 unsigned bestWidth = 0, bestOffset = 0;
5036 if (Lod->isSimple() && Lod->isUnindexed() &&
5037 (Lod->getMemoryVT().isByteSized() ||
5038 isPaddedAtMostSignificantBitsWhenStored(VT: Lod->getMemoryVT()))) {
5039 unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
5040 unsigned origWidth = N0.getValueSizeInBits();
5041 unsigned maskWidth = origWidth;
5042 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
5043 // 8 bits, but have to be careful...
5044 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
5045 origWidth = Lod->getMemoryVT().getSizeInBits();
5046 const APInt &Mask = N0.getConstantOperandAPInt(i: 1);
5047 // Only consider power-of-2 widths (and at least one byte) as candiates
5048 // for the narrowed load.
5049 for (unsigned width = 8; width < origWidth; width *= 2) {
5050 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: width);
5051 APInt newMask = APInt::getLowBitsSet(numBits: maskWidth, loBitsSet: width);
5052 // Avoid accessing any padding here for now (we could use memWidth
5053 // instead of origWidth here otherwise).
5054 unsigned maxOffset = origWidth - width;
5055 for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
5056 if (Mask.isSubsetOf(RHS: newMask)) {
5057 unsigned ptrOffset =
5058 Layout.isLittleEndian() ? offset : memWidth - width - offset;
5059 unsigned IsFast = 0;
5060 assert((ptrOffset % 8) == 0 && "Non-Bytealigned pointer offset");
5061 Align NewAlign = commonAlignment(A: Lod->getAlign(), Offset: ptrOffset / 8);
5062 if (shouldReduceLoadWidth(Load: Lod, ExtTy: ISD::NON_EXTLOAD, NewVT: newVT,
5063 ByteOffset: ptrOffset / 8) &&
5064 allowsMemoryAccess(
5065 Context&: *DAG.getContext(), DL: Layout, VT: newVT, AddrSpace: Lod->getAddressSpace(),
5066 Alignment: NewAlign, Flags: Lod->getMemOperand()->getFlags(), Fast: &IsFast) &&
5067 IsFast) {
5068 bestOffset = ptrOffset / 8;
5069 bestMask = Mask.lshr(shiftAmt: offset);
5070 bestWidth = width;
5071 break;
5072 }
5073 }
5074 newMask <<= 8;
5075 }
5076 if (bestWidth)
5077 break;
5078 }
5079 }
5080 if (bestWidth) {
5081 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: bestWidth);
5082 SDValue Ptr = Lod->getBasePtr();
5083 if (bestOffset != 0)
5084 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: bestOffset));
5085 SDValue NewLoad =
5086 DAG.getLoad(VT: newVT, dl, Chain: Lod->getChain(), Ptr,
5087 PtrInfo: Lod->getPointerInfo().getWithOffset(O: bestOffset),
5088 Alignment: Lod->getBaseAlign());
5089 SDValue And =
5090 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: newVT, N1: NewLoad,
5091 N2: DAG.getConstant(Val: bestMask.trunc(width: bestWidth), DL: dl, VT: newVT));
5092 return DAG.getSetCC(DL: dl, VT, LHS: And, RHS: DAG.getConstant(Val: 0LL, DL: dl, VT: newVT), Cond);
5093 }
5094 }
5095
5096 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
5097 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
5098 unsigned InSize = N0.getOperand(i: 0).getValueSizeInBits();
5099
5100 // If the comparison constant has bits in the upper part, the
5101 // zero-extended value could never match.
5102 if (C1.intersects(RHS: APInt::getHighBitsSet(numBits: C1.getBitWidth(),
5103 hiBitsSet: C1.getBitWidth() - InSize))) {
5104 switch (Cond) {
5105 case ISD::SETUGT:
5106 case ISD::SETUGE:
5107 case ISD::SETEQ:
5108 return DAG.getConstant(Val: 0, DL: dl, VT);
5109 case ISD::SETULT:
5110 case ISD::SETULE:
5111 case ISD::SETNE:
5112 return DAG.getConstant(Val: 1, DL: dl, VT);
5113 case ISD::SETGT:
5114 case ISD::SETGE:
5115 // True if the sign bit of C1 is set.
5116 return DAG.getConstant(Val: C1.isNegative(), DL: dl, VT);
5117 case ISD::SETLT:
5118 case ISD::SETLE:
5119 // True if the sign bit of C1 isn't set.
5120 return DAG.getConstant(Val: C1.isNonNegative(), DL: dl, VT);
5121 default:
5122 break;
5123 }
5124 }
5125
5126 // Otherwise, we can perform the comparison with the low bits.
5127 switch (Cond) {
5128 case ISD::SETEQ:
5129 case ISD::SETNE:
5130 case ISD::SETUGT:
5131 case ISD::SETUGE:
5132 case ISD::SETULT:
5133 case ISD::SETULE: {
5134 EVT newVT = N0.getOperand(i: 0).getValueType();
5135 // FIXME: Should use isNarrowingProfitable.
5136 if (DCI.isBeforeLegalizeOps() ||
5137 (isOperationLegal(Op: ISD::SETCC, VT: newVT) &&
5138 isCondCodeLegal(CC: Cond, VT: newVT.getSimpleVT()) &&
5139 isTypeDesirableForOp(ISD::SETCC, VT: newVT))) {
5140 EVT NewSetCCVT = getSetCCResultType(DL: Layout, Context&: *DAG.getContext(), VT: newVT);
5141 SDValue NewConst = DAG.getConstant(Val: C1.trunc(width: InSize), DL: dl, VT: newVT);
5142
5143 SDValue NewSetCC = DAG.getSetCC(DL: dl, VT: NewSetCCVT, LHS: N0.getOperand(i: 0),
5144 RHS: NewConst, Cond);
5145 return DAG.getBoolExtOrTrunc(Op: NewSetCC, SL: dl, VT, OpVT: N0.getValueType());
5146 }
5147 break;
5148 }
5149 default:
5150 break; // todo, be more careful with signed comparisons
5151 }
5152 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5153 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5154 !isSExtCheaperThanZExt(FromTy: cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT(),
5155 ToTy: OpVT)) {
5156 EVT ExtSrcTy = cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT();
5157 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
5158 EVT ExtDstTy = N0.getValueType();
5159 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
5160
5161 // If the constant doesn't fit into the number of bits for the source of
5162 // the sign extension, it is impossible for both sides to be equal.
5163 if (C1.getSignificantBits() > ExtSrcTyBits)
5164 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
5165
5166 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
5167 ExtDstTy != ExtSrcTy && "Unexpected types!");
5168 APInt Imm = APInt::getLowBitsSet(numBits: ExtDstTyBits, loBitsSet: ExtSrcTyBits);
5169 SDValue ZextOp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ExtDstTy, N1: N0.getOperand(i: 0),
5170 N2: DAG.getConstant(Val: Imm, DL: dl, VT: ExtDstTy));
5171 if (!DCI.isCalledByLegalizer())
5172 DCI.AddToWorklist(N: ZextOp.getNode());
5173 // Otherwise, make this a use of a zext.
5174 return DAG.getSetCC(DL: dl, VT, LHS: ZextOp,
5175 RHS: DAG.getConstant(Val: C1 & Imm, DL: dl, VT: ExtDstTy), Cond);
5176 } else if ((N1C->isZero() || N1C->isOne()) &&
5177 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5178 // SETCC (X), [0|1], [EQ|NE] -> X if X is known 0/1. i1 types are
5179 // excluded as they are handled below whilst checking for foldBooleans.
5180 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
5181 isTypeLegal(VT) && VT.bitsLE(VT: N0.getValueType()) &&
5182 (N0.getValueType() == MVT::i1 ||
5183 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5184 DAG.MaskedValueIsZero(
5185 Op: N0, Mask: APInt::getBitsSetFrom(numBits: N0.getValueSizeInBits(), loBit: 1))) {
5186 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
5187 if (TrueWhenTrue)
5188 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: N0);
5189 // Invert the condition.
5190 if (N0.getOpcode() == ISD::SETCC) {
5191 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
5192 CC = ISD::getSetCCInverse(Operation: CC, Type: N0.getOperand(i: 0).getValueType());
5193 if (DCI.isBeforeLegalizeOps() ||
5194 isCondCodeLegal(CC, VT: N0.getOperand(i: 0).getSimpleValueType()))
5195 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond: CC);
5196 }
5197 }
5198
5199 if ((N0.getOpcode() == ISD::XOR ||
5200 (N0.getOpcode() == ISD::AND &&
5201 N0.getOperand(i: 0).getOpcode() == ISD::XOR &&
5202 N0.getOperand(i: 1) == N0.getOperand(i: 0).getOperand(i: 1))) &&
5203 isOneConstant(V: N0.getOperand(i: 1))) {
5204 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
5205 // can only do this if the top bits are known zero.
5206 unsigned BitWidth = N0.getValueSizeInBits();
5207 if (DAG.MaskedValueIsZero(Op: N0,
5208 Mask: APInt::getHighBitsSet(numBits: BitWidth,
5209 hiBitsSet: BitWidth-1))) {
5210 // Okay, get the un-inverted input value.
5211 SDValue Val;
5212 if (N0.getOpcode() == ISD::XOR) {
5213 Val = N0.getOperand(i: 0);
5214 } else {
5215 assert(N0.getOpcode() == ISD::AND &&
5216 N0.getOperand(0).getOpcode() == ISD::XOR);
5217 // ((X^1)&1)^1 -> X & 1
5218 Val = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N0.getValueType(),
5219 N1: N0.getOperand(i: 0).getOperand(i: 0),
5220 N2: N0.getOperand(i: 1));
5221 }
5222
5223 return DAG.getSetCC(DL: dl, VT, LHS: Val, RHS: N1,
5224 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5225 }
5226 } else if (N1C->isOne()) {
5227 SDValue Op0 = N0;
5228 if (Op0.getOpcode() == ISD::TRUNCATE)
5229 Op0 = Op0.getOperand(i: 0);
5230
5231 if ((Op0.getOpcode() == ISD::XOR) &&
5232 Op0.getOperand(i: 0).getOpcode() == ISD::SETCC &&
5233 Op0.getOperand(i: 1).getOpcode() == ISD::SETCC) {
5234 SDValue XorLHS = Op0.getOperand(i: 0);
5235 SDValue XorRHS = Op0.getOperand(i: 1);
5236 // Ensure that the input setccs return an i1 type or 0/1 value.
5237 if (Op0.getValueType() == MVT::i1 ||
5238 (getBooleanContents(Type: XorLHS.getOperand(i: 0).getValueType()) ==
5239 ZeroOrOneBooleanContent &&
5240 getBooleanContents(Type: XorRHS.getOperand(i: 0).getValueType()) ==
5241 ZeroOrOneBooleanContent)) {
5242 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
5243 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
5244 return DAG.getSetCC(DL: dl, VT, LHS: XorLHS, RHS: XorRHS, Cond);
5245 }
5246 }
5247 if (Op0.getOpcode() == ISD::AND && isOneConstant(V: Op0.getOperand(i: 1))) {
5248 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
5249 if (Op0.getValueType().bitsGT(VT))
5250 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5251 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5252 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5253 else if (Op0.getValueType().bitsLT(VT))
5254 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5255 N1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5256 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5257
5258 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5259 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5260 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5261 }
5262 if (Op0.getOpcode() == ISD::AssertZext &&
5263 cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT() == MVT::i1)
5264 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5265 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5266 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5267 }
5268 }
5269
5270 // Given:
5271 // icmp eq/ne (urem %x, %y), 0
5272 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
5273 // icmp eq/ne %x, 0
5274 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
5275 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5276 KnownBits XKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 0));
5277 KnownBits YKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 1));
5278 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
5279 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
5280 }
5281
5282 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
5283 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
5284 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5285 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
5286 N0.getConstantOperandAPInt(i: 1) == OpVT.getScalarSizeInBits() - 1 &&
5287 N1C->isAllOnes()) {
5288 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
5289 RHS: DAG.getConstant(Val: 0, DL: dl, VT: OpVT),
5290 Cond: Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
5291 }
5292
5293 // fold (setcc (trunc x) c) -> (setcc x c)
5294 if (N0.getOpcode() == ISD::TRUNCATE &&
5295 ((N0->getFlags().hasNoUnsignedWrap() && !ISD::isSignedIntSetCC(Code: Cond)) ||
5296 (N0->getFlags().hasNoSignedWrap() &&
5297 !ISD::isUnsignedIntSetCC(Code: Cond))) &&
5298 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5299 EVT NewVT = N0.getOperand(i: 0).getValueType();
5300 SDValue NewConst = DAG.getConstant(
5301 Val: (N0->getFlags().hasNoSignedWrap() && !ISD::isUnsignedIntSetCC(Code: Cond))
5302 ? C1.sext(width: NewVT.getSizeInBits())
5303 : C1.zext(width: NewVT.getSizeInBits()),
5304 DL: dl, VT: NewVT);
5305 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NewConst, Cond);
5306 }
5307
5308 if (SDValue V =
5309 optimizeSetCCOfSignedTruncationCheck(SCCVT: VT, N0, N1, Cond, DCI, DL: dl))
5310 return V;
5311 }
5312
5313 // These simplifications apply to splat vectors as well.
5314 // TODO: Handle more splat vector cases.
5315 if (auto *N1C = isConstOrConstSplat(N: N1)) {
5316 const APInt &C1 = N1C->getAPIntValue();
5317
5318 APInt MinVal, MaxVal;
5319 unsigned OperandBitSize = N1C->getValueType(ResNo: 0).getScalarSizeInBits();
5320 if (ISD::isSignedIntSetCC(Code: Cond)) {
5321 MinVal = APInt::getSignedMinValue(numBits: OperandBitSize);
5322 MaxVal = APInt::getSignedMaxValue(numBits: OperandBitSize);
5323 } else {
5324 MinVal = APInt::getMinValue(numBits: OperandBitSize);
5325 MaxVal = APInt::getMaxValue(numBits: OperandBitSize);
5326 }
5327
5328 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
5329 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
5330 // X >= MIN --> true
5331 if (C1 == MinVal)
5332 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5333
5334 if (!VT.isVector()) { // TODO: Support this for vectors.
5335 // X >= C0 --> X > (C0 - 1)
5336 APInt C = C1 - 1;
5337 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
5338 if ((DCI.isBeforeLegalizeOps() ||
5339 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5340 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5341 isLegalICmpImmediate(C.getSExtValue())))) {
5342 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5343 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5344 Cond: NewCC);
5345 }
5346 }
5347 }
5348
5349 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
5350 // X <= MAX --> true
5351 if (C1 == MaxVal)
5352 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5353
5354 // X <= C0 --> X < (C0 + 1)
5355 if (!VT.isVector()) { // TODO: Support this for vectors.
5356 APInt C = C1 + 1;
5357 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
5358 if ((DCI.isBeforeLegalizeOps() ||
5359 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5360 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5361 isLegalICmpImmediate(C.getSExtValue())))) {
5362 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5363 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5364 Cond: NewCC);
5365 }
5366 }
5367 }
5368
5369 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
5370 if (C1 == MinVal)
5371 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X < MIN --> false
5372
5373 // TODO: Support this for vectors after legalize ops.
5374 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5375 // Canonicalize setlt X, Max --> setne X, Max
5376 if (C1 == MaxVal)
5377 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5378
5379 // If we have setult X, 1, turn it into seteq X, 0
5380 if (C1 == MinVal+1)
5381 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5382 RHS: DAG.getConstant(Val: MinVal, DL: dl, VT: N0.getValueType()),
5383 Cond: ISD::SETEQ);
5384 }
5385 }
5386
5387 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
5388 if (C1 == MaxVal)
5389 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X > MAX --> false
5390
5391 // TODO: Support this for vectors after legalize ops.
5392 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5393 // Canonicalize setgt X, Min --> setne X, Min
5394 if (C1 == MinVal)
5395 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5396
5397 // If we have setugt X, Max-1, turn it into seteq X, Max
5398 if (C1 == MaxVal-1)
5399 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5400 RHS: DAG.getConstant(Val: MaxVal, DL: dl, VT: N0.getValueType()),
5401 Cond: ISD::SETEQ);
5402 }
5403 }
5404
5405 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
5406 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
5407 if (C1.isZero())
5408 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
5409 SCCVT: VT, N0, N1C: N1, Cond, DCI, DL: dl))
5410 return CC;
5411
5412 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
5413 // For example, when high 32-bits of i64 X are known clear:
5414 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
5415 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
5416 bool CmpZero = N1C->isZero();
5417 bool CmpNegOne = N1C->isAllOnes();
5418 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
5419 // Match or(lo,shl(hi,bw/2)) pattern.
5420 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
5421 unsigned EltBits = V.getScalarValueSizeInBits();
5422 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
5423 return false;
5424 SDValue LHS = V.getOperand(i: 0);
5425 SDValue RHS = V.getOperand(i: 1);
5426 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: EltBits / 2);
5427 // Unshifted element must have zero upperbits.
5428 if (RHS.getOpcode() == ISD::SHL &&
5429 isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)) &&
5430 RHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5431 DAG.MaskedValueIsZero(Op: LHS, Mask: HiBits)) {
5432 Lo = LHS;
5433 Hi = RHS.getOperand(i: 0);
5434 return true;
5435 }
5436 if (LHS.getOpcode() == ISD::SHL &&
5437 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
5438 LHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5439 DAG.MaskedValueIsZero(Op: RHS, Mask: HiBits)) {
5440 Lo = RHS;
5441 Hi = LHS.getOperand(i: 0);
5442 return true;
5443 }
5444 return false;
5445 };
5446
5447 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
5448 unsigned EltBits = N0.getScalarValueSizeInBits();
5449 unsigned HalfBits = EltBits / 2;
5450 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: HalfBits);
5451 SDValue LoBits = DAG.getConstant(Val: ~HiBits, DL: dl, VT: OpVT);
5452 SDValue HiMask = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Hi, N2: LoBits);
5453 SDValue NewN0 =
5454 DAG.getNode(Opcode: CmpZero ? ISD::OR : ISD::AND, DL: dl, VT: OpVT, N1: Lo, N2: HiMask);
5455 SDValue NewN1 = CmpZero ? DAG.getConstant(Val: 0, DL: dl, VT: OpVT) : LoBits;
5456 return DAG.getSetCC(DL: dl, VT, LHS: NewN0, RHS: NewN1, Cond);
5457 };
5458
5459 SDValue Lo, Hi;
5460 if (IsConcat(N0, Lo, Hi))
5461 return MergeConcat(Lo, Hi);
5462
5463 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
5464 SDValue Lo0, Lo1, Hi0, Hi1;
5465 if (IsConcat(N0.getOperand(i: 0), Lo0, Hi0) &&
5466 IsConcat(N0.getOperand(i: 1), Lo1, Hi1)) {
5467 return MergeConcat(DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Lo0, N2: Lo1),
5468 DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Hi0, N2: Hi1));
5469 }
5470 }
5471 }
5472 }
5473
5474 // If we have "setcc X, C0", check to see if we can shrink the immediate
5475 // by changing cc.
5476 // TODO: Support this for vectors after legalize ops.
5477 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5478 // SETUGT X, SINTMAX -> SETLT X, 0
5479 // SETUGE X, SINTMIN -> SETLT X, 0
5480 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
5481 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
5482 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5483 RHS: DAG.getConstant(Val: 0, DL: dl, VT: N1.getValueType()),
5484 Cond: ISD::SETLT);
5485
5486 // SETULT X, SINTMIN -> SETGT X, -1
5487 // SETULE X, SINTMAX -> SETGT X, -1
5488 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
5489 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
5490 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5491 RHS: DAG.getAllOnesConstant(DL: dl, VT: N1.getValueType()),
5492 Cond: ISD::SETGT);
5493 }
5494 }
5495
5496 // Back to non-vector simplifications.
5497 // TODO: Can we do these for vector splats?
5498 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5499 const APInt &C1 = N1C->getAPIntValue();
5500 EVT ShValTy = N0.getValueType();
5501
5502 // Fold bit comparisons when we can. This will result in an
5503 // incorrect value when boolean false is negative one, unless
5504 // the bitsize is 1 in which case the false value is the same
5505 // in practice regardless of the representation.
5506 if ((VT.getSizeInBits() == 1 ||
5507 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5508 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5509 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(VT: ShValTy))) &&
5510 N0.getOpcode() == ISD::AND) {
5511 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5512 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
5513 // Perform the xform if the AND RHS is a single bit.
5514 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5515 if (AndRHS->getAPIntValue().isPowerOf2() &&
5516 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5517 return DAG.getNode(
5518 Opcode: ISD::TRUNCATE, DL: dl, VT,
5519 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5520 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5521 }
5522 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5523 // (X & 8) == 8 --> (X & 8) >> 3
5524 // Perform the xform if C1 is a single bit.
5525 unsigned ShCt = C1.logBase2();
5526 if (C1.isPowerOf2() && !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5527 return DAG.getNode(
5528 Opcode: ISD::TRUNCATE, DL: dl, VT,
5529 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5530 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5531 }
5532 }
5533 }
5534 }
5535
5536 if (C1.getSignificantBits() <= 64 &&
5537 !isLegalICmpImmediate(C1.getSExtValue())) {
5538 // (X & -256) == 256 -> (X >> 8) == 1
5539 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5540 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
5541 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5542 const APInt &AndRHSC = AndRHS->getAPIntValue();
5543 if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(RHS: AndRHSC)) {
5544 unsigned ShiftBits = AndRHSC.countr_zero();
5545 if (!shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5546 // If using an unsigned shift doesn't yield a legal compare
5547 // immediate, try using sra instead.
5548 APInt NewC = C1.lshr(shiftAmt: ShiftBits);
5549 if (NewC.getSignificantBits() <= 64 &&
5550 !isLegalICmpImmediate(NewC.getSExtValue())) {
5551 APInt SignedC = C1.ashr(ShiftAmt: ShiftBits);
5552 if (SignedC.getSignificantBits() <= 64 &&
5553 isLegalICmpImmediate(SignedC.getSExtValue())) {
5554 SDValue Shift = DAG.getNode(
5555 Opcode: ISD::SRA, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5556 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5557 SDValue CmpRHS = DAG.getConstant(Val: SignedC, DL: dl, VT: ShValTy);
5558 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5559 }
5560 }
5561 SDValue Shift = DAG.getNode(
5562 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5563 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5564 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5565 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5566 }
5567 }
5568 }
5569 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
5570 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
5571 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
5572 // X < 0x100000000 -> (X >> 32) < 1
5573 // X >= 0x100000000 -> (X >> 32) >= 1
5574 // X <= 0x0ffffffff -> (X >> 32) < 1
5575 // X > 0x0ffffffff -> (X >> 32) >= 1
5576 unsigned ShiftBits;
5577 APInt NewC = C1;
5578 ISD::CondCode NewCond = Cond;
5579 if (AdjOne) {
5580 ShiftBits = C1.countr_one();
5581 NewC = NewC + 1;
5582 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
5583 } else {
5584 ShiftBits = C1.countr_zero();
5585 }
5586 APInt RangeWidth = NewC;
5587 NewC.lshrInPlace(ShiftAmt: ShiftBits);
5588 if (ShiftBits && NewC.getSignificantBits() <= 64 &&
5589 isLegalICmpImmediate(NewC.getSExtValue()) &&
5590 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5591 // If this is an offset range check, try to move the offset after the
5592 // shift to avoid preserving the pre-shift add with a mask.
5593 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse()) {
5594 if (auto *AddC = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
5595 const APInt &AddVal = AddC->getAPIntValue();
5596 if (AddVal.countr_zero() >= ShiftBits) {
5597 APInt RangeLower = -AddVal;
5598 bool Overflow;
5599 (void)RangeLower.uadd_ov(RHS: RangeWidth, Overflow);
5600 if (!RangeWidth.isZero() && !Overflow) {
5601 SDValue Shift = DAG.getNode(
5602 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5603 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5604 APInt Offset = -RangeLower.lshr(shiftAmt: ShiftBits);
5605 SDValue ShiftedAdd =
5606 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ShValTy, N1: Shift,
5607 N2: DAG.getConstant(Val: Offset, DL: dl, VT: ShValTy));
5608 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5609 return DAG.getSetCC(DL: dl, VT, LHS: ShiftedAdd, RHS: CmpRHS, Cond: NewCond);
5610 }
5611 }
5612 }
5613 }
5614 SDValue Shift =
5615 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5616 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5617 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5618 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond: NewCond);
5619 }
5620 }
5621 }
5622 }
5623
5624 if (!isa<ConstantFPSDNode>(Val: N0) && isa<ConstantFPSDNode>(Val: N1)) {
5625 auto *CFP = cast<ConstantFPSDNode>(Val&: N1);
5626 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
5627
5628 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
5629 // constant if knowing that the operand is non-nan is enough. We prefer to
5630 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
5631 // materialize 0.0.
5632 if (Cond == ISD::SETO || Cond == ISD::SETUO)
5633 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N0, Cond);
5634
5635 // setcc (fneg x), C -> setcc swap(pred) x, -C
5636 if (N0.getOpcode() == ISD::FNEG) {
5637 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Operation: Cond);
5638 if (DCI.isBeforeLegalizeOps() ||
5639 isCondCodeLegal(CC: SwapCond, VT: N0.getSimpleValueType())) {
5640 SDValue NegN1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: N0.getValueType(), Operand: N1);
5641 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NegN1, Cond: SwapCond);
5642 }
5643 }
5644
5645 // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
5646 if (isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT: N0.getValueType()) &&
5647 !isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(ResNo: 0))) {
5648 bool IsFabs = N0.getOpcode() == ISD::FABS;
5649 SDValue Op = IsFabs ? N0.getOperand(i: 0) : N0;
5650 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
5651 FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
5652 : (IsFabs ? fcInf : fcPosInf);
5653 if (Cond == ISD::SETUEQ)
5654 Flag |= fcNan;
5655 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: dl, VT, N1: Op,
5656 N2: DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32));
5657 }
5658 }
5659
5660 // If the condition is not legal, see if we can find an equivalent one
5661 // which is legal.
5662 if (!isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType())) {
5663 // If the comparison was an awkward floating-point == or != and one of
5664 // the comparison operands is infinity or negative infinity, convert the
5665 // condition to a less-awkward <= or >=.
5666 if (CFP->getValueAPF().isInfinity()) {
5667 bool IsNegInf = CFP->getValueAPF().isNegative();
5668 ISD::CondCode NewCond = ISD::SETCC_INVALID;
5669 switch (Cond) {
5670 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
5671 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
5672 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
5673 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
5674 default: break;
5675 }
5676 if (NewCond != ISD::SETCC_INVALID &&
5677 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType()))
5678 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5679 }
5680 }
5681 }
5682
5683 if (N0 == N1) {
5684 // The sext(setcc()) => setcc() optimization relies on the appropriate
5685 // constant being emitted.
5686 assert(!N0.getValueType().isInteger() &&
5687 "Integer types should be handled by FoldSetCC");
5688
5689 bool EqTrue = ISD::isTrueWhenEqual(Cond);
5690 unsigned UOF = ISD::getUnorderedFlavor(Cond);
5691 if (UOF == 2) // FP operators that are undefined on NaNs.
5692 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5693 if (UOF == unsigned(EqTrue))
5694 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5695 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
5696 // if it is not already.
5697 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
5698 if (NewCond != Cond &&
5699 (DCI.isBeforeLegalizeOps() ||
5700 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType())))
5701 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5702 }
5703
5704 // ~X > ~Y --> Y > X
5705 // ~X < ~Y --> Y < X
5706 // ~X < C --> X > ~C
5707 // ~X > C --> X < ~C
5708 if ((isSignedIntSetCC(Code: Cond) || isUnsignedIntSetCC(Code: Cond)) &&
5709 N0.getValueType().isInteger()) {
5710 if (isBitwiseNot(V: N0)) {
5711 if (isBitwiseNot(V: N1))
5712 return DAG.getSetCC(DL: dl, VT, LHS: N1.getOperand(i: 0), RHS: N0.getOperand(i: 0), Cond);
5713
5714 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N1) &&
5715 !DAG.isConstantIntBuildVectorOrConstantInt(N: N0.getOperand(i: 0))) {
5716 SDValue Not = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5717 return DAG.getSetCC(DL: dl, VT, LHS: Not, RHS: N0.getOperand(i: 0), Cond);
5718 }
5719 }
5720 }
5721
5722 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5723 N0.getValueType().isInteger()) {
5724 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
5725 N0.getOpcode() == ISD::XOR) {
5726 // Simplify (X+Y) == (X+Z) --> Y == Z
5727 if (N0.getOpcode() == N1.getOpcode()) {
5728 if (N0.getOperand(i: 0) == N1.getOperand(i: 0))
5729 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 1), Cond);
5730 if (N0.getOperand(i: 1) == N1.getOperand(i: 1))
5731 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
5732 if (isCommutativeBinOp(Opcode: N0.getOpcode())) {
5733 // If X op Y == Y op X, try other combinations.
5734 if (N0.getOperand(i: 0) == N1.getOperand(i: 1))
5735 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 0),
5736 Cond);
5737 if (N0.getOperand(i: 1) == N1.getOperand(i: 0))
5738 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 1),
5739 Cond);
5740 }
5741 }
5742
5743 // If RHS is a legal immediate value for a compare instruction, we need
5744 // to be careful about increasing register pressure needlessly.
5745 bool LegalRHSImm = false;
5746
5747 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: N1)) {
5748 if (auto *LHSR = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5749 // Turn (X+C1) == C2 --> X == C2-C1
5750 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
5751 return DAG.getSetCC(
5752 DL: dl, VT, LHS: N0.getOperand(i: 0),
5753 RHS: DAG.getConstant(Val: RHSC->getAPIntValue() - LHSR->getAPIntValue(),
5754 DL: dl, VT: N0.getValueType()),
5755 Cond);
5756
5757 // Turn (X^C1) == C2 --> X == C1^C2
5758 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
5759 return DAG.getSetCC(
5760 DL: dl, VT, LHS: N0.getOperand(i: 0),
5761 RHS: DAG.getConstant(Val: LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
5762 DL: dl, VT: N0.getValueType()),
5763 Cond);
5764 }
5765
5766 // Turn (C1-X) == C2 --> X == C1-C2
5767 if (auto *SUBC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0)))
5768 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
5769 return DAG.getSetCC(
5770 DL: dl, VT, LHS: N0.getOperand(i: 1),
5771 RHS: DAG.getConstant(Val: SUBC->getAPIntValue() - RHSC->getAPIntValue(),
5772 DL: dl, VT: N0.getValueType()),
5773 Cond);
5774
5775 // Could RHSC fold directly into a compare?
5776 if (RHSC->getValueType(ResNo: 0).getSizeInBits() <= 64)
5777 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
5778 }
5779
5780 // (X+Y) == X --> Y == 0 and similar folds.
5781 // Don't do this if X is an immediate that can fold into a cmp
5782 // instruction and X+Y has other uses. It could be an induction variable
5783 // chain, and the transform would increase register pressure.
5784 if (!LegalRHSImm || N0.hasOneUse())
5785 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, DL: dl, DCI))
5786 return V;
5787 }
5788
5789 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
5790 N1.getOpcode() == ISD::XOR)
5791 if (SDValue V = foldSetCCWithBinOp(VT, N0: N1, N1: N0, Cond, DL: dl, DCI))
5792 return V;
5793
5794 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, DL: dl, DCI))
5795 return V;
5796
5797 if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, DL: dl, DCI))
5798 return V;
5799 }
5800
5801 // Fold remainder of division by a constant.
5802 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
5803 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5804 // When division is cheap or optimizing for minimum size,
5805 // fall through to DIVREM creation by skipping this fold.
5806 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
5807 if (N0.getOpcode() == ISD::UREM) {
5808 if (SDValue Folded = buildUREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5809 return Folded;
5810 } else if (N0.getOpcode() == ISD::SREM) {
5811 if (SDValue Folded = buildSREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5812 return Folded;
5813 }
5814 }
5815 }
5816
5817 // Fold away ALL boolean setcc's.
5818 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
5819 SDValue Temp;
5820 switch (Cond) {
5821 default: llvm_unreachable("Unknown integer setcc!");
5822 case ISD::SETEQ: // X == Y -> ~(X^Y)
5823 Temp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
5824 N0 = DAG.getNOT(DL: dl, Val: Temp, VT: OpVT);
5825 if (!DCI.isCalledByLegalizer())
5826 DCI.AddToWorklist(N: Temp.getNode());
5827 break;
5828 case ISD::SETNE: // X != Y --> (X^Y)
5829 N0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
5830 break;
5831 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
5832 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
5833 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
5834 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1, N2: Temp);
5835 if (!DCI.isCalledByLegalizer())
5836 DCI.AddToWorklist(N: Temp.getNode());
5837 break;
5838 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
5839 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
5840 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5841 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: N0, N2: Temp);
5842 if (!DCI.isCalledByLegalizer())
5843 DCI.AddToWorklist(N: Temp.getNode());
5844 break;
5845 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
5846 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
5847 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
5848 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1, N2: Temp);
5849 if (!DCI.isCalledByLegalizer())
5850 DCI.AddToWorklist(N: Temp.getNode());
5851 break;
5852 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
5853 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
5854 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5855 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: N0, N2: Temp);
5856 break;
5857 }
5858 if (VT.getScalarType() != MVT::i1) {
5859 if (!DCI.isCalledByLegalizer())
5860 DCI.AddToWorklist(N: N0.getNode());
5861 // FIXME: If running after legalize, we probably can't do this.
5862 ISD::NodeType ExtendCode = getExtendForContent(Content: getBooleanContents(Type: OpVT));
5863 N0 = DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: N0);
5864 }
5865 return N0;
5866 }
5867
5868 // Fold (setcc (trunc x) (trunc y)) -> (setcc x y)
5869 if (N0.getOpcode() == ISD::TRUNCATE && N1.getOpcode() == ISD::TRUNCATE &&
5870 N0.getOperand(i: 0).getValueType() == N1.getOperand(i: 0).getValueType() &&
5871 ((!ISD::isSignedIntSetCC(Code: Cond) && N0->getFlags().hasNoUnsignedWrap() &&
5872 N1->getFlags().hasNoUnsignedWrap()) ||
5873 (!ISD::isUnsignedIntSetCC(Code: Cond) && N0->getFlags().hasNoSignedWrap() &&
5874 N1->getFlags().hasNoSignedWrap())) &&
5875 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5876 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
5877 }
5878
5879 // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
5880 // TODO: Remove that .isVector() check
5881 if (VT.isVector() && isZeroOrZeroSplat(N: N1) && N0.getOpcode() == ISD::SUB &&
5882 N0->getFlags().hasNoSignedWrap() && ISD::isSignedIntSetCC(Code: Cond)) {
5883 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond);
5884 }
5885
5886 // Could not fold it.
5887 return SDValue();
5888}
5889
5890/// Returns true (and the GlobalValue and the offset) if the node is a
5891/// GlobalAddress + offset.
5892bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
5893 int64_t &Offset) const {
5894
5895 SDNode *N = unwrapAddress(N: SDValue(WN, 0)).getNode();
5896
5897 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(Val: N)) {
5898 GA = GASD->getGlobal();
5899 Offset += GASD->getOffset();
5900 return true;
5901 }
5902
5903 if (N->isAnyAdd()) {
5904 SDValue N1 = N->getOperand(Num: 0);
5905 SDValue N2 = N->getOperand(Num: 1);
5906 if (isGAPlusOffset(WN: N1.getNode(), GA, Offset)) {
5907 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N2)) {
5908 Offset += V->getSExtValue();
5909 return true;
5910 }
5911 } else if (isGAPlusOffset(WN: N2.getNode(), GA, Offset)) {
5912 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N1)) {
5913 Offset += V->getSExtValue();
5914 return true;
5915 }
5916 }
5917 }
5918
5919 return false;
5920}
5921
5922SDValue TargetLowering::PerformDAGCombine(SDNode *N,
5923 DAGCombinerInfo &DCI) const {
5924 // Default implementation: no optimization.
5925 return SDValue();
5926}
5927
5928//===----------------------------------------------------------------------===//
5929// Inline Assembler Implementation Methods
5930//===----------------------------------------------------------------------===//
5931
5932TargetLowering::ConstraintType
5933TargetLowering::getConstraintType(StringRef Constraint) const {
5934 unsigned S = Constraint.size();
5935
5936 if (S == 1) {
5937 switch (Constraint[0]) {
5938 default: break;
5939 case 'r':
5940 return C_RegisterClass;
5941 case 'm': // memory
5942 case 'o': // offsetable
5943 case 'V': // not offsetable
5944 return C_Memory;
5945 case 'p': // Address.
5946 return C_Address;
5947 case 'n': // Simple Integer
5948 case 'E': // Floating Point Constant
5949 case 'F': // Floating Point Constant
5950 return C_Immediate;
5951 case 'i': // Simple Integer or Relocatable Constant
5952 case 's': // Relocatable Constant
5953 case 'X': // Allow ANY value.
5954 case 'I': // Target registers.
5955 case 'J':
5956 case 'K':
5957 case 'L':
5958 case 'M':
5959 case 'N':
5960 case 'O':
5961 case 'P':
5962 case '<':
5963 case '>':
5964 return C_Other;
5965 }
5966 }
5967
5968 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
5969 if (S == 8 && Constraint.substr(Start: 1, N: 6) == "memory") // "{memory}"
5970 return C_Memory;
5971 return C_Register;
5972 }
5973 return C_Unknown;
5974}
5975
5976/// Try to replace an X constraint, which matches anything, with another that
5977/// has more specific requirements based on the type of the corresponding
5978/// operand.
5979const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
5980 if (ConstraintVT.isInteger())
5981 return "r";
5982 if (ConstraintVT.isFloatingPoint())
5983 return "f"; // works for many targets
5984 return nullptr;
5985}
5986
5987SDValue TargetLowering::LowerAsmOutputForConstraint(
5988 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
5989 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
5990 return SDValue();
5991}
5992
5993/// Lower the specified operand into the Ops vector.
5994/// If it is invalid, don't add anything to Ops.
5995void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
5996 StringRef Constraint,
5997 std::vector<SDValue> &Ops,
5998 SelectionDAG &DAG) const {
5999
6000 if (Constraint.size() > 1)
6001 return;
6002
6003 char ConstraintLetter = Constraint[0];
6004 switch (ConstraintLetter) {
6005 default: break;
6006 case 'X': // Allows any operand
6007 case 'i': // Simple Integer or Relocatable Constant
6008 case 'n': // Simple Integer
6009 case 's': { // Relocatable Constant
6010
6011 ConstantSDNode *C;
6012 uint64_t Offset = 0;
6013
6014 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
6015 // etc., since getelementpointer is variadic. We can't use
6016 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
6017 // while in this case the GA may be furthest from the root node which is
6018 // likely an ISD::ADD.
6019 while (true) {
6020 if ((C = dyn_cast<ConstantSDNode>(Val&: Op)) && ConstraintLetter != 's') {
6021 // gcc prints these as sign extended. Sign extend value to 64 bits
6022 // now; without this it would get ZExt'd later in
6023 // ScheduleDAGSDNodes::EmitNode, which is very generic.
6024 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
6025 BooleanContent BCont = getBooleanContents(Type: MVT::i64);
6026 ISD::NodeType ExtOpc =
6027 IsBool ? getExtendForContent(Content: BCont) : ISD::SIGN_EXTEND;
6028 int64_t ExtVal =
6029 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
6030 Ops.push_back(
6031 x: DAG.getTargetConstant(Val: Offset + ExtVal, DL: SDLoc(C), VT: MVT::i64));
6032 return;
6033 }
6034 if (ConstraintLetter != 'n') {
6035 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
6036 Ops.push_back(x: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(Op),
6037 VT: GA->getValueType(ResNo: 0),
6038 offset: Offset + GA->getOffset()));
6039 return;
6040 }
6041 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
6042 Ops.push_back(x: DAG.getTargetBlockAddress(
6043 BA: BA->getBlockAddress(), VT: BA->getValueType(ResNo: 0),
6044 Offset: Offset + BA->getOffset(), TargetFlags: BA->getTargetFlags()));
6045 return;
6046 }
6047 if (isa<BasicBlockSDNode>(Val: Op)) {
6048 Ops.push_back(x: Op);
6049 return;
6050 }
6051 }
6052 const unsigned OpCode = Op.getOpcode();
6053 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
6054 if ((C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 0))))
6055 Op = Op.getOperand(i: 1);
6056 // Subtraction is not commutative.
6057 else if (OpCode == ISD::ADD &&
6058 (C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))))
6059 Op = Op.getOperand(i: 0);
6060 else
6061 return;
6062 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
6063 continue;
6064 }
6065 return;
6066 }
6067 break;
6068 }
6069 }
6070}
6071
6072void TargetLowering::CollectTargetIntrinsicOperands(
6073 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
6074}
6075
6076std::pair<unsigned, const TargetRegisterClass *>
6077TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
6078 StringRef Constraint,
6079 MVT VT) const {
6080 if (!Constraint.starts_with(Prefix: "{"))
6081 return std::make_pair(x: 0u, y: static_cast<TargetRegisterClass *>(nullptr));
6082 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
6083
6084 // Remove the braces from around the name.
6085 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
6086
6087 std::pair<unsigned, const TargetRegisterClass *> R =
6088 std::make_pair(x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
6089
6090 // Figure out which register class contains this reg.
6091 for (const TargetRegisterClass &RC : RI->regclasses()) {
6092 // If none of the value types for this register class are valid, we
6093 // can't use it. For example, 64-bit reg classes on 32-bit targets.
6094 if (!isLegalRC(TRI: *RI, RC))
6095 continue;
6096
6097 for (const MCPhysReg &PR : RC) {
6098 if (RegName.equals_insensitive(RHS: RI->getRegAsmName(Reg: PR))) {
6099 std::pair<unsigned, const TargetRegisterClass *> S =
6100 std::make_pair(x: PR, y: &RC);
6101
6102 // If this register class has the requested value type, return it,
6103 // otherwise keep searching and return the first class found
6104 // if no other is found which explicitly has the requested type.
6105 if (RI->isTypeLegalForClass(RC, T: VT))
6106 return S;
6107 if (!R.second)
6108 R = S;
6109 }
6110 }
6111 }
6112
6113 return R;
6114}
6115
6116//===----------------------------------------------------------------------===//
6117// Constraint Selection.
6118
6119/// Return true of this is an input operand that is a matching constraint like
6120/// "4".
6121bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
6122 assert(!ConstraintCode.empty() && "No known constraint!");
6123 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
6124}
6125
6126/// If this is an input matching constraint, this method returns the output
6127/// operand it matches.
6128unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
6129 assert(!ConstraintCode.empty() && "No known constraint!");
6130 return atoi(nptr: ConstraintCode.c_str());
6131}
6132
6133/// Split up the constraint string from the inline assembly value into the
6134/// specific constraints and their prefixes, and also tie in the associated
6135/// operand values.
6136/// If this returns an empty vector, and if the constraint string itself
6137/// isn't empty, there was an error parsing.
6138TargetLowering::AsmOperandInfoVector
6139TargetLowering::ParseConstraints(const DataLayout &DL,
6140 const TargetRegisterInfo *TRI,
6141 const CallBase &Call) const {
6142 /// Information about all of the constraints.
6143 AsmOperandInfoVector ConstraintOperands;
6144 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
6145 unsigned maCount = 0; // Largest number of multiple alternative constraints.
6146
6147 // Do a prepass over the constraints, canonicalizing them, and building up the
6148 // ConstraintOperands list.
6149 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6150 unsigned ResNo = 0; // ResNo - The result number of the next output.
6151 unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.
6152
6153 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
6154 ConstraintOperands.emplace_back(args: std::move(CI));
6155 AsmOperandInfo &OpInfo = ConstraintOperands.back();
6156
6157 // Update multiple alternative constraint count.
6158 if (OpInfo.multipleAlternatives.size() > maCount)
6159 maCount = OpInfo.multipleAlternatives.size();
6160
6161 OpInfo.ConstraintVT = MVT::Other;
6162
6163 // Compute the value type for each operand.
6164 switch (OpInfo.Type) {
6165 case InlineAsm::isOutput: {
6166 // Indirect outputs just consume an argument.
6167 if (OpInfo.isIndirect) {
6168 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6169 break;
6170 }
6171
6172 // The return value of the call is this value. As such, there is no
6173 // corresponding argument.
6174 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
6175 EVT VT;
6176 if (auto *STy = dyn_cast<StructType>(Val: Call.getType())) {
6177 VT = getAsmOperandValueType(DL, Ty: STy->getElementType(N: ResNo));
6178 } else {
6179 assert(ResNo == 0 && "Asm only has one result!");
6180 VT = getAsmOperandValueType(DL, Ty: Call.getType());
6181 }
6182 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6183 ++ResNo;
6184 break;
6185 }
6186 case InlineAsm::isInput:
6187 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6188 break;
6189 case InlineAsm::isLabel:
6190 OpInfo.CallOperandVal = cast<CallBrInst>(Val: &Call)->getIndirectDest(i: LabelNo);
6191 ++LabelNo;
6192 continue;
6193 case InlineAsm::isClobber:
6194 // Nothing to do.
6195 break;
6196 }
6197
6198 if (OpInfo.CallOperandVal) {
6199 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
6200 if (OpInfo.isIndirect) {
6201 OpTy = Call.getParamElementType(ArgNo);
6202 assert(OpTy && "Indirect operand must have elementtype attribute");
6203 }
6204
6205 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6206 if (StructType *STy = dyn_cast<StructType>(Val: OpTy))
6207 if (STy->getNumElements() == 1)
6208 OpTy = STy->getElementType(N: 0);
6209
6210 // If OpTy is not a single value, it may be a struct/union that we
6211 // can tile with integers.
6212 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6213 unsigned BitSize = DL.getTypeSizeInBits(Ty: OpTy);
6214 switch (BitSize) {
6215 default: break;
6216 case 1:
6217 case 8:
6218 case 16:
6219 case 32:
6220 case 64:
6221 case 128:
6222 OpTy = IntegerType::get(C&: OpTy->getContext(), NumBits: BitSize);
6223 break;
6224 }
6225 }
6226
6227 EVT VT = getAsmOperandValueType(DL, Ty: OpTy, AllowUnknown: true);
6228 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6229 ArgNo++;
6230 }
6231 }
6232
6233 // If we have multiple alternative constraints, select the best alternative.
6234 if (!ConstraintOperands.empty()) {
6235 if (maCount) {
6236 unsigned bestMAIndex = 0;
6237 int bestWeight = -1;
6238 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
6239 int weight = -1;
6240 unsigned maIndex;
6241 // Compute the sums of the weights for each alternative, keeping track
6242 // of the best (highest weight) one so far.
6243 for (maIndex = 0; maIndex < maCount; ++maIndex) {
6244 int weightSum = 0;
6245 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6246 cIndex != eIndex; ++cIndex) {
6247 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6248 if (OpInfo.Type == InlineAsm::isClobber)
6249 continue;
6250
6251 // If this is an output operand with a matching input operand,
6252 // look up the matching input. If their types mismatch, e.g. one
6253 // is an integer, the other is floating point, or their sizes are
6254 // different, flag it as an maCantMatch.
6255 if (OpInfo.hasMatchingInput()) {
6256 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6257 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6258 if ((OpInfo.ConstraintVT.isInteger() !=
6259 Input.ConstraintVT.isInteger()) ||
6260 (OpInfo.ConstraintVT.getSizeInBits() !=
6261 Input.ConstraintVT.getSizeInBits())) {
6262 weightSum = -1; // Can't match.
6263 break;
6264 }
6265 }
6266 }
6267 weight = getMultipleConstraintMatchWeight(info&: OpInfo, maIndex);
6268 if (weight == -1) {
6269 weightSum = -1;
6270 break;
6271 }
6272 weightSum += weight;
6273 }
6274 // Update best.
6275 if (weightSum > bestWeight) {
6276 bestWeight = weightSum;
6277 bestMAIndex = maIndex;
6278 }
6279 }
6280
6281 // Now select chosen alternative in each constraint.
6282 for (AsmOperandInfo &cInfo : ConstraintOperands)
6283 if (cInfo.Type != InlineAsm::isClobber)
6284 cInfo.selectAlternative(index: bestMAIndex);
6285 }
6286 }
6287
6288 // Check and hook up tied operands, choose constraint code to use.
6289 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6290 cIndex != eIndex; ++cIndex) {
6291 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6292
6293 // If this is an output operand with a matching input operand, look up the
6294 // matching input. If their types mismatch, e.g. one is an integer, the
6295 // other is floating point, or their sizes are different, flag it as an
6296 // error.
6297 if (OpInfo.hasMatchingInput()) {
6298 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6299
6300 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6301 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6302 getRegForInlineAsmConstraint(RI: TRI, Constraint: OpInfo.ConstraintCode,
6303 VT: OpInfo.ConstraintVT);
6304 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6305 getRegForInlineAsmConstraint(RI: TRI, Constraint: Input.ConstraintCode,
6306 VT: Input.ConstraintVT);
6307 const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
6308 OpInfo.ConstraintVT.isFloatingPoint();
6309 const bool InOpIsIntOrFP = Input.ConstraintVT.isInteger() ||
6310 Input.ConstraintVT.isFloatingPoint();
6311 if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
6312 (MatchRC.second != InputRC.second)) {
6313 report_fatal_error(reason: "Unsupported asm: input constraint"
6314 " with a matching output constraint of"
6315 " incompatible type!");
6316 }
6317 }
6318 }
6319 }
6320
6321 return ConstraintOperands;
6322}
6323
6324/// Return a number indicating our preference for chosing a type of constraint
6325/// over another, for the purpose of sorting them. Immediates are almost always
6326/// preferrable (when they can be emitted). A higher return value means a
6327/// stronger preference for one constraint type relative to another.
6328/// FIXME: We should prefer registers over memory but doing so may lead to
6329/// unrecoverable register exhaustion later.
6330/// https://github.com/llvm/llvm-project/issues/20571
6331static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) {
6332 switch (CT) {
6333 case TargetLowering::C_Immediate:
6334 case TargetLowering::C_Other:
6335 return 4;
6336 case TargetLowering::C_Memory:
6337 case TargetLowering::C_Address:
6338 return 3;
6339 case TargetLowering::C_RegisterClass:
6340 return 2;
6341 case TargetLowering::C_Register:
6342 return 1;
6343 case TargetLowering::C_Unknown:
6344 return 0;
6345 }
6346 llvm_unreachable("Invalid constraint type");
6347}
6348
6349/// Examine constraint type and operand type and determine a weight value.
6350/// This object must already have been set up with the operand type
6351/// and the current alternative constraint selected.
6352TargetLowering::ConstraintWeight
6353 TargetLowering::getMultipleConstraintMatchWeight(
6354 AsmOperandInfo &info, int maIndex) const {
6355 InlineAsm::ConstraintCodeVector *rCodes;
6356 if (maIndex >= (int)info.multipleAlternatives.size())
6357 rCodes = &info.Codes;
6358 else
6359 rCodes = &info.multipleAlternatives[maIndex].Codes;
6360 ConstraintWeight BestWeight = CW_Invalid;
6361
6362 // Loop over the options, keeping track of the most general one.
6363 for (const std::string &rCode : *rCodes) {
6364 ConstraintWeight weight =
6365 getSingleConstraintMatchWeight(info, constraint: rCode.c_str());
6366 if (weight > BestWeight)
6367 BestWeight = weight;
6368 }
6369
6370 return BestWeight;
6371}
6372
6373/// Examine constraint type and operand type and determine a weight value.
6374/// This object must already have been set up with the operand type
6375/// and the current alternative constraint selected.
6376TargetLowering::ConstraintWeight
6377 TargetLowering::getSingleConstraintMatchWeight(
6378 AsmOperandInfo &info, const char *constraint) const {
6379 ConstraintWeight weight = CW_Invalid;
6380 Value *CallOperandVal = info.CallOperandVal;
6381 // If we don't have a value, we can't do a match,
6382 // but allow it at the lowest weight.
6383 if (!CallOperandVal)
6384 return CW_Default;
6385 // Look at the constraint type.
6386 switch (*constraint) {
6387 case 'i': // immediate integer.
6388 case 'n': // immediate integer with a known value.
6389 if (isa<ConstantInt>(Val: CallOperandVal))
6390 weight = CW_Constant;
6391 break;
6392 case 's': // non-explicit intregal immediate.
6393 if (isa<GlobalValue>(Val: CallOperandVal))
6394 weight = CW_Constant;
6395 break;
6396 case 'E': // immediate float if host format.
6397 case 'F': // immediate float.
6398 if (isa<ConstantFP>(Val: CallOperandVal))
6399 weight = CW_Constant;
6400 break;
6401 case '<': // memory operand with autodecrement.
6402 case '>': // memory operand with autoincrement.
6403 case 'm': // memory operand.
6404 case 'o': // offsettable memory operand
6405 case 'V': // non-offsettable memory operand
6406 weight = CW_Memory;
6407 break;
6408 case 'r': // general register.
6409 case 'g': // general register, memory operand or immediate integer.
6410 // note: Clang converts "g" to "imr".
6411 if (CallOperandVal->getType()->isIntegerTy())
6412 weight = CW_Register;
6413 break;
6414 case 'X': // any operand.
6415 default:
6416 weight = CW_Default;
6417 break;
6418 }
6419 return weight;
6420}
6421
6422/// If there are multiple different constraints that we could pick for this
6423/// operand (e.g. "imr") try to pick the 'best' one.
6424/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
6425/// into seven classes:
6426/// Register -> one specific register
6427/// RegisterClass -> a group of regs
6428/// Memory -> memory
6429/// Address -> a symbolic memory reference
6430/// Immediate -> immediate values
6431/// Other -> magic values (such as "Flag Output Operands")
6432/// Unknown -> something we don't recognize yet and can't handle
6433/// Ideally, we would pick the most specific constraint possible: if we have
6434/// something that fits into a register, we would pick it. The problem here
6435/// is that if we have something that could either be in a register or in
6436/// memory that use of the register could cause selection of *other*
6437/// operands to fail: they might only succeed if we pick memory. Because of
6438/// this the heuristic we use is:
6439///
6440/// 1) If there is an 'other' constraint, and if the operand is valid for
6441/// that constraint, use it. This makes us take advantage of 'i'
6442/// constraints when available.
6443/// 2) Otherwise, pick the most general constraint present. This prefers
6444/// 'm' over 'r', for example.
6445///
6446TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
6447 TargetLowering::AsmOperandInfo &OpInfo) const {
6448 ConstraintGroup Ret;
6449
6450 Ret.reserve(N: OpInfo.Codes.size());
6451 for (StringRef Code : OpInfo.Codes) {
6452 TargetLowering::ConstraintType CType = getConstraintType(Constraint: Code);
6453
6454 // Indirect 'other' or 'immediate' constraints are not allowed.
6455 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6456 CType == TargetLowering::C_Register ||
6457 CType == TargetLowering::C_RegisterClass))
6458 continue;
6459
6460 // Things with matching constraints can only be registers, per gcc
6461 // documentation. This mainly affects "g" constraints.
6462 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
6463 continue;
6464
6465 Ret.emplace_back(Args&: Code, Args&: CType);
6466 }
6467
6468 llvm::stable_sort(Range&: Ret, C: [](ConstraintPair a, ConstraintPair b) {
6469 return getConstraintPiority(CT: a.second) > getConstraintPiority(CT: b.second);
6470 });
6471
6472 return Ret;
6473}
6474
6475/// If we have an immediate, see if we can lower it. Return true if we can,
6476/// false otherwise.
6477static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P,
6478 SDValue Op, SelectionDAG *DAG,
6479 const TargetLowering &TLI) {
6480
6481 assert((P.second == TargetLowering::C_Other ||
6482 P.second == TargetLowering::C_Immediate) &&
6483 "need immediate or other");
6484
6485 if (!Op.getNode())
6486 return false;
6487
6488 std::vector<SDValue> ResultOps;
6489 TLI.LowerAsmOperandForConstraint(Op, Constraint: P.first, Ops&: ResultOps, DAG&: *DAG);
6490 return !ResultOps.empty();
6491}
6492
6493/// Determines the constraint code and constraint type to use for the specific
6494/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
6495void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
6496 SDValue Op,
6497 SelectionDAG *DAG) const {
6498 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
6499
6500 // Single-letter constraints ('r') are very common.
6501 if (OpInfo.Codes.size() == 1) {
6502 OpInfo.ConstraintCode = OpInfo.Codes[0];
6503 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6504 } else {
6505 ConstraintGroup G = getConstraintPreferences(OpInfo);
6506 if (G.empty())
6507 return;
6508
6509 unsigned BestIdx = 0;
6510 for (const unsigned E = G.size();
6511 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
6512 G[BestIdx].second == TargetLowering::C_Immediate);
6513 ++BestIdx) {
6514 if (lowerImmediateIfPossible(P&: G[BestIdx], Op, DAG, TLI: *this))
6515 break;
6516 // If we're out of constraints, just pick the first one.
6517 if (BestIdx + 1 == E) {
6518 BestIdx = 0;
6519 break;
6520 }
6521 }
6522
6523 OpInfo.ConstraintCode = G[BestIdx].first;
6524 OpInfo.ConstraintType = G[BestIdx].second;
6525 }
6526
6527 // 'X' matches anything.
6528 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
6529 // Constants are handled elsewhere. For Functions, the type here is the
6530 // type of the result, which is not what we want to look at; leave them
6531 // alone.
6532 Value *v = OpInfo.CallOperandVal;
6533 if (isa<ConstantInt>(Val: v) || isa<Function>(Val: v)) {
6534 return;
6535 }
6536
6537 if (isa<BasicBlock>(Val: v) || isa<BlockAddress>(Val: v)) {
6538 OpInfo.ConstraintCode = "i";
6539 return;
6540 }
6541
6542 // Otherwise, try to resolve it to something we know about by looking at
6543 // the actual operand type.
6544 if (const char *Repl = LowerXConstraint(ConstraintVT: OpInfo.ConstraintVT)) {
6545 OpInfo.ConstraintCode = Repl;
6546 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6547 }
6548 }
6549}
6550
6551/// Given an exact SDIV by a constant, create a multiplication
6552/// with the multiplicative inverse of the constant.
6553/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6554static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
6555 const SDLoc &dl, SelectionDAG &DAG,
6556 SmallVectorImpl<SDNode *> &Created) {
6557 SDValue Op0 = N->getOperand(Num: 0);
6558 SDValue Op1 = N->getOperand(Num: 1);
6559 EVT VT = N->getValueType(ResNo: 0);
6560 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6561 EVT ShSVT = ShVT.getScalarType();
6562
6563 bool UseSRA = false;
6564 SmallVector<SDValue, 16> Shifts, Factors;
6565
6566 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6567 if (C->isZero())
6568 return false;
6569
6570 EVT CT = C->getValueType(ResNo: 0);
6571 APInt Divisor = C->getAPIntValue();
6572 unsigned Shift = Divisor.countr_zero();
6573 if (Shift) {
6574 Divisor.ashrInPlace(ShiftAmt: Shift);
6575 UseSRA = true;
6576 }
6577 APInt Factor = Divisor.multiplicativeInverse();
6578 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6579 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6580 return true;
6581 };
6582
6583 // Collect all magic values from the build vector.
6584 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildSDIVPattern))
6585 return SDValue();
6586
6587 SDValue Shift, Factor;
6588 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6589 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6590 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6591 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6592 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6593 "Expected matchUnaryPredicate to return one element for scalable "
6594 "vectors");
6595 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6596 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6597 } else {
6598 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6599 Shift = Shifts[0];
6600 Factor = Factors[0];
6601 }
6602
6603 SDValue Res = Op0;
6604 if (UseSRA) {
6605 Res = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6606 Created.push_back(Elt: Res.getNode());
6607 }
6608
6609 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6610}
6611
6612/// Given an exact UDIV by a constant, create a multiplication
6613/// with the multiplicative inverse of the constant.
6614/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6615static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
6616 const SDLoc &dl, SelectionDAG &DAG,
6617 SmallVectorImpl<SDNode *> &Created) {
6618 EVT VT = N->getValueType(ResNo: 0);
6619 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6620 EVT ShSVT = ShVT.getScalarType();
6621
6622 bool UseSRL = false;
6623 SmallVector<SDValue, 16> Shifts, Factors;
6624
6625 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6626 if (C->isZero())
6627 return false;
6628
6629 EVT CT = C->getValueType(ResNo: 0);
6630 APInt Divisor = C->getAPIntValue();
6631 unsigned Shift = Divisor.countr_zero();
6632 if (Shift) {
6633 Divisor.lshrInPlace(ShiftAmt: Shift);
6634 UseSRL = true;
6635 }
6636 // Calculate the multiplicative inverse modulo BW.
6637 APInt Factor = Divisor.multiplicativeInverse();
6638 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6639 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6640 return true;
6641 };
6642
6643 SDValue Op1 = N->getOperand(Num: 1);
6644
6645 // Collect all magic values from the build vector.
6646 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildUDIVPattern))
6647 return SDValue();
6648
6649 SDValue Shift, Factor;
6650 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6651 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6652 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6653 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6654 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6655 "Expected matchUnaryPredicate to return one element for scalable "
6656 "vectors");
6657 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6658 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6659 } else {
6660 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6661 Shift = Shifts[0];
6662 Factor = Factors[0];
6663 }
6664
6665 SDValue Res = N->getOperand(Num: 0);
6666 if (UseSRL) {
6667 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6668 Created.push_back(Elt: Res.getNode());
6669 }
6670
6671 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6672}
6673
6674SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
6675 SelectionDAG &DAG,
6676 SmallVectorImpl<SDNode *> &Created) const {
6677 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6678 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6679 return SDValue(N, 0); // Lower SDIV as SDIV
6680 return SDValue();
6681}
6682
6683SDValue
6684TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
6685 SelectionDAG &DAG,
6686 SmallVectorImpl<SDNode *> &Created) const {
6687 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6688 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6689 return SDValue(N, 0); // Lower SREM as SREM
6690 return SDValue();
6691}
6692
6693/// Build sdiv by power-of-2 with conditional move instructions
6694/// Ref: "Hacker's Delight" by Henry Warren 10-1
6695/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
6696/// bgez x, label
6697/// add x, x, 2**k-1
6698/// label:
6699/// sra res, x, k
6700/// neg res, res (when the divisor is negative)
6701SDValue TargetLowering::buildSDIVPow2WithCMov(
6702 SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
6703 SmallVectorImpl<SDNode *> &Created) const {
6704 unsigned Lg2 = Divisor.countr_zero();
6705 EVT VT = N->getValueType(ResNo: 0);
6706
6707 SDLoc DL(N);
6708 SDValue N0 = N->getOperand(Num: 0);
6709 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
6710 APInt Lg2Mask = APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: Lg2);
6711 SDValue Pow2MinusOne = DAG.getConstant(Val: Lg2Mask, DL, VT);
6712
6713 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
6714 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
6715 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: N0, RHS: Zero, Cond: ISD::SETLT);
6716 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: Pow2MinusOne);
6717 SDValue CMov = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cmp, N2: Add, N3: N0);
6718
6719 Created.push_back(Elt: Cmp.getNode());
6720 Created.push_back(Elt: Add.getNode());
6721 Created.push_back(Elt: CMov.getNode());
6722
6723 // Divide by pow2.
6724 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CMov,
6725 N2: DAG.getShiftAmountConstant(Val: Lg2, VT, DL));
6726
6727 // If we're dividing by a positive value, we're done. Otherwise, we must
6728 // negate the result.
6729 if (Divisor.isNonNegative())
6730 return SRA;
6731
6732 Created.push_back(Elt: SRA.getNode());
6733 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: SRA);
6734}
6735
6736/// Given an ISD::SDIV node expressing a divide by constant,
6737/// return a DAG expression to select that will generate the same value by
6738/// multiplying by a magic number.
6739/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6740SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
6741 bool IsAfterLegalization,
6742 bool IsAfterLegalTypes,
6743 SmallVectorImpl<SDNode *> &Created) const {
6744 SDLoc dl(N);
6745
6746 // If the sdiv has an 'exact' bit we can use a simpler lowering.
6747 if (N->getFlags().hasExact())
6748 return BuildExactSDIV(TLI: *this, N, dl, DAG, Created);
6749
6750 EVT VT = N->getValueType(ResNo: 0);
6751 EVT SVT = VT.getScalarType();
6752 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6753 EVT ShSVT = ShVT.getScalarType();
6754 unsigned EltBits = VT.getScalarSizeInBits();
6755 EVT MulVT;
6756
6757 // Check to see if we can do this.
6758 // FIXME: We should be more aggressive here.
6759 EVT QueryVT = VT;
6760 if (VT.isVector()) {
6761 // If the vector type will be legalized to a vector type with the same
6762 // element type, allow the transform before type legalization if MULHS or
6763 // SMUL_LOHI are supported.
6764 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
6765 if (!QueryVT.isVector() ||
6766 QueryVT.getVectorElementType() != VT.getVectorElementType())
6767 return SDValue();
6768 } else if (!isTypeLegal(VT)) {
6769 // Limit this to simple scalars for now.
6770 if (!VT.isSimple())
6771 return SDValue();
6772
6773 // If this type will be promoted to a large enough type with a legal
6774 // multiply operation, we can go ahead and do this transform.
6775 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
6776 return SDValue();
6777
6778 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6779 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6780 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
6781 return SDValue();
6782 }
6783
6784 bool HasMULHS =
6785 isOperationLegalOrCustom(Op: ISD::MULHS, VT: QueryVT, LegalOnly: IsAfterLegalization);
6786 bool HasSMUL_LOHI =
6787 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
6788
6789 if (isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
6790 // If type twice as wide legal, widen and use a mul plus a shift.
6791 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
6792 // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
6793 // custom lowered. This is very expensive so avoid it at all costs for
6794 // constant divisors.
6795 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::SDIV, VT) &&
6796 isOperationCustom(Op: ISD::SDIVREM, VT: VT.getScalarType())) ||
6797 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
6798 MulVT = WideVT;
6799 }
6800
6801 if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
6802 return SDValue();
6803
6804 // If we're after type legalization and SVT is not legal, use the
6805 // promoted type for creating constants to avoid creating nodes with
6806 // illegal types.
6807 if (IsAfterLegalTypes && VT.isVector()) {
6808 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
6809 if (SVT.bitsLT(VT: VT.getScalarType()))
6810 return SDValue();
6811 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
6812 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
6813 return SDValue();
6814 }
6815 const unsigned SVTBits = SVT.getSizeInBits();
6816
6817 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
6818
6819 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6820 if (C->isZero())
6821 return false;
6822 // Truncate the divisor to the target scalar type in case it was promoted
6823 // during type legalization.
6824 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
6825 SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(D: Divisor);
6826 int NumeratorFactor = 0;
6827 int ShiftMask = -1;
6828
6829 if (Divisor.isOne() || Divisor.isAllOnes()) {
6830 // If d is +1/-1, we just multiply the numerator by +1/-1.
6831 NumeratorFactor = Divisor.getSExtValue();
6832 magics.Magic = 0;
6833 magics.ShiftAmount = 0;
6834 ShiftMask = 0;
6835 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
6836 // If d > 0 and m < 0, add the numerator.
6837 NumeratorFactor = 1;
6838 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
6839 // If d < 0 and m > 0, subtract the numerator.
6840 NumeratorFactor = -1;
6841 }
6842
6843 MagicFactors.push_back(
6844 Elt: DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT));
6845 Factors.push_back(Elt: DAG.getSignedConstant(Val: NumeratorFactor, DL: dl, VT: SVT));
6846 Shifts.push_back(Elt: DAG.getConstant(Val: magics.ShiftAmount, DL: dl, VT: ShSVT));
6847 ShiftMasks.push_back(Elt: DAG.getSignedConstant(Val: ShiftMask, DL: dl, VT: SVT));
6848 return true;
6849 };
6850
6851 SDValue N0 = N->getOperand(Num: 0);
6852 SDValue N1 = N->getOperand(Num: 1);
6853
6854 // Collect the shifts / magic values from each element.
6855 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildSDIVPattern, /*AllowUndefs=*/false,
6856 /*AllowTruncation=*/true))
6857 return SDValue();
6858
6859 SDValue MagicFactor, Factor, Shift, ShiftMask;
6860 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
6861 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
6862 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6863 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6864 ShiftMask = DAG.getBuildVector(VT, DL: dl, Ops: ShiftMasks);
6865 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
6866 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
6867 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
6868 "Expected matchUnaryPredicate to return one element for scalable "
6869 "vectors");
6870 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
6871 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6872 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6873 ShiftMask = DAG.getSplatVector(VT, DL: dl, Op: ShiftMasks[0]);
6874 } else {
6875 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
6876 MagicFactor = MagicFactors[0];
6877 Factor = Factors[0];
6878 Shift = Shifts[0];
6879 ShiftMask = ShiftMasks[0];
6880 }
6881
6882 // Multiply the numerator (operand 0) by the magic value.
6883 auto GetMULHS = [&](SDValue X, SDValue Y) {
6884 if (HasMULHS)
6885 return DAG.getNode(Opcode: ISD::MULHS, DL: dl, VT, N1: X, N2: Y);
6886 if (HasSMUL_LOHI) {
6887 SDValue LoHi =
6888 DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
6889 return LoHi.getValue(R: 1);
6890 }
6891
6892 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: X);
6893 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: Y);
6894 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
6895 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
6896 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
6897 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
6898 };
6899
6900 SDValue Q = GetMULHS(N0, MagicFactor);
6901 if (!Q)
6902 return SDValue();
6903
6904 Created.push_back(Elt: Q.getNode());
6905
6906 // (Optionally) Add/subtract the numerator using Factor.
6907 Factor = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: N0, N2: Factor);
6908 Created.push_back(Elt: Factor.getNode());
6909 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: Factor);
6910 Created.push_back(Elt: Q.getNode());
6911
6912 // Shift right algebraic by shift value.
6913 Q = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Q, N2: Shift);
6914 Created.push_back(Elt: Q.getNode());
6915
6916 // Extract the sign bit, mask it and add it to the quotient.
6917 SDValue SignShift = DAG.getConstant(Val: EltBits - 1, DL: dl, VT: ShVT);
6918 SDValue T = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: SignShift);
6919 Created.push_back(Elt: T.getNode());
6920 T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: ShiftMask);
6921 Created.push_back(Elt: T.getNode());
6922 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: T);
6923}
6924
6925/// Given an ISD::UDIV node expressing a divide by constant,
6926/// return a DAG expression to select that will generate the same value by
6927/// multiplying by a magic number.
6928/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6929SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
6930 bool IsAfterLegalization,
6931 bool IsAfterLegalTypes,
6932 SmallVectorImpl<SDNode *> &Created) const {
6933 SDLoc dl(N);
6934
6935 // If the udiv has an 'exact' bit we can use a simpler lowering.
6936 if (N->getFlags().hasExact())
6937 return BuildExactUDIV(TLI: *this, N, dl, DAG, Created);
6938
6939 EVT VT = N->getValueType(ResNo: 0);
6940 EVT SVT = VT.getScalarType();
6941 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6942 EVT ShSVT = ShVT.getScalarType();
6943 unsigned EltBits = VT.getScalarSizeInBits();
6944 EVT MulVT;
6945
6946 // Check to see if we can do this.
6947 // FIXME: We should be more aggressive here.
6948 EVT QueryVT = VT;
6949 if (VT.isVector()) {
6950 // If the vector type will be legalized to a vector type with the same
6951 // element type, allow the transform before type legalization if MULHU or
6952 // UMUL_LOHI are supported.
6953 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
6954 if (!QueryVT.isVector() ||
6955 QueryVT.getVectorElementType() != VT.getVectorElementType())
6956 return SDValue();
6957 } else if (!isTypeLegal(VT)) {
6958 // Limit this to simple scalars for now.
6959 if (!VT.isSimple())
6960 return SDValue();
6961
6962 // If this type will be promoted to a large enough type with a legal
6963 // multiply operation, we can go ahead and do this transform.
6964 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
6965 return SDValue();
6966
6967 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6968 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6969 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
6970 return SDValue();
6971 }
6972
6973 bool HasMULHU =
6974 isOperationLegalOrCustom(Op: ISD::MULHU, VT: QueryVT, LegalOnly: IsAfterLegalization);
6975 bool HasUMUL_LOHI =
6976 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
6977
6978 if (isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
6979 // If type twice as wide legal, widen and use a mul plus a shift.
6980 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
6981 // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
6982 // custom lowered. This is very expensive so avoid it at all costs for
6983 // constant divisors.
6984 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::UDIV, VT) &&
6985 isOperationCustom(Op: ISD::UDIVREM, VT: VT.getScalarType())) ||
6986 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
6987 MulVT = WideVT;
6988 }
6989
6990 if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
6991 return SDValue();
6992
6993 SDValue N0 = N->getOperand(Num: 0);
6994 SDValue N1 = N->getOperand(Num: 1);
6995
6996 // Try to use leading zeros of the dividend to reduce the multiplier and
6997 // avoid expensive fixups.
6998 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
6999
7000 // If we're after type legalization and SVT is not legal, use the
7001 // promoted type for creating constants to avoid creating nodes with
7002 // illegal types.
7003 if (IsAfterLegalTypes && VT.isVector()) {
7004 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
7005 if (SVT.bitsLT(VT: VT.getScalarType()))
7006 return SDValue();
7007 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
7008 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
7009 return SDValue();
7010 }
7011 const unsigned SVTBits = SVT.getSizeInBits();
7012
7013 // Allow i32 to be widened to i64 for uncooperative divisors if i64 MULHU or
7014 // UMUL_LOHI is supported.
7015 const EVT WideSVT = MVT::i64;
7016 const bool HasWideMULHU =
7017 VT == MVT::i32 &&
7018 isOperationLegalOrCustom(Op: ISD::MULHU, VT: WideSVT, LegalOnly: IsAfterLegalization);
7019 const bool HasWideUMUL_LOHI =
7020 VT == MVT::i32 &&
7021 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: WideSVT, LegalOnly: IsAfterLegalization);
7022 const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
7023
7024 // For even divisors with a 33-bit magic number, the widened high-multiply
7025 // path is only worthwhile over the even-divisor rewrite on targets that
7026 // zero-extend i32 to i64 for free (e.g. x86-64 and AArch64). Elsewhere (e.g.
7027 // RISC-V) keep the even-divisor rewrite, which avoids the explicit extension.
7028 const bool AllowEvenToWiden = AllowWiden && isZExtFree(FromTy: VT, ToTy: WideSVT);
7029
7030 bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
7031 bool UseWiden = false;
7032 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
7033
7034 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
7035 if (C->isZero())
7036 return false;
7037 // Truncate the divisor to the target scalar type in case it was promoted
7038 // during type legalization.
7039 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
7040
7041 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
7042
7043 // Magic algorithm doesn't work for division by 1. We need to emit a select
7044 // at the end.
7045 if (Divisor.isOne()) {
7046 PreShift = PostShift = DAG.getUNDEF(VT: ShSVT);
7047 MagicFactor = NPQFactor = DAG.getUNDEF(VT: SVT);
7048 } else {
7049 UnsignedDivisionByConstantInfo magics =
7050 UnsignedDivisionByConstantInfo::get(
7051 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()),
7052 /*AllowEvenDivisorOptimization=*/!AllowEvenToWiden,
7053 /*AllowWidenOptimization=*/AllowWiden);
7054
7055 if (magics.Widen) {
7056 UseWiden = true;
7057 MagicFactor = DAG.getConstant(Val: magics.Magic, DL: dl, VT: WideSVT);
7058 } else {
7059 MagicFactor = DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT);
7060 }
7061
7062 assert(magics.PreShift < Divisor.getBitWidth() &&
7063 "We shouldn't generate an undefined shift!");
7064 assert(magics.PostShift < Divisor.getBitWidth() &&
7065 "We shouldn't generate an undefined shift!");
7066 assert((!magics.IsAdd || magics.PreShift == 0) &&
7067 "Unexpected pre-shift");
7068 PreShift = DAG.getConstant(Val: magics.PreShift, DL: dl, VT: ShSVT);
7069 PostShift = DAG.getConstant(Val: magics.PostShift, DL: dl, VT: ShSVT);
7070 NPQFactor = DAG.getConstant(
7071 Val: magics.IsAdd ? APInt::getOneBitSet(numBits: SVTBits, BitNo: EltBits - 1)
7072 : APInt::getZero(numBits: SVTBits),
7073 DL: dl, VT: SVT);
7074 UseNPQ |= magics.IsAdd;
7075 UsePreShift |= magics.PreShift != 0;
7076 UsePostShift |= magics.PostShift != 0;
7077 }
7078
7079 PreShifts.push_back(Elt: PreShift);
7080 MagicFactors.push_back(Elt: MagicFactor);
7081 NPQFactors.push_back(Elt: NPQFactor);
7082 PostShifts.push_back(Elt: PostShift);
7083 return true;
7084 };
7085
7086 // Collect the shifts/magic values from each element.
7087 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildUDIVPattern, /*AllowUndefs=*/false,
7088 /*AllowTruncation=*/true))
7089 return SDValue();
7090
7091 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
7092 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7093 PreShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PreShifts);
7094 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
7095 NPQFactor = DAG.getBuildVector(VT, DL: dl, Ops: NPQFactors);
7096 PostShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PostShifts);
7097 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7098 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
7099 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
7100 "Expected matchUnaryPredicate to return one for scalable vectors");
7101 PreShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PreShifts[0]);
7102 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
7103 NPQFactor = DAG.getSplatVector(VT, DL: dl, Op: NPQFactors[0]);
7104 PostShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PostShifts[0]);
7105 } else {
7106 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7107 PreShift = PreShifts[0];
7108 MagicFactor = MagicFactors[0];
7109 PostShift = PostShifts[0];
7110 }
7111
7112 if (UseWiden) {
7113 // Compute: (WideSVT(x) * MagicFactor) >> WideSVTBits.
7114 SDValue WideN0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: WideSVT, Operand: N0);
7115
7116 // Perform WideSVTxWideSVT -> 2*WideSVT multiplication and extract high
7117 // WideSVT bits
7118 SDValue High;
7119 if (HasWideMULHU) {
7120 High = DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT: WideSVT, N1: WideN0, N2: MagicFactor);
7121 } else {
7122 assert(HasWideUMUL_LOHI);
7123 SDValue LoHi =
7124 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: WideSVT, VT2: WideSVT),
7125 N1: WideN0, N2: MagicFactor);
7126 High = LoHi.getValue(R: 1);
7127 }
7128
7129 Created.push_back(Elt: High.getNode());
7130 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: High);
7131 }
7132
7133 SDValue Q = N0;
7134 if (UsePreShift) {
7135 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PreShift);
7136 Created.push_back(Elt: Q.getNode());
7137 }
7138
7139 auto GetMULHU = [&](SDValue X, SDValue Y) {
7140 if (HasMULHU)
7141 return DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT, N1: X, N2: Y);
7142 if (HasUMUL_LOHI) {
7143 SDValue LoHi =
7144 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7145 return LoHi.getValue(R: 1);
7146 }
7147
7148 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: X);
7149 Y = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7150 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7151 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7152 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7153 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7154 };
7155
7156 // Multiply the numerator (operand 0) by the magic value.
7157 Q = GetMULHU(Q, MagicFactor);
7158 if (!Q)
7159 return SDValue();
7160
7161 Created.push_back(Elt: Q.getNode());
7162
7163 if (UseNPQ) {
7164 SDValue NPQ = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: N0, N2: Q);
7165 Created.push_back(Elt: NPQ.getNode());
7166
7167 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
7168 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
7169 if (VT.isVector())
7170 NPQ = GetMULHU(NPQ, NPQFactor);
7171 else
7172 NPQ = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NPQ, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT));
7173
7174 Created.push_back(Elt: NPQ.getNode());
7175
7176 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: NPQ, N2: Q);
7177 Created.push_back(Elt: Q.getNode());
7178 }
7179
7180 if (UsePostShift) {
7181 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PostShift);
7182 Created.push_back(Elt: Q.getNode());
7183 }
7184
7185 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7186
7187 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
7188 SDValue IsOne = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N1, RHS: One, Cond: ISD::SETEQ);
7189 return DAG.getSelect(DL: dl, VT, Cond: IsOne, LHS: N0, RHS: Q);
7190}
7191
7192/// If all values in Values that *don't* match the predicate are same 'splat'
7193/// value, then replace all values with that splat value.
7194/// Else, if AlternativeReplacement was provided, then replace all values that
7195/// do match predicate with AlternativeReplacement value.
7196static void
7197turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
7198 std::function<bool(SDValue)> Predicate,
7199 SDValue AlternativeReplacement = SDValue()) {
7200 SDValue Replacement;
7201 // Is there a value for which the Predicate does *NOT* match? What is it?
7202 auto SplatValue = llvm::find_if_not(Range&: Values, P: Predicate);
7203 if (SplatValue != Values.end()) {
7204 // Does Values consist only of SplatValue's and values matching Predicate?
7205 if (llvm::all_of(Range&: Values, P: [Predicate, SplatValue](SDValue Value) {
7206 return Value == *SplatValue || Predicate(Value);
7207 })) // Then we shall replace values matching predicate with SplatValue.
7208 Replacement = *SplatValue;
7209 }
7210 if (!Replacement) {
7211 // Oops, we did not find the "baseline" splat value.
7212 if (!AlternativeReplacement)
7213 return; // Nothing to do.
7214 // Let's replace with provided value then.
7215 Replacement = AlternativeReplacement;
7216 }
7217 std::replace_if(first: Values.begin(), last: Values.end(), pred: Predicate, new_value: Replacement);
7218}
7219
7220/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
7221/// where the divisor and comparison target are constants,
7222/// return a DAG expression that will generate the same comparison result
7223/// using only multiplications, additions and shifts/rotations.
7224/// Ref: "Hacker's Delight" 10-17.
7225SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
7226 SDValue CompTargetNode,
7227 ISD::CondCode Cond,
7228 DAGCombinerInfo &DCI,
7229 const SDLoc &DL) const {
7230 SmallVector<SDNode *, 5> Built;
7231 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7232 DCI, DL, Created&: Built)) {
7233 for (SDNode *N : Built)
7234 DCI.AddToWorklist(N);
7235 return Folded;
7236 }
7237
7238 return SDValue();
7239}
7240
7241SDValue
7242TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
7243 SDValue CompTargetNode, ISD::CondCode Cond,
7244 DAGCombinerInfo &DCI, const SDLoc &DL,
7245 SmallVectorImpl<SDNode *> &Created) const {
7246 // fold (seteq/ne (urem N, D), C) ->
7247 // (setule/ugt (rotr (mul (sub N, C), P), K), Q)
7248 // - D must be constant, with D = D0 * 2^K where D0 is odd
7249 // - P is the multiplicative inverse of D0 modulo 2^W
7250 // - Q = floor(((2^W) - 1) / D)
7251 // where W is the width of the common type of N and D.
7252 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7253 "Only applicable for (in)equality comparisons.");
7254
7255 SelectionDAG &DAG = DCI.DAG;
7256
7257 EVT VT = REMNode.getValueType();
7258 EVT SVT = VT.getScalarType();
7259 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7260 EVT ShSVT = ShVT.getScalarType();
7261
7262 // If MUL is unavailable, we cannot proceed in any case.
7263 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7264 return SDValue();
7265
7266 bool ComparingWithAllZeros = true;
7267 bool AllComparisonsWithNonZerosAreTautological = true;
7268 bool HadTautologicalLanes = false;
7269 bool AllLanesAreTautological = true;
7270 bool HadEvenDivisor = false;
7271 bool AllDivisorsArePowerOfTwo = true;
7272 bool HadTautologicalInvertedLanes = false;
7273 SmallVector<SDValue, 16> PAmts, KAmts, QAmts;
7274
7275 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
7276 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7277 if (CDiv->isZero())
7278 return false;
7279
7280 const APInt &D = CDiv->getAPIntValue();
7281 const APInt &Cmp = CCmp->getAPIntValue();
7282
7283 ComparingWithAllZeros &= Cmp.isZero();
7284
7285 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7286 // if C2 is not less than C1, the comparison is always false.
7287 // But we will only be able to produce the comparison that will give the
7288 // opposive tautological answer. So this lane would need to be fixed up.
7289 bool TautologicalInvertedLane = D.ule(RHS: Cmp);
7290 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
7291
7292 // If all lanes are tautological (either all divisors are ones, or divisor
7293 // is not greater than the constant we are comparing with),
7294 // we will prefer to avoid the fold.
7295 bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
7296 HadTautologicalLanes |= TautologicalLane;
7297 AllLanesAreTautological &= TautologicalLane;
7298
7299 // If we are comparing with non-zero, we need'll need to subtract said
7300 // comparison value from the LHS. But there is no point in doing that if
7301 // every lane where we are comparing with non-zero is tautological..
7302 if (!Cmp.isZero())
7303 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
7304
7305 // Decompose D into D0 * 2^K
7306 unsigned K = D.countr_zero();
7307 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7308 APInt D0 = D.lshr(shiftAmt: K);
7309
7310 // D is even if it has trailing zeros.
7311 HadEvenDivisor |= (K != 0);
7312 // D is a power-of-two if D0 is one.
7313 // If all divisors are power-of-two, we will prefer to avoid the fold.
7314 AllDivisorsArePowerOfTwo &= D0.isOne();
7315
7316 // P = inv(D0, 2^W)
7317 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7318 unsigned W = D.getBitWidth();
7319 APInt P = D0.multiplicativeInverse();
7320 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7321
7322 // Q = floor((2^W - 1) u/ D)
7323 // R = ((2^W - 1) u% D)
7324 APInt Q, R;
7325 APInt::udivrem(LHS: APInt::getAllOnes(numBits: W), RHS: D, Quotient&: Q, Remainder&: R);
7326
7327 // If we are comparing with zero, then that comparison constant is okay,
7328 // else it may need to be one less than that.
7329 if (Cmp.ugt(RHS: R))
7330 Q -= 1;
7331
7332 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7333 "We are expecting that K is always less than all-ones for ShSVT");
7334
7335 // If the lane is tautological the result can be constant-folded.
7336 if (TautologicalLane) {
7337 // Set P and K amount to a bogus values so we can try to splat them.
7338 P = 0;
7339 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7340 // And ensure that comparison constant is tautological,
7341 // it will always compare true/false.
7342 Q.setAllBits();
7343 } else {
7344 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7345 }
7346
7347 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7348 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7349 return true;
7350 };
7351
7352 SDValue N = REMNode.getOperand(i: 0);
7353 SDValue D = REMNode.getOperand(i: 1);
7354
7355 // Collect the values from each element.
7356 if (!ISD::matchBinaryPredicate(LHS: D, RHS: CompTargetNode, Match: BuildUREMPattern))
7357 return SDValue();
7358
7359 // If all lanes are tautological, the result can be constant-folded.
7360 if (AllLanesAreTautological)
7361 return SDValue();
7362
7363 // If this is a urem by a powers-of-two, avoid the fold since it can be
7364 // best implemented as a bit test.
7365 if (AllDivisorsArePowerOfTwo)
7366 return SDValue();
7367
7368 SDValue PVal, KVal, QVal;
7369 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7370 if (HadTautologicalLanes) {
7371 // Try to turn PAmts into a splat, since we don't care about the values
7372 // that are currently '0'. If we can't, just keep '0'`s.
7373 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7374 // Try to turn KAmts into a splat, since we don't care about the values
7375 // that are currently '-1'. If we can't, change them to '0'`s.
7376 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7377 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7378 }
7379
7380 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7381 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7382 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7383 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7384 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
7385 "Expected matchBinaryPredicate to return one element for "
7386 "SPLAT_VECTORs");
7387 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7388 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7389 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7390 } else {
7391 PVal = PAmts[0];
7392 KVal = KAmts[0];
7393 QVal = QAmts[0];
7394 }
7395
7396 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
7397 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::SUB, VT))
7398 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
7399 assert(CompTargetNode.getValueType() == N.getValueType() &&
7400 "Expecting that the types on LHS and RHS of comparisons match.");
7401 N = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N, N2: CompTargetNode);
7402 }
7403
7404 // (mul N, P)
7405 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7406 Created.push_back(Elt: Op0.getNode());
7407
7408 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7409 // divisors as a performance improvement, since rotating by 0 is a no-op.
7410 if (HadEvenDivisor) {
7411 // We need ROTR to do this.
7412 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7413 return SDValue();
7414 // UREM: (rotr (mul N, P), K)
7415 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7416 Created.push_back(Elt: Op0.getNode());
7417 }
7418
7419 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
7420 SDValue NewCC =
7421 DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7422 Cond: ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
7423 if (!HadTautologicalInvertedLanes)
7424 return NewCC;
7425
7426 // If any lanes previously compared always-false, the NewCC will give
7427 // always-true result for them, so we need to fixup those lanes.
7428 // Or the other way around for inequality predicate.
7429 assert(VT.isVector() && "Can/should only get here for vectors.");
7430 Created.push_back(Elt: NewCC.getNode());
7431
7432 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7433 // if C2 is not less than C1, the comparison is always false.
7434 // But we have produced the comparison that will give the
7435 // opposive tautological answer. So these lanes would need to be fixed up.
7436 SDValue TautologicalInvertedChannels =
7437 DAG.getSetCC(DL, VT: SETCCVT, LHS: D, RHS: CompTargetNode, Cond: ISD::SETULE);
7438 Created.push_back(Elt: TautologicalInvertedChannels.getNode());
7439
7440 // NOTE: we avoid letting illegal types through even if we're before legalize
7441 // ops – legalization has a hard time producing good code for this.
7442 if (isOperationLegalOrCustom(Op: ISD::VSELECT, VT: SETCCVT)) {
7443 // If we have a vector select, let's replace the comparison results in the
7444 // affected lanes with the correct tautological result.
7445 SDValue Replacement = DAG.getBoolConstant(V: Cond == ISD::SETEQ ? false : true,
7446 DL, VT: SETCCVT, OpVT: SETCCVT);
7447 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: SETCCVT, N1: TautologicalInvertedChannels,
7448 N2: Replacement, N3: NewCC);
7449 }
7450
7451 // Else, we can just invert the comparison result in the appropriate lanes.
7452 //
7453 // NOTE: see the note above VSELECT above.
7454 if (isOperationLegalOrCustom(Op: ISD::XOR, VT: SETCCVT))
7455 return DAG.getNode(Opcode: ISD::XOR, DL, VT: SETCCVT, N1: NewCC,
7456 N2: TautologicalInvertedChannels);
7457
7458 return SDValue(); // Don't know how to lower.
7459}
7460
7461/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
7462/// where the divisor is constant and the comparison target is zero,
7463/// return a DAG expression that will generate the same comparison result
7464/// using only multiplications, additions and shifts/rotations.
7465/// Ref: "Hacker's Delight" 10-17.
7466SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
7467 SDValue CompTargetNode,
7468 ISD::CondCode Cond,
7469 DAGCombinerInfo &DCI,
7470 const SDLoc &DL) const {
7471 SmallVector<SDNode *, 7> Built;
7472 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7473 DCI, DL, Created&: Built)) {
7474 assert(Built.size() <= 7 && "Max size prediction failed.");
7475 for (SDNode *N : Built)
7476 DCI.AddToWorklist(N);
7477 return Folded;
7478 }
7479
7480 return SDValue();
7481}
7482
7483SDValue
7484TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
7485 SDValue CompTargetNode, ISD::CondCode Cond,
7486 DAGCombinerInfo &DCI, const SDLoc &DL,
7487 SmallVectorImpl<SDNode *> &Created) const {
7488 // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
7489 // Fold:
7490 // (seteq/ne (srem N, D), 0)
7491 // To:
7492 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
7493 //
7494 // - D must be constant, with D = D0 * 2^K where D0 is odd
7495 // - P is the multiplicative inverse of D0 modulo 2^W
7496 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
7497 // - Q = floor((2 * A) / (2^K))
7498 // where W is the width of the common type of N and D.
7499 //
7500 // When D is a power of two (and thus D0 is 1), the normal
7501 // formula for A and Q don't apply, because the derivation
7502 // depends on D not dividing 2^(W-1), and thus theorem ZRS
7503 // does not apply. This specifically fails when N = INT_MIN.
7504 //
7505 // Instead, for power-of-two D, we use:
7506 // - A = 0
7507 // | -> No offset needed. We're effectively treating it the same as urem.
7508 // - Q = 2^(W-K) - 1
7509 // |-> Test that the top K bits are zero after rotation
7510 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7511 "Only applicable for (in)equality comparisons.");
7512
7513 SelectionDAG &DAG = DCI.DAG;
7514
7515 EVT VT = REMNode.getValueType();
7516 EVT SVT = VT.getScalarType();
7517 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7518 EVT ShSVT = ShVT.getScalarType();
7519
7520 // If we are after ops legalization, and MUL is unavailable, we can not
7521 // proceed.
7522 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7523 return SDValue();
7524
7525 // TODO: Could support comparing with non-zero too.
7526 ConstantSDNode *CompTarget = isConstOrConstSplat(N: CompTargetNode);
7527 if (!CompTarget || !CompTarget->isZero())
7528 return SDValue();
7529
7530 bool HadOneDivisor = false;
7531 bool AllDivisorsAreOnes = true;
7532 bool HadEvenDivisor = false;
7533 bool AllDivisorsArePowerOfTwo = true;
7534 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
7535
7536 auto BuildSREMPattern = [&](ConstantSDNode *C) {
7537 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7538 if (C->isZero())
7539 return false;
7540
7541 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
7542
7543 // WARNING: this fold is only valid for positive divisors!
7544 // `rem %X, -C` is equivalent to `rem %X, C`
7545 APInt D = C->getAPIntValue().abs();
7546
7547 // If all divisors are ones, we will prefer to avoid the fold.
7548 HadOneDivisor |= D.isOne();
7549 AllDivisorsAreOnes &= D.isOne();
7550
7551 // Decompose D into D0 * 2^K
7552 unsigned K = D.countr_zero();
7553 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7554 APInt D0 = D.lshr(shiftAmt: K);
7555
7556 // D is even if it has trailing zeros.
7557 HadEvenDivisor |= (K != 0);
7558
7559 // D is a power-of-two if D0 is one. This includes INT_MIN.
7560 // If all divisors are power-of-two, we will prefer to avoid the fold.
7561 AllDivisorsArePowerOfTwo &= D0.isOne();
7562
7563 // P = inv(D0, 2^W)
7564 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7565 unsigned W = D.getBitWidth();
7566 APInt P = D0.multiplicativeInverse();
7567 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7568
7569 // A = floor((2^(W - 1) - 1) / D0) & -2^K
7570 APInt A = APInt::getSignedMaxValue(numBits: W).udiv(RHS: D0);
7571 A.clearLowBits(loBits: K);
7572
7573 // Q = floor((2 * A) / (2^K))
7574 APInt Q = (2 * A).udiv(RHS: APInt::getOneBitSet(numBits: W, BitNo: K));
7575
7576 assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
7577 "We are expecting that A is always less than all-ones for SVT");
7578 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7579 "We are expecting that K is always less than all-ones for ShSVT");
7580
7581 // If D was a power of two, apply the alternate constant derivation.
7582 if (D0.isOne()) {
7583 // A = 0
7584 A = APInt(W, 0);
7585 // - Q = 2^(W-K) - 1
7586 Q = APInt::getLowBitsSet(numBits: W, loBitsSet: W - K);
7587 }
7588
7589 // If the divisor is 1 the result can be constant-folded.
7590 if (D.isOne()) {
7591 // Set P, A and K to a bogus values so we can try to splat them.
7592 P = 0;
7593 A.setAllBits();
7594 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7595
7596 // x ?% 1 == 0 <--> true <--> x u<= -1
7597 Q.setAllBits();
7598 } else {
7599 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7600 }
7601
7602 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7603 AAmts.push_back(Elt: DAG.getConstant(Val: A, DL, VT: SVT));
7604 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7605 return true;
7606 };
7607
7608 SDValue N = REMNode.getOperand(i: 0);
7609 SDValue D = REMNode.getOperand(i: 1);
7610
7611 // Collect the values from each element.
7612 if (!ISD::matchUnaryPredicate(Op: D, Match: BuildSREMPattern))
7613 return SDValue();
7614
7615 // If this is a srem by a one, avoid the fold since it can be constant-folded.
7616 if (AllDivisorsAreOnes)
7617 return SDValue();
7618
7619 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
7620 // since it can be best implemented as a bit test.
7621 if (AllDivisorsArePowerOfTwo)
7622 return SDValue();
7623
7624 SDValue PVal, AVal, KVal, QVal;
7625 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7626 if (HadOneDivisor) {
7627 // Try to turn PAmts into a splat, since we don't care about the values
7628 // that are currently '0'. If we can't, just keep '0'`s.
7629 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7630 // Try to turn AAmts into a splat, since we don't care about the
7631 // values that are currently '-1'. If we can't, change them to '0'`s.
7632 turnVectorIntoSplatVector(Values: AAmts, Predicate: isAllOnesConstant,
7633 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: SVT));
7634 // Try to turn KAmts into a splat, since we don't care about the values
7635 // that are currently '-1'. If we can't, change them to '0'`s.
7636 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7637 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7638 }
7639
7640 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7641 AVal = DAG.getBuildVector(VT, DL, Ops: AAmts);
7642 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7643 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7644 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7645 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
7646 QAmts.size() == 1 &&
7647 "Expected matchUnaryPredicate to return one element for scalable "
7648 "vectors");
7649 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7650 AVal = DAG.getSplatVector(VT, DL, Op: AAmts[0]);
7651 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7652 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7653 } else {
7654 assert(isa<ConstantSDNode>(D) && "Expected a constant");
7655 PVal = PAmts[0];
7656 AVal = AAmts[0];
7657 KVal = KAmts[0];
7658 QVal = QAmts[0];
7659 }
7660
7661 // (mul N, P)
7662 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7663 Created.push_back(Elt: Op0.getNode());
7664
7665 // We need ADD to do this.
7666 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ADD, VT))
7667 return SDValue();
7668
7669 // (add (mul N, P), A)
7670 Op0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: AVal);
7671 Created.push_back(Elt: Op0.getNode());
7672
7673 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7674 // divisors as a performance improvement, since rotating by 0 is a no-op.
7675 if (HadEvenDivisor) {
7676 // We need ROTR to do this.
7677 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7678 return SDValue();
7679 // SREM: (rotr (add (mul N, P), A), K)
7680 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7681 Created.push_back(Elt: Op0.getNode());
7682 }
7683
7684 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
7685 return DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7686 Cond: (Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT);
7687}
7688
7689SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
7690 const DenormalMode &Mode,
7691 SDNodeFlags Flags) const {
7692 SDLoc DL(Op);
7693 EVT VT = Op.getValueType();
7694 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7695 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL, VT);
7696
7697 // This is specifically a check for the handling of denormal inputs, not the
7698 // result.
7699 if (Mode.Input == DenormalMode::PreserveSign ||
7700 Mode.Input == DenormalMode::PositiveZero) {
7701 // Test = X == 0.0
7702 return DAG.getSetCC(DL, VT: CCVT, LHS: Op, RHS: FPZero, Cond: ISD::SETEQ, /*Chain=*/{},
7703 /*Signaling=*/IsSignaling: false, Flags);
7704 }
7705
7706 // Testing it with denormal inputs to avoid wrong estimate.
7707 //
7708 // Test = fabs(X) < SmallestNormal
7709 const fltSemantics &FltSem = VT.getFltSemantics();
7710 APFloat SmallestNorm = APFloat::getSmallestNormalized(Sem: FltSem);
7711 SDValue NormC = DAG.getConstantFP(Val: SmallestNorm, DL, VT);
7712 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: Op, Flags);
7713 return DAG.getSetCC(DL, VT: CCVT, LHS: Fabs, RHS: NormC, Cond: ISD::SETLT, /*Chain=*/{},
7714 /*Signaling=*/IsSignaling: false, Flags);
7715}
7716
7717SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
7718 bool LegalOps, bool OptForSize,
7719 NegatibleCost &Cost,
7720 unsigned Depth) const {
7721 // fneg is removable even if it has multiple uses.
7722 if (Op.getOpcode() == ISD::FNEG) {
7723 Cost = NegatibleCost::Cheaper;
7724 return Op.getOperand(i: 0);
7725 }
7726
7727 // Don't recurse exponentially.
7728 if (Depth > SelectionDAG::MaxRecursionDepth)
7729 return SDValue();
7730
7731 // Pre-increment recursion depth for use in recursive calls.
7732 ++Depth;
7733 const SDNodeFlags Flags = Op->getFlags();
7734 EVT VT = Op.getValueType();
7735 unsigned Opcode = Op.getOpcode();
7736
7737 // Don't allow anything with multiple uses unless we know it is free.
7738 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
7739 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
7740 isFPExtFree(DestVT: VT, SrcVT: Op.getOperand(i: 0).getValueType());
7741 if (!IsFreeExtend)
7742 return SDValue();
7743 }
7744
7745 auto RemoveDeadNode = [&](SDValue N) {
7746 if (N && N.getNode()->use_empty())
7747 DAG.RemoveDeadNode(N: N.getNode());
7748 };
7749
7750 SDLoc DL(Op);
7751
7752 // Because getNegatedExpression can delete nodes we need a handle to keep
7753 // temporary nodes alive in case the recursion manages to create an identical
7754 // node.
7755 std::list<HandleSDNode> Handles;
7756
7757 switch (Opcode) {
7758 case ISD::ConstantFP: {
7759 // Don't invert constant FP values after legalization unless the target says
7760 // the negated constant is legal.
7761 bool IsOpLegal =
7762 isOperationLegal(Op: ISD::ConstantFP, VT) ||
7763 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF()), VT,
7764 ForCodeSize: OptForSize);
7765
7766 if (LegalOps && !IsOpLegal)
7767 break;
7768
7769 APFloat V = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
7770 V.changeSign();
7771 SDValue CFP = DAG.getConstantFP(Val: V, DL, VT);
7772
7773 // If we already have the use of the negated floating constant, it is free
7774 // to negate it even it has multiple uses.
7775 if (!Op.hasOneUse() && CFP.use_empty())
7776 break;
7777 Cost = NegatibleCost::Neutral;
7778 return CFP;
7779 }
7780 case ISD::SPLAT_VECTOR: {
7781 // fold splat_vector(fneg(X)) -> splat_vector(-X)
7782 SDValue X = Op.getOperand(i: 0);
7783 if (!isOperationLegal(Op: ISD::SPLAT_VECTOR, VT))
7784 break;
7785
7786 SDValue NegX = getCheaperNegatedExpression(Op: X, DAG, LegalOps, OptForSize);
7787 if (!NegX)
7788 break;
7789 Cost = NegatibleCost::Cheaper;
7790 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: NegX);
7791 }
7792 case ISD::BUILD_VECTOR: {
7793 // Only permit BUILD_VECTOR of constants.
7794 if (llvm::any_of(Range: Op->op_values(), P: [&](SDValue N) {
7795 return !N.isUndef() && !isa<ConstantFPSDNode>(Val: N);
7796 }))
7797 break;
7798
7799 bool IsOpLegal =
7800 (isOperationLegal(Op: ISD::ConstantFP, VT) &&
7801 isOperationLegal(Op: ISD::BUILD_VECTOR, VT)) ||
7802 llvm::all_of(Range: Op->op_values(), P: [&](SDValue N) {
7803 return N.isUndef() ||
7804 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: N)->getValueAPF()), VT,
7805 ForCodeSize: OptForSize);
7806 });
7807
7808 if (LegalOps && !IsOpLegal)
7809 break;
7810
7811 SmallVector<SDValue, 4> Ops;
7812 for (SDValue C : Op->op_values()) {
7813 if (C.isUndef()) {
7814 Ops.push_back(Elt: C);
7815 continue;
7816 }
7817 APFloat V = cast<ConstantFPSDNode>(Val&: C)->getValueAPF();
7818 V.changeSign();
7819 Ops.push_back(Elt: DAG.getConstantFP(Val: V, DL, VT: C.getValueType()));
7820 }
7821 Cost = NegatibleCost::Neutral;
7822 return DAG.getBuildVector(VT, DL, Ops);
7823 }
7824 case ISD::FADD: {
7825 if (!Flags.hasNoSignedZeros())
7826 break;
7827
7828 // After operation legalization, it might not be legal to create new FSUBs.
7829 if (LegalOps && !isOperationLegalOrCustom(Op: ISD::FSUB, VT))
7830 break;
7831 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7832
7833 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
7834 NegatibleCost CostX = NegatibleCost::Expensive;
7835 SDValue NegX =
7836 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
7837 // Prevent this node from being deleted by the next call.
7838 if (NegX)
7839 Handles.emplace_back(args&: NegX);
7840
7841 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
7842 NegatibleCost CostY = NegatibleCost::Expensive;
7843 SDValue NegY =
7844 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
7845
7846 // We're done with the handles.
7847 Handles.clear();
7848
7849 // Negate the X if its cost is less or equal than Y.
7850 if (NegX && (CostX <= CostY)) {
7851 Cost = CostX;
7852 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegX, N2: Y, Flags);
7853 if (NegY != N)
7854 RemoveDeadNode(NegY);
7855 return N;
7856 }
7857
7858 // Negate the Y if it is not expensive.
7859 if (NegY) {
7860 Cost = CostY;
7861 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegY, N2: X, Flags);
7862 if (NegX != N)
7863 RemoveDeadNode(NegX);
7864 return N;
7865 }
7866 break;
7867 }
7868 case ISD::FSUB: {
7869 // We can't turn -(A-B) into B-A when we honor signed zeros.
7870 if (!Flags.hasNoSignedZeros())
7871 break;
7872
7873 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7874 // fold (fneg (fsub 0, Y)) -> Y
7875 if (ConstantFPSDNode *C = isConstOrConstSplatFP(N: X, /*AllowUndefs*/ true))
7876 if (C->isZero()) {
7877 Cost = NegatibleCost::Cheaper;
7878 return Y;
7879 }
7880
7881 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
7882 Cost = NegatibleCost::Neutral;
7883 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: X, Flags);
7884 }
7885 case ISD::FMUL:
7886 case ISD::FDIV: {
7887 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7888
7889 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
7890 NegatibleCost CostX = NegatibleCost::Expensive;
7891 SDValue NegX =
7892 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
7893 // Prevent this node from being deleted by the next call.
7894 if (NegX)
7895 Handles.emplace_back(args&: NegX);
7896
7897 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
7898 NegatibleCost CostY = NegatibleCost::Expensive;
7899 SDValue NegY =
7900 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
7901
7902 // We're done with the handles.
7903 Handles.clear();
7904
7905 // Negate the X if its cost is less or equal than Y.
7906 if (NegX && (CostX <= CostY)) {
7907 Cost = CostX;
7908 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, Flags);
7909 if (NegY != N)
7910 RemoveDeadNode(NegY);
7911 return N;
7912 }
7913
7914 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
7915 if (auto *C = isConstOrConstSplatFP(N: Op.getOperand(i: 1)))
7916 if (C->isExactlyValue(V: 2.0) && Op.getOpcode() == ISD::FMUL)
7917 break;
7918
7919 // Negate the Y if it is not expensive.
7920 if (NegY) {
7921 Cost = CostY;
7922 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, Flags);
7923 if (NegX != N)
7924 RemoveDeadNode(NegX);
7925 return N;
7926 }
7927 break;
7928 }
7929 case ISD::FMA:
7930 case ISD::FMULADD:
7931 case ISD::FMAD: {
7932 if (!Flags.hasNoSignedZeros())
7933 break;
7934
7935 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), Z = Op.getOperand(i: 2);
7936 NegatibleCost CostZ = NegatibleCost::Expensive;
7937 SDValue NegZ =
7938 getNegatedExpression(Op: Z, DAG, LegalOps, OptForSize, Cost&: CostZ, Depth);
7939 // Give up if fail to negate the Z.
7940 if (!NegZ)
7941 break;
7942
7943 // Prevent this node from being deleted by the next two calls.
7944 Handles.emplace_back(args&: NegZ);
7945
7946 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
7947 NegatibleCost CostX = NegatibleCost::Expensive;
7948 SDValue NegX =
7949 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
7950 // Prevent this node from being deleted by the next call.
7951 if (NegX)
7952 Handles.emplace_back(args&: NegX);
7953
7954 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
7955 NegatibleCost CostY = NegatibleCost::Expensive;
7956 SDValue NegY =
7957 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
7958
7959 // We're done with the handles.
7960 Handles.clear();
7961
7962 // Negate the X if its cost is less or equal than Y.
7963 if (NegX && (CostX <= CostY)) {
7964 Cost = std::min(a: CostX, b: CostZ);
7965 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, N3: NegZ, Flags);
7966 if (NegY != N)
7967 RemoveDeadNode(NegY);
7968 return N;
7969 }
7970
7971 // Negate the Y if it is not expensive.
7972 if (NegY) {
7973 Cost = std::min(a: CostY, b: CostZ);
7974 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, N3: NegZ, Flags);
7975 if (NegX != N)
7976 RemoveDeadNode(NegX);
7977 return N;
7978 }
7979 break;
7980 }
7981
7982 case ISD::FP_EXTEND:
7983 case ISD::FSIN:
7984 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
7985 OptForSize, Cost, Depth))
7986 return DAG.getNode(Opcode, DL, VT, Operand: NegV);
7987 break;
7988 case ISD::FP_ROUND:
7989 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
7990 OptForSize, Cost, Depth))
7991 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NegV, N2: Op.getOperand(i: 1));
7992 break;
7993 case ISD::SELECT:
7994 case ISD::VSELECT: {
7995 // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
7996 // iff at least one cost is cheaper and the other is neutral/cheaper
7997 SDValue LHS = Op.getOperand(i: 1);
7998 NegatibleCost CostLHS = NegatibleCost::Expensive;
7999 SDValue NegLHS =
8000 getNegatedExpression(Op: LHS, DAG, LegalOps, OptForSize, Cost&: CostLHS, Depth);
8001 if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
8002 RemoveDeadNode(NegLHS);
8003 break;
8004 }
8005
8006 // Prevent this node from being deleted by the next call.
8007 Handles.emplace_back(args&: NegLHS);
8008
8009 SDValue RHS = Op.getOperand(i: 2);
8010 NegatibleCost CostRHS = NegatibleCost::Expensive;
8011 SDValue NegRHS =
8012 getNegatedExpression(Op: RHS, DAG, LegalOps, OptForSize, Cost&: CostRHS, Depth);
8013
8014 // We're done with the handles.
8015 Handles.clear();
8016
8017 if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
8018 (CostLHS != NegatibleCost::Cheaper &&
8019 CostRHS != NegatibleCost::Cheaper)) {
8020 RemoveDeadNode(NegLHS);
8021 RemoveDeadNode(NegRHS);
8022 break;
8023 }
8024
8025 Cost = std::min(a: CostLHS, b: CostRHS);
8026 return DAG.getSelect(DL, VT, Cond: Op.getOperand(i: 0), LHS: NegLHS, RHS: NegRHS);
8027 }
8028 }
8029
8030 return SDValue();
8031}
8032
8033//===----------------------------------------------------------------------===//
8034// Legalization Utilities
8035//===----------------------------------------------------------------------===//
8036
8037bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
8038 SDValue LHS, SDValue RHS,
8039 SmallVectorImpl<SDValue> &Result,
8040 EVT HiLoVT, SelectionDAG &DAG,
8041 MulExpansionKind Kind, SDValue LL,
8042 SDValue LH, SDValue RL, SDValue RH) const {
8043 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
8044 Opcode == ISD::SMUL_LOHI);
8045
8046 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
8047 isOperationLegalOrCustom(Op: ISD::MULHS, VT: HiLoVT);
8048 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
8049 isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT);
8050 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8051 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: HiLoVT);
8052 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8053 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT);
8054
8055 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
8056 return false;
8057
8058 unsigned OuterBitSize = VT.getScalarSizeInBits();
8059 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
8060
8061 // LL, LH, RL, and RH must be either all NULL or all set to a value.
8062 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
8063 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
8064
8065 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
8066 bool Signed) -> bool {
8067 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
8068 SDVTList VTs = DAG.getVTList(VT1: HiLoVT, VT2: HiLoVT);
8069 Lo = DAG.getNode(Opcode: Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, DL: dl, VTList: VTs, N1: L, N2: R);
8070 Hi = Lo.getValue(R: 1);
8071 return true;
8072 }
8073 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
8074 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: L, N2: R);
8075 Hi = DAG.getNode(Opcode: Signed ? ISD::MULHS : ISD::MULHU, DL: dl, VT: HiLoVT, N1: L, N2: R);
8076 return true;
8077 }
8078 return false;
8079 };
8080
8081 SDValue Lo, Hi;
8082
8083 if (!LL.getNode() && !RL.getNode() &&
8084 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8085 LL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LHS);
8086 RL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RHS);
8087 }
8088
8089 if (!LL.getNode())
8090 return false;
8091
8092 APInt HighMask = APInt::getHighBitsSet(numBits: OuterBitSize, hiBitsSet: InnerBitSize);
8093 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask) &&
8094 DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask)) {
8095 // The inputs are both zero-extended.
8096 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
8097 Result.push_back(Elt: Lo);
8098 Result.push_back(Elt: Hi);
8099 if (Opcode != ISD::MUL) {
8100 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8101 Result.push_back(Elt: Zero);
8102 Result.push_back(Elt: Zero);
8103 }
8104 return true;
8105 }
8106 }
8107
8108 if (!VT.isVector() && Opcode == ISD::MUL &&
8109 DAG.ComputeMaxSignificantBits(Op: LHS) <= InnerBitSize &&
8110 DAG.ComputeMaxSignificantBits(Op: RHS) <= InnerBitSize) {
8111 // The input values are both sign-extended.
8112 // TODO non-MUL case?
8113 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
8114 Result.push_back(Elt: Lo);
8115 Result.push_back(Elt: Hi);
8116 return true;
8117 }
8118 }
8119
8120 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
8121 SDValue Shift = DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
8122
8123 if (!LH.getNode() && !RH.getNode() &&
8124 isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
8125 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8126 LH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS, N2: Shift);
8127 LH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LH);
8128 RH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: RHS, N2: Shift);
8129 RH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RH);
8130 }
8131
8132 if (!LH.getNode())
8133 return false;
8134
8135 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
8136 return false;
8137
8138 Result.push_back(Elt: Lo);
8139
8140 if (Opcode == ISD::MUL) {
8141 RH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LL, N2: RH);
8142 LH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LH, N2: RL);
8143 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: RH);
8144 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: LH);
8145 Result.push_back(Elt: Hi);
8146 return true;
8147 }
8148
8149 // Compute the full width result.
8150 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
8151 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
8152 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8153 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
8154 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi);
8155 };
8156
8157 SDValue Next = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8158 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
8159 return false;
8160
8161 // This is effectively the add part of a multiply-add of half-sized operands,
8162 // so it cannot overflow.
8163 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8164
8165 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
8166 return false;
8167
8168 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8169 EVT BoolType = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
8170
8171 bool UseGlue = (isOperationLegalOrCustom(Op: ISD::ADDC, VT) &&
8172 isOperationLegalOrCustom(Op: ISD::ADDE, VT));
8173 if (UseGlue)
8174 Next = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Glue), N1: Next,
8175 N2: Merge(Lo, Hi));
8176 else
8177 Next = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolType), N1: Next,
8178 N2: Merge(Lo, Hi), N3: DAG.getConstant(Val: 0, DL: dl, VT: BoolType));
8179
8180 SDValue Carry = Next.getValue(R: 1);
8181 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8182 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8183
8184 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
8185 return false;
8186
8187 if (UseGlue)
8188 Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::Glue), N1: Hi, N2: Zero,
8189 N3: Carry);
8190 else
8191 Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: BoolType), N1: Hi,
8192 N2: Zero, N3: Carry);
8193
8194 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8195
8196 if (Opcode == ISD::SMUL_LOHI) {
8197 SDValue NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8198 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RL));
8199 Next = DAG.getSelectCC(DL: dl, LHS: LH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8200
8201 NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8202 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LL));
8203 Next = DAG.getSelectCC(DL: dl, LHS: RH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8204 }
8205
8206 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8207 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8208 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8209 return true;
8210}
8211
8212bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
8213 SelectionDAG &DAG, MulExpansionKind Kind,
8214 SDValue LL, SDValue LH, SDValue RL,
8215 SDValue RH) const {
8216 SmallVector<SDValue, 2> Result;
8217 bool Ok = expandMUL_LOHI(Opcode: N->getOpcode(), VT: N->getValueType(ResNo: 0), dl: SDLoc(N),
8218 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Result, HiLoVT,
8219 DAG, Kind, LL, LH, RL, RH);
8220 if (Ok) {
8221 assert(Result.size() == 2);
8222 Lo = Result[0];
8223 Hi = Result[1];
8224 }
8225 return Ok;
8226}
8227
8228// Optimize unsigned division or remainder by constants for types twice as large
8229// as a legal VT.
8230//
8231// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
8232// can be computed
8233// as:
8234// Sum = __builtin_uadd_overflow(Lo, High, &Sum);
8235// Remainder = Sum % Constant;
8236//
8237// If (1 << (BitWidth / 2)) % Constant != 1, we can search for a smaller value
8238// W such that W != (BitWidth / 2) and (1 << W) % Constant == 1. We can break
8239// High:Low into 3 chunks of W bits and compute remainder as
8240// Sum = Chunk0 + Chunk1 + Chunk2;
8241// Remainder = Sum % Constant;
8242//
8243// This is based on "Remainder by Summing Digits" from Hacker's Delight.
8244//
8245// For division, we can compute the remainder using the algorithm described
8246// above, subtract it from the dividend to get an exact multiple of Constant.
8247// Then multiply that exact multiply by the multiplicative inverse modulo
8248// (1 << (BitWidth / 2)) to get the quotient.
8249
8250// If Constant is even, we can shift right the dividend and the divisor by the
8251// number of trailing zeros in Constant before applying the remainder algorithm.
8252// If we're after the quotient, we can subtract this value from the shifted
8253// dividend and multiply by the multiplicative inverse of the shifted divisor.
8254// If we want the remainder, we shift the value left by the number of trailing
8255// zeros and add the bits that were shifted out of the dividend.
8256bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
8257 SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
8258 SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8259 unsigned Opcode = N->getOpcode();
8260 EVT VT = N->getValueType(ResNo: 0);
8261
8262 unsigned BitWidth = Divisor.getBitWidth();
8263 unsigned HBitWidth = BitWidth / 2;
8264 assert(VT.getScalarSizeInBits() == BitWidth &&
8265 HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");
8266
8267 // If the divisor is even, shift it until it becomes odd.
8268 unsigned TrailingZeros = 0;
8269 if (!Divisor[0]) {
8270 TrailingZeros = Divisor.countr_zero();
8271 Divisor.lshrInPlace(ShiftAmt: TrailingZeros);
8272 }
8273
8274 // After removing trailing zeros, the divisor needs to be less than
8275 // (1 << HBitWidth).
8276 APInt HalfMaxPlus1 = APInt::getOneBitSet(numBits: BitWidth, BitNo: HBitWidth);
8277 if (Divisor.uge(RHS: HalfMaxPlus1))
8278 return false;
8279
8280 // Look for the largest chunk width W such that (1 << W) % Divisor == 1 or
8281 // (1 << W) % Divisor == -1.
8282 unsigned BestChunkWidth = 0, AltChunkWidth = 0;
8283 for (unsigned I = HBitWidth, E = HBitWidth / 2; I > E; --I) {
8284 // Skip HBitWidth-1, it doesn't have enough bits for carries.
8285 if (I == HBitWidth - 1)
8286 continue;
8287
8288 APInt Mod = APInt::getOneBitSet(numBits: Divisor.getBitWidth(), BitNo: I).urem(RHS: Divisor);
8289
8290 if (Mod.isOne()) {
8291 BestChunkWidth = I;
8292 break;
8293 }
8294
8295 // We have an alternate strategy for Remainder == Divisor - 1.
8296 // FIXME: Support HBitWidth.
8297 if (I != HBitWidth && Mod == Divisor - 1)
8298 AltChunkWidth = I;
8299 }
8300
8301 bool Alternate = false;
8302 if (!BestChunkWidth) {
8303 if (!AltChunkWidth)
8304 return false;
8305 Alternate = true;
8306 BestChunkWidth = AltChunkWidth;
8307 }
8308
8309 SDLoc dl(N);
8310
8311 assert(!LL == !LH && "Expected both input halves or no input halves!");
8312 if (!LL)
8313 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8314
8315 bool HasFSHR = isOperationLegal(Op: ISD::FSHR, VT: HiLoVT);
8316
8317 auto GetFSHR = [&](SDValue Lo, SDValue Hi, unsigned ShiftAmt) {
8318 assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
8319 if (HasFSHR)
8320 return DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: HiLoVT, N1: Hi, N2: Lo,
8321 N3: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8322 return DAG.getNode(
8323 Opcode: ISD::OR, DL: dl, VT: HiLoVT,
8324 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Lo,
8325 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl)),
8326 N2: DAG.getNode(
8327 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: Hi,
8328 N2: DAG.getShiftAmountConstant(Val: HBitWidth - ShiftAmt, VT: HiLoVT, DL: dl)));
8329 };
8330
8331 // Helper to perform a right shift on a 128-bit value split into two halves.
8332 // Handles shifts >= HBitWidth by moving Hi to Lo and shifting Hi.
8333 auto ShiftRight = [&](SDValue &Lo, SDValue &Hi, unsigned ShiftAmt) {
8334 if (ShiftAmt == 0)
8335 return;
8336 if (ShiftAmt < HBitWidth) {
8337 Lo = GetFSHR(Lo, Hi, ShiftAmt);
8338 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8339 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8340 } else if (ShiftAmt == HBitWidth) {
8341 Lo = Hi;
8342 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8343 } else {
8344 Lo = DAG.getNode(
8345 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8346 N2: DAG.getShiftAmountConstant(Val: ShiftAmt - HBitWidth, VT: HiLoVT, DL: dl));
8347 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8348 }
8349 };
8350
8351 // Shift the input by the number of TrailingZeros in the divisor. The
8352 // shifted out bits will be added to the remainder later.
8353 SDValue PartialRemL, PartialRemH;
8354 if (TrailingZeros && Opcode != ISD::UDIV) {
8355 // Save the shifted off bits if we need the remainder.
8356 if (TrailingZeros < HBitWidth) {
8357 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros);
8358 PartialRemL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LL,
8359 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8360 } else if (TrailingZeros == HBitWidth) {
8361 // All of LL is part of the remainder.
8362 PartialRemL = LL;
8363 } else {
8364 // TrailingZeros > HBitWidth: LL and part of LH are the remainder.
8365 PartialRemL = LL;
8366 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros - HBitWidth);
8367 PartialRemH = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LH,
8368 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8369 }
8370 }
8371
8372 SDValue Sum;
8373 // If BestChunkWidth is HBitWidth add low and high half. If there is a carry
8374 // out, add that to the final sum.
8375 if (BestChunkWidth == HBitWidth) {
8376 assert(!Alternate);
8377 // Shift LH:LL right if there were trailing zeros in the divisor.
8378 ShiftRight(LL, LH, TrailingZeros);
8379
8380 // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
8381 EVT SetCCType =
8382 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: HiLoVT);
8383 if (isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT: HiLoVT)) {
8384 SDVTList VTList = DAG.getVTList(VT1: HiLoVT, VT2: SetCCType);
8385 Sum = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, N1: LL, N2: LH);
8386 Sum = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, N1: Sum,
8387 N2: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT), N3: Sum.getValue(R: 1));
8388 } else {
8389 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: LL, N2: LH);
8390 SDValue Carry = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LL, Cond: ISD::SETULT);
8391 // If the boolean for the target is 0 or 1, we can add the setcc result
8392 // directly.
8393 if (getBooleanContents(Type: HiLoVT) ==
8394 TargetLoweringBase::ZeroOrOneBooleanContent)
8395 Carry = DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: HiLoVT);
8396 else
8397 Carry = DAG.getSelect(DL: dl, VT: HiLoVT, Cond: Carry, LHS: DAG.getConstant(Val: 1, DL: dl, VT: HiLoVT),
8398 RHS: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT));
8399 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum, N2: Carry);
8400 }
8401 } else {
8402 // Otherwise split into multple chunks and add them together. We chose
8403 // BestChunkWidth so that the sum will not overflow.
8404 SDValue Mask = DAG.getConstant(
8405 Val: APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: BestChunkWidth), DL: dl, VT: HiLoVT);
8406
8407 for (unsigned I = 0; I < BitWidth - TrailingZeros; I += BestChunkWidth) {
8408 // If there were trailing zeros in the divisor, increase the shift amount.
8409 unsigned Shift = I + TrailingZeros;
8410 SDValue Chunk;
8411 if (Shift == 0)
8412 Chunk = LL;
8413 else if (Shift >= HBitWidth)
8414 Chunk = DAG.getNode(
8415 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: LH,
8416 N2: DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL: dl));
8417 else
8418 Chunk = GetFSHR(LL, LH, Shift);
8419 // If we're on the last chunk, we don't need an AND.
8420 if (I + BestChunkWidth < BitWidth - TrailingZeros)
8421 Chunk = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: Chunk, N2: Mask);
8422 if (!Sum) {
8423 Sum = Chunk;
8424 } else {
8425 // For Alternate, we need to subtract odd chunks.
8426 unsigned ChunkNum = I / BestChunkWidth;
8427 unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ? ISD::SUB : ISD::ADD;
8428 Sum = DAG.getNode(Opcode: Opc, DL: dl, VT: HiLoVT, N1: Sum, N2: Chunk);
8429 }
8430 }
8431
8432 // For Alternate, the sum may be negative, but we need a positive sum. We
8433 // can increase it by a multiple of the divisor to make it positive. For 3
8434 // chunks the largest negative value is -(2^BestChunkWidth - 1). For 4
8435 // chunks, it's 2*-(2^BestChunkWidth - 1). We know that 2^BestChunkWidth + 1
8436 // is a multiple of the divisor. Add that 1 or 2 times to make the sum
8437 // positive.
8438 if (Alternate) {
8439 unsigned NumChunks = divideCeil(Numerator: BitWidth - TrailingZeros, Denominator: BestChunkWidth);
8440 assert(NumChunks <= 4);
8441
8442 APInt Adjust = APInt::getOneBitSet(numBits: HBitWidth, BitNo: BestChunkWidth);
8443 Adjust.setBit(0);
8444 // If there are 4 chunks, we need to adjust twice.
8445 if (NumChunks == 4)
8446 Adjust <<= 1;
8447 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum,
8448 N2: DAG.getConstant(Val: Adjust, DL: dl, VT: HiLoVT));
8449 }
8450 }
8451
8452 // Perform a HiLoVT urem on the Sum using truncated divisor.
8453 SDValue RemL =
8454 DAG.getNode(Opcode: ISD::UREM, DL: dl, VT: HiLoVT, N1: Sum,
8455 N2: DAG.getConstant(Val: Divisor.trunc(width: HBitWidth), DL: dl, VT: HiLoVT));
8456 SDValue RemH = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8457
8458 if (Opcode != ISD::UREM) {
8459 // If we didn't shift LH/LR earlier, do it now.
8460 if (BestChunkWidth != HBitWidth)
8461 ShiftRight(LL, LH, TrailingZeros);
8462
8463 // Subtract the remainder from the shifted dividend.
8464 SDValue Dividend = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: LL, N2: LH);
8465 SDValue Rem = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: RemL, N2: RemH);
8466
8467 Dividend = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Rem);
8468
8469 // Multiply by the multiplicative inverse of the divisor modulo
8470 // (1 << BitWidth).
8471 APInt MulFactor = Divisor.multiplicativeInverse();
8472
8473 SDValue Quotient = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Dividend,
8474 N2: DAG.getConstant(Val: MulFactor, DL: dl, VT));
8475
8476 // Split the quotient into low and high parts.
8477 SDValue QuotL, QuotH;
8478 std::tie(args&: QuotL, args&: QuotH) = DAG.SplitScalar(N: Quotient, DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8479 Result.push_back(Elt: QuotL);
8480 Result.push_back(Elt: QuotH);
8481 }
8482
8483 if (Opcode != ISD::UDIV) {
8484 // If we shifted the input, shift the remainder left and add the bits we
8485 // shifted off the input.
8486 if (TrailingZeros) {
8487 if (TrailingZeros < HBitWidth) {
8488 // Shift RemH:RemL left by TrailingZeros.
8489 // RemH gets the high bits shifted out of RemL.
8490 RemH = DAG.getNode(
8491 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: RemL,
8492 N2: DAG.getShiftAmountConstant(Val: HBitWidth - TrailingZeros, VT: HiLoVT, DL: dl));
8493 RemL =
8494 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8495 N2: DAG.getShiftAmountConstant(Val: TrailingZeros, VT: HiLoVT, DL: dl));
8496 // OR in the partial remainder.
8497 RemL = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemL, N2: PartialRemL,
8498 Flags: SDNodeFlags::Disjoint);
8499 } else if (TrailingZeros == HBitWidth) {
8500 // Shift left by exactly HBitWidth: RemH becomes RemL, RemL becomes
8501 // PartialRemL.
8502 RemH = RemL;
8503 RemL = PartialRemL;
8504 } else {
8505 // Shift left by more than HBitWidth.
8506 RemH = DAG.getNode(
8507 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8508 N2: DAG.getShiftAmountConstant(Val: TrailingZeros - HBitWidth, VT: HiLoVT, DL: dl));
8509 RemH = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemH, N2: PartialRemH,
8510 Flags: SDNodeFlags::Disjoint);
8511 RemL = PartialRemL;
8512 }
8513 }
8514 Result.push_back(Elt: RemL);
8515 Result.push_back(Elt: RemH);
8516 }
8517
8518 return true;
8519}
8520
8521bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
8522 SDNode *N, const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
8523 EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8524
8525 SDValue N0 = N->getOperand(Num: 0);
8526 EVT VT = N0->getValueType(ResNo: 0);
8527 SDLoc DL{N};
8528
8529 assert(!Divisor.isOne() && "Magic algorithm does not work for division by 1");
8530
8531 // This helper creates a MUL_LOHI of the pair (LL, LH) by a constant.
8532 auto MakeMUL_LOHIByConst = [&](unsigned Opc, SDValue LL, SDValue LH,
8533 const APInt &Const,
8534 SmallVectorImpl<SDValue> &Result) {
8535 SDValue LHS = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT, N1: LL, N2: LH);
8536 SDValue RHS = DAG.getConstant(Val: Const, DL, VT);
8537 auto [RL, RH] = DAG.SplitScalar(N: RHS, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8538 return expandMUL_LOHI(Opcode: Opc, VT, dl: DL, LHS, RHS, Result, HiLoVT, DAG,
8539 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
8540 LL, LH, RL, RH);
8541 };
8542
8543 // This helper creates an ADD/SUB of the pairs (LL, LH) and (RL, RH).
8544 auto MakeAddSubLong = [&](unsigned Opc, SDValue LL, SDValue LH, SDValue RL,
8545 SDValue RH) {
8546 SDValue AddSubNode =
8547 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO : ISD::USUBO, DL,
8548 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LL, N2: RL);
8549 SDValue OutL = AddSubNode.getValue(R: 0);
8550 SDValue Overflow = AddSubNode.getValue(R: 1);
8551 SDValue AddSubWithOverflow =
8552 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, DL,
8553 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LH, N2: RH, N3: Overflow);
8554 SDValue OutH = AddSubWithOverflow.getValue(R: 0);
8555 return std::make_pair(x&: OutL, y&: OutH);
8556 };
8557
8558 // This helper creates a SRL of the pair (LL, LH) by Shift.
8559 auto MakeSRLLong = [&](SDValue LL, SDValue LH, unsigned Shift) {
8560 unsigned HBitWidth = HiLoVT.getScalarSizeInBits();
8561 if (Shift < HBitWidth) {
8562 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift, VT: HiLoVT, DL);
8563 SDValue ResL = DAG.getNode(Opcode: ISD::FSHR, DL, VT: HiLoVT, N1: LH, N2: LL, N3: ShAmt);
8564 SDValue ResH = DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt);
8565 return std::make_pair(x&: ResL, y&: ResH);
8566 }
8567 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HiLoVT);
8568 if (Shift == HBitWidth)
8569 return std::make_pair(x&: LH, y&: Zero);
8570 assert(Shift - HBitWidth < HBitWidth &&
8571 "We shouldn't generate an undefined shift");
8572 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL);
8573 return std::make_pair(x: DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt), y&: Zero);
8574 };
8575
8576 // Knowledge of leading zeros may help to reduce the multiplier.
8577 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
8578
8579 UnsignedDivisionByConstantInfo Magics = UnsignedDivisionByConstantInfo::get(
8580 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
8581
8582 assert(!LL == !LH && "Expected both input halves or no input halves!");
8583 if (!LL)
8584 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N0, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8585 SDValue QL = LL;
8586 SDValue QH = LH;
8587 if (Magics.PreShift != 0)
8588 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PreShift);
8589
8590 SmallVector<SDValue, 4> UMulResult;
8591 if (!MakeMUL_LOHIByConst(ISD::UMUL_LOHI, QL, QH, Magics.Magic, UMulResult))
8592 return false;
8593
8594 QL = UMulResult[2];
8595 QH = UMulResult[3];
8596
8597 if (Magics.IsAdd) {
8598 auto [NPQL, NPQH] = MakeAddSubLong(ISD::SUB, LL, LH, QL, QH);
8599 std::tie(args&: NPQL, args&: NPQH) = MakeSRLLong(NPQL, NPQH, 1);
8600 std::tie(args&: QL, args&: QH) = MakeAddSubLong(ISD::ADD, NPQL, NPQH, QL, QH);
8601 }
8602
8603 if (Magics.PostShift != 0)
8604 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PostShift);
8605
8606 unsigned Opcode = N->getOpcode();
8607 if (Opcode != ISD::UREM) {
8608 Result.push_back(Elt: QL);
8609 Result.push_back(Elt: QH);
8610 }
8611
8612 if (Opcode != ISD::UDIV) {
8613 SmallVector<SDValue, 2> MulResult;
8614 if (!MakeMUL_LOHIByConst(ISD::MUL, QL, QH, Divisor, MulResult))
8615 return false;
8616
8617 assert(MulResult.size() == 2);
8618
8619 auto [RemL, RemH] =
8620 MakeAddSubLong(ISD::SUB, LL, LH, MulResult[0], MulResult[1]);
8621
8622 Result.push_back(Elt: RemL);
8623 Result.push_back(Elt: RemH);
8624 }
8625
8626 return true;
8627}
8628
8629bool TargetLowering::expandDIVREMByConstant(SDNode *N,
8630 SmallVectorImpl<SDValue> &Result,
8631 EVT HiLoVT, SelectionDAG &DAG,
8632 SDValue LL, SDValue LH) const {
8633 unsigned Opcode = N->getOpcode();
8634
8635 // TODO: Support signed division/remainder.
8636 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
8637 return false;
8638 assert(
8639 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
8640 "Unexpected opcode");
8641
8642 auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8643 if (!CN)
8644 return false;
8645
8646 APInt Divisor = CN->getAPIntValue();
8647
8648 // The generated half-width UREM is normally optimized using high multiply.
8649 // If the wide UREM libcall is unavailable, a legal or custom half-width
8650 // UDIVREM can lower it instead.
8651 bool CanDecomposeUREMWithoutMulHi =
8652 Opcode == ISD::UREM &&
8653 getLibcallImpl(Call: RTLIB::getUREM(VT: N->getValueType(ResNo: 0))) ==
8654 RTLIB::Unsupported &&
8655 isOperationLegalOrCustom(Op: ISD::UDIVREM, VT: HiLoVT);
8656 if (!CanDecomposeUREMWithoutMulHi &&
8657 !isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT) &&
8658 !isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT))
8659 return false;
8660
8661 // Prefer the smaller libcall when one is available.
8662 if (DAG.shouldOptForSize() && !CanDecomposeUREMWithoutMulHi)
8663 return false;
8664
8665 // Early out for 0 or 1 divisors.
8666 if (Divisor.ule(RHS: 1))
8667 return false;
8668
8669 if (expandUDIVREMByConstantViaUREMDecomposition(N, Divisor, Result, HiLoVT,
8670 DAG, LL, LH))
8671 return true;
8672
8673 if (expandUDIVREMByConstantViaUMulHiMagic(N, Divisor, Result, HiLoVT, DAG, LL,
8674 LH))
8675 return true;
8676
8677 return false;
8678}
8679
8680// Check that (every element of) Z is undef or not an exact multiple of BW.
8681static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
8682 return ISD::matchUnaryPredicate(
8683 Op: Z,
8684 Match: [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(RHS: BW) != 0; },
8685 /*AllowUndefs=*/true, /*AllowTruncation=*/true);
8686}
8687
8688SDValue TargetLowering::expandFunnelShift(SDNode *Node,
8689 SelectionDAG &DAG) const {
8690 EVT VT = Node->getValueType(ResNo: 0);
8691
8692 if (VT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8693 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8694 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8695 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
8696 return SDValue();
8697
8698 SDValue X = Node->getOperand(Num: 0);
8699 SDValue Y = Node->getOperand(Num: 1);
8700 SDValue Z = Node->getOperand(Num: 2);
8701
8702 unsigned BW = VT.getScalarSizeInBits();
8703 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
8704 SDLoc DL(SDValue(Node, 0));
8705
8706 EVT ShVT = Z.getValueType();
8707
8708 // If a funnel shift in the other direction is more supported, use it.
8709 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
8710 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8711 isOperationLegalOrCustom(Op: RevOpcode, VT) && isPowerOf2_32(Value: BW)) {
8712 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8713 // fshl X, Y, Z -> fshr X, Y, -Z
8714 // fshr X, Y, Z -> fshl X, Y, -Z
8715 Z = DAG.getNegative(Val: Z, DL, VT: ShVT);
8716 } else {
8717 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8718 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8719 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8720 if (IsFSHL) {
8721 Y = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8722 X = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: One);
8723 } else {
8724 X = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8725 Y = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Y, N2: One);
8726 }
8727 Z = DAG.getNOT(DL, Val: Z, VT: ShVT);
8728 }
8729 return DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: Z);
8730 }
8731
8732 SDValue ShX, ShY;
8733 SDValue ShAmt, InvShAmt;
8734 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8735 // fshl: X << C | Y >> (BW - C)
8736 // fshr: X << (BW - C) | Y >> C
8737 // where C = Z % BW is not zero
8738 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8739 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8740 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthC, N2: ShAmt);
8741 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: IsFSHL ? ShAmt : InvShAmt);
8742 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: IsFSHL ? InvShAmt : ShAmt);
8743 } else {
8744 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8745 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8746 SDValue Mask = DAG.getConstant(Val: BW - 1, DL, VT: ShVT);
8747 if (isPowerOf2_32(Value: BW)) {
8748 // Z % BW -> Z & (BW - 1)
8749 ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Z, N2: Mask);
8750 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8751 InvShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: DAG.getNOT(DL, Val: Z, VT: ShVT), N2: Mask);
8752 } else {
8753 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8754 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8755 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Mask, N2: ShAmt);
8756 }
8757
8758 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8759 if (IsFSHL) {
8760 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShAmt);
8761 SDValue ShY1 = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: One);
8762 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShY1, N2: InvShAmt);
8763 } else {
8764 SDValue ShX1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: One);
8765 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShX1, N2: InvShAmt);
8766 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt);
8767 }
8768 }
8769 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShX, N2: ShY);
8770}
8771
8772// TODO: Merge with expandFunnelShift.
8773SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
8774 SelectionDAG &DAG) const {
8775 EVT VT = Node->getValueType(ResNo: 0);
8776 unsigned EltSizeInBits = VT.getScalarSizeInBits();
8777 bool IsLeft = Node->getOpcode() == ISD::ROTL;
8778 SDValue Op0 = Node->getOperand(Num: 0);
8779 SDValue Op1 = Node->getOperand(Num: 1);
8780 SDLoc DL(SDValue(Node, 0));
8781
8782 EVT ShVT = Op1.getValueType();
8783 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: ShVT);
8784
8785 // If a rotate in the other direction is more supported, use it.
8786 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
8787 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8788 isOperationLegalOrCustom(Op: RevRot, VT) && isPowerOf2_32(Value: EltSizeInBits)) {
8789 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8790 return DAG.getNode(Opcode: RevRot, DL, VT, N1: Op0, N2: Sub);
8791 }
8792
8793 if (!AllowVectorOps && VT.isVector() &&
8794 (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8795 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8796 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8797 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT) ||
8798 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT)))
8799 return SDValue();
8800
8801 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
8802 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
8803 SDValue BitWidthMinusOneC = DAG.getConstant(Val: EltSizeInBits - 1, DL, VT: ShVT);
8804 SDValue ShVal;
8805 SDValue HsVal;
8806 if (isPowerOf2_32(Value: EltSizeInBits)) {
8807 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8808 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8809 SDValue NegOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8810 SDValue ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Op1, N2: BitWidthMinusOneC);
8811 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
8812 SDValue HsAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: NegOp1, N2: BitWidthMinusOneC);
8813 HsVal = DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: HsAmt);
8814 } else {
8815 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8816 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8817 SDValue BitWidthC = DAG.getConstant(Val: EltSizeInBits, DL, VT: ShVT);
8818 SDValue ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Op1, N2: BitWidthC);
8819 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
8820 SDValue HsAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthMinusOneC, N2: ShAmt);
8821 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8822 HsVal =
8823 DAG.getNode(Opcode: HsOpc, DL, VT, N1: DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: One), N2: HsAmt);
8824 }
8825 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShVal, N2: HsVal);
8826}
8827
8828/// Check if CLMUL on VT can eventually reach a type with legal CLMUL through
8829/// a chain of halving decompositions (halving element width) and/or vector
8830/// widening (doubling element count). This guides expansion strategy selection:
8831/// if true, the halving/widening path produces better code than bit-by-bit.
8832///
8833/// HalveDepth tracks halving steps only (each creates ~4x more operations).
8834/// Widening steps are cheap (O(1) pad/extract) and don't count.
8835/// Limiting halvings to 2 prevents exponential blowup:
8836/// 1 halving: ~4 sub-CLMULs (good, e.g. v8i16 -> v8i8)
8837/// 2 halvings: ~16 sub-CLMULs (acceptable, e.g. v4i32 -> v4i16 -> v8i8)
8838/// 3 halvings: ~64 sub-CLMULs (worse than bit-by-bit expansion)
8839static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx,
8840 EVT VT, unsigned HalveDepth = 0,
8841 unsigned TotalDepth = 0) {
8842 if (HalveDepth > 2 || TotalDepth > 8 || !VT.isFixedLengthVector())
8843 return false;
8844 if (TLI.isOperationLegalOrCustom(Op: ISD::CLMUL, VT))
8845 return true;
8846 if (!TLI.isTypeLegal(VT))
8847 return false;
8848
8849 unsigned BW = VT.getScalarSizeInBits();
8850
8851 // Halve: halve element width, same element count.
8852 // This is the expensive step -- each halving creates ~4x more operations.
8853 if (BW % 2 == 0) {
8854 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: BW / 2);
8855 EVT HalfVT = VT.changeVectorElementType(Context&: Ctx, EltVT: HalfEltVT);
8856 if (TLI.isTypeLegal(VT: HalfVT) &&
8857 canNarrowCLMULToLegal(TLI, Ctx, VT: HalfVT, HalveDepth: HalveDepth + 1, TotalDepth: TotalDepth + 1))
8858 return true;
8859 }
8860
8861 // Widen: double element count (fixed-width vectors only).
8862 // This is cheap -- just INSERT_SUBVECTOR + EXTRACT_SUBVECTOR.
8863 EVT WideVT = VT.getDoubleNumVectorElementsVT(Context&: Ctx);
8864 if (TLI.isTypeLegal(VT: WideVT) &&
8865 canNarrowCLMULToLegal(TLI, Ctx, VT: WideVT, HalveDepth, TotalDepth: TotalDepth + 1))
8866 return true;
8867
8868 return false;
8869}
8870
8871SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
8872 SDLoc DL(Node);
8873 EVT VT = Node->getValueType(ResNo: 0);
8874 SDValue X = Node->getOperand(Num: 0);
8875 SDValue Y = Node->getOperand(Num: 1);
8876 unsigned BW = VT.getScalarSizeInBits();
8877 unsigned Opcode = Node->getOpcode();
8878 LLVMContext &Ctx = *DAG.getContext();
8879
8880 switch (Opcode) {
8881 case ISD::CLMUL: {
8882 // For vector types, try decomposition strategies that leverage legal
8883 // CLMUL on narrower or wider element types, avoiding the expensive
8884 // bit-by-bit expansion.
8885 if (VT.isVector()) {
8886 // Strategy 1: Halving decomposition to half-element-width CLMUL.
8887 // Applies ExpandIntRes_CLMUL's identity element-wise:
8888 // CLMUL(X, Y) = (Hi << HalfBW) | Lo
8889 // where:
8890 // Lo = CLMUL(XLo, YLo)
8891 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
8892 unsigned HalfBW = BW / 2;
8893 if (BW % 2 == 0) {
8894 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: HalfBW);
8895 EVT HalfVT =
8896 EVT::getVectorVT(Context&: Ctx, VT: HalfEltVT, EC: VT.getVectorElementCount());
8897 if (isTypeLegal(VT: HalfVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: HalfVT,
8898 /*HalveDepth=*/1)) {
8899 SDValue ShAmt = DAG.getShiftAmountConstant(Val: HalfBW, VT, DL);
8900
8901 // Extract low and high halves of each element.
8902 SDValue XLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: X);
8903 SDValue XHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
8904 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: ShAmt));
8905 SDValue YLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: Y);
8906 SDValue YHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
8907 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt));
8908
8909 // Lo = CLMUL(XLo, YLo)
8910 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YLo);
8911
8912 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
8913 SDValue LoH = DAG.getNode(Opcode: ISD::CLMULH, DL, VT: HalfVT, N1: XLo, N2: YLo);
8914 SDValue Cross1 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YHi);
8915 SDValue Cross2 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XHi, N2: YLo);
8916 SDValue Cross = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: Cross1, N2: Cross2);
8917 SDValue Hi = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: LoH, N2: Cross);
8918
8919 // Reassemble: Result = ZExt(Lo) | (AnyExt(Hi) << HalfBW)
8920 SDValue LoExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Lo);
8921 SDValue HiExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Hi);
8922 SDValue HiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: HiExt, N2: ShAmt);
8923 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoExt, N2: HiShifted);
8924 }
8925 }
8926
8927 // Strategy 2: Promote to double-element-width CLMUL.
8928 // CLMUL(X, Y) = Trunc(CLMUL(AnyExt(X), AnyExt(Y)))
8929 {
8930 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
8931 if (isTypeLegal(VT: ExtVT) && isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT)) {
8932 // If CLMUL on ExtVT is Custom (not Legal), the target may
8933 // scalarize it, costing O(NumElements) scalar ops. The bit-by-bit
8934 // fallback costs O(BW) vectorized iterations. Only widen when
8935 // element count is small enough that scalarization is cheaper.
8936 unsigned NumElts = VT.getVectorMinNumElements();
8937 if (isOperationLegal(Op: ISD::CLMUL, VT: ExtVT) || NumElts < BW) {
8938 SDValue XExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: X);
8939 SDValue YExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: Y);
8940 SDValue Mul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
8941 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Mul);
8942 }
8943 }
8944 }
8945
8946 // Strategy 3: Widen element count (pad with undef, do CLMUL on wider
8947 // vector, extract lower result). CLMUL is element-wise, so upper
8948 // (undef) lanes don't affect the lower results.
8949 // e.g. v4i16 => pad to v8i16 => halve to v8i8 PMUL => extract v4i16.
8950 if (auto EC = VT.getVectorElementCount(); EC.isFixed()) {
8951 EVT WideVT = EVT::getVectorVT(Context&: Ctx, VT: VT.getVectorElementType(), EC: EC * 2);
8952 if (isTypeLegal(VT: WideVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: WideVT)) {
8953 SDValue Undef = DAG.getUNDEF(VT: WideVT);
8954 SDValue XWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
8955 N2: X, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8956 SDValue YWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
8957 N2: Y, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8958 SDValue WideRes = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: WideVT, N1: XWide, N2: YWide);
8959 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WideRes,
8960 N2: DAG.getVectorIdxConstant(Val: 0, DL));
8961 }
8962 }
8963 }
8964
8965 // Special case: clmul(X, Y) where Y is a known constant (splat) that forms
8966 // a contiguous block of trailing ones whose length N is a power of two
8967 // (e.g. i8 0xFF, i8 0x0F, ...) or equal to the operand width. In this
8968 // special case, clmul(X, Y) is equivalent to a "parallel prefix XOR" or
8969 // "bitwise parity" operation on X.
8970 //
8971 // Note: This special currently dose NOT apply when the mask is neither a
8972 // power of two nor equal to the operand width because the loop inside
8973 // behaves as if the mask was bit-ceiled, and "undoing" the XOR with parts
8974 // of that CLMUL is a recursive problem (e.g. CLMUL with a 20-bit mask
8975 // requires correction XOR with CLMUL with 12-bit mask).
8976 if (auto *C = isConstOrConstSplat(N: Y, /*AllowUndefs=*/true)) {
8977 const APInt &YVal = C->getAPIntValue();
8978 unsigned N = YVal.countr_one();
8979 if (YVal.isAllOnes() || (YVal.isMask() && isPowerOf2_32(Value: N))) {
8980 SDValue R = X;
8981 for (unsigned I = 1; I < N; I <<= 1) {
8982 SDValue ShAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
8983 SDValue Shifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: R, N2: ShAmt);
8984 R = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: R, N2: Shifted);
8985 }
8986 return R;
8987 }
8988 }
8989
8990 // NOTE: If you change this expansion, please update the cost model
8991 // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
8992 // Intrinsic::clmul.
8993
8994 // Strategy 4: multiplication with holes.
8995 //
8996 // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
8997 // do occur, they wind up in a "hole" and are subsequently masked out of the
8998 // result.
8999 //
9000 // A hole of 3 bits is optimal for 32-bit and 64-bit inputs. 128-bit
9001 // integers need a larger hole, and for smaller integers the fallback below
9002 // is more efficient.
9003 //
9004 // Based on bmul64 in bearssl and bmul in the rust polyval crate.
9005 if (BW >= 32 && BW <= 64 &&
9006 isOperationLegalOrCustom(Op: ISD::MUL, VT: getTypeToTransformTo(Context&: Ctx, VT))) {
9007
9008 // Set every fourth bit of each nibble, equivalent to 0b00010001...0001.
9009 APInt MaskVal = APInt::getSplat(NewLen: BW, V: APInt(4, 0b0001));
9010
9011 // Create versions of X and Y that keep only the I-th bit of
9012 // each nibble.
9013 SDValue M[4], Xp[4], Yp[4];
9014 for (unsigned I = 0; I < 4; ++I) {
9015 M[I] = DAG.getConstant(Val: MaskVal.shl(shiftAmt: I), DL, VT);
9016 Xp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: M[I]);
9017 Yp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: M[I]);
9018 }
9019
9020 // Codegens these expressions (16 multiplications):
9021 //
9022 // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
9023 // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
9024 // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
9025 // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
9026 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9027 for (unsigned I = 0; I < 4; ++I) {
9028 SDValue Zi = DAG.getConstant(Val: 0, DL, VT);
9029 for (unsigned J = 0; J < 4; ++J) {
9030 unsigned K = (I + 4 - J) % 4;
9031 SDValue P = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Xp[J], N2: Yp[K]);
9032 Zi = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Zi, N2: P);
9033 }
9034
9035 // Keep only the bits belonging to this iteration, and bitwise or it all
9036 // together.
9037 Zi = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Zi, N2: M[I]);
9038 Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Res, N2: Zi, Flags: SDNodeFlags::Disjoint);
9039 }
9040 return Res;
9041 }
9042
9043 // Strategy 5: the naive fallback.
9044 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
9045
9046 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9047 for (unsigned I = 0; I < BW; ++I) {
9048 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9049 SDValue Mask = DAG.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: I), DL, VT);
9050 SDValue YMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: Mask);
9051
9052 // For targets with a fast bit test instruction (e.g., x86 BT) or without
9053 // multiply, use a shift-based expansion to avoid expensive MUL
9054 // instructions.
9055 SDValue Part;
9056 if (!hasBitTest(X: Y, Y: ShiftAmt) &&
9057 isOperationLegalOrCustom(
9058 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
9059 Part = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: YMasked);
9060 } else {
9061 // Canonical bit test: (Y & (1 << I)) != 0
9062 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
9063 SDValue Cond = DAG.getSetCC(DL, VT: SetCCVT, LHS: YMasked, RHS: Zero, Cond: ISD::SETEQ);
9064 SDValue XShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftAmt);
9065 Part = DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: XShifted);
9066 }
9067 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Res, N2: Part);
9068 }
9069 return Res;
9070 }
9071 case ISD::CLMULR:
9072 // If we have CLMUL/CLMULH, merge the shifted results to form CLMULR.
9073 if (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) &&
9074 isOperationLegalOrCustom(Op: ISD::CLMULH, VT)) {
9075 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
9076 SDValue Hi = DAG.getNode(Opcode: ISD::CLMULH, DL, VT, N1: X, N2: Y);
9077 Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo,
9078 N2: DAG.getShiftAmountConstant(Val: BW - 1, VT, DL));
9079 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi,
9080 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9081 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Lo, N2: Hi);
9082 }
9083 [[fallthrough]];
9084 case ISD::CLMULH: {
9085 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9086 // Use bitreverse-based lowering (CLMULR/H = rev(CLMUL(rev,rev)) >> S)
9087 // when any of these hold:
9088 // (a) ZERO_EXTEND to ExtVT or SRL on ExtVT isn't legal.
9089 // (b) CLMUL is legal on VT but not on ExtVT (e.g. v8i8 on AArch64).
9090 // (c) CLMUL on ExtVT isn't legal, but CLMUL on VT can be efficiently
9091 // expanded via halving/widening to reach legal CLMUL. The bitreverse
9092 // path creates CLMUL(VT) which will be expanded efficiently. The
9093 // promote path would create CLMUL(ExtVT) => halving => CLMULH(VT),
9094 // causing a cycle.
9095 // Note: when CLMUL is legal on ExtVT, the zext => CLMUL(ExtVT) => shift
9096 // => trunc path is preferred over the bitreverse path, as it avoids the
9097 // cost of 3 bitreverse operations.
9098 if (!isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT: ExtVT) ||
9099 !isOperationLegalOrCustom(Op: ISD::SRL, VT: ExtVT) ||
9100 (!isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT) &&
9101 (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) ||
9102 canNarrowCLMULToLegal(TLI: *this, Ctx, VT)))) {
9103 SDValue XRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: X);
9104 SDValue YRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: Y);
9105 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: XRev, N2: YRev);
9106 SDValue Res = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: ClMul);
9107 if (Opcode == ISD::CLMULH)
9108 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res,
9109 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9110 return Res;
9111 }
9112 SDValue XExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: X);
9113 SDValue YExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: Y);
9114 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9115 unsigned ShAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
9116 SDValue HiBits = DAG.getNode(Opcode: ISD::SRL, DL, VT: ExtVT, N1: ClMul,
9117 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: ExtVT, DL));
9118 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: HiBits);
9119 }
9120 }
9121 llvm_unreachable("Expected CLMUL, CLMULR, or CLMULH");
9122}
9123
9124SDValue TargetLowering::expandPEXT(SDNode *Node, SelectionDAG &DAG) const {
9125 SDLoc DL(Node);
9126 EVT VT = Node->getValueType(ResNo: 0);
9127 SDValue Val = Node->getOperand(Num: 0);
9128 SDValue Msk = Node->getOperand(Num: 1);
9129 unsigned BW = VT.getScalarSizeInBits();
9130
9131 // Just scalarize if scalar PEXT is legal
9132 if (VT.isVector() && isOperationLegal(Op: ISD::PEXT, VT: VT.getVectorElementType()))
9133 return DAG.UnrollVectorOp(N: Node);
9134
9135 // Hacker's Delight §7-4: Compress, or Generalized Extract
9136 SDValue X = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Val, N2: Msk);
9137 SDValue M = Msk;
9138 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9139 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: M, VT), N2: One);
9140
9141 // Repeatedly compute which bits would shift to the right by an odd amount,
9142 // shift all such bits in parallel using a mask, and double the shift amount.
9143 for (unsigned I = 1; I < BW; I *= 2) {
9144 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9145 SDValue Mp =
9146 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9147 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: M);
9148 SDValue ShiftI = DAG.getShiftAmountConstant(Val: I, VT, DL);
9149 SDValue MvS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mv, N2: ShiftI);
9150 M = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: M, N2: Mv), N2: MvS,
9151 Flags: SDNodeFlags::Disjoint);
9152 SDValue T = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Mv);
9153 SDValue TS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: T, N2: ShiftI);
9154 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: X, N2: T), N2: TS,
9155 Flags: SDNodeFlags::Disjoint);
9156 if (I * 2 < BW)
9157 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9158 }
9159
9160 return X;
9161}
9162
9163SDValue TargetLowering::expandPDEP(SDNode *Node, SelectionDAG &DAG) const {
9164 SDLoc DL(Node);
9165 EVT VT = Node->getValueType(ResNo: 0);
9166 SDValue Val = Node->getOperand(Num: 0);
9167 SDValue Msk = Node->getOperand(Num: 1);
9168 unsigned BW = VT.getScalarSizeInBits();
9169
9170 // Just scalarize if scalar PDEP is legal
9171 if (VT.isVector() && isOperationLegal(Op: ISD::PDEP, VT: VT.getVectorElementType()))
9172 return DAG.UnrollVectorOp(N: Node);
9173
9174 // Hacker's Delight §7-5: Expand, or Generalized Insert.
9175 unsigned LogBW = Log2_32_Ceil(Value: BW);
9176 SmallVector<SDValue, 8> MvArray(LogBW);
9177 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9178 SDValue Mc = Msk;
9179 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: Msk, VT), N2: One);
9180
9181 // First pass: compute move masks for each power of two that a bit moves by.
9182 for (unsigned S = 0; S < LogBW; ++S) {
9183 unsigned ShiftS = 1u << S;
9184 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9185 SDValue Mp =
9186 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9187 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: Mc);
9188 MvArray[S] = Mv;
9189 if (S + 1 < LogBW) {
9190 SDValue McXorMv = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Mc, N2: Mv);
9191 SDValue MvShifted = DAG.getNode(
9192 Opcode: ISD::SRL, DL, VT, N1: Mv, N2: DAG.getShiftAmountConstant(Val: ShiftS, VT, DL));
9193 Mc = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: McXorMv, N2: MvShifted,
9194 Flags: SDNodeFlags::Disjoint);
9195 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9196 }
9197 }
9198
9199 // Second pass: move bits by 32, 16, 8, 4, 2, 1, using masks, in parallel.
9200 // Each pass handles half the shift amount of the previous pass.
9201 SDValue X = Val;
9202 for (int S = (int)LogBW - 1; S >= 0; --S) {
9203 SDValue ShiftSv = DAG.getShiftAmountConstant(Val: 1ull << S, VT, DL);
9204 SDValue T = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftSv);
9205 SDValue UnshiftedBits =
9206 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: DAG.getNOT(DL, Val: MvArray[S], VT));
9207 SDValue ShiftedBits = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T, N2: MvArray[S]);
9208 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: UnshiftedBits, N2: ShiftedBits,
9209 Flags: SDNodeFlags::Disjoint);
9210 }
9211
9212 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Msk);
9213}
9214
9215void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
9216 SelectionDAG &DAG) const {
9217 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
9218 EVT VT = Node->getValueType(ResNo: 0);
9219 unsigned VTBits = VT.getScalarSizeInBits();
9220 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
9221
9222 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
9223 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
9224 SDValue ShOpLo = Node->getOperand(Num: 0);
9225 SDValue ShOpHi = Node->getOperand(Num: 1);
9226 SDValue ShAmt = Node->getOperand(Num: 2);
9227 EVT ShAmtVT = ShAmt.getValueType();
9228 EVT ShAmtCCVT =
9229 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ShAmtVT);
9230 SDLoc dl(Node);
9231
9232 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
9233 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
9234 // away during isel.
9235 SDValue SafeShAmt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9236 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT));
9237 SDValue Tmp1 = IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: ShOpHi,
9238 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT))
9239 : DAG.getConstant(Val: 0, DL: dl, VT);
9240
9241 SDValue Tmp2, Tmp3;
9242 if (IsSHL) {
9243 Tmp2 = DAG.getNode(Opcode: ISD::FSHL, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9244 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: SafeShAmt);
9245 } else {
9246 Tmp2 = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9247 Tmp3 = DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: ShOpHi, N2: SafeShAmt);
9248 }
9249
9250 // If the shift amount is larger or equal than the width of a part we don't
9251 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
9252 // values for large shift amounts.
9253 SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9254 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: ShAmtVT));
9255 SDValue Cond = DAG.getSetCC(DL: dl, VT: ShAmtCCVT, LHS: AndNode,
9256 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShAmtVT), Cond: ISD::SETNE);
9257
9258 if (IsSHL) {
9259 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9260 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9261 } else {
9262 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9263 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9264 }
9265}
9266
9267SDValue TargetLowering::expandFCANONICALIZE(SDNode *Node,
9268 SelectionDAG &DAG) const {
9269 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
9270 // suggested in
9271 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
9272 // It uses strict_fp operations even outside a strict_fp context in order
9273 // to guarantee that the canonicalization is not optimized away by later
9274 // passes. The result chain introduced by that is intentionally ignored
9275 // since no ordering requirement is intended here.
9276 EVT VT = Node->getValueType(ResNo: 0);
9277 SDLoc DL(Node);
9278 SDNodeFlags Flags = Node->getFlags();
9279 Flags.setNoFPExcept(true);
9280 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT);
9281 SDValue Mul =
9282 DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {VT, MVT::Other},
9283 Ops: {DAG.getEntryNode(), Node->getOperand(Num: 0), One}, Flags);
9284 return Mul;
9285}
9286
9287SDValue TargetLowering::expandCONVERT_TO_ARBITRARY_FP(SDNode *Node,
9288 SelectionDAG &DAG) const {
9289 // Expand conversion from a native IEEE float type to an arbitrary FP format
9290 // returning the result as an integer using bit manipulation.
9291 EVT ResVT = Node->getValueType(ResNo: 0);
9292 SDLoc dl(Node);
9293
9294 SDValue FloatVal = Node->getOperand(Num: 0);
9295 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9296 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9297 const auto RoundMode =
9298 static_cast<RoundingMode>(Node->getConstantOperandVal(Num: 2));
9299 const bool Saturate = Node->getConstantOperandVal(Num: 3) != 0;
9300
9301 // Supported destination formats.
9302 switch (Sem) {
9303 case APFloatBase::S_Float8E5M2:
9304 case APFloatBase::S_Float8E4M3FN:
9305 case APFloatBase::S_Float8E5M3FNU:
9306 case APFloatBase::S_Float6E3M2FN:
9307 case APFloatBase::S_Float6E2M3FN:
9308 case APFloatBase::S_Float4E2M1FN:
9309 break;
9310 default:
9311 DAG.getContext()->emitError(ErrorStr: "CONVERT_TO_ARBITRARY_FP: not implemented "
9312 "destination format (semantics enum " +
9313 Twine(SemEnum) + ")");
9314 return SDValue();
9315 }
9316
9317 // Supported rounding modes.
9318 switch (RoundMode) {
9319 case RoundingMode::NearestTiesToEven:
9320 case RoundingMode::TowardZero:
9321 case RoundingMode::TowardPositive:
9322 case RoundingMode::TowardNegative:
9323 case RoundingMode::NearestTiesToAway:
9324 break;
9325 default:
9326 DAG.getContext()->emitError(
9327 ErrorStr: "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
9328 Twine(static_cast<int>(RoundMode)) + ")");
9329 return SDValue();
9330 }
9331
9332 // Destination format parameters.
9333 const fltSemantics &DstSem = APFloatBase::EnumToSemantics(S: Sem);
9334 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9335 const unsigned DstPrecision = APFloat::semanticsPrecision(DstSem);
9336 const unsigned DstMant = DstPrecision - 1;
9337 // Unsigned formats spend no bit on the sign.
9338 const bool DstHasSign = APFloat::semanticsHasSignedRepr(DstSem);
9339 const unsigned DstExpBits = DstBits - (DstHasSign ? 1 : 0) - DstMant;
9340 const int DstBias = 1 - APFloat::semanticsMinExponent(DstSem);
9341 const unsigned DstExpMax = (1U << DstExpBits) - 1;
9342 const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
9343 const fltNonfiniteBehavior DstNFBehavior = DstSem.nonFiniteBehavior;
9344 const fltNanEncoding DstNanEnc = DstSem.nanEncoding;
9345
9346 // Compute the maximum normal exponent for the destination format.
9347 const unsigned DstExpMaxNormal =
9348 DstNFBehavior == fltNonfiniteBehavior::IEEE754 ? DstExpMax - 1
9349 : DstExpMax;
9350
9351 // For NanOnly formats the max exponent field for finite values
9352 // is DstExpMax, but the encoding with exp = DstExpMax and
9353 // mant = all-ones is NaN. So DstExpMaxNormal = DstExpMax, but max
9354 // mantissa at that exponent is DstMantMask - 1 (if NanEnc == AllOnes) to
9355 // avoid the NaN encoding.
9356 uint64_t DstMaxMantAtMaxExp = DstMantMask;
9357 if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9358 DstNanEnc == fltNanEncoding::AllOnes)
9359 DstMaxMantAtMaxExp = DstMantMask - 1;
9360
9361 // Source format parameters.
9362 EVT SrcVT = FloatVal.getValueType();
9363 const fltSemantics &SrcSem = SrcVT.getScalarType().getFltSemantics();
9364 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9365 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9366 const unsigned SrcMant = SrcPrecision - 1;
9367 const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;
9368
9369 // Work in the source integer type. Match the destination shape so the
9370 // expansion stays vector when ResVT is a vector.
9371 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcBits);
9372 EVT IntVT = ResVT.changeElementType(Context&: *DAG.getContext(), EltVT: IntScalarVT);
9373 EVT SetCCVT =
9374 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9375 EVT FPSetCCVT =
9376 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
9377
9378 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9379 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9380
9381 // Bitcast source float to integer to extract the sign bit.
9382 SDValue Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FloatVal);
9383 SDValue SignBit =
9384 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9385 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
9386
9387 // Classify the input.
9388 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT);
9389 SDValue FPInf = DAG.getConstantFP(Val: APFloat::getInf(Sem: SrcSem), DL: dl, VT: SrcVT);
9390 SDValue AbsVal = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: SrcVT, Operand: FloatVal);
9391 SDValue IsNaN = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETUO);
9392 SDValue IsInf = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: AbsVal, RHS: FPInf, Cond: ISD::SETOEQ);
9393 SDValue IsZero = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOEQ);
9394
9395 // Split into a normalized fraction and unbiased exponent. FFREXP normalizes
9396 // source denormals automatically. The result is unspecified for Inf/NaN, but
9397 // those inputs are detected above and override the final result.
9398 EVT FrexpExpScalarVT =
9399 getValueType(DL: DAG.getDataLayout(), Ty: Type::getInt32Ty(C&: *DAG.getContext()));
9400 EVT FrexpExpVT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: FrexpExpScalarVT);
9401 SDValue Frexp =
9402 DAG.getNode(Opcode: ISD::FFREXP, DL: dl, VTList: DAG.getVTList(VT1: SrcVT, VT2: FrexpExpVT), N: FloatVal);
9403 SDValue FrexpFrac = Frexp.getValue(R: 0);
9404 SDValue FrexpExp = Frexp.getValue(R: 1);
9405
9406 SDValue FrexpFracInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FrexpFrac);
9407 SDValue EffSrcMant = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FrexpFracInt,
9408 N2: DAG.getConstant(Val: SrcMantMask, DL: dl, VT: IntVT));
9409
9410 SDValue FrexpExpExt = DAG.getSExtOrTrunc(Op: FrexpExp, DL: dl, VT: IntVT);
9411 SDValue NewExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: FrexpExpExt,
9412 N2: DAG.getConstant(Val: DstBias - 1, DL: dl, VT: IntVT));
9413
9414 // Compute rounding increment given the round bit, sticky bits, and LSB
9415 // of the truncated mantissa.
9416 auto ComputeRoundUp = [&](SDValue RoundBit, SDValue StickyBits,
9417 SDValue LSB) -> SDValue {
9418 switch (RoundMode) {
9419 case RoundingMode::NearestTiesToEven: {
9420 // Round up if round_bit && (sticky || lsb)
9421 SDValue StickyOrLSB = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: StickyBits, N2: LSB);
9422 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyOrLSB);
9423 }
9424 case RoundingMode::TowardZero:
9425 return Zero;
9426 case RoundingMode::TowardPositive: {
9427 // Round up if positive and any truncated bits are set.
9428 SDValue AnyTruncBits =
9429 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9430 SDValue HasTruncBits =
9431 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9432 SDValue IsPositive = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETEQ);
9433 SDValue DoRound =
9434 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsPositive);
9435 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9436 }
9437 case RoundingMode::TowardNegative: {
9438 // Round up if negative and any truncated bits are set (to -Inf).
9439 SDValue AnyTruncBits =
9440 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9441 SDValue HasTruncBits =
9442 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9443 SDValue IsNegative = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETNE);
9444 SDValue DoRound =
9445 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsNegative);
9446 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9447 }
9448 case RoundingMode::NearestTiesToAway:
9449 return RoundBit;
9450 default:
9451 llvm_unreachable("unsupported rounding mode");
9452 }
9453 };
9454
9455 // Round mantissa from SrcMant bits to DstMant bits.
9456 SDValue TruncMant;
9457 SDValue RoundUp;
9458 if (SrcMant > DstMant) {
9459 const unsigned Shift = SrcMant - DstMant;
9460 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: Shift, VT: IntVT, DL: dl);
9461 TruncMant = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ShiftConst);
9462
9463 // Check bit at position Shift - 1 aka the round bit.
9464 SDValue RoundBit;
9465 if (Shift >= 1) {
9466 SDValue RoundBitShift = DAG.getShiftAmountConstant(Val: Shift - 1, VT: IntVT, DL: dl);
9467 SDValue ShiftedMant =
9468 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: RoundBitShift);
9469 RoundBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: ShiftedMant, N2: One);
9470 } else {
9471 RoundBit = Zero;
9472 }
9473
9474 // OR of all bits below the round bit to get sticky bits.
9475 SDValue StickyBits;
9476 if (Shift >= 2) {
9477 uint64_t StickyMask = maskTrailingOnes<uint64_t>(N: Shift - 1);
9478 StickyBits = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: EffSrcMant,
9479 N2: DAG.getConstant(Val: StickyMask, DL: dl, VT: IntVT));
9480 StickyBits = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: StickyBits, RHS: Zero, Cond: ISD::SETNE);
9481 StickyBits = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: StickyBits);
9482 } else {
9483 StickyBits = Zero;
9484 }
9485
9486 // LSB of truncated mantissa.
9487 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: TruncMant, N2: One);
9488
9489 RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
9490 } else {
9491 // If DstMant >= SrcMant, then no rounding needed, just shift left.
9492 SDValue MantShift =
9493 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
9494 TruncMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: MantShift);
9495 RoundUp = Zero;
9496 }
9497
9498 // Apply rounding.
9499 SDValue RoundedMant = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: TruncMant, N2: RoundUp);
9500
9501 // Handle mantissa overflow from rounding.
9502 // If rounded_mant > DstMantMask, carry into exponent.
9503 SDValue MantOverflow =
9504 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: RoundedMant,
9505 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9506 // On overflow: mant = 0, exp += 1.
9507 SDValue AdjMant = DAG.getSelect(DL: dl, VT: IntVT, Cond: MantOverflow, LHS: Zero, RHS: RoundedMant);
9508 SDValue AdjExp =
9509 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: NewExp,
9510 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: MantOverflow));
9511
9512 // Precompute sign shifted to MSB of destination. Unsigned formats have no
9513 // sign bit to merge in.
9514 SDValue SignShifted =
9515 DstHasSign
9516 ? DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
9517 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
9518 : Zero;
9519
9520 // Destination denormal conversion (when new_exp <= 0).
9521 // Shift the mantissa right by 1 - new_exp additional bits and set the
9522 // exponent field to 0.
9523 SDValue ExpIsNeg = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9524 RHS: DAG.getConstant(Val: 1, DL: dl, VT: IntVT), Cond: ISD::SETLT);
9525
9526 SDValue DenormResult;
9527 {
9528 // denorm_shift = 1 - NewExp.
9529 SDValue DenormShift = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: One, N2: NewExp);
9530
9531 // full_src_mant = (1 << SrcMant) | EffSrcMant.
9532 SDValue ImplicitOne =
9533 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One,
9534 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl));
9535 SDValue FullSrcMant =
9536 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ImplicitOne);
9537
9538 // Total right shift = DenormShift + (SrcMant - DstMant).
9539 int64_t MantDelta = static_cast<int64_t>(SrcMant) - DstMant;
9540 SDValue TotalShift =
9541 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormShift,
9542 N2: DAG.getSignedConstant(Val: MantDelta, DL: dl, VT: IntVT));
9543
9544 // Clamp total shift to avoid UB, then truncate denorm mantissa.
9545 EVT ShiftVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
9546 SDValue MaxShift = DAG.getConstant(Val: SrcBits - 1, DL: dl, VT: IntVT);
9547 SDValue ClampedShift =
9548 DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IntVT, N1: TotalShift, N2: MaxShift);
9549 SDValue DenormTruncMant =
9550 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant,
9551 N2: DAG.getZExtOrTrunc(Op: ClampedShift, DL: dl, VT: ShiftVT));
9552
9553 // Rounding for denorm path.
9554 SDValue DenormRoundUp;
9555 {
9556 // Round bit is at position TotalShift - 1 of FullSrcMant.
9557 // Clamp to at least 1 so the subtraction doesn't underflow and create
9558 // shift nodes with invalid shift amounts.
9559 SDValue SafeShift = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT: IntVT, N1: ClampedShift, N2: One);
9560 SDValue RoundBitPos = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: SafeShift, N2: One);
9561 SDValue RoundBitPosAmt = DAG.getZExtOrTrunc(Op: RoundBitPos, DL: dl, VT: ShiftVT);
9562 SDValue DenormRoundBit = DAG.getNode(
9563 Opcode: ISD::AND, DL: dl, VT: IntVT,
9564 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant, N2: RoundBitPosAmt), N2: One);
9565
9566 // Sticky: all bits below round bit.
9567 // sticky_mask = (1 << RoundBitPos) - 1
9568 SDValue StickyMask = DAG.getNode(
9569 Opcode: ISD::SUB, DL: dl, VT: IntVT,
9570 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: RoundBitPosAmt), N2: One);
9571 SDValue DenormStickyBits =
9572 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FullSrcMant, N2: StickyMask);
9573 SDValue HasSticky = DAG.getNode(
9574 Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT,
9575 Operand: DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormStickyBits, RHS: Zero, Cond: ISD::SETNE));
9576
9577 SDValue DenormLSB =
9578 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: One);
9579
9580 DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);
9581
9582 // Only apply rounding if TotalShift >= 1 (i.e., there are bits to round).
9583 SDValue ShiftGEOne =
9584 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ClampedShift, RHS: One, Cond: ISD::SETUGE);
9585 DenormRoundUp = DAG.getSelect(DL: dl, VT: IntVT, Cond: ShiftGEOne, LHS: DenormRoundUp, RHS: Zero);
9586 }
9587
9588 SDValue DenormRoundedMant =
9589 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: DenormRoundUp);
9590
9591 // If rounding caused overflow into the normal range, then we get the
9592 // smallest normal number.
9593 SDValue DenormMantOF =
9594 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormRoundedMant,
9595 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9596 SDValue DenormFinalMant =
9597 DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: Zero, RHS: DenormRoundedMant);
9598 SDValue DenormFinalExp = DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: One, RHS: Zero);
9599
9600 // Assemble: sign | (exp << DstMant) | mant
9601 SDValue DenormExpShifted =
9602 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormFinalExp,
9603 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9604 DenormResult = DAG.getNode(
9605 Opcode: ISD::OR, DL: dl, VT: IntVT,
9606 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
9607 N2: DenormFinalMant);
9608 }
9609
9610 // Exponent overflow detection.
9611 SDValue ExpOF =
9612 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9613 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9614
9615 // Also check if AdjExp == DstExpMaxNormal and mantissa overflow into
9616 // a value that exceeds the max allowed mantissa at that exponent.
9617 SDValue ExpAtMax =
9618 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9619 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETEQ);
9620 SDValue MantExceedsMax =
9621 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjMant,
9622 RHS: DAG.getConstant(Val: DstMaxMantAtMaxExp, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9623 SDValue ExpMantOF =
9624 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: ExpAtMax, N2: MantExceedsMax);
9625 SDValue IsOverflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCVT, N1: ExpOF, N2: ExpMantOF);
9626
9627 // Build overflow result.
9628 SDValue OverflowResult;
9629
9630 if (Saturate) {
9631 // Clamp to max finite value:
9632 // sign | (DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp
9633 uint64_t MaxFinite =
9634 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9635 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9636 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9637 } else if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9638 // Produce infinity.
9639 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9640 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9641 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9642 } else {
9643 // Emit poison if no Inf in format and not saturating.
9644 OverflowResult = DAG.getPOISON(VT: IntVT);
9645 }
9646
9647 // Assemble normal result: sign | (AdjExp << DstMant) | AdjMant
9648 SDValue NormExpShifted =
9649 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: AdjExp,
9650 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9651 SDValue NormResult = DAG.getNode(
9652 Opcode: ISD::OR, DL: dl, VT: IntVT,
9653 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted), N2: AdjMant);
9654
9655 // Build special-value results.
9656 SDValue NaNResult;
9657 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9658 // Produce canonical NaN.
9659 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
9660 NaNResult =
9661 DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | QNaNBit, DL: dl, VT: IntVT);
9662 } else if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9663 DstNanEnc == fltNanEncoding::AllOnes) {
9664 // E4M3FN-style: NaN is exp=all-ones, mant=all-ones.
9665 NaNResult = DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | DstMantMask,
9666 DL: dl, VT: IntVT);
9667 } else {
9668 // NaN -> poison for finite only values.
9669 NaNResult = DAG.getPOISON(VT: IntVT);
9670 }
9671
9672 // Inf handling.
9673 SDValue InfResult;
9674 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9675 // Produce signed infinity.
9676 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9677 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9678 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9679 } else if (Saturate) {
9680 // Inf saturates to max finite.
9681 uint64_t MaxFinite =
9682 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9683 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9684 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9685 } else {
9686 // No Inf and not saturating -> poison.
9687 InfResult = DAG.getPOISON(VT: IntVT);
9688 }
9689
9690 SDValue ZeroResult = SignShifted;
9691
9692 // Final selection in an order: NaN takes priority, then Inf, then Zero.
9693 SDValue FiniteResult =
9694 DAG.getSelect(DL: dl, VT: IntVT, Cond: ExpIsNeg, LHS: DenormResult, RHS: NormResult);
9695 FiniteResult =
9696 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: OverflowResult, RHS: FiniteResult);
9697
9698 SDValue Result = FiniteResult;
9699 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
9700 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
9701
9702 // Negative values are unrepresentable in an unsigned format: clamp to zero
9703 // when saturating, poison otherwise so no select is needed. -0.0 is handled
9704 // by IsZero above. Run before the NaN case so a negative NaN still yields
9705 // NaN.
9706 if (!DstHasSign && Saturate) {
9707 SDValue IsNegative =
9708 DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOLT);
9709 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNegative, LHS: Zero, RHS: Result);
9710 }
9711
9712 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
9713
9714 // Truncate to destination integer type.
9715 return DAG.getZExtOrTrunc(Op: Result, DL: dl, VT: ResVT);
9716}
9717
9718SDValue
9719TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
9720 SelectionDAG &DAG) const {
9721 SDLoc dl(Node);
9722 EVT DstVT = Node->getValueType(ResNo: 0);
9723 EVT DstScalarVT = DstVT.getScalarType();
9724
9725 SDValue IntVal = Node->getOperand(Num: 0);
9726 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9727 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9728
9729 // Supported source formats.
9730 switch (Sem) {
9731 case APFloatBase::S_Float8E5M2:
9732 case APFloatBase::S_Float8E4M3FN:
9733 case APFloatBase::S_Float8E5M3FNU:
9734 case APFloatBase::S_Float6E3M2FN:
9735 case APFloatBase::S_Float6E2M3FN:
9736 case APFloatBase::S_Float4E2M1FN:
9737 break;
9738 default:
9739 DAG.getContext()->emitError(ErrorStr: "CONVERT_FROM_ARBITRARY_FP: not implemented "
9740 "source format (semantics enum " +
9741 Twine(SemEnum) + ")");
9742 return SDValue();
9743 }
9744
9745 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(S: Sem);
9746 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9747 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9748 const unsigned SrcMant = SrcPrecision - 1;
9749 // Unsigned formats spend no bit on the sign.
9750 const bool SrcHasSign = APFloat::semanticsHasSignedRepr(SrcSem);
9751 const unsigned SrcExp = SrcBits - (SrcHasSign ? 1 : 0) - SrcMant;
9752 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
9753 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
9754
9755 // Destination format parameters.
9756 const fltSemantics &DstSem = DstScalarVT.getFltSemantics();
9757 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9758 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
9759 const unsigned DstExpBits = DstBits - DstMant - 1;
9760 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
9761 const int DstBias = 1 - DstMinExp;
9762 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
9763
9764 // Work in an integer type matching the destination float width.
9765 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstBits);
9766 EVT IntVT = IntScalarVT;
9767 if (DstVT.isVector()) {
9768 IntVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: IntScalarVT,
9769 EC: DstVT.getVectorElementCount());
9770 } else if (!isTypeLegal(VT: IntScalarVT)) {
9771 // Avoid generating illegal type as there is no other places that'll
9772 // legalize it. Vector types don't have this problem because they
9773 // are subject to LegalizeVectorOps and another type legalization phase
9774 // will follow.
9775 if (getTypeAction(Context&: *DAG.getContext(), VT: IntScalarVT) != TypePromoteInteger) {
9776 // We only know how to handle situations where the legal type is wider.
9777 DAG.getContext()->emitError(
9778 ErrorStr: "CONVERT_FROM_ARBITRARY_FP: the requested integer value type for its "
9779 "legalization is not supported");
9780 return SDValue();
9781 }
9782 IntVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: IntScalarVT);
9783 }
9784
9785 SDValue Src = DAG.getZExtOrTrunc(Op: IntVal, DL: dl, VT: IntVT);
9786
9787 EVT SetCCVT =
9788 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9789
9790 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9791 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9792
9793 // Extract bit fields.
9794 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
9795 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
9796
9797 SDValue MantField = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Src,
9798 N2: DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT));
9799
9800 SDValue ExpField =
9801 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT,
9802 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9803 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl)),
9804 N2: DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT));
9805
9806 // An unsigned source has no sign bit; bit SrcBits - 1 is part of the
9807 // exponent.
9808 SDValue SignShifted =
9809 SrcHasSign
9810 ? DAG.getNode(
9811 Opcode: ISD::SHL, DL: dl, VT: IntVT,
9812 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9813 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl)),
9814 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl))
9815 : Zero;
9816
9817 // Classify the input.
9818 SDValue ExpAllOnes = DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT);
9819 SDValue IsExpAllOnes =
9820 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: ExpAllOnes, Cond: ISD::SETEQ);
9821 SDValue IsExpZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: Zero, Cond: ISD::SETEQ);
9822 SDValue IsMantZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETEQ);
9823 SDValue IsMantNonZero =
9824 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETNE);
9825
9826 SDValue IsNaN;
9827 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
9828 IsNaN = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
9829 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
9830 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantNonZero);
9831 } else {
9832 assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
9833 SDValue MantAllOnes = DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT);
9834 SDValue IsMantAllOnes =
9835 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: MantAllOnes, Cond: ISD::SETEQ);
9836 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantAllOnes);
9837 }
9838
9839 SDValue IsInf;
9840 if (NFBehavior == fltNonfiniteBehavior::IEEE754)
9841 IsInf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantZero);
9842 else
9843 IsInf = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
9844
9845 SDValue IsZero = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantZero);
9846 SDValue IsDenorm =
9847 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantNonZero);
9848
9849 // Normal value conversion.
9850 const int BiasAdjust = DstBias - SrcBias;
9851 SDValue NormDstExp = DAG.getNode(
9852 Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: ExpField,
9853 N2: DAG.getConstant(Val: APInt(IntVT.getScalarSizeInBits(), BiasAdjust, true), DL: dl,
9854 VT: IntVT));
9855
9856 SDValue NormDstMant;
9857 if (DstMant > SrcMant) {
9858 SDValue NormDstMantShift =
9859 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
9860 NormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: MantField, N2: NormDstMantShift);
9861 } else {
9862 NormDstMant = MantField;
9863 }
9864
9865 SDValue DstMantShift = DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl);
9866 SDValue NormExpShifted =
9867 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: NormDstExp, N2: DstMantShift);
9868 SDValue NormResult =
9869 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
9870 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted),
9871 N2: NormDstMant);
9872
9873 // Denormal value conversion.
9874 SDValue DenormResult;
9875 {
9876 const unsigned IntVTBits = IntVT.getScalarSizeInBits();
9877 SDValue LeadingZeros =
9878 DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT: IntVT, Operand: MantField);
9879
9880 const int DenormExpConst =
9881 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
9882 SDValue DenormDstExp = DAG.getNode(
9883 Opcode: ISD::SUB, DL: dl, VT: IntVT,
9884 N1: DAG.getConstant(Val: APInt(IntVTBits, DenormExpConst, true), DL: dl, VT: IntVT),
9885 N2: LeadingZeros);
9886
9887 SDValue MantMSB =
9888 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT,
9889 N1: DAG.getConstant(Val: IntVTBits - 1, DL: dl, VT: IntVT), N2: LeadingZeros);
9890
9891 SDValue LeadingOne = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: MantMSB);
9892 SDValue Frac = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: IntVT, N1: MantField, N2: LeadingOne);
9893
9894 const unsigned ShiftSub = IntVTBits - 1 - DstMant;
9895 SDValue ShiftAmount = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: LeadingZeros,
9896 N2: DAG.getConstant(Val: ShiftSub, DL: dl, VT: IntVT));
9897
9898 SDValue DenormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: Frac, N2: ShiftAmount);
9899
9900 SDValue DenormExpShifted =
9901 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormDstExp, N2: DstMantShift);
9902 DenormResult = DAG.getNode(
9903 Opcode: ISD::OR, DL: dl, VT: IntVT,
9904 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
9905 N2: DenormDstMant);
9906 }
9907
9908 SDValue FiniteResult =
9909 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsDenorm, LHS: DenormResult, RHS: NormResult);
9910
9911 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
9912 SDValue NaNResult =
9913 DAG.getConstant(Val: (DstExpAllOnes << DstMant) | QNaNBit, DL: dl, VT: IntVT);
9914
9915 SDValue InfResult =
9916 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9917 N2: DAG.getConstant(Val: DstExpAllOnes << DstMant, DL: dl, VT: IntVT));
9918
9919 SDValue ZeroResult = SignShifted;
9920
9921 SDValue Result = FiniteResult;
9922 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
9923 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
9924 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
9925
9926 if (!DstVT.bitsEq(VT: IntVT)) {
9927 // Store to stack before loading it back.
9928 assert(!IntVT.isVector() && IntVT.bitsGT(DstVT));
9929 // IntScalarVT is the original type that has the same width as DstVT.
9930 Align Alignment = DAG.getReducedAlign(VT: IntScalarVT, /*UseABI=*/false);
9931 SDValue StackPtr =
9932 DAG.CreateStackTemporary(Bytes: IntScalarVT.getStoreSize(), Alignment);
9933 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
9934 MachineFunction &MF = DAG.getMachineFunction();
9935 MachinePointerInfo PtrInfo =
9936 MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
9937 SDValue Store = DAG.getTruncStore(Chain: DAG.getEntryNode(), dl, Val: Result, Ptr: StackPtr,
9938 PtrInfo, SVT: IntScalarVT, Alignment);
9939
9940 SDValue Load = DAG.getLoad(VT: DstVT, dl, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment);
9941 return DAG.getMergeValues(Ops: {Load, Load.getValue(R: 1)}, dl);
9942 }
9943
9944 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT, Operand: Result);
9945}
9946
9947bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
9948 SelectionDAG &DAG) const {
9949 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
9950 SDValue Src = Node->getOperand(Num: OpNo);
9951 EVT SrcVT = Src.getValueType();
9952 EVT DstVT = Node->getValueType(ResNo: 0);
9953 SDLoc dl(SDValue(Node, 0));
9954
9955 // FIXME: Only f32 to i64 conversions are supported.
9956 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
9957 return false;
9958
9959 if (Node->isStrictFPOpcode())
9960 // When a NaN is converted to an integer a trap is allowed. We can't
9961 // use this expansion here because it would eliminate that trap. Other
9962 // traps are also allowed and cannot be eliminated. See
9963 // IEEE 754-2008 sec 5.8.
9964 return false;
9965
9966 // Expand f32 -> i64 conversion
9967 // This algorithm comes from compiler-rt's implementation of fixsfdi:
9968 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
9969 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9970 EVT IntVT = SrcVT.changeTypeToInteger();
9971 EVT IntShVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
9972
9973 SDValue ExponentMask = DAG.getConstant(Val: 0x7F800000, DL: dl, VT: IntVT);
9974 SDValue ExponentLoBit = DAG.getConstant(Val: 23, DL: dl, VT: IntVT);
9975 SDValue Bias = DAG.getConstant(Val: 127, DL: dl, VT: IntVT);
9976 SDValue SignMask = DAG.getConstant(Val: APInt::getSignMask(BitWidth: SrcEltBits), DL: dl, VT: IntVT);
9977 SDValue SignLowBit = DAG.getConstant(Val: SrcEltBits - 1, DL: dl, VT: IntVT);
9978 SDValue MantissaMask = DAG.getConstant(Val: 0x007FFFFF, DL: dl, VT: IntVT);
9979
9980 SDValue Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: Src);
9981
9982 SDValue ExponentBits = DAG.getNode(
9983 Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: ExponentMask),
9984 N2: DAG.getZExtOrTrunc(Op: ExponentLoBit, DL: dl, VT: IntShVT));
9985 SDValue Exponent = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentBits, N2: Bias);
9986
9987 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: IntVT,
9988 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: SignMask),
9989 N2: DAG.getZExtOrTrunc(Op: SignLowBit, DL: dl, VT: IntShVT));
9990 Sign = DAG.getSExtOrTrunc(Op: Sign, DL: dl, VT: DstVT);
9991
9992 SDValue R = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
9993 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: MantissaMask),
9994 N2: DAG.getConstant(Val: 0x00800000, DL: dl, VT: IntVT));
9995
9996 R = DAG.getZExtOrTrunc(Op: R, DL: dl, VT: DstVT);
9997
9998 R = DAG.getSelectCC(
9999 DL: dl, LHS: Exponent, RHS: ExponentLoBit,
10000 True: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: DstVT, N1: R,
10001 N2: DAG.getZExtOrTrunc(
10002 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: Exponent, N2: ExponentLoBit),
10003 DL: dl, VT: IntShVT)),
10004 False: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: DstVT, N1: R,
10005 N2: DAG.getZExtOrTrunc(
10006 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentLoBit, N2: Exponent),
10007 DL: dl, VT: IntShVT)),
10008 Cond: ISD::SETGT);
10009
10010 SDValue Ret = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: DstVT,
10011 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: R, N2: Sign), N2: Sign);
10012
10013 Result = DAG.getSelectCC(DL: dl, LHS: Exponent, RHS: DAG.getConstant(Val: 0, DL: dl, VT: IntVT),
10014 True: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), False: Ret, Cond: ISD::SETLT);
10015 return true;
10016}
10017
10018bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
10019 SDValue &Chain,
10020 SelectionDAG &DAG) const {
10021 SDLoc dl(SDValue(Node, 0));
10022 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
10023 SDValue Src = Node->getOperand(Num: OpNo);
10024
10025 EVT SrcVT = Src.getValueType();
10026 EVT DstVT = Node->getValueType(ResNo: 0);
10027 EVT SetCCVT =
10028 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
10029 EVT DstSetCCVT =
10030 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: DstVT);
10031
10032 // Only expand vector types if we have the appropriate vector bit operations.
10033 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
10034 ISD::FP_TO_SINT;
10035 if (DstVT.isVector() && (!isOperationLegalOrCustom(Op: SIntOpcode, VT: DstVT) ||
10036 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT: SrcVT)))
10037 return false;
10038
10039 // If the maximum float value is smaller then the signed integer range,
10040 // the destination signmask can't be represented by the float, so we can
10041 // just use FP_TO_SINT directly.
10042 const fltSemantics &APFSem = SrcVT.getFltSemantics();
10043 APFloat APF(APFSem, APInt::getZero(numBits: SrcVT.getScalarSizeInBits()));
10044 APInt SignMask = APInt::getSignMask(BitWidth: DstVT.getScalarSizeInBits());
10045 if (APFloat::opOverflow &
10046 APF.convertFromAPInt(Input: SignMask, IsSigned: false, RM: APFloat::rmNearestTiesToEven)) {
10047 if (Node->isStrictFPOpcode()) {
10048 Result = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10049 Ops: { Node->getOperand(Num: 0), Src });
10050 Chain = Result.getValue(R: 1);
10051 } else
10052 Result = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10053 return true;
10054 }
10055
10056 // Don't expand it if there isn't cheap fsub instruction.
10057 if (!isOperationLegalOrCustom(
10058 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, VT: SrcVT))
10059 return false;
10060
10061 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
10062 SDValue Sel;
10063
10064 if (Node->isStrictFPOpcode()) {
10065 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
10066 Chain: Node->getOperand(Num: 0), /*IsSignaling*/ true);
10067 Chain = Sel.getValue(R: 1);
10068 } else {
10069 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT);
10070 }
10071
10072 bool Strict = Node->isStrictFPOpcode() ||
10073 shouldUseStrictFP_TO_INT(FpVT: SrcVT, IntVT: DstVT, /*IsSigned*/ false);
10074
10075 if (Strict) {
10076 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
10077 // signmask then offset (the result of which should be fully representable).
10078 // Sel = Src < 0x8000000000000000
10079 // FltOfs = select Sel, 0, 0x8000000000000000
10080 // IntOfs = select Sel, 0, 0x8000000000000000
10081 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
10082
10083 // TODO: Should any fast-math-flags be set for the FSUB?
10084 SDValue FltOfs = DAG.getSelect(DL: dl, VT: SrcVT, Cond: Sel,
10085 LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
10086 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10087 SDValue IntOfs = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel,
10088 LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT),
10089 RHS: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10090 SDValue SInt;
10091 if (Node->isStrictFPOpcode()) {
10092 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: { SrcVT, MVT::Other },
10093 Ops: { Chain, Src, FltOfs });
10094 SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10095 Ops: { Val.getValue(R: 1), Val });
10096 Chain = SInt.getValue(R: 1);
10097 } else {
10098 SDValue Val = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: FltOfs);
10099 SInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Val);
10100 }
10101 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
10102 } else {
10103 // Expand based on maximum range of FP_TO_SINT:
10104 // True = fp_to_sint(Src)
10105 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
10106 // Result = select (Src < 0x8000000000000000), True, False
10107
10108 SDValue True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10109 // TODO: Should any fast-math-flags be set for the FSUB?
10110 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT,
10111 Operand: DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: Cst));
10112 False = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: False,
10113 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10114 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10115 Result = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel, LHS: True, RHS: False);
10116 }
10117 return true;
10118}
10119
10120bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
10121 SDValue &Chain, SelectionDAG &DAG) const {
10122 // This transform is not correct for converting 0 when rounding mode is set
10123 // to round toward negative infinity which will produce -0.0. So disable
10124 // under strictfp.
10125 if (Node->isStrictFPOpcode())
10126 return false;
10127
10128 SDValue Src = Node->getOperand(Num: 0);
10129 EVT SrcVT = Src.getValueType();
10130 EVT DstVT = Node->getValueType(ResNo: 0);
10131
10132 // If the input is known to be non-negative and SINT_TO_FP is legal then use
10133 // it.
10134 if (Node->getFlags().hasNonNeg() &&
10135 isOperationLegalOrCustom(Op: ISD::SINT_TO_FP, VT: SrcVT)) {
10136 Result =
10137 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Node->getOperand(Num: 0));
10138 return true;
10139 }
10140
10141 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
10142 return false;
10143
10144 // Only expand vector types if we have the appropriate vector bit
10145 // operations.
10146 if (SrcVT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SRL, VT: SrcVT) ||
10147 !isOperationLegalOrCustom(Op: ISD::FADD, VT: DstVT) ||
10148 !isOperationLegalOrCustom(Op: ISD::FSUB, VT: DstVT) ||
10149 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: SrcVT) ||
10150 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: SrcVT)))
10151 return false;
10152
10153 SDLoc dl(SDValue(Node, 0));
10154
10155 // Implementation of unsigned i64 to f64 following the algorithm in
10156 // __floatundidf in compiler_rt. This implementation performs rounding
10157 // correctly in all rounding modes with the exception of converting 0
10158 // when rounding toward negative infinity. In that case the fsub will
10159 // produce -0.0. This will be added to +0.0 and produce -0.0 which is
10160 // incorrect.
10161 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), DL: dl, VT: SrcVT);
10162 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
10163 Val: llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), DL: dl, VT: DstVT);
10164 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), DL: dl, VT: SrcVT);
10165 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), DL: dl, VT: SrcVT);
10166 SDValue HiShift = DAG.getShiftAmountConstant(Val: 32, VT: SrcVT, DL: dl);
10167
10168 SDValue Lo = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Src, N2: LoMask);
10169 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Src, N2: HiShift);
10170 SDValue LoOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Lo, N2: TwoP52);
10171 SDValue HiOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Hi, N2: TwoP84);
10172 SDValue LoFlt = DAG.getBitcast(VT: DstVT, V: LoOr);
10173 SDValue HiFlt = DAG.getBitcast(VT: DstVT, V: HiOr);
10174 SDValue HiSub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: DstVT, N1: HiFlt, N2: TwoP84PlusTwoP52);
10175 Result = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DstVT, N1: LoFlt, N2: HiSub);
10176 return true;
10177}
10178
10179SDValue
10180TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node,
10181 SelectionDAG &DAG) const {
10182 unsigned Opcode = Node->getOpcode();
10183 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
10184 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
10185 "Wrong opcode");
10186
10187 if (Node->getFlags().hasNoNaNs()) {
10188 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
10189 EVT VT = Node->getValueType(ResNo: 0);
10190 if ((!isCondCodeLegal(CC: Pred, VT: VT.getSimpleVT()) ||
10191 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)) &&
10192 VT.isVector())
10193 return SDValue();
10194 SDValue Op1 = Node->getOperand(Num: 0);
10195 SDValue Op2 = Node->getOperand(Num: 1);
10196 return DAG.getSelectCC(DL: SDLoc(Node), LHS: Op1, RHS: Op2, True: Op1, False: Op2, Cond: Pred,
10197 Flags: Node->getFlags());
10198 }
10199
10200 return SDValue();
10201}
10202
10203SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
10204 SelectionDAG &DAG) const {
10205 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node, DAG))
10206 return Expanded;
10207
10208 EVT VT = Node->getValueType(ResNo: 0);
10209 if (VT.isScalableVector())
10210 report_fatal_error(
10211 reason: "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
10212
10213 SDLoc dl(Node);
10214 unsigned NewOp =
10215 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10216
10217 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10218 SDValue Quiet0 = Node->getOperand(Num: 0);
10219 SDValue Quiet1 = Node->getOperand(Num: 1);
10220
10221 if (!Node->getFlags().hasNoNaNs()) {
10222 // Insert canonicalizes if it's possible we need to quiet to get correct
10223 // sNaN behavior.
10224 if (!DAG.isKnownNeverSNaN(Op: Quiet0)) {
10225 Quiet0 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet0,
10226 Flags: Node->getFlags());
10227 }
10228 if (!DAG.isKnownNeverSNaN(Op: Quiet1)) {
10229 Quiet1 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet1,
10230 Flags: Node->getFlags());
10231 }
10232 }
10233
10234 return DAG.getNode(Opcode: NewOp, DL: dl, VT, N1: Quiet0, N2: Quiet1, Flags: Node->getFlags());
10235 }
10236
10237 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
10238 // instead if there are no NaNs.
10239 if (Node->getFlags().hasNoNaNs() ||
10240 (DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 0)) &&
10241 DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 1)))) {
10242 unsigned IEEE2018Op =
10243 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10244 if (isOperationLegalOrCustom(Op: IEEE2018Op, VT))
10245 return DAG.getNode(Opcode: IEEE2018Op, DL: dl, VT, N1: Node->getOperand(Num: 0),
10246 N2: Node->getOperand(Num: 1), Flags: Node->getFlags());
10247 }
10248
10249 if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
10250 return SelCC;
10251
10252 return SDValue();
10253}
10254
10255static SDValue isSpecificZeroAfterMaybeRounding(SelectionDAG &DAG,
10256 const TargetLowering &TLI,
10257 const SDLoc &DL, SDValue Val,
10258 FPClassTest FPClass) {
10259 EVT VT = Val.getValueType();
10260 EVT CCVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10261 EVT IntVT = VT.changeTypeToInteger();
10262 EVT FloatVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
10263 SDValue TestZero = DAG.getTargetConstant(Val: FPClass, DL, VT: MVT::i32);
10264 if (!TLI.isTypeLegal(VT: IntVT) &&
10265 !TLI.isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT))
10266 Val = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: FloatVT, N1: Val,
10267 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
10268 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: CCVT, N1: Val, N2: TestZero);
10269}
10270
10271SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
10272 SelectionDAG &DAG) const {
10273 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node: N, DAG))
10274 return Expanded;
10275
10276 SDLoc DL(N);
10277 SDValue LHS = N->getOperand(Num: 0);
10278 SDValue RHS = N->getOperand(Num: 1);
10279 unsigned Opc = N->getOpcode();
10280 EVT VT = N->getValueType(ResNo: 0);
10281 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10282 bool IsMax = Opc == ISD::FMAXIMUM;
10283 SDNodeFlags Flags = N->getFlags();
10284
10285 // First, implement comparison not propagating NaN. If no native fmin or fmax
10286 // available, use plain select with setcc instead.
10287 SDValue MinMax;
10288 unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
10289 unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;
10290
10291 // FIXME: We should probably define fminnum/fmaxnum variants with correct
10292 // signed zero behavior.
10293 bool MinMaxMustRespectOrderedZero = false;
10294
10295 if (isOperationLegalOrCustom(Op: CompOpcIeee, VT)) {
10296 MinMax = DAG.getNode(Opcode: CompOpcIeee, DL, VT, N1: LHS, N2: RHS, Flags);
10297 MinMaxMustRespectOrderedZero = true;
10298 } else if (isOperationLegalOrCustom(Op: CompOpc, VT)) {
10299 MinMax = DAG.getNode(Opcode: CompOpc, DL, VT, N1: LHS, N2: RHS, Flags);
10300 } else {
10301 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
10302 return DAG.UnrollVectorOp(N);
10303
10304 // NaN (if exists) will be propagated later, so orderness doesn't matter.
10305 SDValue Compare =
10306 DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: IsMax ? ISD::SETOGT : ISD::SETOLT);
10307 MinMax = DAG.getSelect(DL, VT, Cond: Compare, LHS, RHS, Flags);
10308 }
10309
10310 // Propagate any NaN of both operands
10311 if (!N->getFlags().hasNoNaNs() &&
10312 (!DAG.isKnownNeverNaN(Op: RHS) || !DAG.isKnownNeverNaN(Op: LHS))) {
10313 ConstantFP *FPNaN = ConstantFP::get(Context&: *DAG.getContext(),
10314 V: APFloat::getNaN(Sem: VT.getFltSemantics()));
10315 MinMax = DAG.getSelect(DL, VT, Cond: DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETUO),
10316 LHS: DAG.getConstantFP(V: *FPNaN, DL, VT), RHS: MinMax, Flags);
10317 }
10318
10319 // fminimum/fmaximum requires -0.0 less than +0.0
10320 if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
10321 !DAG.isKnownNeverLogicalZero(Op: RHS) && !DAG.isKnownNeverLogicalZero(Op: LHS)) {
10322 SDValue IsEqual = DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETOEQ);
10323 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10324 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10325 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS, Flags);
10326 MinMax = DAG.getSelect(DL, VT, Cond: IsEqual, LHS: RetZero, RHS: MinMax, Flags);
10327 }
10328
10329 return MinMax;
10330}
10331
10332SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
10333 SelectionDAG &DAG) const {
10334 SDLoc DL(Node);
10335 SDValue LHS = Node->getOperand(Num: 0);
10336 SDValue RHS = Node->getOperand(Num: 1);
10337 unsigned Opc = Node->getOpcode();
10338 EVT VT = Node->getValueType(ResNo: 0);
10339 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10340 bool IsMax = Opc == ISD::FMAXIMUMNUM;
10341 SDNodeFlags Flags = Node->getFlags();
10342
10343 unsigned NewOp =
10344 Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10345
10346 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10347 if (!Flags.hasNoNaNs()) {
10348 // Insert canonicalizes if it's possible we need to quiet to get correct
10349 // sNaN behavior.
10350 if (!DAG.isKnownNeverSNaN(Op: LHS)) {
10351 LHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: LHS, Flags);
10352 }
10353 if (!DAG.isKnownNeverSNaN(Op: RHS)) {
10354 RHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: RHS, Flags);
10355 }
10356 }
10357
10358 return DAG.getNode(Opcode: NewOp, DL, VT, N1: LHS, N2: RHS, Flags);
10359 }
10360
10361 // We can use FMINIMUM/FMAXIMUM if there is no NaN, since it has
10362 // same behaviors for all of other cases: +0.0 vs -0.0 included.
10363 if (Flags.hasNoNaNs() ||
10364 (DAG.isKnownNeverNaN(Op: LHS) && DAG.isKnownNeverNaN(Op: RHS))) {
10365 unsigned IEEE2019Op =
10366 Opc == ISD::FMINIMUMNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10367 if (isOperationLegalOrCustom(Op: IEEE2019Op, VT))
10368 return DAG.getNode(Opcode: IEEE2019Op, DL, VT, N1: LHS, N2: RHS, Flags);
10369 }
10370
10371 // FMINNUM/FMAXMUM returns qNaN if either operand is sNaN, and it may return
10372 // either one for +0.0 vs -0.0.
10373 if ((Flags.hasNoNaNs() ||
10374 (DAG.isKnownNeverSNaN(Op: LHS) && DAG.isKnownNeverSNaN(Op: RHS))) &&
10375 (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10376 DAG.isKnownNeverLogicalZero(Op: RHS))) {
10377 unsigned IEEE2008Op = Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM : ISD::FMAXNUM;
10378 if (isOperationLegalOrCustom(Op: IEEE2008Op, VT))
10379 return DAG.getNode(Opcode: IEEE2008Op, DL, VT, N1: LHS, N2: RHS, Flags);
10380 }
10381
10382 if (VT.isVector() &&
10383 (isOperationLegalOrCustomOrPromote(Op: Opc, VT: VT.getVectorElementType()) ||
10384 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)))
10385 return DAG.UnrollVectorOp(N: Node);
10386
10387 // If only one operand is NaN, override it with another operand.
10388 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: LHS)) {
10389 LHS = DAG.getSelectCC(DL, LHS, RHS: LHS, True: RHS, False: LHS, Cond: ISD::SETUO);
10390 }
10391 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: RHS)) {
10392 RHS = DAG.getSelectCC(DL, LHS: RHS, RHS, True: LHS, False: RHS, Cond: ISD::SETUO);
10393 }
10394
10395 // Always prefer RHS if equal.
10396 SDValue MinMax =
10397 DAG.getSelectCC(DL, LHS, RHS, True: LHS, False: RHS, Cond: IsMax ? ISD::SETGT : ISD::SETLT);
10398
10399 // TODO: We need quiet sNaN if strictfp.
10400
10401 // Fixup signed zero behavior.
10402 if (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10403 DAG.isKnownNeverLogicalZero(Op: RHS)) {
10404 return MinMax;
10405 }
10406 SDValue IsZero = DAG.getSetCC(DL, VT: CCVT, LHS: MinMax,
10407 RHS: DAG.getConstantFP(Val: 0.0, DL, VT), Cond: ISD::SETEQ);
10408 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10409 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10410 // It's OK to select from LHS and MinMax, with only one ISD::IS_FPCLASS, as
10411 // we preferred RHS when generate MinMax, if the operands are equal.
10412 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS: MinMax, Flags);
10413 return DAG.getSelect(DL, VT, Cond: IsZero, LHS: RetZero, RHS: MinMax, Flags);
10414}
10415
10416/// Returns a true value if if this FPClassTest can be performed with an ordered
10417/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
10418/// std::nullopt if it cannot be performed as a compare with 0.
10419static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
10420 const fltSemantics &Semantics,
10421 const MachineFunction &MF) {
10422 FPClassTest OrderedMask = Test & ~fcNan;
10423 FPClassTest NanTest = Test & fcNan;
10424 bool IsOrdered = NanTest == fcNone;
10425 bool IsUnordered = NanTest == fcNan;
10426
10427 // Skip cases that are testing for only a qnan or snan.
10428 if (!IsOrdered && !IsUnordered)
10429 return std::nullopt;
10430
10431 if (OrderedMask == fcZero &&
10432 MF.getDenormalMode(FPType: Semantics).Input == DenormalMode::IEEE)
10433 return IsOrdered;
10434 if (OrderedMask == (fcZero | fcSubnormal) &&
10435 MF.getDenormalMode(FPType: Semantics).inputsAreZero())
10436 return IsOrdered;
10437 return std::nullopt;
10438}
10439
10440SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
10441 const FPClassTest OrigTestMask,
10442 SDNodeFlags Flags, const SDLoc &DL,
10443 SelectionDAG &DAG) const {
10444 EVT OperandVT = Op.getValueType();
10445 assert(OperandVT.isFloatingPoint());
10446 FPClassTest Test = OrigTestMask;
10447
10448 // Degenerated cases.
10449 if (Test == fcNone)
10450 return DAG.getBoolConstant(V: false, DL, VT: ResultVT, OpVT: OperandVT);
10451 if (Test == fcAllFlags)
10452 return DAG.getBoolConstant(V: true, DL, VT: ResultVT, OpVT: OperandVT);
10453
10454 // PPC double double is a pair of doubles, of which the higher part determines
10455 // the value class.
10456 if (OperandVT == MVT::ppcf128) {
10457 Op = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::f64, N1: Op,
10458 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
10459 OperandVT = MVT::f64;
10460 }
10461
10462 // Floating-point type properties.
10463 EVT ScalarFloatVT = OperandVT.getScalarType();
10464 const Type *FloatTy = ScalarFloatVT.getTypeForEVT(Context&: *DAG.getContext());
10465 const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
10466 bool IsF80 = (ScalarFloatVT == MVT::f80);
10467
10468 // Some checks can be implemented using float comparisons, if floating point
10469 // exceptions are ignored.
10470 if (Flags.hasNoFPExcept() &&
10471 isOperationLegalOrCustom(Op: ISD::SETCC, VT: OperandVT.getScalarType())) {
10472 FPClassTest FPTestMask = Test;
10473 bool IsInvertedFP = false;
10474
10475 if (FPClassTest InvertedFPCheck =
10476 invertFPClassTestIfSimpler(Test: FPTestMask, UseFCmp: true)) {
10477 FPTestMask = InvertedFPCheck;
10478 IsInvertedFP = true;
10479 }
10480
10481 ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ;
10482 ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ;
10483
10484 // See if we can fold an | fcNan into an unordered compare.
10485 FPClassTest OrderedFPTestMask = FPTestMask & ~fcNan;
10486
10487 // Can't fold the ordered check if we're only testing for snan or qnan
10488 // individually.
10489 if ((FPTestMask & fcNan) != fcNan)
10490 OrderedFPTestMask = FPTestMask;
10491
10492 const bool IsOrdered = FPTestMask == OrderedFPTestMask;
10493
10494 if (std::optional<bool> IsCmp0 =
10495 isFCmpEqualZero(Test: FPTestMask, Semantics, MF: DAG.getMachineFunction());
10496 IsCmp0 && (isCondCodeLegalOrCustom(
10497 CC: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
10498 VT: OperandVT.getScalarType().getSimpleVT()))) {
10499
10500 // If denormals could be implicitly treated as 0, this is not equivalent
10501 // to a compare with 0 since it will also be true for denormals.
10502 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op,
10503 RHS: DAG.getConstantFP(Val: 0.0, DL, VT: OperandVT),
10504 Cond: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
10505 }
10506
10507 if (FPTestMask == fcNan &&
10508 isCondCodeLegalOrCustom(CC: IsInvertedFP ? ISD::SETO : ISD::SETUO,
10509 VT: OperandVT.getScalarType().getSimpleVT()))
10510 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Op,
10511 Cond: IsInvertedFP ? ISD::SETO : ISD::SETUO);
10512
10513 bool IsOrderedInf = FPTestMask == fcInf;
10514 if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) &&
10515 isCondCodeLegalOrCustom(CC: IsOrderedInf ? OrderedCmpOpcode
10516 : UnorderedCmpOpcode,
10517 VT: OperandVT.getScalarType().getSimpleVT()) &&
10518 isOperationLegalOrCustom(Op: ISD::FABS, VT: OperandVT.getScalarType()) &&
10519 (isOperationLegal(Op: ISD::ConstantFP, VT: OperandVT.getScalarType()) ||
10520 (OperandVT.isVector() &&
10521 isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: OperandVT)))) {
10522 // isinf(x) --> fabs(x) == inf
10523 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10524 SDValue Inf =
10525 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10526 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf,
10527 Cond: IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
10528 }
10529
10530 if ((OrderedFPTestMask == fcPosInf || OrderedFPTestMask == fcNegInf) &&
10531 isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedCmpOpcode
10532 : UnorderedCmpOpcode,
10533 VT: OperandVT.getSimpleVT())) {
10534 // isposinf(x) --> x == inf
10535 // isneginf(x) --> x == -inf
10536 // isposinf(x) || nan --> x u== inf
10537 // isneginf(x) || nan --> x u== -inf
10538
10539 SDValue Inf = DAG.getConstantFP(
10540 Val: APFloat::getInf(Sem: Semantics, Negative: OrderedFPTestMask == fcNegInf), DL,
10541 VT: OperandVT);
10542 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Inf,
10543 Cond: IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
10544 }
10545
10546 if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
10547 // TODO: Could handle ordered case, but it produces worse code for
10548 // x86. Maybe handle ordered if fabs is free?
10549
10550 ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10551 ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;
10552
10553 if (isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedOp : UnorderedOp,
10554 VT: OperandVT.getScalarType().getSimpleVT())) {
10555 // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal
10556
10557 // TODO: Maybe only makes sense if fabs is free. Integer test of
10558 // exponent bits seems better for x86.
10559 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10560 SDValue SmallestNormal = DAG.getConstantFP(
10561 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10562 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal,
10563 Cond: IsOrdered ? OrderedOp : UnorderedOp);
10564 }
10565 }
10566
10567 if (FPTestMask == fcNormal) {
10568 // TODO: Handle unordered
10569 ISD::CondCode IsFiniteOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10570 ISD::CondCode IsNormalOp = IsInvertedFP ? ISD::SETOLT : ISD::SETUGE;
10571
10572 if (isCondCodeLegalOrCustom(CC: IsFiniteOp,
10573 VT: OperandVT.getScalarType().getSimpleVT()) &&
10574 isCondCodeLegalOrCustom(CC: IsNormalOp,
10575 VT: OperandVT.getScalarType().getSimpleVT()) &&
10576 isFAbsFree(VT: OperandVT)) {
10577 // isnormal(x) --> fabs(x) < infinity && !(fabs(x) < smallest_normal)
10578 SDValue Inf =
10579 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10580 SDValue SmallestNormal = DAG.getConstantFP(
10581 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10582
10583 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10584 SDValue IsFinite = DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf, Cond: IsFiniteOp);
10585 SDValue IsNormal =
10586 DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal, Cond: IsNormalOp);
10587 unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
10588 return DAG.getNode(Opcode: LogicOp, DL, VT: ResultVT, N1: IsFinite, N2: IsNormal);
10589 }
10590 }
10591 }
10592
10593 // Some checks may be represented as inversion of simpler check, for example
10594 // "inf|normal|subnormal|zero" => !"nan".
10595 bool IsInverted = false;
10596
10597 if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, UseFCmp: false)) {
10598 Test = InvertedCheck;
10599 IsInverted = true;
10600 }
10601
10602 // In the general case use integer operations.
10603 unsigned BitSize = OperandVT.getScalarSizeInBits();
10604 EVT IntVT = OperandVT.changeElementType(
10605 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BitSize));
10606 SDValue OpAsInt = DAG.getBitcast(VT: IntVT, V: Op);
10607
10608 // Various masks.
10609 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10610 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10611 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10612 const unsigned ExplicitIntBitInF80 = 63;
10613 APInt ExpMask = Inf;
10614 if (IsF80)
10615 ExpMask.clearBit(BitPosition: ExplicitIntBitInF80);
10616 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10617 APInt QNaNBitMask =
10618 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10619 APInt InversionMask = APInt::getAllOnes(numBits: ResultVT.getScalarSizeInBits());
10620
10621 SDValue ValueMaskV = DAG.getConstant(Val: ValueMask, DL, VT: IntVT);
10622 SDValue SignBitV = DAG.getConstant(Val: SignBit, DL, VT: IntVT);
10623 SDValue ExpMaskV = DAG.getConstant(Val: ExpMask, DL, VT: IntVT);
10624 SDValue ZeroV = DAG.getConstant(Val: 0, DL, VT: IntVT);
10625 SDValue InfV = DAG.getConstant(Val: Inf, DL, VT: IntVT);
10626 SDValue ResultInversionMask = DAG.getConstant(Val: InversionMask, DL, VT: ResultVT);
10627
10628 SDValue Res;
10629 const auto appendResult = [&](SDValue PartialRes) {
10630 if (PartialRes) {
10631 if (Res)
10632 Res = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: Res, N2: PartialRes);
10633 else
10634 Res = PartialRes;
10635 }
10636 };
10637
10638 SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
10639 const auto getIntBitIsSet = [&]() -> SDValue {
10640 if (!IntBitIsSetV) {
10641 APInt IntBitMask(BitSize, 0);
10642 IntBitMask.setBit(ExplicitIntBitInF80);
10643 SDValue IntBitMaskV = DAG.getConstant(Val: IntBitMask, DL, VT: IntVT);
10644 SDValue IntBitV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: IntBitMaskV);
10645 IntBitIsSetV = DAG.getSetCC(DL, VT: ResultVT, LHS: IntBitV, RHS: ZeroV, Cond: ISD::SETNE);
10646 }
10647 return IntBitIsSetV;
10648 };
10649
10650 // Split the value into sign bit and absolute value.
10651 SDValue AbsV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ValueMaskV);
10652 SDValue SignV = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt,
10653 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETLT);
10654
10655 // Tests that involve more than one class should be processed first.
10656 SDValue PartialRes;
10657
10658 if (IsF80)
10659 ; // Detect finite numbers of f80 by checking individual classes because
10660 // they have different settings of the explicit integer bit.
10661 else if ((Test & fcFinite) == fcFinite) {
10662 // finite(V) ==> (a << 1) < (inf << 1)
10663 //
10664 // See https://github.com/llvm/llvm-project/issues/169270, this is slightly
10665 // shorter than the `finite(V) ==> abs(V) < exp_mask` formula used before.
10666
10667 assert(APFloat::isIEEELikeFP(OperandVT.getFltSemantics()) &&
10668 "finite check requires IEEE-like FP");
10669
10670 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: IntVT, DL);
10671 SDValue TwiceOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: OpAsInt, N2: One);
10672 SDValue TwiceInf = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: ExpMaskV, N2: One);
10673
10674 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: TwiceOp, RHS: TwiceInf, Cond: ISD::SETULT);
10675 Test &= ~fcFinite;
10676 } else if ((Test & fcFinite) == fcPosFinite) {
10677 // finite(V) && V > 0 ==> V < exp_mask
10678 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ExpMaskV, Cond: ISD::SETULT);
10679 Test &= ~fcPosFinite;
10680 } else if ((Test & fcFinite) == fcNegFinite) {
10681 // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
10682 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ExpMaskV, Cond: ISD::SETLT);
10683 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10684 Test &= ~fcNegFinite;
10685 }
10686 appendResult(PartialRes);
10687
10688 if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
10689 // fcZero | fcSubnormal => test all exponent bits are 0
10690 // TODO: Handle sign bit specific cases
10691 if (PartialCheck == (fcZero | fcSubnormal)) {
10692 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ExpMaskV);
10693 SDValue ExpIsZero =
10694 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
10695 appendResult(ExpIsZero);
10696 Test &= ~PartialCheck & fcAllFlags;
10697 }
10698 }
10699
10700 // Check for individual classes.
10701
10702 if (unsigned PartialCheck = Test & fcZero) {
10703 if (PartialCheck == fcPosZero)
10704 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ZeroV, Cond: ISD::SETEQ);
10705 else if (PartialCheck == fcZero)
10706 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ZeroV, Cond: ISD::SETEQ);
10707 else // ISD::fcNegZero
10708 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: SignBitV, Cond: ISD::SETEQ);
10709 appendResult(PartialRes);
10710 }
10711
10712 if (unsigned PartialCheck = Test & fcSubnormal) {
10713 // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
10714 // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
10715 SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
10716 SDValue MantissaV = DAG.getConstant(Val: AllOneMantissa, DL, VT: IntVT);
10717 SDValue VMinusOneV =
10718 DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: V, N2: DAG.getConstant(Val: 1, DL, VT: IntVT));
10719 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: VMinusOneV, RHS: MantissaV, Cond: ISD::SETULT);
10720 if (PartialCheck == fcNegSubnormal)
10721 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10722 appendResult(PartialRes);
10723 }
10724
10725 if (unsigned PartialCheck = Test & fcInf) {
10726 if (PartialCheck == fcPosInf)
10727 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: InfV, Cond: ISD::SETEQ);
10728 else if (PartialCheck == fcInf)
10729 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETEQ);
10730 else { // ISD::fcNegInf
10731 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10732 SDValue NegInfV = DAG.getConstant(Val: NegInf, DL, VT: IntVT);
10733 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: NegInfV, Cond: ISD::SETEQ);
10734 }
10735 appendResult(PartialRes);
10736 }
10737
10738 if (unsigned PartialCheck = Test & fcNan) {
10739 APInt InfWithQnanBit = Inf | QNaNBitMask;
10740 SDValue InfWithQnanBitV = DAG.getConstant(Val: InfWithQnanBit, DL, VT: IntVT);
10741 if (PartialCheck == fcNan) {
10742 // isnan(V) ==> abs(V) > int(inf)
10743 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
10744 if (IsF80) {
10745 // Recognize unsupported values as NaNs for compatibility with glibc.
10746 // In them (exp(V)==0) == int_bit.
10747 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: AbsV, N2: ExpMaskV);
10748 SDValue ExpIsZero =
10749 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
10750 SDValue IsPseudo =
10751 DAG.getSetCC(DL, VT: ResultVT, LHS: getIntBitIsSet(), RHS: ExpIsZero, Cond: ISD::SETEQ);
10752 PartialRes = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: PartialRes, N2: IsPseudo);
10753 }
10754 } else if (PartialCheck == fcQNan) {
10755 // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
10756 PartialRes =
10757 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETGE);
10758 } else { // ISD::fcSNan
10759 // issignaling(V) ==> abs(V) > unsigned(Inf) &&
10760 // abs(V) < (unsigned(Inf) | quiet_bit)
10761 SDValue IsNan = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
10762 SDValue IsNotQnan =
10763 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETLT);
10764 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: IsNan, N2: IsNotQnan);
10765 }
10766 appendResult(PartialRes);
10767 }
10768
10769 if (unsigned PartialCheck = Test & fcNormal) {
10770 // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
10771 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10772 SDValue ExpLSBV = DAG.getConstant(Val: ExpLSB, DL, VT: IntVT);
10773 SDValue ExpMinus1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: AbsV, N2: ExpLSBV);
10774 APInt ExpLimit = ExpMask - ExpLSB;
10775 SDValue ExpLimitV = DAG.getConstant(Val: ExpLimit, DL, VT: IntVT);
10776 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: ExpMinus1, RHS: ExpLimitV, Cond: ISD::SETULT);
10777 if (PartialCheck == fcNegNormal)
10778 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10779 else if (PartialCheck == fcPosNormal) {
10780 SDValue PosSignV =
10781 DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: SignV, N2: ResultInversionMask);
10782 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: PosSignV);
10783 }
10784 if (IsF80)
10785 PartialRes =
10786 DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: getIntBitIsSet());
10787 appendResult(PartialRes);
10788 }
10789
10790 if (!Res)
10791 return DAG.getConstant(Val: IsInverted, DL, VT: ResultVT);
10792 if (IsInverted)
10793 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: Res, N2: ResultInversionMask);
10794 return Res;
10795}
10796
10797// Only expand vector types if we have the appropriate vector bit operations.
10798static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
10799 assert(VT.isVector() && "Expected vector type");
10800 unsigned Len = VT.getScalarSizeInBits();
10801 return TLI.isOperationLegalOrCustom(Op: ISD::ADD, VT) &&
10802 TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT) &&
10803 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
10804 (Len == 8 || TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT)) &&
10805 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT);
10806}
10807
10808SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
10809 SDLoc dl(Node);
10810 EVT VT = Node->getValueType(ResNo: 0);
10811 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
10812 SDValue Op = Node->getOperand(Num: 0);
10813 unsigned Len = VT.getScalarSizeInBits();
10814
10815 // Compute effective bit width from known bits, allowing us to shift the
10816 // active bits down if necessary to fit into smaller specialized expansions.
10817 KnownBits Known = DAG.computeKnownBits(Op);
10818 unsigned LZ = Known.countMinLeadingZeros();
10819 unsigned TZ = Known.countMinTrailingZeros();
10820 unsigned ShiftedActiveBits = Known.getBitWidth() - (LZ + TZ);
10821
10822 // Round up to 8-bit boundary for byte-oriented SWAR algorithm
10823 unsigned EffectiveLen = Len;
10824 if (ShiftedActiveBits > 0 && ShiftedActiveBits < Len)
10825 EffectiveLen = std::min(a: alignTo(Value: ShiftedActiveBits, Align: 8), b: Len);
10826
10827 assert(VT.isInteger() && "CTPOP not implemented for this type.");
10828
10829 // TODO: Add support for irregular type lengths.
10830 if (!(Len <= 128 && Len % 8 == 0))
10831 return SDValue();
10832
10833 // Only expand vector types if we have the appropriate vector bit operations.
10834 if (VT.isVector() && !canExpandVectorCTPOP(TLI: *this, VT))
10835 return SDValue();
10836
10837 // If the active bits are not at the low end, shift them down
10838 if (EffectiveLen < Len && TZ > 0) {
10839 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10840 N2: DAG.getShiftAmountConstant(Val: TZ, VT, DL: dl));
10841 }
10842
10843 // This is the "best" algorithm from
10844 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
10845 SDValue Mask55 =
10846 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x55)), DL: dl, VT);
10847 SDValue Mask33 =
10848 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x33)), DL: dl, VT);
10849 SDValue Mask0F =
10850 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x0F)), DL: dl, VT);
10851
10852 // v = v - ((v >> 1) & 0x55555555...)
10853 Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op,
10854 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10855 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10856 N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT)),
10857 N2: Mask55));
10858 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
10859 Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: Mask33),
10860 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10861 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10862 N2: DAG.getConstant(Val: 2, DL: dl, VT: ShVT)),
10863 N2: Mask33));
10864 // v = (v + (v >> 4)) & 0x0F0F0F0F...
10865 Op = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10866 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
10867 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10868 N2: DAG.getConstant(Val: 4, DL: dl, VT: ShVT))),
10869 N2: Mask0F);
10870
10871 if (EffectiveLen <= 8)
10872 return Op;
10873
10874 // Avoid the multiply if we only have 2 bytes to add.
10875 // TODO: Only doing this for scalars because vectors weren't as obviously
10876 // improved.
10877 if (EffectiveLen == 16 && !VT.isVector()) {
10878 // v = (v + (v >> 8)) & 0x00FF;
10879 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10880 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
10881 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10882 N2: DAG.getConstant(Val: 8, DL: dl, VT: ShVT))),
10883 N2: DAG.getConstant(Val: 0xFF, DL: dl, VT));
10884 }
10885
10886 // v = (v * 0x01010101...) >> (Len - 8)
10887 SDValue V;
10888 if (isOperationLegalOrCustomOrPromote(
10889 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
10890 SDValue Mask01 =
10891 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x01)), DL: dl, VT);
10892 V = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Op, N2: Mask01);
10893 } else {
10894 V = Op;
10895 for (unsigned Shift = 8; Shift < EffectiveLen; Shift *= 2) {
10896 SDValue ShiftC = DAG.getShiftAmountConstant(Val: Shift, VT, DL: dl);
10897 V = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: V,
10898 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: ShiftC));
10899 }
10900 }
10901 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: V, N2: DAG.getConstant(Val: Len - 8, DL: dl, VT: ShVT));
10902}
10903
10904SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
10905 SDLoc dl(Node);
10906 EVT VT = Node->getValueType(ResNo: 0);
10907 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
10908 SDValue Op = Node->getOperand(Num: 0);
10909 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
10910
10911 // If the non-ZERO_POISON version is supported we can use that instead.
10912 if (Node->getOpcode() == ISD::CTLZ_ZERO_POISON &&
10913 isOperationLegalOrCustom(Op: ISD::CTLZ, VT))
10914 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Op);
10915
10916 // If the ZERO_POISON version is supported use that and handle the zero case.
10917 if (isOperationLegalOrCustom(Op: ISD::CTLZ_ZERO_POISON, VT)) {
10918 EVT SetCCVT =
10919 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10920 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Op);
10921 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
10922 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
10923 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
10924 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTLZ);
10925 }
10926
10927 // Only expand vector types if we have the appropriate vector bit operations.
10928 // This includes the operations needed to expand CTPOP if it isn't supported.
10929 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
10930 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
10931 !canExpandVectorCTPOP(TLI: *this, VT)) ||
10932 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
10933 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
10934 return SDValue();
10935
10936 // for now, we do this:
10937 // x = x | (x >> 1);
10938 // x = x | (x >> 2);
10939 // ...
10940 // x = x | (x >>16);
10941 // x = x | (x >>32); // for 64-bit input
10942 // return popcount(~x);
10943 //
10944 // Ref: "Hacker's Delight" by Henry Warren
10945 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
10946 SDValue Tmp = DAG.getConstant(Val: 1ULL << i, DL: dl, VT: ShVT);
10947 Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op,
10948 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: Tmp));
10949 }
10950 Op = DAG.getNOT(DL: dl, Val: Op, VT);
10951 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
10952}
10953
10954SDValue TargetLowering::expandCTLS(SDNode *Node, SelectionDAG &DAG) const {
10955 SDLoc dl(Node);
10956 EVT VT = Node->getValueType(ResNo: 0);
10957 SDValue Op = DAG.getFreeze(V: Node->getOperand(Num: 0));
10958 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
10959
10960 // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
10961 // This transforms the sign bits into leading zeros that can be counted.
10962 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: NumBitsPerElt - 1, VT, DL: dl);
10963 SDValue SignBit = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Op, N2: ShiftAmt);
10964 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: SignBit);
10965 SDValue Shl =
10966 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
10967 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Shl, N2: DAG.getConstant(Val: 1, DL: dl, VT));
10968 return DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Or);
10969}
10970
10971SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG,
10972 const SDLoc &DL, EVT VT, SDValue Op,
10973 unsigned BitWidth) const {
10974 if (BitWidth != 32 && BitWidth != 64)
10975 return SDValue();
10976
10977 const DataLayout &TD = DAG.getDataLayout();
10978 if (!isOperationCustom(Op: ISD::ConstantPool, VT: getPointerTy(DL: TD)))
10979 return SDValue();
10980
10981 APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
10982 : APInt(64, 0x0218A392CD3D5DBFULL);
10983 MachinePointerInfo PtrInfo =
10984 MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction());
10985 unsigned ShiftAmt = BitWidth - Log2_32(Value: BitWidth);
10986 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
10987 SDValue Lookup = DAG.getNode(
10988 Opcode: ISD::SRL, DL, VT,
10989 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op, N2: Neg),
10990 N2: DAG.getConstant(Val: DeBruijn, DL, VT)),
10991 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
10992 Lookup = DAG.getSExtOrTrunc(Op: Lookup, DL, VT: getPointerTy(DL: TD));
10993
10994 SmallVector<uint8_t> Table(BitWidth, 0);
10995 for (unsigned i = 0; i < BitWidth; i++) {
10996 APInt Shl = DeBruijn.shl(shiftAmt: i);
10997 APInt Lshr = Shl.lshr(shiftAmt: ShiftAmt);
10998 Table[Lshr.getZExtValue()] = i;
10999 }
11000
11001 // Create a ConstantArray in Constant Pool
11002 auto *CA = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: Table);
11003 SDValue CPIdx = DAG.getConstantPool(C: CA, VT: getPointerTy(DL: TD),
11004 Align: TD.getPrefTypeAlign(Ty: CA->getType()));
11005 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: DL, VT, Chain: DAG.getEntryNode(),
11006 Ptr: DAG.getMemBasePlusOffset(Base: CPIdx, Offset: Lookup, DL),
11007 PtrInfo, MemVT: MVT::i8);
11008 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
11009 return ExtLoad;
11010
11011 EVT SetCCVT =
11012 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11013 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
11014 SDValue SrcIsZero = DAG.getSetCC(DL, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11015 return DAG.getSelect(DL, VT, Cond: SrcIsZero,
11016 LHS: DAG.getConstant(Val: BitWidth, DL, VT), RHS: ExtLoad);
11017}
11018
11019SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
11020 SDLoc dl(Node);
11021 EVT VT = Node->getValueType(ResNo: 0);
11022 SDValue Op = Node->getOperand(Num: 0);
11023 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11024
11025 // If the non-ZERO_POISON version is supported we can use that instead.
11026 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON &&
11027 isOperationLegalOrCustom(Op: ISD::CTTZ, VT))
11028 return DAG.getNode(Opcode: ISD::CTTZ, DL: dl, VT, Operand: Op);
11029
11030 // If the ZERO_POISON version is supported use that and handle the zero case.
11031 if (isOperationLegalOrCustom(Op: ISD::CTTZ_ZERO_POISON, VT)) {
11032 EVT SetCCVT =
11033 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11034 SDValue CTTZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11035 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11036 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11037 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11038 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTTZ);
11039 }
11040
11041 // Only expand vector types if we have the appropriate vector bit operations.
11042 // This includes the operations needed to expand CTPOP if it isn't supported.
11043 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11044 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11045 !isOperationLegalOrCustom(Op: ISD::CTLZ, VT) &&
11046 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11047 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
11048 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) ||
11049 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11050 return SDValue();
11051
11052 // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
11053 // to be expanded or converted to a libcall.
11054 if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT) &&
11055 !isOperationLegal(Op: ISD::CTLZ, VT))
11056 if (SDValue V = CTTZTableLookup(Node, DAG, DL: dl, VT, Op, BitWidth: NumBitsPerElt))
11057 return V;
11058
11059 bool UseCTLZ =
11060 isOperationLegal(Op: ISD::CTLZ, VT) && !isOperationLegal(Op: ISD::CTPOP, VT);
11061
11062 // When only ctlz is available and the operand is nonzero we can use:
11063 // { return nlz(x & -x) ^ 31; }
11064 // which is more efficient than:
11065 // { return 32 - nlz(~x & (x - 1)); }.
11066 if (UseCTLZ && Node->getOpcode() == ISD::CTTZ_ZERO_POISON) {
11067 SDValue LowestBit =
11068 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: DAG.getNegative(Val: Op, DL: dl, VT));
11069 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT,
11070 N1: DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: LowestBit),
11071 N2: DAG.getConstant(Val: NumBitsPerElt - 1, DL: dl, VT));
11072 }
11073
11074 // If ctpop is available, we use:
11075 // { return popcount(~x & (x-1)); }
11076 // If the target has ctlz but not ctpop, we use:
11077 // { return 32 - nlz(~x & (x-1)); }
11078 // Ref: "Hacker's Delight" by Henry Warren
11079 SDValue Tmp = DAG.getNode(
11080 Opcode: ISD::AND, DL: dl, VT, N1: DAG.getNOT(DL: dl, Val: Op, VT),
11081 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 1, DL: dl, VT)));
11082
11083 if (UseCTLZ)
11084 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT),
11085 N2: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Tmp));
11086
11087 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Tmp);
11088}
11089
11090SDValue TargetLowering::expandVPCTTZElements(SDNode *N,
11091 SelectionDAG &DAG) const {
11092 // %cond = to_bool_vec %source
11093 // %splat = splat /*val=*/VL
11094 // %tz = step_vector
11095 // %v = select %cond, /*true=*/tz, /*false=*/%splat
11096 // %r = vp.reduce.umin %v
11097 SDLoc DL(N);
11098 SDValue Source = N->getOperand(Num: 0);
11099 SDValue Mask = N->getOperand(Num: 1);
11100 SDValue EVL = N->getOperand(Num: 2);
11101 EVT SrcVT = Source.getValueType();
11102 EVT ResVT = N->getValueType(ResNo: 0);
11103 EVT ResVecVT =
11104 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT, EC: SrcVT.getVectorElementCount());
11105
11106 // Convert to boolean vector.
11107 if (SrcVT.getScalarType() != MVT::i1) {
11108 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
11109 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
11110 EC: SrcVT.getVectorElementCount());
11111 Source = DAG.getSetCC(DL, VT: SrcVT, LHS: Source, RHS: AllZero, Cond: ISD::SETNE);
11112 }
11113
11114 SDValue ExtEVL = DAG.getZExtOrTrunc(Op: EVL, DL, VT: ResVT);
11115 SDValue Splat = DAG.getSplat(VT: ResVecVT, DL, Op: ExtEVL);
11116 SDValue StepVec = DAG.getStepVector(DL, ResVT: ResVecVT);
11117 SDValue Select = DAG.getSelect(DL, VT: ResVecVT, Cond: Source, LHS: StepVec, RHS: Splat);
11118 return DAG.getNode(Opcode: ISD::VP_REDUCE_UMIN, DL, VT: ResVT, N1: ExtEVL, N2: Select, N3: Mask, N4: EVL);
11119}
11120
11121/// Returns a type-legalized version of \p Mask as the first item in the
11122/// pair. The second item contains a type-legalized step vector that's
11123/// guaranteed to fit the number of elements in \p Mask.
11124/// If the stepvector would require splitting, returns an empty SDValue
11125/// as the second item to signal that the operation should be split instead.
11126static std::pair<SDValue, SDValue>
11127getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL,
11128 SelectionDAG &DAG) {
11129 EVT MaskVT = Mask.getValueType();
11130 EVT BoolVT = MaskVT.getScalarType();
11131
11132 // Find a suitable type for a stepvector.
11133 // If zero is poison, we can assume the upper limit of the result is VF-1.
11134 ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
11135 if (MaskVT.isScalableVector())
11136 VScaleRange = getVScaleRange(F: &DAG.getMachineFunction().getFunction(), BitWidth: 64);
11137 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11138 uint64_t EltWidth = TLI.getBitWidthForCttzElements(
11139 RetVT: EVT(TLI.getVectorIdxTy(DL: DAG.getDataLayout())),
11140 EC: MaskVT.getVectorElementCount(), ZeroIsPoison, VScaleRange: &VScaleRange);
11141 // If the step vector element type is smaller than the mask element type,
11142 // use the mask type directly to avoid widening issues.
11143 EltWidth = std::max(a: EltWidth, b: BoolVT.getFixedSizeInBits());
11144 EVT StepVT = MVT::getIntegerVT(BitWidth: EltWidth);
11145 EVT StepVecVT = MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: StepVT);
11146
11147 // If promotion or widening is required to make the type legal, do it here.
11148 // Promotion of integers within LegalizeVectorOps is looking for types of
11149 // the same size but with a smaller number of larger elements, not the usual
11150 // larger size with the same number of larger elements.
11151 TargetLowering::LegalizeTypeAction TypeAction =
11152 TLI.getTypeAction(Context&: *DAG.getContext(), VT: StepVecVT);
11153 SDValue StepVec;
11154 if (TypeAction == TargetLowering::TypePromoteInteger) {
11155 StepVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11156 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11157 } else if (TypeAction == TargetLowering::TypeWidenVector) {
11158 // For widening, the element count changes. Create a step vector with only
11159 // the original elements valid and zeros for padding. Also widen the mask.
11160 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11161 unsigned WideNumElts = WideVecVT.getVectorNumElements();
11162
11163 // Build widened step vector: <0, 1, ..., OrigNumElts-1, poison, poison, ..>
11164 SDValue OrigStepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11165 SDValue UndefStep = DAG.getPOISON(VT: WideVecVT);
11166 StepVec = DAG.getInsertSubvector(DL, Vec: UndefStep, SubVec: OrigStepVec, Idx: 0);
11167
11168 // Widen mask: pad with zeros.
11169 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: BoolVT, NumElements: WideNumElts);
11170 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
11171 Mask = DAG.getInsertSubvector(DL, Vec: ZeroMask, SubVec: Mask, Idx: 0);
11172 } else if (TypeAction == TargetLowering::TypeSplitVector) {
11173 // The stepvector type would require splitting. Signal to the caller
11174 // that the operation should be split instead of expanded.
11175 return {Mask, SDValue()};
11176 } else {
11177 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11178 }
11179
11180 return {Mask, StepVec};
11181}
11182
11183SDValue TargetLowering::expandVectorFindLastActive(SDNode *N,
11184 SelectionDAG &DAG) const {
11185 SDLoc DL(N);
11186 auto [Mask, StepVec] = getLegalMaskAndStepVector(
11187 Mask: N->getOperand(Num: 0), /*ZeroIsPoison=*/true, DL, DAG);
11188
11189 // If StepVec is empty, the stepvector would require splitting.
11190 // Split the operation instead and let it be recursively legalized.
11191 if (!StepVec) {
11192 EVT MaskVT = N->getOperand(Num: 0).getValueType();
11193 EVT ResVT = N->getValueType(ResNo: 0);
11194
11195 // Split the mask
11196 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: MaskVT);
11197 auto [MaskLo, MaskHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
11198
11199 // Create split VECTOR_FIND_LAST_ACTIVE operations
11200 SDValue LoResult =
11201 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskLo);
11202 SDValue HiResult =
11203 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskHi);
11204
11205 // Check if any lane is active in the high mask.
11206 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: MaskHi);
11207 SDValue Cond = DAG.getBoolExtOrTrunc(
11208 Op: AnyHiActive, SL: DL,
11209 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i1),
11210 OpVT: MVT::i1);
11211
11212 // Adjust HiResult by adding the number of elements in Lo
11213 SDValue LoNumElts =
11214 DAG.getElementCount(DL, VT: ResVT, EC: LoVT.getVectorElementCount());
11215 SDValue AdjustedHiResult =
11216 DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: HiResult, N2: LoNumElts);
11217
11218 // Return: AnyHiActive ? AdjustedHiResult : LoResult;
11219 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: ResVT, N1: Cond, N2: AdjustedHiResult,
11220 N3: LoResult);
11221 }
11222
11223 EVT StepVecVT = StepVec.getValueType();
11224 EVT StepVT = StepVec.getValueType().getVectorElementType();
11225
11226 // Zero out lanes with inactive elements, then find the highest remaining
11227 // value from the stepvector.
11228 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
11229 SDValue ActiveElts = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
11230 SDValue HighestIdx = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL, VT: StepVT, Operand: ActiveElts);
11231 return DAG.getZExtOrTrunc(Op: HighestIdx, DL, VT: N->getValueType(ResNo: 0));
11232}
11233
11234SDValue TargetLowering::expandLoopDependenceMask(SDNode *N,
11235 SelectionDAG &DAG) const {
11236 SDLoc DL(N);
11237 EVT VT = N->getValueType(ResNo: 0);
11238 SDValue SourceValue = N->getOperand(Num: 0);
11239 SDValue SinkValue = N->getOperand(Num: 1);
11240 SDValue EltSizeInBytes = N->getOperand(Num: 2);
11241
11242 // Note: The lane offset is scalable if the mask is scalable.
11243 ElementCount LaneOffsetEC =
11244 ElementCount::get(MinVal: N->getConstantOperandVal(Num: 3), Scalable: VT.isScalableVT());
11245
11246 EVT AddrVT = SourceValue->getValueType(ResNo: 0);
11247 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
11248
11249 EVT CmpVT =
11250 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: AddrVT);
11251
11252 // Unsigned compare: Source >= Sink.
11253 SDValue SourceAheadOfOrEqualToSink =
11254 DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETUGE);
11255
11256 // Take the difference between the pointers and divided by the element size,
11257 // to see how many lanes separate them.
11258 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: AddrVT, N1: SinkValue, N2: SourceValue);
11259
11260 // RAW_MASK: Diff = Source >= Sink ? (Source - Sink) : (Sink - Source)
11261 if (IsReadAfterWrite)
11262 Diff = DAG.getSelect(DL, VT: AddrVT, Cond: SourceAheadOfOrEqualToSink,
11263 LHS: DAG.getNegative(Val: Diff, DL, VT: AddrVT), RHS: Diff);
11264
11265 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: AddrVT, N1: Diff, N2: EltSizeInBytes);
11266
11267 // The pointers do not alias if:
11268 // - Source >= Sink (WAR_MASK)
11269 // - Source == Sink (RAW_MASK)
11270 SDValue NoAlias = SourceAheadOfOrEqualToSink;
11271 if (IsReadAfterWrite)
11272 NoAlias = DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETEQ);
11273
11274 // The pointers do not alias if:
11275 // Lane + LaneOffset < Diff (WAR/RAW_MASK)
11276 SDValue LaneOffset = DAG.getElementCount(DL, VT: AddrVT, EC: LaneOffsetEC);
11277 SDValue MaskN = DAG.getSelect(
11278 DL, VT: AddrVT, Cond: NoAlias,
11279 LHS: DAG.getConstant(Val: APInt::getMaxValue(numBits: AddrVT.getScalarSizeInBits()), DL,
11280 VT: AddrVT),
11281 RHS: Diff);
11282
11283 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT, N1: LaneOffset, N2: MaskN);
11284}
11285
11286SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
11287 bool IsNegative) const {
11288 SDLoc dl(N);
11289 EVT VT = N->getValueType(ResNo: 0);
11290 SDValue Op = N->getOperand(Num: 0);
11291
11292 // If expanding ABS_MIN_POISON, fall back to ABS if the target supports it.
11293 if (N->getOpcode() == ISD::ABS_MIN_POISON &&
11294 isOperationLegalOrCustom(Op: ISD::ABS, VT)) {
11295 SDValue AbsVal = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT, Operand: Op);
11296 if (IsNegative)
11297 return DAG.getNegative(Val: AbsVal, DL: dl, VT);
11298 return AbsVal;
11299 }
11300
11301 // abs(x) -> smax(x,sub(0,x))
11302 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11303 isOperationLegal(Op: ISD::SMAX, VT)) {
11304 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11305 Op = DAG.getFreeze(V: Op);
11306 return DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: Op,
11307 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11308 }
11309
11310 // abs(x) -> umin(x,sub(0,x))
11311 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11312 isOperationLegal(Op: ISD::UMIN, VT)) {
11313 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11314 Op = DAG.getFreeze(V: Op);
11315 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: Op,
11316 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11317 }
11318
11319 // 0 - abs(x) -> smin(x, sub(0,x))
11320 if (IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11321 isOperationLegal(Op: ISD::SMIN, VT)) {
11322 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11323 Op = DAG.getFreeze(V: Op);
11324 return DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: Op,
11325 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11326 }
11327
11328 // Only expand vector types if we have the appropriate vector operations.
11329 if (VT.isVector() &&
11330 (!isOperationLegalOrCustom(Op: ISD::SRA, VT) ||
11331 (!IsNegative && !isOperationLegalOrCustom(Op: ISD::ADD, VT)) ||
11332 (IsNegative && !isOperationLegalOrCustom(Op: ISD::SUB, VT)) ||
11333 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11334 return SDValue();
11335
11336 Op = DAG.getFreeze(V: Op);
11337 SDValue Shift = DAG.getNode(
11338 Opcode: ISD::SRA, DL: dl, VT, N1: Op,
11339 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11340 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Shift);
11341
11342 // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
11343 if (!IsNegative)
11344 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Shift);
11345
11346 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
11347 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Shift, N2: Xor);
11348}
11349
11350SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
11351 SDLoc dl(N);
11352 EVT VT = N->getValueType(ResNo: 0);
11353 SDValue LHS = N->getOperand(Num: 0);
11354 SDValue RHS = N->getOperand(Num: 1);
11355 bool IsSigned = N->getOpcode() == ISD::ABDS;
11356
11357 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
11358 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
11359 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
11360 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
11361 if (isOperationLegal(Op: MaxOpc, VT) && isOperationLegal(Op: MinOpc, VT)) {
11362 LHS = DAG.getFreeze(V: LHS);
11363 RHS = DAG.getFreeze(V: RHS);
11364 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
11365 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
11366 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
11367 }
11368
11369 // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
11370 if (!IsSigned && isOperationLegal(Op: ISD::USUBSAT, VT)) {
11371 LHS = DAG.getFreeze(V: LHS);
11372 RHS = DAG.getFreeze(V: RHS);
11373 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT,
11374 N1: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: LHS, N2: RHS),
11375 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: RHS, N2: LHS));
11376 }
11377
11378 // If the subtract doesn't overflow then just use abs(sub())
11379 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
11380
11381 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: LHS, N1: RHS))
11382 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11383 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS));
11384
11385 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: RHS, N1: LHS))
11386 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11387 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11388
11389 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11390 ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
11391 LHS = DAG.getFreeze(V: LHS);
11392 RHS = DAG.getFreeze(V: RHS);
11393 SDValue Cmp = DAG.getSetCC(DL: dl, VT: CCVT, LHS, RHS, Cond: CC);
11394
11395 // Branchless expansion iff cmp result is allbits:
11396 // abds(lhs, rhs) -> sub(sgt(lhs, rhs), xor(sgt(lhs, rhs), sub(lhs, rhs)))
11397 // abdu(lhs, rhs) -> sub(ugt(lhs, rhs), xor(ugt(lhs, rhs), sub(lhs, rhs)))
11398 if (CCVT == VT && getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
11399 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
11400 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Diff, N2: Cmp);
11401 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Cmp, N2: Xor);
11402 }
11403
11404 // Similar to the branchless expansion, if we don't prefer selects, use the
11405 // (sign-extended) usubo overflow flag if the (scalar) type is illegal as this
11406 // is more likely to legalize cleanly: abdu(lhs, rhs) -> sub(xor(sub(lhs,
11407 // rhs), uof(lhs, rhs)), uof(lhs, rhs))
11408 if (!IsSigned && VT.isScalarInteger() && !isTypeLegal(VT) &&
11409 !preferSelectsOverBooleanArithmetic(VT)) {
11410 SDValue USubO =
11411 DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {LHS, RHS});
11412 SDValue Cmp = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT, Operand: USubO.getValue(R: 1));
11413 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: USubO.getValue(R: 0), N2: Cmp);
11414 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Cmp);
11415 }
11416
11417 // FIXME: Should really try to split the vector in case it's legal on a
11418 // subvector.
11419 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
11420 return DAG.UnrollVectorOp(N);
11421
11422 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11423 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11424 return DAG.getSelect(DL: dl, VT, Cond: Cmp, LHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS),
11425 RHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11426}
11427
11428SDValue TargetLowering::expandAVG(SDNode *N, SelectionDAG &DAG) const {
11429 SDLoc dl(N);
11430 EVT VT = N->getValueType(ResNo: 0);
11431 SDValue LHS = N->getOperand(Num: 0);
11432 SDValue RHS = N->getOperand(Num: 1);
11433
11434 unsigned Opc = N->getOpcode();
11435 bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
11436 bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
11437 unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
11438 unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
11439 unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
11440 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
11441 assert((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS ||
11442 Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
11443 "Unknown AVG node");
11444
11445 // If the operands are already extended, we can add+shift.
11446 bool IsExt =
11447 (IsSigned && DAG.ComputeNumSignBits(Op: LHS) >= 2 &&
11448 DAG.ComputeNumSignBits(Op: RHS) >= 2) ||
11449 (!IsSigned && DAG.computeKnownBits(Op: LHS).countMinLeadingZeros() >= 1 &&
11450 DAG.computeKnownBits(Op: RHS).countMinLeadingZeros() >= 1);
11451 if (IsExt) {
11452 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
11453 if (!IsFloor)
11454 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Sum, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11455 return DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Sum,
11456 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11457 }
11458
11459 // For scalars, see if we can efficiently extend/truncate to use add+shift.
11460 if (VT.isScalarInteger()) {
11461 EVT ExtVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
11462 if (isTypeLegal(VT: ExtVT) && isTruncateFree(FromVT: ExtVT, ToVT: VT)) {
11463 LHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: LHS);
11464 RHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: RHS);
11465 SDValue Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: LHS, N2: RHS);
11466 if (!IsFloor)
11467 Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: Avg,
11468 N2: DAG.getConstant(Val: 1, DL: dl, VT: ExtVT));
11469 // Just use SRL as we will be truncating away the extended sign bits.
11470 Avg = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ExtVT, N1: Avg,
11471 N2: DAG.getShiftAmountConstant(Val: 1, VT: ExtVT, DL: dl));
11472 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Avg);
11473 }
11474 }
11475
11476 // avgflooru(lhs, rhs) -> or(lshr(add(lhs, rhs),1),shl(overflow, typesize-1))
11477 if (Opc == ISD::AVGFLOORU && VT.isScalarInteger() && !isTypeLegal(VT) &&
11478 isOperationLegalOrCustom(
11479 Op: ISD::UADDO, VT: getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11480 SDValue UAddWithOverflow =
11481 DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {RHS, LHS});
11482
11483 SDValue Sum = UAddWithOverflow.getValue(R: 0);
11484 SDValue Overflow = UAddWithOverflow.getValue(R: 1);
11485
11486 // Right shift the sum by 1
11487 SDValue LShrVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Sum,
11488 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11489
11490 SDValue ZeroExtOverflow = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Overflow);
11491 SDValue OverflowShl = DAG.getNode(
11492 Opcode: ISD::SHL, DL: dl, VT, N1: ZeroExtOverflow,
11493 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11494
11495 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: LShrVal, N2: OverflowShl);
11496 }
11497
11498 // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
11499 // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
11500 // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
11501 // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
11502 LHS = DAG.getFreeze(V: LHS);
11503 RHS = DAG.getFreeze(V: RHS);
11504 SDValue Sign = DAG.getNode(Opcode: SignOpc, DL: dl, VT, N1: LHS, N2: RHS);
11505 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
11506 SDValue Shift =
11507 DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11508 return DAG.getNode(Opcode: SumOpc, DL: dl, VT, N1: Sign, N2: Shift);
11509}
11510
11511SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
11512 SDLoc dl(N);
11513 EVT VT = N->getValueType(ResNo: 0);
11514 SDValue Op = N->getOperand(Num: 0);
11515
11516 if (!VT.isSimple())
11517 return SDValue();
11518
11519 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11520 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
11521 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
11522 default:
11523 return SDValue();
11524 case MVT::i16:
11525 // Use a rotate by 8. This can be further expanded if necessary.
11526 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11527 case MVT::i32:
11528 // This is meant for ARM specifically, which has ROTR but no ROTL.
11529 // t = x ^ rotr(x, 16)
11530 // t = bic(t, 0x00ff0000)
11531 // t = lshr(t, 8)
11532 // x = t ^ rotr(x, 8)
11533 if (isOperationLegalOrCustom(Op: ISD::ROTR, VT)) {
11534 SDValue Rotr16 =
11535 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 16, DL: dl, VT: SHVT));
11536 SDValue Tmp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Rotr16);
11537 Tmp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp,
11538 N2: DAG.getConstant(Val: 0xFF00FFFF, DL: dl, VT));
11539 Tmp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11540 SDValue Rotr8 =
11541 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11542 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Tmp, N2: Rotr8);
11543 }
11544 Tmp4 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11545 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11546 N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11547 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11548 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11549 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11550 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11551 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11552 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11553 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11554 case MVT::i64:
11555 Tmp8 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11556 Tmp7 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11557 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11558 Tmp7 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp7, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11559 Tmp6 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11560 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11561 Tmp6 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp6, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11562 Tmp5 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11563 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11564 Tmp5 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp5, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11565 Tmp4 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11566 Tmp4 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp4,
11567 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11568 Tmp3 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11569 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp3,
11570 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11571 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11572 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2,
11573 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11574 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11575 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp7);
11576 Tmp6 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp6, N2: Tmp5);
11577 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11578 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11579 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp6);
11580 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11581 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp4);
11582 }
11583}
11584
11585SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
11586 SDLoc dl(N);
11587 EVT VT = N->getValueType(ResNo: 0);
11588 SDValue Op = N->getOperand(Num: 0);
11589 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11590 unsigned Sz = VT.getScalarSizeInBits();
11591
11592 SDValue Tmp, Tmp2, Tmp3;
11593
11594 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
11595 // and finally the i1 pairs.
11596 // TODO: We can easily support i4/i2 legal types if any target ever does.
11597 if (Sz >= 8 && isPowerOf2_32(Value: Sz)) {
11598 // Create the masks - repeating the pattern every byte.
11599 APInt Mask4 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x0F));
11600 APInt Mask2 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x33));
11601 APInt Mask1 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x55));
11602
11603 // BSWAP if the type is wider than a single byte.
11604 Tmp = (Sz > 8 ? DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT, Operand: Op) : Op);
11605
11606 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
11607 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11608 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11609 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11610 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11611 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11612
11613 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
11614 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11615 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11616 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11617 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11618 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11619
11620 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
11621 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11622 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11623 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11624 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11625 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11626 return Tmp;
11627 }
11628
11629 Tmp = DAG.getConstant(Val: 0, DL: dl, VT);
11630 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
11631 if (I < J)
11632 Tmp2 =
11633 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: J - I, DL: dl, VT: SHVT));
11634 else
11635 Tmp2 =
11636 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: I - J, DL: dl, VT: SHVT));
11637
11638 APInt Shift = APInt::getOneBitSet(numBits: Sz, BitNo: J);
11639 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Shift, DL: dl, VT));
11640 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp, N2: Tmp2);
11641 }
11642
11643 return Tmp;
11644}
11645
11646std::pair<SDValue, SDValue>
11647TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
11648 SelectionDAG &DAG) const {
11649 SDLoc SL(LD);
11650 SDValue Chain = LD->getChain();
11651 SDValue BasePTR = LD->getBasePtr();
11652 EVT SrcVT = LD->getMemoryVT();
11653 EVT DstVT = LD->getValueType(ResNo: 0);
11654 ISD::LoadExtType ExtType = LD->getExtensionType();
11655
11656 if (SrcVT.isScalableVector())
11657 report_fatal_error(reason: "Cannot scalarize scalable vector loads");
11658
11659 unsigned NumElem = SrcVT.getVectorNumElements();
11660
11661 EVT SrcEltVT = SrcVT.getScalarType();
11662 EVT DstEltVT = DstVT.getScalarType();
11663
11664 // A vector must always be stored in memory as-is, i.e. without any padding
11665 // between the elements, since various code depend on it, e.g. in the
11666 // handling of a bitcast of a vector type to int, which may be done with a
11667 // vector store followed by an integer load. A vector that does not have
11668 // elements that are byte-sized must therefore be stored as an integer
11669 // built out of the extracted vector elements.
11670 if (!SrcEltVT.isByteSized()) {
11671 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
11672 EVT LoadVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumLoadBits);
11673
11674 unsigned NumSrcBits = SrcVT.getSizeInBits();
11675 EVT SrcIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumSrcBits);
11676
11677 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
11678 SDValue SrcEltBitMask = DAG.getConstant(
11679 Val: APInt::getLowBitsSet(numBits: NumLoadBits, loBitsSet: SrcEltBits), DL: SL, VT: LoadVT);
11680
11681 // Load the whole vector and avoid masking off the top bits as it makes
11682 // the codegen worse.
11683 SDValue Load =
11684 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: SL, VT: LoadVT, Chain, Ptr: BasePTR,
11685 PtrInfo: LD->getPointerInfo(), MemVT: SrcIntVT, Alignment: LD->getBaseAlign(),
11686 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
11687
11688 SmallVector<SDValue, 8> Vals;
11689 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11690 unsigned ShiftIntoIdx =
11691 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
11692 SDValue ShiftAmount = DAG.getShiftAmountConstant(
11693 Val: ShiftIntoIdx * SrcEltVT.getSizeInBits(), VT: LoadVT, DL: SL);
11694 SDValue ShiftedElt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: LoadVT, N1: Load, N2: ShiftAmount);
11695 SDValue Elt =
11696 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: LoadVT, N1: ShiftedElt, N2: SrcEltBitMask);
11697 SDValue Scalar = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: SrcEltVT, Operand: Elt);
11698
11699 if (ExtType != ISD::NON_EXTLOAD) {
11700 unsigned ExtendOp = ISD::getExtForLoadExtType(IsFP: false, ExtType);
11701 Scalar = DAG.getNode(Opcode: ExtendOp, DL: SL, VT: DstEltVT, Operand: Scalar);
11702 }
11703
11704 Vals.push_back(Elt: Scalar);
11705 }
11706
11707 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
11708 return std::make_pair(x&: Value, y: Load.getValue(R: 1));
11709 }
11710
11711 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
11712 assert(SrcEltVT.isByteSized());
11713
11714 SmallVector<SDValue, 8> Vals;
11715 SmallVector<SDValue, 8> LoadChains;
11716
11717 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11718 SDValue ScalarLoad = DAG.getExtLoad(
11719 ExtType, dl: SL, VT: DstEltVT, Chain, Ptr: BasePTR,
11720 PtrInfo: LD->getPointerInfo().getWithOffset(O: Idx * Stride), MemVT: SrcEltVT,
11721 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
11722
11723 BasePTR = DAG.getObjectPtrOffset(SL, Ptr: BasePTR, Offset: TypeSize::getFixed(ExactSize: Stride));
11724
11725 Vals.push_back(Elt: ScalarLoad.getValue(R: 0));
11726 LoadChains.push_back(Elt: ScalarLoad.getValue(R: 1));
11727 }
11728
11729 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: LoadChains);
11730 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
11731
11732 return std::make_pair(x&: Value, y&: NewChain);
11733}
11734
11735SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
11736 SelectionDAG &DAG) const {
11737 SDLoc SL(ST);
11738
11739 SDValue Chain = ST->getChain();
11740 SDValue BasePtr = ST->getBasePtr();
11741 SDValue Value = ST->getValue();
11742 EVT StVT = ST->getMemoryVT();
11743
11744 if (StVT.isScalableVector())
11745 report_fatal_error(reason: "Cannot scalarize scalable vector stores");
11746
11747 // The type of the data we want to save
11748 EVT RegVT = Value.getValueType();
11749 EVT RegSclVT = RegVT.getScalarType();
11750
11751 // The type of data as saved in memory.
11752 EVT MemSclVT = StVT.getScalarType();
11753
11754 unsigned NumElem = StVT.getVectorNumElements();
11755
11756 // A vector must always be stored in memory as-is, i.e. without any padding
11757 // between the elements, since various code depend on it, e.g. in the
11758 // handling of a bitcast of a vector type to int, which may be done with a
11759 // vector store followed by an integer load. A vector that does not have
11760 // elements that are byte-sized must therefore be stored as an integer
11761 // built out of the extracted vector elements.
11762 if (!MemSclVT.isByteSized()) {
11763 unsigned NumBits = StVT.getSizeInBits();
11764 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
11765
11766 SDValue CurrVal = DAG.getConstant(Val: 0, DL: SL, VT: IntVT);
11767
11768 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11769 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
11770 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MemSclVT, Operand: Elt);
11771 SDValue ExtElt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: IntVT, Operand: Trunc);
11772 unsigned ShiftIntoIdx =
11773 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
11774 SDValue ShiftAmount =
11775 DAG.getConstant(Val: ShiftIntoIdx * MemSclVT.getSizeInBits(), DL: SL, VT: IntVT);
11776 SDValue ShiftedElt =
11777 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT, N1: ExtElt, N2: ShiftAmount);
11778 CurrVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: CurrVal, N2: ShiftedElt);
11779 }
11780
11781 return DAG.getStore(Chain, dl: SL, Val: CurrVal, Ptr: BasePtr, PtrInfo: ST->getPointerInfo(),
11782 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
11783 Metadata: ST->getAAInfo());
11784 }
11785
11786 // Store Stride in bytes
11787 unsigned Stride = MemSclVT.getSizeInBits() / 8;
11788 assert(Stride && "Zero stride!");
11789 // Extract each of the elements from the original vector and save them into
11790 // memory individually.
11791 SmallVector<SDValue, 8> Stores;
11792 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11793 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
11794
11795 SDValue Ptr =
11796 DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Idx * Stride));
11797
11798 // This scalar TruncStore may be illegal, but we legalize it later.
11799 SDValue Store = DAG.getTruncStore(
11800 Chain, dl: SL, Val: Elt, Ptr, PtrInfo: ST->getPointerInfo().getWithOffset(O: Idx * Stride),
11801 SVT: MemSclVT, Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
11802 Metadata: ST->getAAInfo());
11803
11804 Stores.push_back(Elt: Store);
11805 }
11806
11807 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: Stores);
11808}
11809
11810std::pair<SDValue, SDValue>
11811TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
11812 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
11813 "unaligned indexed loads not implemented!");
11814 SDValue Chain = LD->getChain();
11815 SDValue Ptr = LD->getBasePtr();
11816 EVT VT = LD->getValueType(ResNo: 0);
11817 EVT LoadedVT = LD->getMemoryVT();
11818 SDLoc dl(LD);
11819 auto &MF = DAG.getMachineFunction();
11820
11821 if (VT.isFloatingPoint() || VT.isVector()) {
11822 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadedVT.getSizeInBits());
11823 if (isTypeLegal(VT: intVT) && isTypeLegal(VT: LoadedVT)) {
11824 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: intVT) &&
11825 LoadedVT.isVector()) {
11826 // Scalarize the load and let the individual components be handled.
11827 return scalarizeVectorLoad(LD, DAG);
11828 }
11829
11830 // Expand to a (misaligned) integer load of the same size,
11831 // then bitconvert to floating point or vector.
11832 SDValue newLoad = DAG.getLoad(VT: intVT, dl, Chain, Ptr,
11833 MMO: LD->getMemOperand());
11834 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoadedVT, Operand: newLoad);
11835 if (LoadedVT != VT)
11836 Result = DAG.getNode(Opcode: VT.isFloatingPoint() ? ISD::FP_EXTEND :
11837 ISD::ANY_EXTEND, DL: dl, VT, Operand: Result);
11838
11839 return std::make_pair(x&: Result, y: newLoad.getValue(R: 1));
11840 }
11841
11842 // Copy the value to a (aligned) stack slot using (unaligned) integer
11843 // loads and stores, then do a (aligned) load from the stack slot.
11844 MVT RegVT = getRegisterType(Context&: *DAG.getContext(), VT: intVT);
11845 unsigned LoadedBytes = LoadedVT.getStoreSize();
11846 unsigned RegBytes = RegVT.getSizeInBits() / 8;
11847 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
11848
11849 // Make sure the stack slot is also aligned for the register type.
11850 SDValue StackBase = DAG.CreateStackTemporary(VT1: LoadedVT, VT2: RegVT);
11851 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackBase.getNode())->getIndex();
11852 SmallVector<SDValue, 8> Stores;
11853 SDValue StackPtr = StackBase;
11854 unsigned Offset = 0;
11855
11856 EVT PtrVT = Ptr.getValueType();
11857 EVT StackPtrVT = StackPtr.getValueType();
11858
11859 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
11860 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
11861
11862 // Do all but one copies using the full register width.
11863 for (unsigned i = 1; i < NumRegs; i++) {
11864 // Load one integer register's worth from the original location.
11865 SDValue Load = DAG.getLoad(
11866 VT: RegVT, dl, Chain, Ptr, PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset),
11867 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
11868 // Follow the load with a store to the stack slot. Remember the store.
11869 Stores.push_back(Elt: DAG.getStore(
11870 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
11871 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset)));
11872 // Increment the pointers.
11873 Offset += RegBytes;
11874
11875 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
11876 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
11877 }
11878
11879 // The last copy may be partial. Do an extending load.
11880 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
11881 BitWidth: 8 * (LoadedBytes - Offset));
11882 SDValue Load = DAG.getExtLoad(
11883 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain, Ptr,
11884 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT, Alignment: LD->getBaseAlign(),
11885 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
11886 // Follow the load with a store to the stack slot. Remember the store.
11887 // On big-endian machines this requires a truncating store to ensure
11888 // that the bits end up in the right place.
11889 Stores.push_back(Elt: DAG.getTruncStore(
11890 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
11891 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), SVT: MemVT));
11892
11893 // The order of the stores doesn't matter - say it with a TokenFactor.
11894 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
11895
11896 // Finally, perform the original load only redirected to the stack slot.
11897 Load = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT, Chain: TF, Ptr: StackBase,
11898 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0),
11899 MemVT: LoadedVT);
11900
11901 // Callers expect a MERGE_VALUES node.
11902 return std::make_pair(x&: Load, y&: TF);
11903 }
11904
11905 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
11906 "Unaligned load of unsupported type.");
11907
11908 // Compute the new VT that is half the size of the old one. This is an
11909 // integer MVT.
11910 unsigned NumBits = LoadedVT.getSizeInBits();
11911 EVT NewLoadedVT;
11912 NewLoadedVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits/2);
11913 NumBits >>= 1;
11914
11915 Align Alignment = LD->getBaseAlign();
11916 unsigned IncrementSize = NumBits / 8;
11917 ISD::LoadExtType HiExtType = LD->getExtensionType();
11918
11919 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
11920 if (HiExtType == ISD::NON_EXTLOAD)
11921 HiExtType = ISD::ZEXTLOAD;
11922
11923 // Load the value in two parts
11924 SDValue Lo, Hi;
11925 if (DAG.getDataLayout().isLittleEndian()) {
11926 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
11927 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
11928 Metadata: LD->getAAInfo());
11929
11930 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
11931 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr,
11932 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
11933 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
11934 Metadata: LD->getAAInfo());
11935 } else {
11936 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
11937 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
11938 Metadata: LD->getAAInfo());
11939
11940 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
11941 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
11942 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
11943 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
11944 Metadata: LD->getAAInfo());
11945 }
11946
11947 // aggregate the two parts
11948 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: NumBits, VT, DL: dl);
11949 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: ShiftAmount);
11950 Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Result, N2: Lo);
11951
11952 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
11953 N2: Hi.getValue(R: 1));
11954
11955 return std::make_pair(x&: Result, y&: TF);
11956}
11957
11958SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
11959 SelectionDAG &DAG) const {
11960 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
11961 "unaligned indexed stores not implemented!");
11962 SDValue Chain = ST->getChain();
11963 SDValue Ptr = ST->getBasePtr();
11964 SDValue Val = ST->getValue();
11965 EVT VT = Val.getValueType();
11966 Align Alignment = ST->getBaseAlign();
11967 auto &MF = DAG.getMachineFunction();
11968 EVT StoreMemVT = ST->getMemoryVT();
11969
11970 SDLoc dl(ST);
11971 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
11972 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
11973 if (isTypeLegal(VT: intVT)) {
11974 if (!isOperationLegalOrCustom(Op: ISD::STORE, VT: intVT) &&
11975 StoreMemVT.isVector()) {
11976 // Scalarize the store and let the individual components be handled.
11977 SDValue Result = scalarizeVectorStore(ST, DAG);
11978 return Result;
11979 }
11980 // Expand to a bitconvert of the value to the integer type of the
11981 // same size, then a (misaligned) int store.
11982 // FIXME: Does not handle truncating floating point stores!
11983 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: intVT, Operand: Val);
11984 Result = DAG.getStore(Chain, dl, Val: Result, Ptr, PtrInfo: ST->getPointerInfo(),
11985 Alignment, MMOFlags: ST->getMemOperand()->getFlags());
11986 return Result;
11987 }
11988 // Do a (aligned) store to a stack slot, then copy from the stack slot
11989 // to the final destination using (unaligned) integer loads and stores.
11990 MVT RegVT = getRegisterType(
11991 Context&: *DAG.getContext(),
11992 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreMemVT.getSizeInBits()));
11993 EVT PtrVT = Ptr.getValueType();
11994 unsigned StoredBytes = StoreMemVT.getStoreSize();
11995 unsigned RegBytes = RegVT.getSizeInBits() / 8;
11996 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
11997
11998 // Make sure the stack slot is also aligned for the register type.
11999 SDValue StackPtr = DAG.CreateStackTemporary(VT1: StoreMemVT, VT2: RegVT);
12000 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
12001
12002 // Perform the original store, only redirected to the stack slot.
12003 SDValue Store = DAG.getTruncStore(
12004 Chain, dl, Val, Ptr: StackPtr,
12005 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0), SVT: StoreMemVT);
12006
12007 EVT StackPtrVT = StackPtr.getValueType();
12008
12009 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12010 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12011 SmallVector<SDValue, 8> Stores;
12012 unsigned Offset = 0;
12013
12014 // Do all but one copies using the full register width.
12015 for (unsigned i = 1; i < NumRegs; i++) {
12016 // Load one integer register's worth from the stack slot.
12017 SDValue Load = DAG.getLoad(
12018 VT: RegVT, dl, Chain: Store, Ptr: StackPtr,
12019 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset));
12020 // Store it to the final location. Remember the store.
12021 Stores.push_back(Elt: DAG.getStore(Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12022 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset),
12023 Alignment: ST->getBaseAlign(),
12024 MMOFlags: ST->getMemOperand()->getFlags()));
12025 // Increment the pointers.
12026 Offset += RegBytes;
12027 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12028 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12029 }
12030
12031 // The last store may be partial. Do a truncating store. On big-endian
12032 // machines this requires an extending load from the stack slot to ensure
12033 // that the bits are in the right place.
12034 EVT LoadMemVT =
12035 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: 8 * (StoredBytes - Offset));
12036
12037 // Load from the stack slot.
12038 SDValue Load = DAG.getExtLoad(
12039 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain: Store, Ptr: StackPtr,
12040 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), MemVT: LoadMemVT);
12041
12042 Stores.push_back(Elt: DAG.getTruncStore(
12043 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12044 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), SVT: LoadMemVT,
12045 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getAAInfo()));
12046 // The order of the stores doesn't matter - say it with a TokenFactor.
12047 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12048 return Result;
12049 }
12050
12051 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
12052 "Unaligned store of unknown type.");
12053 // Get the half-size VT
12054 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
12055 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
12056 unsigned IncrementSize = NumBits / 8;
12057
12058 // Divide the stored value in two parts.
12059 SDValue ShiftAmount =
12060 DAG.getShiftAmountConstant(Val: NumBits, VT: Val.getValueType(), DL: dl);
12061 SDValue Lo = Val;
12062 // If Val is a constant, replace the upper bits with 0. The SRL will constant
12063 // fold and not use the upper bits. A smaller constant may be easier to
12064 // materialize.
12065 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Lo); C && !C->isOpaque())
12066 Lo = DAG.getNode(
12067 Opcode: ISD::AND, DL: dl, VT, N1: Lo,
12068 N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: NumBits), DL: dl,
12069 VT));
12070 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Val, N2: ShiftAmount);
12071
12072 // Store the two parts
12073 SDValue Store1, Store2;
12074 Store1 = DAG.getTruncStore(Chain, dl,
12075 Val: DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
12076 Ptr, PtrInfo: ST->getPointerInfo(), SVT: NewStoredVT, Alignment,
12077 MMOFlags: ST->getMemOperand()->getFlags());
12078
12079 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12080 Store2 = DAG.getTruncStore(
12081 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
12082 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize), SVT: NewStoredVT, Alignment,
12083 MMOFlags: ST->getMemOperand()->getFlags(), Metadata: ST->getAAInfo());
12084
12085 SDValue Result =
12086 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
12087 return Result;
12088}
12089
12090SDValue
12091TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
12092 const SDLoc &DL, EVT DataVT,
12093 SelectionDAG &DAG,
12094 bool IsCompressedMemory) const {
12095 SDValue Increment;
12096 EVT AddrVT = Addr.getValueType();
12097 EVT MaskVT = Mask.getValueType();
12098 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
12099 "Incompatible types of Data and Mask");
12100 if (IsCompressedMemory) {
12101 // Incrementing the pointer according to number of '1's in the mask.
12102 if (DataVT.isScalableVector()) {
12103 EVT MaskExtVT = MaskVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
12104 SDValue MaskExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MaskExtVT, Operand: Mask);
12105 Increment = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: MaskExt);
12106 } else {
12107 EVT MaskIntVT =
12108 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MaskVT.getSizeInBits());
12109 SDValue MaskInIntReg = DAG.getBitcast(VT: MaskIntVT, V: Mask);
12110 if (MaskIntVT.getSizeInBits() < 32) {
12111 MaskInIntReg =
12112 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: MaskInIntReg);
12113 MaskIntVT = MVT::i32;
12114 }
12115 Increment = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MaskIntVT, Operand: MaskInIntReg);
12116 }
12117 // Scale is an element size in bytes.
12118 SDValue Scale = DAG.getConstant(Val: DataVT.getScalarSizeInBits() / 8, DL,
12119 VT: AddrVT);
12120 Increment = DAG.getZExtOrTrunc(Op: Increment, DL, VT: AddrVT);
12121 Increment = DAG.getNode(Opcode: ISD::MUL, DL, VT: AddrVT, N1: Increment, N2: Scale);
12122 } else
12123 Increment = DAG.getTypeSize(DL, VT: AddrVT, TS: DataVT.getStoreSize());
12124
12125 return DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: Addr, N2: Increment);
12126}
12127
12128static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
12129 EVT VecVT, const SDLoc &dl,
12130 ElementCount SubEC) {
12131 assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
12132 "Cannot index a scalable vector within a fixed-width vector");
12133
12134 unsigned NElts = VecVT.getVectorMinNumElements();
12135 unsigned NumSubElts = SubEC.getKnownMinValue();
12136 EVT IdxVT = Idx.getValueType();
12137
12138 if (VecVT.isScalableVector() && !SubEC.isScalable()) {
12139 // If this is a constant index and we know the value plus the number of the
12140 // elements in the subvector minus one is less than the minimum number of
12141 // elements then it's safe to return Idx.
12142 if (auto *IdxCst = dyn_cast<ConstantSDNode>(Val&: Idx))
12143 if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
12144 return Idx;
12145 SDValue VS =
12146 DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getFixedSizeInBits(), NElts));
12147 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
12148 SDValue Sub = DAG.getNode(Opcode: SubOpcode, DL: dl, VT: IdxVT, N1: VS,
12149 N2: DAG.getConstant(Val: NumSubElts, DL: dl, VT: IdxVT));
12150 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx, N2: Sub);
12151 }
12152 if (isPowerOf2_32(Value: NElts) && NumSubElts == 1) {
12153 APInt Imm = APInt::getLowBitsSet(numBits: IdxVT.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
12154 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IdxVT, N1: Idx,
12155 N2: DAG.getConstant(Val: Imm, DL: dl, VT: IdxVT));
12156 }
12157 unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
12158 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx,
12159 N2: DAG.getConstant(Val: MaxIndex, DL: dl, VT: IdxVT));
12160}
12161
12162SDValue
12163TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr,
12164 EVT VecVT, SDValue Index,
12165 const SDNodeFlags PtrArithFlags) const {
12166 return getVectorSubVecPointer(
12167 DAG, VecPtr, VecVT,
12168 SubVecVT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(), NumElements: 1),
12169 Index, PtrArithFlags);
12170}
12171
12172SDValue
12173TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr,
12174 EVT VecVT, EVT SubVecVT, SDValue Index,
12175 const SDNodeFlags PtrArithFlags) const {
12176 SDLoc dl(Index);
12177 // Make sure the index type is big enough to compute in.
12178 Index = DAG.getZExtOrTrunc(Op: Index, DL: dl, VT: VecPtr.getValueType());
12179
12180 EVT EltVT = VecVT.getVectorElementType();
12181
12182 // Calculate the element offset and add it to the pointer.
12183 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
12184 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
12185 "Converting bits to bytes lost precision");
12186 assert(SubVecVT.getVectorElementType() == EltVT &&
12187 "Sub-vector must be a vector with matching element type");
12188 Index = clampDynamicVectorIndex(DAG, Idx: Index, VecVT, dl,
12189 SubEC: SubVecVT.getVectorElementCount());
12190
12191 EVT IdxVT = Index.getValueType();
12192 if (SubVecVT.isScalableVector())
12193 Index =
12194 DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12195 N2: DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getSizeInBits(), 1)));
12196
12197 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12198 N2: DAG.getConstant(Val: EltSize, DL: dl, VT: IdxVT));
12199 return DAG.getMemBasePlusOffset(Base: VecPtr, Offset: Index, DL: dl, Flags: PtrArithFlags);
12200}
12201
12202//===----------------------------------------------------------------------===//
12203// Implementation of Emulated TLS Model
12204//===----------------------------------------------------------------------===//
12205
12206SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
12207 SelectionDAG &DAG) const {
12208 // Access to address of TLS varialbe xyz is lowered to a function call:
12209 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
12210 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12211 PointerType *VoidPtrType = PointerType::get(C&: *DAG.getContext(), AddressSpace: 0);
12212 SDLoc dl(GA);
12213
12214 ArgListTy Args;
12215 const GlobalValue *GV =
12216 cast<GlobalValue>(Val: GA->getGlobal()->stripPointerCastsAndAliases());
12217 SmallString<32> NameString("__emutls_v.");
12218 NameString += GV->getName();
12219 StringRef EmuTlsVarName(NameString);
12220 const GlobalVariable *EmuTlsVar =
12221 GV->getParent()->getNamedGlobal(Name: EmuTlsVarName);
12222 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
12223 Args.emplace_back(args: DAG.getGlobalAddress(GV: EmuTlsVar, DL: dl, VT: PtrVT), args&: VoidPtrType);
12224
12225 SDValue EmuTlsGetAddr = DAG.getExternalSymbol(Sym: "__emutls_get_address", VT: PtrVT);
12226
12227 TargetLowering::CallLoweringInfo CLI(DAG);
12228 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
12229 CLI.setLibCallee(CC: CallingConv::C, ResultType: VoidPtrType, Target: EmuTlsGetAddr, ArgsList: std::move(Args));
12230 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
12231
12232 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
12233 // At last for X86 targets, maybe good for other targets too?
12234 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
12235 MFI.setAdjustsStack(true); // Is this only for X86 target?
12236 MFI.setHasCalls(true);
12237
12238 assert((GA->getOffset() == 0) &&
12239 "Emulated TLS must have zero offset in GlobalAddressSDNode");
12240 return CallResult.first;
12241}
12242
12243SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
12244 SelectionDAG &DAG) const {
12245 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
12246 if (!isCtlzFast())
12247 return SDValue();
12248 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
12249 SDLoc dl(Op);
12250 if (isNullConstant(V: Op.getOperand(i: 1)) && CC == ISD::SETEQ) {
12251 EVT VT = Op.getOperand(i: 0).getValueType();
12252 SDValue Zext = Op.getOperand(i: 0);
12253 if (VT.bitsLT(VT: MVT::i32)) {
12254 VT = MVT::i32;
12255 Zext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Op.getOperand(i: 0));
12256 }
12257 unsigned Log2b = Log2_32(Value: VT.getSizeInBits());
12258 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Zext);
12259 SDValue Scc = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Clz,
12260 N2: DAG.getConstant(Val: Log2b, DL: dl, VT: MVT::i32));
12261 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Scc);
12262 }
12263 return SDValue();
12264}
12265
12266SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
12267 SDValue Op0 = Node->getOperand(Num: 0);
12268 SDValue Op1 = Node->getOperand(Num: 1);
12269 EVT VT = Op0.getValueType();
12270 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12271 unsigned Opcode = Node->getOpcode();
12272 SDLoc DL(Node);
12273
12274 // If both sign bits are zero, flip UMIN/UMAX <-> SMIN/SMAX if legal.
12275 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(MinMaxOpc: Opcode);
12276 if (isOperationLegal(Op: AltOpcode, VT) && DAG.SignBitIsZero(Op: Op0) &&
12277 DAG.SignBitIsZero(Op: Op1))
12278 return DAG.getNode(Opcode: AltOpcode, DL, VT, N1: Op0, N2: Op1);
12279
12280 // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
12281 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(V: Op1, AllowUndefs: true) && BoolVT == VT &&
12282 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12283 Op0 = DAG.getFreeze(V: Op0);
12284 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
12285 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12286 N2: DAG.getSetCC(DL, VT, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ));
12287 }
12288
12289 // umin(x,y) -> sub(x,usubsat(x,y))
12290 // TODO: Missing freeze(Op0)?
12291 if (Opcode == ISD::UMIN && isOperationLegal(Op: ISD::SUB, VT) &&
12292 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12293 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12294 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op0, N2: Op1));
12295 }
12296
12297 // umax(x,y) -> add(x,usubsat(y,x))
12298 // TODO: Missing freeze(Op0)?
12299 if (Opcode == ISD::UMAX && isOperationLegal(Op: ISD::ADD, VT) &&
12300 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12301 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0,
12302 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op1, N2: Op0));
12303 }
12304
12305 // FIXME: Should really try to split the vector in case it's legal on a
12306 // subvector.
12307 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12308 return DAG.UnrollVectorOp(N: Node);
12309
12310 // Attempt to find an existing SETCC node that we can reuse.
12311 // TODO: Do we need a generic doesSETCCNodeExist?
12312 // TODO: Missing freeze(Op0)/freeze(Op1)?
12313 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
12314 ISD::CondCode PrefCommuteCC,
12315 ISD::CondCode AltCommuteCC) {
12316 SDVTList BoolVTList = DAG.getVTList(VT: BoolVT);
12317 for (ISD::CondCode CC : {PrefCC, AltCC}) {
12318 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12319 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12320 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12321 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12322 }
12323 }
12324 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
12325 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12326 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12327 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12328 return DAG.getSelect(DL, VT, Cond, LHS: Op1, RHS: Op0);
12329 }
12330 }
12331 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: PrefCC);
12332 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12333 };
12334
12335 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
12336 // -> Y = (A < B) ? B : A
12337 // -> Y = (A >= B) ? A : B
12338 // -> Y = (A <= B) ? B : A
12339 switch (Opcode) {
12340 case ISD::SMAX:
12341 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
12342 case ISD::SMIN:
12343 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
12344 case ISD::UMAX:
12345 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
12346 case ISD::UMIN:
12347 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
12348 }
12349
12350 llvm_unreachable("How did we get here?");
12351}
12352
12353SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
12354 unsigned Opcode = Node->getOpcode();
12355 SDValue LHS = Node->getOperand(Num: 0);
12356 SDValue RHS = Node->getOperand(Num: 1);
12357 EVT VT = LHS.getValueType();
12358 SDLoc dl(Node);
12359
12360 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
12361 assert(VT.isInteger() && "Expected operands to be integers");
12362
12363 // usub.sat(a, b) -> umax(a, b) - b
12364 if (Opcode == ISD::USUBSAT && isOperationLegal(Op: ISD::UMAX, VT)) {
12365 SDValue Max = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT, N1: LHS, N2: RHS);
12366 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: RHS);
12367 }
12368
12369 // usub.sat(a, 1) -> sub(a, zext(a != 0))
12370 // Prefer this on targets without legal/cost-effective overflow-carry nodes.
12371 if (Opcode == ISD::USUBSAT && isOneOrOneSplat(V: RHS) &&
12372 !isOperationLegalOrCustom(Op: ISD::USUBO_CARRY, VT)) {
12373 LHS = DAG.getFreeze(V: LHS);
12374 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12375 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12376 SDValue IsNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETNE);
12377 SDValue Subtrahend = DAG.getBoolExtOrTrunc(Op: IsNonZero, SL: dl, VT, OpVT: BoolVT);
12378 Subtrahend =
12379 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Subtrahend, N2: DAG.getConstant(Val: 1, DL: dl, VT));
12380 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: Subtrahend);
12381 }
12382
12383 // uadd.sat(a, b) -> umin(a, ~b) + b
12384 if (Opcode == ISD::UADDSAT && isOperationLegal(Op: ISD::UMIN, VT)) {
12385 SDValue InvRHS = DAG.getNOT(DL: dl, Val: RHS, VT);
12386 SDValue Min = DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: LHS, N2: InvRHS);
12387 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Min, N2: RHS);
12388 }
12389
12390 unsigned OverflowOp;
12391 switch (Opcode) {
12392 case ISD::SADDSAT:
12393 OverflowOp = ISD::SADDO;
12394 break;
12395 case ISD::UADDSAT:
12396 OverflowOp = ISD::UADDO;
12397 break;
12398 case ISD::SSUBSAT:
12399 OverflowOp = ISD::SSUBO;
12400 break;
12401 case ISD::USUBSAT:
12402 OverflowOp = ISD::USUBO;
12403 break;
12404 default:
12405 llvm_unreachable("Expected method to receive signed or unsigned saturation "
12406 "addition or subtraction node.");
12407 }
12408
12409 // FIXME: Should really try to split the vector in case it's legal on a
12410 // subvector.
12411 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12412 return DAG.UnrollVectorOp(N: Node);
12413
12414 unsigned BitWidth = LHS.getScalarValueSizeInBits();
12415 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12416 SDValue Result = DAG.getNode(Opcode: OverflowOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12417 SDValue SumDiff = Result.getValue(R: 0);
12418 SDValue Overflow = Result.getValue(R: 1);
12419 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12420 SDValue AllOnes = DAG.getAllOnesConstant(DL: dl, VT);
12421
12422 if (Opcode == ISD::UADDSAT) {
12423 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12424 // (LHS + RHS) | OverflowMask
12425 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12426 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SumDiff, N2: OverflowMask);
12427 }
12428 // Overflow ? 0xffff.... : (LHS + RHS)
12429 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: AllOnes, RHS: SumDiff);
12430 }
12431
12432 if (Opcode == ISD::USUBSAT) {
12433 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12434 // (LHS - RHS) & ~OverflowMask
12435 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12436 SDValue Not = DAG.getNOT(DL: dl, Val: OverflowMask, VT);
12437 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SumDiff, N2: Not);
12438 }
12439 // Overflow ? 0 : (LHS - RHS)
12440 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Zero, RHS: SumDiff);
12441 }
12442
12443 assert((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
12444 "Expected signed saturating add/sub opcode");
12445
12446 const APInt MinVal = APInt::getSignedMinValue(numBits: BitWidth);
12447 const APInt MaxVal = APInt::getSignedMaxValue(numBits: BitWidth);
12448
12449 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
12450 KnownBits KnownRHS = DAG.computeKnownBits(Op: RHS);
12451
12452 // If either of the operand signs are known, then they are guaranteed to
12453 // only saturate in one direction. If non-negative they will saturate
12454 // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
12455 //
12456 // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
12457 // sign of 'y' has to be flipped.
12458
12459 bool LHSIsNonNegative = KnownLHS.isNonNegative();
12460 bool RHSIsNonNegative =
12461 Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() : KnownRHS.isNegative();
12462 if (LHSIsNonNegative || RHSIsNonNegative) {
12463 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12464 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: SumDiff);
12465 }
12466
12467 bool LHSIsNegative = KnownLHS.isNegative();
12468 bool RHSIsNegative =
12469 Opcode == ISD::SADDSAT ? KnownRHS.isNegative() : KnownRHS.isNonNegative();
12470 if (LHSIsNegative || RHSIsNegative) {
12471 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12472 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMin, RHS: SumDiff);
12473 }
12474
12475 // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
12476 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12477 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SumDiff,
12478 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT));
12479 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Shift, N2: SatMin);
12480 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: SumDiff);
12481}
12482
12483SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
12484 unsigned Opcode = Node->getOpcode();
12485 SDValue LHS = Node->getOperand(Num: 0);
12486 SDValue RHS = Node->getOperand(Num: 1);
12487 EVT VT = LHS.getValueType();
12488 EVT ResVT = Node->getValueType(ResNo: 0);
12489 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12490 SDLoc dl(Node);
12491
12492 auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT);
12493 auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT);
12494 SDValue IsLT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: LTPredicate);
12495 SDValue IsGT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: GTPredicate);
12496
12497 // We can't perform arithmetic on i1 values. Extending them would
12498 // probably result in worse codegen, so let's just use two selects instead.
12499 // Some targets are also just better off using selects rather than subtraction
12500 // because one of the conditions can be merged with one of the selects.
12501 // And finally, if we don't know the contents of high bits of a boolean value
12502 // we can't perform any arithmetic either.
12503 if (preferSelectsOverBooleanArithmetic(VT) ||
12504 BoolVT.getScalarSizeInBits() == 1 ||
12505 getBooleanContents(Type: BoolVT) == UndefinedBooleanContent) {
12506 SDValue SelectZeroOrOne =
12507 DAG.getSelect(DL: dl, VT: ResVT, Cond: IsGT, LHS: DAG.getConstant(Val: 1, DL: dl, VT: ResVT),
12508 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ResVT));
12509 return DAG.getSelect(DL: dl, VT: ResVT, Cond: IsLT, LHS: DAG.getAllOnesConstant(DL: dl, VT: ResVT),
12510 RHS: SelectZeroOrOne);
12511 }
12512
12513 if (getBooleanContents(Type: BoolVT) == ZeroOrNegativeOneBooleanContent)
12514 std::swap(a&: IsGT, b&: IsLT);
12515 return DAG.getSExtOrTrunc(Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: BoolVT, N1: IsGT, N2: IsLT), DL: dl,
12516 VT: ResVT);
12517}
12518
12519SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
12520 unsigned Opcode = Node->getOpcode();
12521 bool IsSigned = Opcode == ISD::SSHLSAT;
12522 SDValue LHS = Node->getOperand(Num: 0);
12523 SDValue RHS = Node->getOperand(Num: 1);
12524 EVT VT = LHS.getValueType();
12525 SDLoc dl(Node);
12526
12527 assert((Node->getOpcode() == ISD::SSHLSAT ||
12528 Node->getOpcode() == ISD::USHLSAT) &&
12529 "Expected a SHLSAT opcode");
12530 assert(VT.isInteger() && "Expected operands to be integers");
12531
12532 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12533 return DAG.UnrollVectorOp(N: Node);
12534
12535 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
12536
12537 unsigned BW = VT.getScalarSizeInBits();
12538 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12539 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: RHS);
12540 SDValue Orig =
12541 DAG.getNode(Opcode: IsSigned ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: Result, N2: RHS);
12542
12543 SDValue SatVal;
12544 if (IsSigned) {
12545 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: BW), DL: dl, VT);
12546 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: BW), DL: dl, VT);
12547 SDValue Cond =
12548 DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT);
12549 SatVal = DAG.getSelect(DL: dl, VT, Cond, LHS: SatMin, RHS: SatMax);
12550 } else {
12551 SatVal = DAG.getConstant(Val: APInt::getMaxValue(numBits: BW), DL: dl, VT);
12552 }
12553 SDValue Cond = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Orig, Cond: ISD::SETNE);
12554 return DAG.getSelect(DL: dl, VT, Cond, LHS: SatVal, RHS: Result);
12555}
12556
12557void TargetLowering::forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl,
12558 bool Signed, SDValue &Lo, SDValue &Hi,
12559 SDValue LHS, SDValue RHS,
12560 SDValue HiLHS, SDValue HiRHS) const {
12561 EVT VT = LHS.getValueType();
12562 assert(RHS.getValueType() == VT && "Mismatching operand types");
12563
12564 assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
12565 assert((!Signed || !HiLHS) &&
12566 "Signed flag should only be set when HiLHS and RiRHS are null");
12567
12568 // We'll expand the multiplication by brute force because we have no other
12569 // options. This is a trivially-generalized version of the code from
12570 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
12571 // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
12572 // sign bits while calculating the Hi half.
12573 unsigned Bits = VT.getScalarSizeInBits();
12574 unsigned HalfBits = Bits / 2;
12575 SDValue Mask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: Bits, loBitsSet: HalfBits), DL: dl, VT);
12576 SDValue LL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Mask);
12577 SDValue RL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: RHS, N2: Mask);
12578
12579 SDValue T = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RL);
12580 SDValue TL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: Mask);
12581
12582 SDValue Shift = DAG.getShiftAmountConstant(Val: HalfBits, VT, DL: dl);
12583 // This is always an unsigned shift.
12584 SDValue TH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: T, N2: Shift);
12585
12586 unsigned ShiftOpc = Signed ? ISD::SRA : ISD::SRL;
12587 SDValue LH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: LHS, N2: Shift);
12588 SDValue RH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: RHS, N2: Shift);
12589
12590 SDValue U =
12591 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RL), N2: TH);
12592 SDValue UL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: U, N2: Mask);
12593 SDValue UH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: U, N2: Shift);
12594
12595 SDValue V =
12596 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RH), N2: UL);
12597 SDValue VH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: V, N2: Shift);
12598
12599 Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: TL,
12600 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: Shift));
12601
12602 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RH),
12603 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: UH, N2: VH));
12604
12605 // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
12606 // the products to Hi.
12607 if (HiLHS) {
12608 SDValue RHLL = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: HiRHS, N2: LHS);
12609 SDValue RLLH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: RHS, N2: HiLHS);
12610 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Hi,
12611 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: RHLL, N2: RLLH));
12612 }
12613}
12614
12615void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
12616 bool Signed, const SDValue LHS,
12617 const SDValue RHS, SDValue &Lo,
12618 SDValue &Hi) const {
12619 EVT VT = LHS.getValueType();
12620 assert(RHS.getValueType() == VT && "Mismatching operand types");
12621 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
12622 // We can fall back to a libcall with an illegal type for the MUL if we
12623 // have a libcall big enough.
12624 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
12625 if (WideVT == MVT::i16)
12626 LC = RTLIB::MUL_I16;
12627 else if (WideVT == MVT::i32)
12628 LC = RTLIB::MUL_I32;
12629 else if (WideVT == MVT::i64)
12630 LC = RTLIB::MUL_I64;
12631 else if (WideVT == MVT::i128)
12632 LC = RTLIB::MUL_I128;
12633
12634 RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(Call: LC);
12635 if (LibcallImpl == RTLIB::Unsupported) {
12636 forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
12637 return;
12638 }
12639
12640 SDValue HiLHS, HiRHS;
12641 if (Signed) {
12642 // The high part is obtained by SRA'ing all but one of the bits of low
12643 // part.
12644 unsigned LoSize = VT.getFixedSizeInBits();
12645 SDValue Shift = DAG.getShiftAmountConstant(Val: LoSize - 1, VT, DL: dl);
12646 HiLHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS, N2: Shift);
12647 HiRHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: RHS, N2: Shift);
12648 } else {
12649 HiLHS = DAG.getConstant(Val: 0, DL: dl, VT);
12650 HiRHS = DAG.getConstant(Val: 0, DL: dl, VT);
12651 }
12652
12653 // Attempt a libcall.
12654 SDValue Ret;
12655 TargetLowering::MakeLibCallOptions CallOptions;
12656 CallOptions.setIsSigned(Signed);
12657 CallOptions.setIsPostTypeLegalization(true);
12658 if (shouldSplitFunctionArgumentsAsLittleEndian(DL: DAG.getDataLayout())) {
12659 // Halves of WideVT are packed into registers in different order
12660 // depending on platform endianness. This is usually handled by
12661 // the C calling convention, but we can't defer to it in
12662 // the legalizer.
12663 SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
12664 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12665 } else {
12666 SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
12667 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12668 }
12669 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
12670 "Ret value is a collection of constituent nodes holding result.");
12671 if (DAG.getDataLayout().isLittleEndian()) {
12672 // Same as above.
12673 Lo = Ret.getOperand(i: 0);
12674 Hi = Ret.getOperand(i: 1);
12675 } else {
12676 Lo = Ret.getOperand(i: 1);
12677 Hi = Ret.getOperand(i: 0);
12678 }
12679}
12680
12681SDValue
12682TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
12683 assert((Node->getOpcode() == ISD::SMULFIX ||
12684 Node->getOpcode() == ISD::UMULFIX ||
12685 Node->getOpcode() == ISD::SMULFIXSAT ||
12686 Node->getOpcode() == ISD::UMULFIXSAT) &&
12687 "Expected a fixed point multiplication opcode");
12688
12689 SDLoc dl(Node);
12690 SDValue LHS = Node->getOperand(Num: 0);
12691 SDValue RHS = Node->getOperand(Num: 1);
12692 EVT VT = LHS.getValueType();
12693 unsigned Scale = Node->getConstantOperandVal(Num: 2);
12694 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
12695 Node->getOpcode() == ISD::UMULFIXSAT);
12696 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
12697 Node->getOpcode() == ISD::SMULFIXSAT);
12698 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12699 unsigned VTSize = VT.getScalarSizeInBits();
12700
12701 if (!Scale) {
12702 // [us]mul.fix(a, b, 0) -> mul(a, b)
12703 if (!Saturating) {
12704 if (isOperationLegalOrCustom(Op: ISD::MUL, VT))
12705 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
12706 } else if (Signed && isOperationLegalOrCustom(Op: ISD::SMULO, VT)) {
12707 SDValue Result =
12708 DAG.getNode(Opcode: ISD::SMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12709 SDValue Product = Result.getValue(R: 0);
12710 SDValue Overflow = Result.getValue(R: 1);
12711 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12712
12713 APInt MinVal = APInt::getSignedMinValue(numBits: VTSize);
12714 APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize);
12715 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12716 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12717 // Xor the inputs, if resulting sign bit is 0 the product will be
12718 // positive, else negative.
12719 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
12720 SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT);
12721 Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax);
12722 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product);
12723 } else if (!Signed && isOperationLegalOrCustom(Op: ISD::UMULO, VT)) {
12724 SDValue Result =
12725 DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12726 SDValue Product = Result.getValue(R: 0);
12727 SDValue Overflow = Result.getValue(R: 1);
12728
12729 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
12730 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12731 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product);
12732 }
12733 }
12734
12735 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
12736 "Expected scale to be less than the number of bits if signed or at "
12737 "most the number of bits if unsigned.");
12738 assert(LHS.getValueType() == RHS.getValueType() &&
12739 "Expected both operands to be the same type");
12740
12741 // Select the saturated value when Cond0 <CC> Cond1, keeping it vectorized:
12742 // SELECT_CC is scalarized for vector types, so build SETCC + VSELECT there.
12743 auto getSaturatingSelect = [&](SDValue Cond0, SDValue Cond1, SDValue Sat,
12744 SDValue Val, ISD::CondCode CC) {
12745 if (VT.isVector())
12746 return DAG.getSelect(DL: dl, VT, Cond: DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Cond0, RHS: Cond1, Cond: CC),
12747 LHS: Sat, RHS: Val);
12748 return DAG.getSelectCC(DL: dl, LHS: Cond0, RHS: Cond1, True: Sat, False: Val, Cond: CC);
12749 };
12750
12751 // Get the upper and lower bits of the result.
12752 SDValue Lo, Hi;
12753 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
12754 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
12755 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
12756 if (isOperationLegalOrCustom(Op: LoHiOp, VT)) {
12757 SDValue Result = DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
12758 Lo = Result.getValue(R: 0);
12759 Hi = Result.getValue(R: 1);
12760 } else if (isOperationLegalOrCustom(Op: HiOp, VT)) {
12761 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
12762 Hi = DAG.getNode(Opcode: HiOp, DL: dl, VT, N1: LHS, N2: RHS);
12763 } else if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
12764 // Try for a multiplication using a wider type.
12765 unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
12766 SDValue LHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: LHS);
12767 SDValue RHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: RHS);
12768 SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHSExt, N2: RHSExt);
12769 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Res);
12770 SDValue Shifted =
12771 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: WideVT, N1: Res,
12772 N2: DAG.getShiftAmountConstant(Val: VTSize, VT: WideVT, DL: dl));
12773 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Shifted);
12774 } else if (VT.isVector()) {
12775 return SDValue();
12776 } else {
12777 forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
12778 }
12779
12780 if (Scale == VTSize)
12781 // Result is just the top half since we'd be shifting by the width of the
12782 // operand. Overflow impossible so this works for both UMULFIX and
12783 // UMULFIXSAT.
12784 return Hi;
12785
12786 // The result will need to be shifted right by the scale since both operands
12787 // are scaled. The result is given to us in 2 halves, so we only want part of
12788 // both in the result.
12789 SDValue Result = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: Hi, N2: Lo,
12790 N3: DAG.getShiftAmountConstant(Val: Scale, VT, DL: dl));
12791 if (!Saturating)
12792 return Result;
12793
12794 if (!Signed) {
12795 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
12796 // widened multiplication) aren't all zeroes.
12797
12798 // Saturate to max if ((Hi >> Scale) != 0),
12799 // which is the same as if (Hi > ((1 << Scale) - 1))
12800 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
12801 SDValue LowMask =
12802 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale), DL: dl, VT);
12803 return getSaturatingSelect(Hi, LowMask, DAG.getConstant(Val: MaxVal, DL: dl, VT),
12804 Result, ISD::SETUGT);
12805 }
12806
12807 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
12808 // widened multiplication) aren't all ones or all zeroes.
12809
12810 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: VTSize), DL: dl, VT);
12811 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: VTSize), DL: dl, VT);
12812
12813 if (Scale == 0) {
12814 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Lo,
12815 N2: DAG.getShiftAmountConstant(Val: VTSize - 1, VT, DL: dl));
12816 SDValue Overflow = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Hi, RHS: Sign, Cond: ISD::SETNE);
12817 // Saturated to SatMin if wide product is negative, and SatMax if wide
12818 // product is positive ...
12819 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12820 SDValue ResultIfOverflow =
12821 getSaturatingSelect(Hi, Zero, SatMin, SatMax, ISD::SETLT);
12822 // ... but only if we overflowed.
12823 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: ResultIfOverflow, RHS: Result);
12824 }
12825
12826 // We handled Scale==0 above so all the bits to examine is in Hi.
12827
12828 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
12829 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
12830 SDValue LowMask =
12831 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale - 1), DL: dl, VT);
12832 // Saturate to min if (Hi >> (Scale - 1)) < -1),
12833 // which is the same as if (HI < (-1 << (Scale - 1))
12834 SDValue HighMask = DAG.getConstant(
12835 Val: APInt::getHighBitsSet(numBits: VTSize, hiBitsSet: VTSize - Scale + 1), DL: dl, VT);
12836 Result = getSaturatingSelect(Hi, LowMask, SatMax, Result, ISD::SETGT);
12837 Result = getSaturatingSelect(Hi, HighMask, SatMin, Result, ISD::SETLT);
12838 return Result;
12839}
12840
12841SDValue
12842TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
12843 SDValue LHS, SDValue RHS,
12844 unsigned Scale, SelectionDAG &DAG) const {
12845 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
12846 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
12847 "Expected a fixed point division opcode");
12848
12849 EVT VT = LHS.getValueType();
12850 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
12851 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
12852 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12853
12854 // If there is enough room in the type to upscale the LHS or downscale the
12855 // RHS before the division, we can perform it in this type without having to
12856 // resize. For signed operations, the LHS headroom is the number of
12857 // redundant sign bits, and for unsigned ones it is the number of zeroes.
12858 // The headroom for the RHS is the number of trailing zeroes.
12859 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(Op: LHS) - 1
12860 : DAG.computeKnownBits(Op: LHS).countMinLeadingZeros();
12861 unsigned RHSTrail = DAG.computeKnownBits(Op: RHS).countMinTrailingZeros();
12862
12863 // For signed saturating operations, we need to be able to detect true integer
12864 // division overflow; that is, when you have MIN / -EPS. However, this
12865 // is undefined behavior and if we emit divisions that could take such
12866 // values it may cause undesired behavior (arithmetic exceptions on x86, for
12867 // example).
12868 // Avoid this by requiring an extra bit so that we never get this case.
12869 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
12870 // signed saturating division, we need to emit a whopping 32-bit division.
12871 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
12872 return SDValue();
12873
12874 unsigned LHSShift = std::min(a: LHSLead, b: Scale);
12875 unsigned RHSShift = Scale - LHSShift;
12876
12877 // At this point, we know that if we shift the LHS up by LHSShift and the
12878 // RHS down by RHSShift, we can emit a regular division with a final scaling
12879 // factor of Scale.
12880
12881 if (LHSShift)
12882 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS,
12883 N2: DAG.getShiftAmountConstant(Val: LHSShift, VT, DL: dl));
12884 if (RHSShift)
12885 RHS = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: RHS,
12886 N2: DAG.getShiftAmountConstant(Val: RHSShift, VT, DL: dl));
12887
12888 SDValue Quot;
12889 if (Signed) {
12890 // For signed operations, if the resulting quotient is negative and the
12891 // remainder is nonzero, subtract 1 from the quotient to round towards
12892 // negative infinity.
12893 SDValue Rem;
12894 // FIXME: Ideally we would always produce an SDIVREM here, but if the
12895 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
12896 // we couldn't just form a libcall, but the type legalizer doesn't do it.
12897 if (isTypeLegal(VT) &&
12898 isOperationLegalOrCustom(Op: ISD::SDIVREM, VT)) {
12899 Quot = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl,
12900 VTList: DAG.getVTList(VT1: VT, VT2: VT),
12901 N1: LHS, N2: RHS);
12902 Rem = Quot.getValue(R: 1);
12903 Quot = Quot.getValue(R: 0);
12904 } else {
12905 Quot = DAG.getNode(Opcode: ISD::SDIV, DL: dl, VT,
12906 N1: LHS, N2: RHS);
12907 Rem = DAG.getNode(Opcode: ISD::SREM, DL: dl, VT,
12908 N1: LHS, N2: RHS);
12909 }
12910 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12911 SDValue RemNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Rem, RHS: Zero, Cond: ISD::SETNE);
12912 SDValue LHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETLT);
12913 SDValue RHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
12914 SDValue QuotNeg = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: BoolVT, N1: LHSNeg, N2: RHSNeg);
12915 SDValue Sub1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Quot,
12916 N2: DAG.getConstant(Val: 1, DL: dl, VT));
12917 Quot = DAG.getSelect(DL: dl, VT,
12918 Cond: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolVT, N1: RemNonZero, N2: QuotNeg),
12919 LHS: Sub1, RHS: Quot);
12920 } else
12921 Quot = DAG.getNode(Opcode: ISD::UDIV, DL: dl, VT,
12922 N1: LHS, N2: RHS);
12923
12924 return Quot;
12925}
12926
12927void TargetLowering::expandUADDSUBO(
12928 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
12929 SDLoc dl(Node);
12930 SDValue LHS = Node->getOperand(Num: 0);
12931 SDValue RHS = Node->getOperand(Num: 1);
12932 bool IsAdd = Node->getOpcode() == ISD::UADDO;
12933
12934 // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
12935 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
12936 if (isOperationLegalOrCustom(Op: OpcCarry, VT: Node->getValueType(ResNo: 0))) {
12937 SDValue CarryIn = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 1));
12938 SDValue NodeCarry = DAG.getNode(Opcode: OpcCarry, DL: dl, VTList: Node->getVTList(),
12939 Ops: { LHS, RHS, CarryIn });
12940 Result = SDValue(NodeCarry.getNode(), 0);
12941 Overflow = SDValue(NodeCarry.getNode(), 1);
12942 return;
12943 }
12944
12945 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
12946 VT: LHS.getValueType(), N1: LHS, N2: RHS);
12947
12948 EVT ResultType = Node->getValueType(ResNo: 1);
12949 EVT SetCCType = getSetCCResultType(
12950 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
12951 SDValue SetCC;
12952 if (IsAdd && isOneConstant(V: RHS)) {
12953 // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
12954 // the live range of X. We assume comparing with 0 is cheap.
12955 // The general case (X + C) < C is not necessarily beneficial. Although we
12956 // reduce the live range of X, we may introduce the materialization of
12957 // constant C.
12958 SetCC =
12959 DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result,
12960 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETEQ);
12961 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
12962 // Special case: uaddo X, -1 overflows if X != 0.
12963 SetCC =
12964 DAG.getSetCC(DL: dl, VT: SetCCType, LHS,
12965 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETNE);
12966 } else {
12967 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
12968 SetCC = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result, RHS: LHS, Cond: CC);
12969 }
12970 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
12971}
12972
12973void TargetLowering::expandSADDSUBO(
12974 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
12975 SDLoc dl(Node);
12976 SDValue LHS = Node->getOperand(Num: 0);
12977 SDValue RHS = Node->getOperand(Num: 1);
12978 bool IsAdd = Node->getOpcode() == ISD::SADDO;
12979
12980 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
12981 VT: LHS.getValueType(), N1: LHS, N2: RHS);
12982
12983 EVT ResultType = Node->getValueType(ResNo: 1);
12984 EVT OType = getSetCCResultType(
12985 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
12986
12987 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
12988 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
12989 if (isOperationLegal(Op: OpcSat, VT: LHS.getValueType())) {
12990 SDValue Sat = DAG.getNode(Opcode: OpcSat, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS);
12991 SDValue SetCC = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Sat, Cond: ISD::SETNE);
12992 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
12993 return;
12994 }
12995
12996 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
12997
12998 if (IsAdd) {
12999 // For an addition, the result should be less than one of the operands (LHS)
13000 // if and only if the other operand (RHS) is negative, otherwise there will
13001 // be overflow.
13002 SDValue ResultLowerThanLHS =
13003 DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: LHS, Cond: ISD::SETLT);
13004 SDValue RHSNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13005 Overflow = DAG.getBoolExtOrTrunc(
13006 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: RHSNegative, N2: ResultLowerThanLHS), SL: dl,
13007 VT: ResultType, OpVT: ResultType);
13008 } else {
13009 // For subtraction, overflow occurs when the signed comparison of operands
13010 // doesn't match the sign of the result.
13011 SDValue LHSLessThanRHS = DAG.getSetCC(DL: dl, VT: OType, LHS, RHS, Cond: ISD::SETLT);
13012 SDValue ResultNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Zero, Cond: ISD::SETLT);
13013 Overflow = DAG.getBoolExtOrTrunc(
13014 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: LHSLessThanRHS, N2: ResultNegative), SL: dl,
13015 VT: ResultType, OpVT: ResultType);
13016 }
13017}
13018
13019bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
13020 SDValue &Overflow, SelectionDAG &DAG) const {
13021 SDLoc dl(Node);
13022 EVT VT = Node->getValueType(ResNo: 0);
13023 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13024 SDValue LHS = Node->getOperand(Num: 0);
13025 SDValue RHS = Node->getOperand(Num: 1);
13026 bool isSigned = Node->getOpcode() == ISD::SMULO;
13027
13028 // For power-of-two multiplications we can use a simpler shift expansion.
13029 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
13030 const APInt &C = RHSC->getAPIntValue();
13031 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
13032 if (C.isPowerOf2()) {
13033 // smulo(x, signed_min) is same as umulo(x, signed_min).
13034 bool UseArithShift = isSigned && !C.isMinSignedValue();
13035 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: C.logBase2(), VT, DL: dl);
13036 Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: ShiftAmt);
13037 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT,
13038 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL,
13039 DL: dl, VT, N1: Result, N2: ShiftAmt),
13040 RHS: LHS, Cond: ISD::SETNE);
13041 return true;
13042 }
13043 }
13044
13045 SDValue BottomHalf;
13046 SDValue TopHalf;
13047 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13048
13049 static const unsigned Ops[2][3] =
13050 { { ISD::UMUL_LOHI, ISD::MULHU, ISD::ZERO_EXTEND },
13051 { ISD::SMUL_LOHI, ISD::MULHS, ISD::SIGN_EXTEND }};
13052 if (isOperationLegalOrCustom(Op: Ops[isSigned][0], VT)) {
13053 BottomHalf = DAG.getNode(Opcode: Ops[isSigned][0], DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS,
13054 N2: RHS);
13055 TopHalf = BottomHalf.getValue(R: 1);
13056 } else if (isOperationLegalOrCustom(Op: Ops[isSigned][1], VT)) {
13057 BottomHalf = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13058 TopHalf = DAG.getNode(Opcode: Ops[isSigned][1], DL: dl, VT, N1: LHS, N2: RHS);
13059 } else if (isTypeLegal(VT: WideVT)) {
13060 LHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: LHS);
13061 RHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: RHS);
13062 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS);
13063 BottomHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Mul);
13064 SDValue ShiftAmt =
13065 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits(), VT: WideVT, DL: dl);
13066 TopHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13067 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT, N1: Mul, N2: ShiftAmt));
13068 } else {
13069 if (VT.isVector())
13070 return false;
13071
13072 forceExpandWideMUL(DAG, dl, Signed: isSigned, LHS, RHS, Lo&: BottomHalf, Hi&: TopHalf);
13073 }
13074
13075 Result = BottomHalf;
13076 if (isSigned) {
13077 SDValue ShiftAmt = DAG.getShiftAmountConstant(
13078 Val: VT.getScalarSizeInBits() - 1, VT: BottomHalf.getValueType(), DL: dl);
13079 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: BottomHalf, N2: ShiftAmt);
13080 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf, RHS: Sign, Cond: ISD::SETNE);
13081 } else {
13082 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf,
13083 RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
13084 }
13085
13086 // Truncate the result if SetCC returns a larger type than needed.
13087 EVT RType = Node->getValueType(ResNo: 1);
13088 if (RType.bitsLT(VT: Overflow.getValueType()))
13089 Overflow = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RType, Operand: Overflow);
13090
13091 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
13092 "Unexpected result type for S/UMULO legalization");
13093 return true;
13094}
13095
13096SDValue TargetLowering::expandMULH(SDNode *Node, SelectionDAG &DAG) const {
13097 SDLoc dl(Node);
13098 EVT VT = Node->getValueType(ResNo: 0);
13099 SDValue LHS = Node->getOperand(Num: 0);
13100 SDValue RHS = Node->getOperand(Num: 1);
13101 bool IsSigned = Node->getOpcode() == ISD::MULHS;
13102
13103 // Use MUL_LOHI if legal/custom for the original type.
13104 unsigned LoHiOp = IsSigned ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
13105 if (isOperationLegalOrCustom(Op: LoHiOp, VT))
13106 return DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS).getValue(R: 1);
13107
13108 // Use a wide multiply if available.
13109 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13110 if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
13111 unsigned BW = VT.getScalarSizeInBits();
13112 LHS = DAG.getExtOrTrunc(IsSigned, Op: LHS, DL: dl, VT: WideVT);
13113 RHS = DAG.getExtOrTrunc(IsSigned, Op: RHS, DL: dl, VT: WideVT);
13114 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13115 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT,
13116 N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS),
13117 N2: DAG.getShiftAmountConstant(Val: BW, VT: WideVT, DL: dl)));
13118 }
13119
13120 // Let fixed-length vectors be scalarised by the caller.
13121 // Expand everything else with a wide multiply.
13122 if (!VT.isFixedLengthVector()) {
13123 SDValue Lo, Hi;
13124 forceExpandWideMUL(DAG, dl, Signed: IsSigned, LHS, RHS, Lo, Hi);
13125 return Hi;
13126 }
13127
13128 return SDValue();
13129}
13130
13131SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
13132 SDLoc dl(Node);
13133 ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13134 SDValue Op = Node->getOperand(Num: 0);
13135 SDNodeFlags Flags = Node->getFlags();
13136 EVT VT = Op.getValueType();
13137
13138 // Try to use a shuffle reduction for power of two vectors.
13139 if (VT.isPow2VectorType()) {
13140 // See if the reduction opcode is safe to use with widened types.
13141 bool WidenSrc = false;
13142 switch (Node->getOpcode()) {
13143 case ISD::VECREDUCE_FADD:
13144 case ISD::VECREDUCE_FMUL:
13145 case ISD::VECREDUCE_ADD:
13146 case ISD::VECREDUCE_MUL:
13147 case ISD::VECREDUCE_AND:
13148 case ISD::VECREDUCE_OR:
13149 case ISD::VECREDUCE_XOR:
13150 case ISD::VECREDUCE_SMAX:
13151 case ISD::VECREDUCE_SMIN:
13152 case ISD::VECREDUCE_UMAX:
13153 case ISD::VECREDUCE_UMIN:
13154 WidenSrc = VT.isFixedLengthVector();
13155 break;
13156 }
13157
13158 while (VT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
13159 EVT HalfVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
13160 if (!isOperationLegalOrCustom(Op: BaseOpcode, VT: HalfVT)) {
13161 if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
13162 // Attempt to widen the source vectors to a legal op.
13163 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: HalfVT);
13164 if (WideVT.isVector() &&
13165 WideVT.getScalarType() == HalfVT.getScalarType() &&
13166 WideVT.getVectorNumElements() >= HalfVT.getVectorNumElements() &&
13167 isOperationLegalOrCustom(Op: BaseOpcode, VT: WideVT)) {
13168 SDValue Lo, Hi;
13169 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13170 Lo = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Lo, Idx: 0);
13171 Hi = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Hi, Idx: 0);
13172 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: WideVT, N1: Lo, N2: Hi, Flags);
13173 Op = DAG.getExtractSubvector(DL: dl, VT: HalfVT, Vec: Op, Idx: 0);
13174 VT = HalfVT;
13175 continue;
13176 }
13177 }
13178 break;
13179 }
13180
13181 SDValue Lo, Hi;
13182 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13183 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: HalfVT, N1: Lo, N2: Hi, Flags);
13184 VT = HalfVT;
13185
13186 // Stop if splitting is enough to make the reduction legal.
13187 if (isOperationLegalOrCustom(Op: Node->getOpcode(), VT: HalfVT))
13188 return DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op,
13189 Flags);
13190 }
13191 }
13192
13193 if (VT.isScalableVector())
13194 reportFatalInternalError(
13195 reason: "Expanding reductions for scalable vectors is undefined.");
13196
13197 EVT EltVT = VT.getVectorElementType();
13198 unsigned NumElts = VT.getVectorNumElements();
13199
13200 SmallVector<SDValue, 8> Ops;
13201 DAG.ExtractVectorElements(Op, Args&: Ops, Start: 0, Count: NumElts);
13202
13203 SDValue Res = Ops[0];
13204 for (unsigned i = 1; i < NumElts; i++)
13205 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13206
13207 // Result type may be wider than element type.
13208 if (EltVT != Node->getValueType(ResNo: 0))
13209 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res);
13210 return Res;
13211}
13212
13213SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
13214 SDLoc dl(Node);
13215 SDValue AccOp = Node->getOperand(Num: 0);
13216 SDValue VecOp = Node->getOperand(Num: 1);
13217 SDNodeFlags Flags = Node->getFlags();
13218
13219 EVT VT = VecOp.getValueType();
13220 EVT EltVT = VT.getVectorElementType();
13221
13222 if (VT.isScalableVector())
13223 report_fatal_error(
13224 reason: "Expanding reductions for scalable vectors is undefined.");
13225
13226 unsigned NumElts = VT.getVectorNumElements();
13227
13228 SmallVector<SDValue, 8> Ops;
13229 DAG.ExtractVectorElements(Op: VecOp, Args&: Ops, Start: 0, Count: NumElts);
13230
13231 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13232
13233 SDValue Res = AccOp;
13234 for (unsigned i = 0; i < NumElts; i++)
13235 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13236
13237 return Res;
13238}
13239
13240bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
13241 SelectionDAG &DAG) const {
13242 EVT VT = Node->getValueType(ResNo: 0);
13243 SDLoc dl(Node);
13244 bool isSigned = Node->getOpcode() == ISD::SREM;
13245 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
13246 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
13247 SDValue Dividend = Node->getOperand(Num: 0);
13248 SDValue Divisor = Node->getOperand(Num: 1);
13249 if (isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
13250 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
13251 Result = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Dividend, N2: Divisor).getValue(R: 1);
13252 return true;
13253 }
13254 if (isOperationLegalOrCustom(Op: DivOpc, VT)) {
13255 // X % Y -> X-X/Y*Y
13256 SDValue Divide = DAG.getNode(Opcode: DivOpc, DL: dl, VT, N1: Dividend, N2: Divisor);
13257 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Divide, N2: Divisor);
13258 Result = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
13259 return true;
13260 }
13261 return false;
13262}
13263
13264SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
13265 SelectionDAG &DAG) const {
13266 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
13267 SDLoc dl(SDValue(Node, 0));
13268 SDValue Src = Node->getOperand(Num: 0);
13269
13270 // DstVT is the result type, while SatVT is the size to which we saturate
13271 EVT SrcVT = Src.getValueType();
13272 EVT DstVT = Node->getValueType(ResNo: 0);
13273
13274 EVT SatVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
13275 unsigned SatWidth = SatVT.getScalarSizeInBits();
13276 unsigned DstWidth = DstVT.getScalarSizeInBits();
13277 assert(SatWidth <= DstWidth &&
13278 "Expected saturation width smaller than result width");
13279
13280 // Determine minimum and maximum integer values and their corresponding
13281 // floating-point values.
13282 APInt MinInt, MaxInt;
13283 if (IsSigned) {
13284 MinInt = APInt::getSignedMinValue(numBits: SatWidth).sext(width: DstWidth);
13285 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth).sext(width: DstWidth);
13286 } else {
13287 MinInt = APInt::getMinValue(numBits: SatWidth).zext(width: DstWidth);
13288 MaxInt = APInt::getMaxValue(numBits: SatWidth).zext(width: DstWidth);
13289 }
13290
13291 // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
13292 // libcall emission cannot handle this. Large result types will fail.
13293 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
13294 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f32, Operand: Src);
13295 SrcVT = Src.getValueType();
13296 }
13297
13298 const fltSemantics &Sem = SrcVT.getFltSemantics();
13299 APFloat MinFloat(Sem);
13300 APFloat MaxFloat(Sem);
13301
13302 APFloat::opStatus MinStatus =
13303 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
13304 APFloat::opStatus MaxStatus =
13305 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
13306 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
13307 !(MaxStatus & APFloat::opStatus::opInexact);
13308
13309 SDValue MinFloatNode = DAG.getConstantFP(Val: MinFloat, DL: dl, VT: SrcVT);
13310 SDValue MaxFloatNode = DAG.getConstantFP(Val: MaxFloat, DL: dl, VT: SrcVT);
13311
13312 // If the integer bounds are exactly representable as floats and min/max are
13313 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
13314 // of comparisons and selects.
13315 auto EmitMinMax = [&](unsigned MinOpcode, unsigned MaxOpcode,
13316 bool MayPropagateNaN) {
13317 bool MinMaxLegal = isOperationLegalOrCustom(Op: MinOpcode, VT: SrcVT) &&
13318 isOperationLegalOrCustom(Op: MaxOpcode, VT: SrcVT);
13319 if (!MinMaxLegal)
13320 return SDValue();
13321
13322 SDValue Clamped = Src;
13323
13324 // Clamp Src by MinFloat from below. If !MayPropagateNaN and Src is NaN
13325 // then the result is MinFloat.
13326 Clamped = DAG.getNode(Opcode: MaxOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MinFloatNode);
13327 // Clamp by MaxFloat from above. If !MayPropagateNaN then NaN cannot occur.
13328 Clamped = DAG.getNode(Opcode: MinOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MaxFloatNode);
13329 // Convert clamped value to integer.
13330 SDValue FpToInt = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
13331 DL: dl, VT: DstVT, Operand: Clamped);
13332
13333 // If !MayPropagateNan and the conversion is unsigned case we're done,
13334 // because we mapped NaN to MinFloat, which will cast to zero.
13335 if (!MayPropagateNaN && !IsSigned)
13336 return FpToInt;
13337
13338 // Otherwise, select 0 if Src is NaN.
13339 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13340 EVT SetCCVT =
13341 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13342 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13343 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: FpToInt);
13344 };
13345 if (AreExactFloatBounds) {
13346 if (SDValue Res = EmitMinMax(ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
13347 /*MayPropagateNaN=*/false))
13348 return Res;
13349 // These may propagate NaN for sNaN operands.
13350 if (SDValue Res =
13351 EmitMinMax(ISD::FMINNUM, ISD::FMAXNUM, /*MayPropagateNaN=*/true))
13352 return Res;
13353 // These always propagate NaN.
13354 if (SDValue Res =
13355 EmitMinMax(ISD::FMINIMUM, ISD::FMAXIMUM, /*MayPropagateNaN=*/true))
13356 return Res;
13357 }
13358
13359 SDValue MinIntNode = DAG.getConstant(Val: MinInt, DL: dl, VT: DstVT);
13360 SDValue MaxIntNode = DAG.getConstant(Val: MaxInt, DL: dl, VT: DstVT);
13361
13362 // Result of direct conversion. The assumption here is that the operation is
13363 // non-trapping and it's fine to apply it to an out-of-range value if we
13364 // select it away later.
13365 SDValue FpToInt =
13366 DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL: dl, VT: DstVT, Operand: Src);
13367
13368 SDValue Select = FpToInt;
13369
13370 EVT SetCCVT =
13371 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13372
13373 // If Src ULT MinFloat, select MinInt. In particular, this also selects
13374 // MinInt if Src is NaN.
13375 SDValue ULT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MinFloatNode, Cond: ISD::SETULT);
13376 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: ULT, LHS: MinIntNode, RHS: Select);
13377 // If Src OGT MaxFloat, select MaxInt.
13378 SDValue OGT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MaxFloatNode, Cond: ISD::SETOGT);
13379 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: OGT, LHS: MaxIntNode, RHS: Select);
13380
13381 // In the unsigned case we are done, because we mapped NaN to MinInt, which
13382 // is already zero.
13383 if (!IsSigned)
13384 return Select;
13385
13386 // Otherwise, select 0 if Src is NaN.
13387 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13388 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13389 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: Select);
13390}
13391
13392SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op,
13393 const SDLoc &dl,
13394 SelectionDAG &DAG) const {
13395 EVT OperandVT = Op.getValueType();
13396 if (OperandVT.getScalarType() == ResultVT.getScalarType())
13397 return Op;
13398 EVT ResultIntVT = ResultVT.changeTypeToInteger();
13399 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13400 // can induce double-rounding which may alter the results. We can
13401 // correct for this using a trick explained in: Boldo, Sylvie, and
13402 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13403 // World Congress. 2005.
13404 SDValue Narrow = DAG.getFPExtendOrRound(Op, DL: dl, VT: ResultVT);
13405 SDValue NarrowAsWide = DAG.getFPExtendOrRound(Op: Narrow, DL: dl, VT: OperandVT);
13406
13407 // We can keep the narrow value as-is if narrowing was exact (no
13408 // rounding error), the wide value was NaN (the narrow value is also
13409 // NaN and should be preserved) or if we rounded to the odd value.
13410 SDValue NarrowBits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultIntVT, Operand: Narrow);
13411 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResultIntVT);
13412 SDValue NegativeOne = DAG.getAllOnesConstant(DL: dl, VT: ResultIntVT);
13413 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: One);
13414 EVT ResultIntVTCCVT = getSetCCResultType(
13415 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: And.getValueType());
13416 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ResultIntVT);
13417 // The result is already odd so we don't need to do anything.
13418 SDValue AlreadyOdd = DAG.getSetCC(DL: dl, VT: ResultIntVTCCVT, LHS: And, RHS: Zero, Cond: ISD::SETNE);
13419
13420 EVT WideSetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
13421 VT: Op.getValueType());
13422 // We keep results which are exact, odd or NaN.
13423 SDValue KeepNarrow =
13424 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: Op, RHS: NarrowAsWide, Cond: ISD::SETUEQ);
13425 KeepNarrow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: WideSetCCVT, N1: KeepNarrow, N2: AlreadyOdd);
13426 // We morally performed a round-down if AbsNarrow is smaller than
13427 // AbsWide.
13428 SDValue AbsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: Op);
13429 SDValue AbsNarrowAsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: NarrowAsWide);
13430 SDValue NarrowIsRd =
13431 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: AbsWide, RHS: AbsNarrowAsWide, Cond: ISD::SETOGT);
13432 // If the narrow value is odd or exact, pick it.
13433 // Otherwise, narrow is even and corresponds to either the rounded-up
13434 // or rounded-down value. If narrow is the rounded-down value, we want
13435 // the rounded-up value as it will be odd.
13436 SDValue Adjust = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: NarrowIsRd, LHS: One, RHS: NegativeOne);
13437 SDValue Adjusted = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: Adjust);
13438 Op = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: KeepNarrow, LHS: NarrowBits, RHS: Adjusted);
13439 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultVT, Operand: Op);
13440}
13441
13442SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const {
13443 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
13444 SDValue Op = Node->getOperand(Num: 0);
13445 EVT VT = Node->getValueType(ResNo: 0);
13446 SDLoc dl(Node);
13447 if (VT.getScalarType() == MVT::bf16) {
13448 if (Node->getConstantOperandVal(Num: 1) == 1) {
13449 return DAG.getNode(Opcode: ISD::FP_TO_BF16, DL: dl, VT, Operand: Node->getOperand(Num: 0));
13450 }
13451 EVT OperandVT = Op.getValueType();
13452 SDValue IsNaN = DAG.getSetCC(
13453 DL: dl,
13454 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT),
13455 LHS: Op, RHS: Op, Cond: ISD::SETUO);
13456
13457 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13458 // can induce double-rounding which may alter the results. We can
13459 // correct for this using a trick explained in: Boldo, Sylvie, and
13460 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13461 // World Congress. 2005.
13462 EVT F32 = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
13463 EVT I32 = F32.changeTypeToInteger();
13464 Op = expandRoundInexactToOdd(ResultVT: F32, Op, dl, DAG);
13465 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: I32, Operand: Op);
13466
13467 // Conversions should set NaN's quiet bit. This also prevents NaNs from
13468 // turning into infinities.
13469 SDValue NaN =
13470 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: I32, N1: Op, N2: DAG.getConstant(Val: 0x400000, DL: dl, VT: I32));
13471
13472 // Factor in the contribution of the low 16 bits.
13473 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: I32);
13474 SDValue Lsb = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13475 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13476 Lsb = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: I32, N1: Lsb, N2: One);
13477 SDValue RoundingBias =
13478 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Lsb, N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: I32));
13479 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Op, N2: RoundingBias);
13480
13481 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
13482 // 0x80000000.
13483 Op = DAG.getSelect(DL: dl, VT: I32, Cond: IsNaN, LHS: NaN, RHS: Add);
13484
13485 // Now that we have rounded, shift the bits into position.
13486 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13487 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13488 EVT I16 = I32.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i16);
13489 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: I16, Operand: Op);
13490 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
13491 }
13492 return SDValue();
13493}
13494
13495SDValue TargetLowering::expandVectorSplice(SDNode *Node,
13496 SelectionDAG &DAG) const {
13497 assert((Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT ||
13498 Node->getOpcode() == ISD::VECTOR_SPLICE_RIGHT) &&
13499 "Unexpected opcode!");
13500 assert((Node->getValueType(0).isScalableVector() ||
13501 !isa<ConstantSDNode>(Node->getOperand(2))) &&
13502 "Fixed length vector types with constant offsets expected to use "
13503 "SHUFFLE_VECTOR!");
13504
13505 EVT VT = Node->getValueType(ResNo: 0);
13506 SDValue V1 = Node->getOperand(Num: 0);
13507 SDValue V2 = Node->getOperand(Num: 1);
13508 SDValue Offset = Node->getOperand(Num: 2);
13509 SDLoc DL(Node);
13510
13511 // Expand through memory thusly:
13512 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
13513 // Store V1, Ptr
13514 // Store V2, Ptr + sizeof(V1)
13515 // if (VECTOR_SPLICE_LEFT)
13516 // Ptr = Ptr + (Offset * sizeof(VT.Elt))
13517 // else
13518 // Ptr = Ptr + sizeof(V1) - (Offset * size(VT.Elt))
13519 // Res = Load Ptr
13520
13521 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
13522
13523 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
13524 EC: VT.getVectorElementCount() * 2);
13525 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
13526 EVT PtrVT = StackPtr.getValueType();
13527 auto &MF = DAG.getMachineFunction();
13528 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13529 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13530
13531 // Store the lo part of CONCAT_VECTORS(V1, V2)
13532 SDValue StoreV1 =
13533 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, PtrInfo, Alignment);
13534 // Store the hi part of CONCAT_VECTORS(V1, V2)
13535 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
13536 SDValue StackPtr2 = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: VTBytes);
13537 SDValue StoreV2 =
13538 DAG.getStore(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, PtrInfo, Alignment);
13539
13540 // NOTE: TrailingBytes must be clamped so as not to read outside of V1:V2.
13541 SDValue EltByteSize =
13542 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
13543 Offset = DAG.getZExtOrTrunc(Op: Offset, DL, VT: PtrVT);
13544 SDValue TrailingBytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: Offset, N2: EltByteSize);
13545
13546 TrailingBytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: VTBytes);
13547
13548 if (Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT)
13549 StackPtr = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TrailingBytes, DL);
13550 else
13551 StackPtr = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
13552
13553 // Load the spliced result
13554 return DAG.getLoad(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr,
13555 PtrInfo: MachinePointerInfo::getUnknownStack(MF), Alignment);
13556}
13557
13558SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
13559 SelectionDAG &DAG) const {
13560 SDLoc DL(Node);
13561 SDValue Vec = Node->getOperand(Num: 0);
13562 SDValue Mask = Node->getOperand(Num: 1);
13563 SDValue Passthru = Node->getOperand(Num: 2);
13564
13565 EVT VecVT = Vec.getValueType();
13566 EVT ScalarVT = VecVT.getScalarType();
13567 EVT MaskVT = Mask.getValueType();
13568 EVT MaskScalarVT = MaskVT.getScalarType();
13569
13570 // Needs to be handled by targets that have scalable vector types.
13571 if (VecVT.isScalableVector())
13572 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
13573
13574 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
13575 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment);
13576 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13577 MachinePointerInfo PtrInfo =
13578 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
13579
13580 MVT PositionVT = getVectorIdxTy(DL: DAG.getDataLayout());
13581 SDValue Chain = DAG.getEntryNode();
13582 SDValue OutPos = DAG.getConstant(Val: 0, DL, VT: PositionVT);
13583
13584 bool HasPassthru = !Passthru.isUndef();
13585
13586 // If we have a passthru vector, store it on the stack, overwrite the matching
13587 // positions and then re-write the last element that was potentially
13588 // overwritten even though mask[i] = false.
13589 if (HasPassthru)
13590 Chain = DAG.getStore(Chain, dl: DL, Val: Passthru, Ptr: StackPtr, PtrInfo, Alignment);
13591
13592 SDValue LastWriteVal;
13593 APInt PassthruSplatVal;
13594 bool IsSplatPassthru =
13595 ISD::isConstantSplatVector(N: Passthru.getNode(), SplatValue&: PassthruSplatVal);
13596
13597 if (IsSplatPassthru) {
13598 // As we do not know which position we wrote to last, we cannot simply
13599 // access that index from the passthru vector. So we first check if passthru
13600 // is a splat vector, to use any element ...
13601 LastWriteVal = DAG.getConstant(Val: PassthruSplatVal, DL, VT: ScalarVT);
13602 } else if (HasPassthru) {
13603 // ... if it is not a splat vector, we need to get the passthru value at
13604 // position = popcount(mask) and re-load it from the stack before it is
13605 // overwritten in the loop below.
13606 EVT PopcountVT = ScalarVT.changeTypeToInteger();
13607 SDValue Popcount = DAG.getNode(
13608 Opcode: ISD::TRUNCATE, DL,
13609 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1), Operand: Mask);
13610 Popcount = DAG.getNode(
13611 Opcode: ISD::ZERO_EXTEND, DL,
13612 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: PopcountVT),
13613 Operand: Popcount);
13614 Popcount = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: PopcountVT, Operand: Popcount);
13615 SDValue LastElmtPtr =
13616 getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Popcount);
13617 LastWriteVal = DAG.getLoad(
13618 VT: ScalarVT, dl: DL, Chain, Ptr: LastElmtPtr,
13619 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13620 Chain = LastWriteVal.getValue(R: 1);
13621 }
13622
13623 unsigned NumElms = VecVT.getVectorNumElements();
13624 for (unsigned I = 0; I < NumElms; I++) {
13625 SDValue ValI = DAG.getExtractVectorElt(DL, VT: ScalarVT, Vec, Idx: I);
13626 SDValue OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
13627 Chain = DAG.getStore(
13628 Chain, dl: DL, Val: ValI, Ptr: OutPtr,
13629 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13630
13631 // Get the mask value and add it to the current output position. This
13632 // either increments by 1 if MaskI is true or adds 0 otherwise.
13633 // Freeze in case we have poison/undef mask entries.
13634 SDValue MaskI = DAG.getExtractVectorElt(DL, VT: MaskScalarVT, Vec: Mask, Idx: I);
13635 MaskI = DAG.getFreeze(V: MaskI);
13636 MaskI = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: MaskI);
13637 MaskI = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PositionVT, Operand: MaskI);
13638 OutPos = DAG.getNode(Opcode: ISD::ADD, DL, VT: PositionVT, N1: OutPos, N2: MaskI);
13639
13640 if (HasPassthru && I == NumElms - 1) {
13641 SDValue EndOfVector =
13642 DAG.getConstant(Val: VecVT.getVectorNumElements() - 1, DL, VT: PositionVT);
13643 SDValue AllLanesSelected =
13644 DAG.getSetCC(DL, VT: MVT::i1, LHS: OutPos, RHS: EndOfVector, Cond: ISD::CondCode::SETUGT);
13645 OutPos = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PositionVT, N1: OutPos, N2: EndOfVector);
13646 OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
13647
13648 // Re-write the last ValI if all lanes were selected. Otherwise,
13649 // overwrite the last write it with the passthru value.
13650 LastWriteVal = DAG.getSelect(DL, VT: ScalarVT, Cond: AllLanesSelected, LHS: ValI,
13651 RHS: LastWriteVal, Flags: SDNodeFlags::Unpredictable);
13652 Chain = DAG.getStore(
13653 Chain, dl: DL, Val: LastWriteVal, Ptr: OutPtr,
13654 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13655 }
13656 }
13657
13658 return DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo, Alignment);
13659}
13660
13661SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
13662 SDLoc DL(Node);
13663 EVT VT = Node->getValueType(ResNo: 0);
13664 SDValue Op = Node->getOperand(Num: 0);
13665 ElementCount EC = Op.getValueType().getVectorElementCount();
13666
13667 bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
13668 auto [Mask, StepVec] = getLegalMaskAndStepVector(Mask: Op, ZeroIsPoison, DL, DAG);
13669
13670 // No legal step vector: split mask in half and recombine results.
13671 // LoNumElts uses the non-poison CTTZ_ELTS so its result is well-defined
13672 // (== LoNumElts when no active lane), allowing the SETNE comparison.
13673 // Result: (ResLo != LoNumElts) ? ResLo : (LoNumElts + ResHi)
13674 if (!StepVec) {
13675 EVT ResVT = Node->getValueType(ResNo: 0);
13676 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Op, DL);
13677 SDValue LoNumElts = DAG.getElementCount(
13678 DL, VT: ResVT, EC: MaskLo.getValueType().getVectorElementCount());
13679 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: MaskLo);
13680 SDValue ResHi = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Operand: MaskHi);
13681 SDValue ResLoNotNumElts = DAG.getSetCC(
13682 DL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ResVT),
13683 LHS: ResLo, RHS: LoNumElts, Cond: ISD::SETNE);
13684 // Per LangRef, ResVT must be wide enough to hold the total element count,
13685 // so the sum cannot wrap as an unsigned add. NSW is not guaranteed since
13686 // the count is only required to fit unsigned.
13687 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: LoNumElts, N2: ResHi,
13688 Flags: SDNodeFlags::NoUnsignedWrap);
13689 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotNumElts, LHS: ResLo, RHS: Sum);
13690 }
13691
13692 EVT StepVecVT = StepVec.getValueType();
13693 EVT StepVT = StepVecVT.getVectorElementType();
13694
13695 // Promote the scalar result type early to avoid redundant zexts.
13696 if (getTypeAction(VT: StepVT.getSimpleVT()) == TypePromoteInteger)
13697 StepVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVT);
13698
13699 SDValue VL = DAG.getElementCount(DL, VT: StepVT, EC);
13700 SDValue SplatVL = DAG.getSplat(VT: StepVecVT, DL, Op: VL);
13701 StepVec = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVecVT, N1: SplatVL, N2: StepVec);
13702 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
13703 SDValue Select = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
13704 SDValue Max = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL,
13705 VT: StepVecVT.getVectorElementType(), Operand: Select);
13706 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVT, N1: VL,
13707 N2: DAG.getZExtOrTrunc(Op: Max, DL, VT: StepVT));
13708
13709 return DAG.getZExtOrTrunc(Op: Sub, DL, VT);
13710}
13711
13712SDValue TargetLowering::expandVectorMatch(SDNode *N, SelectionDAG &DAG) const {
13713 SDLoc DL(N);
13714 SDValue Source = N->getOperand(Num: 0);
13715 SDValue Needle = N->getOperand(Num: 1);
13716 SDValue Mask = N->getOperand(Num: 2);
13717 EVT SourceVT = Source.getValueType();
13718 EVT NeedleVT = Needle.getValueType();
13719 EVT ResVT = N->getValueType(ResNo: 0);
13720 EVT CmpVT =
13721 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SourceVT);
13722
13723 assert(NeedleVT.isFixedLengthVector() && "Needle must be a fixed vector");
13724
13725 SDValue Ret = DAG.getConstant(Val: 0, DL, VT: CmpVT);
13726 EVT NeedleEltVT = NeedleVT.getVectorElementType();
13727 for (unsigned I = 0, E = NeedleVT.getVectorNumElements(); I != E; ++I) {
13728 SDValue Splat;
13729 if (NeedleVT == SourceVT) {
13730 // Prefer a shuffle over scalar extracts + splat for fixed vectors.
13731 Splat = DAG.getVectorShuffle(
13732 VT: SourceVT, dl: DL, N1: Needle, N2: DAG.getUNDEF(VT: SourceVT),
13733 Mask: SmallVector<int>(NeedleVT.getVectorNumElements(), I));
13734 } else {
13735 SDValue NeedleElt = DAG.getExtractVectorElt(DL, VT: NeedleEltVT, Vec: Needle, Idx: I);
13736 Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: SourceVT, Operand: NeedleElt);
13737 }
13738 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: Source, RHS: Splat, Cond: ISD::SETEQ);
13739 Ret = DAG.getNode(Opcode: ISD::OR, DL, VT: CmpVT, N1: Ret, N2: Cmp);
13740 }
13741
13742 EVT UseVT = ResVT;
13743 // If the result is immediately truncated, only extend to that type (to avoid
13744 // unnecessary sign/zero extends).
13745 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TRUNCATE)
13746 UseVT = N->user_begin()->getValueType(ResNo: 0);
13747
13748 Mask = DAG.getBoolExtOrTrunc(Op: Mask, SL: DL, VT: UseVT, OpVT: Mask.getValueType());
13749 Ret = DAG.getBoolExtOrTrunc(Op: Ret, SL: DL, VT: UseVT, OpVT: Ret.getValueType());
13750
13751 Ret = DAG.getNode(Opcode: ISD::AND, DL, VT: UseVT, N1: Ret, N2: Mask);
13752 if (UseVT != ResVT)
13753 Ret = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ResVT, Operand: Ret);
13754 return Ret;
13755}
13756
13757SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
13758 SelectionDAG &DAG) const {
13759 SDLoc DL(N);
13760 SDValue Acc = N->getOperand(Num: 0);
13761 SDValue MulLHS = N->getOperand(Num: 1);
13762 SDValue MulRHS = N->getOperand(Num: 2);
13763 EVT AccVT = Acc.getValueType();
13764 EVT MulOpVT = MulLHS.getValueType();
13765
13766 EVT ExtMulOpVT =
13767 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccVT.getVectorElementType(),
13768 EC: MulOpVT.getVectorElementCount());
13769
13770 unsigned ExtOpcLHS, ExtOpcRHS;
13771 switch (N->getOpcode()) {
13772 default:
13773 llvm_unreachable("Unexpected opcode");
13774 case ISD::PARTIAL_REDUCE_UMLA:
13775 ExtOpcLHS = ExtOpcRHS = ISD::ZERO_EXTEND;
13776 break;
13777 case ISD::PARTIAL_REDUCE_SMLA:
13778 ExtOpcLHS = ExtOpcRHS = ISD::SIGN_EXTEND;
13779 break;
13780 case ISD::PARTIAL_REDUCE_SUMLA:
13781 ExtOpcLHS = ISD::SIGN_EXTEND;
13782 ExtOpcRHS = ISD::ZERO_EXTEND;
13783 break;
13784 case ISD::PARTIAL_REDUCE_FMLA:
13785 ExtOpcLHS = ExtOpcRHS = ISD::FP_EXTEND;
13786 break;
13787 }
13788
13789 // A wide partial reduction is built from a ladder of narrower ones, a rung
13790 // at a time, each halving the element count and doubling the width.
13791 unsigned Opc = N->getOpcode();
13792 ElementCount MulEC = MulOpVT.getVectorElementCount();
13793 ElementCount AccEC = AccVT.getVectorElementCount();
13794 unsigned CountRatio =
13795 MulEC.hasKnownScalarFactor(RHS: AccEC) ? MulEC.getKnownScalarFactor(RHS: AccEC) : 0;
13796 unsigned WidthRatio =
13797 AccVT.getScalarSizeInBits() / MulOpVT.getScalarSizeInBits();
13798 if (Opc != ISD::PARTIAL_REDUCE_FMLA && CountRatio > 2 && WidthRatio >= 2) {
13799 LLVMContext &Ctx = *DAG.getContext();
13800 EVT ProdVT = MulOpVT.widenIntegerVectorElementType(Context&: Ctx);
13801
13802 // A pure reduction peels one rung and re-enters.
13803 if (llvm::isOneOrOneSplat(V: MulRHS)) {
13804 EVT RungVT = ProdVT.getHalfNumVectorElementsVT(Context&: Ctx);
13805 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc,
13806 N2: DAG.getNode(Opcode: Opc, DL, VT: RungVT,
13807 N1: DAG.getConstant(Val: 0, DL, VT: RungVT), N2: MulLHS,
13808 N3: MulRHS),
13809 N3: DAG.getConstant(Val: 1, DL, VT: RungVT));
13810 }
13811
13812 // A multiply widens the products by one rung, which legalizes back into a
13813 // widening multiply per half, and the ladder re-enters as a plain sum.
13814 SDValue Prod = DAG.getNode(Opcode: ISD::MUL, DL, VT: ProdVT,
13815 N1: DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ProdVT, Operand: MulLHS),
13816 N2: DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ProdVT, Operand: MulRHS));
13817 auto [Lo, Hi] = DAG.SplitVector(N: Prod, DL);
13818 SDValue One = DAG.getConstant(Val: 1, DL, VT: Lo.getValueType());
13819
13820 // The halves meet at the narrowest rung, so the accumulator is added once.
13821 EVT MidVT = Lo.getValueType()
13822 .widenIntegerVectorElementType(Context&: Ctx)
13823 .getHalfNumVectorElementsVT(Context&: Ctx);
13824 if (ElementCount::isKnownLE(LHS: MidVT.getVectorElementCount(), RHS: AccEC))
13825 return DAG.getNode(Opcode: Opc, DL, VT: AccVT,
13826 N1: DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Lo, N3: One), N2: Hi, N3: One);
13827 SDValue Mid =
13828 DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: DAG.getConstant(Val: 0, DL, VT: MidVT), N2: Lo, N3: One);
13829 Mid = DAG.getNode(Opcode: Opc, DL, VT: MidVT, N1: Mid, N2: Hi, N3: One);
13830 return DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: Mid, N3: DAG.getConstant(Val: 1, DL, VT: MidVT));
13831 }
13832
13833 if (ExtMulOpVT != MulOpVT) {
13834 MulLHS = DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ExtMulOpVT, Operand: MulLHS);
13835 MulRHS = DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ExtMulOpVT, Operand: MulRHS);
13836 }
13837 SDValue Input = MulLHS;
13838 if (N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA) {
13839 if (!llvm::isOneOrOneSplatFP(V: MulRHS))
13840 Input = DAG.getNode(Opcode: ISD::FMUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
13841 } else if (!llvm::isOneOrOneSplat(V: MulRHS)) {
13842 Input = DAG.getNode(Opcode: ISD::MUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
13843 }
13844
13845 unsigned Stride = AccVT.getVectorMinNumElements();
13846 unsigned ScaleFactor = MulOpVT.getVectorMinNumElements() / Stride;
13847
13848 // Collect all of the subvectors
13849 std::deque<SDValue> Subvectors = {Acc};
13850 for (unsigned I = 0; I < ScaleFactor; I++)
13851 Subvectors.push_back(x: DAG.getExtractSubvector(DL, VT: AccVT, Vec: Input, Idx: I * Stride));
13852
13853 unsigned FlatNode =
13854 N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA ? ISD::FADD : ISD::ADD;
13855
13856 // Flatten the subvector tree
13857 while (Subvectors.size() > 1) {
13858 Subvectors.push_back(
13859 x: DAG.getNode(Opcode: FlatNode, DL, VT: AccVT, Ops: {Subvectors[0], Subvectors[1]}));
13860 Subvectors.pop_front();
13861 Subvectors.pop_front();
13862 }
13863
13864 assert(Subvectors.size() == 1 &&
13865 "There should only be one subvector after tree flattening");
13866
13867 return Subvectors[0];
13868}
13869
13870/// Given a store node \p StoreNode, return true if it is safe to fold that node
13871/// into \p FPNode, which expands to a library call with output pointers.
13872static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
13873 SDNode *FPNode) {
13874 SmallVector<const SDNode *, 8> Worklist;
13875 SmallVector<const SDNode *, 8> DeferredNodes;
13876 SmallPtrSet<const SDNode *, 16> Visited;
13877
13878 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
13879 for (SDValue Op : StoreNode->ops())
13880 if (Op.getNode() != FPNode)
13881 Worklist.push_back(Elt: Op.getNode());
13882
13883 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
13884 while (!Worklist.empty()) {
13885 const SDNode *Node = Worklist.pop_back_val();
13886 auto [_, Inserted] = Visited.insert(Ptr: Node);
13887 if (!Inserted)
13888 continue;
13889
13890 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
13891 return false;
13892
13893 // Reached the FPNode (would result in a cycle).
13894 // OR Reached CALLSEQ_START (would result in nested call sequences).
13895 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
13896 return false;
13897
13898 if (Node->getOpcode() == ISD::CALLSEQ_END) {
13899 // Defer looking into call sequences (so we can check we're outside one).
13900 // We still need to look through these for the predecessor check.
13901 DeferredNodes.push_back(Elt: Node);
13902 continue;
13903 }
13904
13905 for (SDValue Op : Node->ops())
13906 Worklist.push_back(Elt: Op.getNode());
13907 }
13908
13909 // True if we're outside a call sequence and don't have the FPNode as a
13910 // predecessor. No cycles or nested call sequences possible.
13911 return !SDNode::hasPredecessorHelper(N: FPNode, Visited, Worklist&: DeferredNodes,
13912 MaxSteps);
13913}
13914
13915bool TargetLowering::expandMultipleResultFPLibCall(
13916 SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
13917 SmallVectorImpl<SDValue> &Results,
13918 std::optional<unsigned> CallRetResNo) const {
13919 if (LC == RTLIB::UNKNOWN_LIBCALL)
13920 return false;
13921
13922 RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(Call: LC);
13923 if (LibcallImpl == RTLIB::Unsupported)
13924 return false;
13925
13926 LLVMContext &Ctx = *DAG.getContext();
13927 EVT VT = Node->getValueType(ResNo: 0);
13928 unsigned NumResults = Node->getNumValues();
13929
13930 // Find users of the node that store the results (and share input chains). The
13931 // destination pointers can be used instead of creating stack allocations.
13932 SDValue StoresInChain;
13933 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
13934 for (SDNode *User : Node->users()) {
13935 if (!ISD::isNormalStore(N: User))
13936 continue;
13937 auto *ST = cast<StoreSDNode>(Val: User);
13938 SDValue StoreValue = ST->getValue();
13939 unsigned ResNo = StoreValue.getResNo();
13940 // Ensure the store corresponds to an output pointer.
13941 if (CallRetResNo == ResNo)
13942 continue;
13943 // Ensure the store to the default address space and not atomic or volatile.
13944 if (!ST->isSimple() || ST->getAddressSpace() != 0)
13945 continue;
13946 // Ensure all store chains are the same (so they don't alias).
13947 if (StoresInChain && ST->getChain() != StoresInChain)
13948 continue;
13949 // Ensure the store is properly aligned.
13950 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Context&: Ctx);
13951 if (ST->getAlign() <
13952 DAG.getDataLayout().getABITypeAlign(Ty: StoreType->getScalarType()))
13953 continue;
13954 // Avoid:
13955 // 1. Creating cyclic dependencies.
13956 // 2. Expanding the node to a call within a call sequence.
13957 if (!canFoldStoreIntoLibCallOutputPointers(StoreNode: ST, FPNode: Node))
13958 continue;
13959 ResultStores[ResNo] = ST;
13960 StoresInChain = ST->getChain();
13961 }
13962
13963 ArgListTy Args;
13964
13965 // Pass the arguments.
13966 for (const SDValue &Op : Node->op_values()) {
13967 EVT ArgVT = Op.getValueType();
13968 Type *ArgTy = ArgVT.getTypeForEVT(Context&: Ctx);
13969 Args.emplace_back(args: Op, args&: ArgTy);
13970 }
13971
13972 // Pass the output pointers.
13973 SmallVector<SDValue, 2> ResultPtrs(NumResults);
13974 Type *PointerTy = PointerType::getUnqual(C&: Ctx);
13975 for (auto [ResNo, ST] : llvm::enumerate(First&: ResultStores)) {
13976 if (ResNo == CallRetResNo)
13977 continue;
13978 EVT ResVT = Node->getValueType(ResNo);
13979 SDValue ResultPtr = ST ? ST->getBasePtr() : DAG.CreateStackTemporary(VT: ResVT);
13980 ResultPtrs[ResNo] = ResultPtr;
13981 Args.emplace_back(args&: ResultPtr, args&: PointerTy);
13982 }
13983
13984 SDLoc DL(Node);
13985
13986 if (RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(Impl: LibcallImpl)) {
13987 // Pass the vector mask (if required).
13988 EVT MaskVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
13989 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT);
13990 Args.emplace_back(args&: Mask, args: MaskVT.getTypeForEVT(Context&: Ctx));
13991 }
13992
13993 Type *RetType = CallRetResNo.has_value()
13994 ? Node->getValueType(ResNo: *CallRetResNo).getTypeForEVT(Context&: Ctx)
13995 : Type::getVoidTy(C&: Ctx);
13996 SDValue InChain = StoresInChain ? StoresInChain : DAG.getEntryNode();
13997 SDValue Callee =
13998 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
13999 TargetLowering::CallLoweringInfo CLI(DAG);
14000 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
14001 CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetType, Target: Callee, ArgsList: std::move(Args));
14002
14003 auto [Call, CallChain] = LowerCallTo(CLI);
14004
14005 for (auto [ResNo, ResultPtr] : llvm::enumerate(First&: ResultPtrs)) {
14006 if (ResNo == CallRetResNo) {
14007 Results.push_back(Elt: Call);
14008 continue;
14009 }
14010 MachinePointerInfo PtrInfo;
14011 SDValue LoadResult = DAG.getLoad(VT: Node->getValueType(ResNo), dl: DL, Chain: CallChain,
14012 Ptr: ResultPtr, PtrInfo);
14013 SDValue OutChain = LoadResult.getValue(R: 1);
14014
14015 if (StoreSDNode *ST = ResultStores[ResNo]) {
14016 // Replace store with the library call.
14017 DAG.ReplaceAllUsesOfValueWith(From: SDValue(ST, 0), To: OutChain);
14018 PtrInfo = ST->getPointerInfo();
14019 } else {
14020 PtrInfo = MachinePointerInfo::getFixedStack(
14021 MF&: DAG.getMachineFunction(),
14022 FI: cast<FrameIndexSDNode>(Val&: ResultPtr)->getIndex());
14023 }
14024
14025 Results.push_back(Elt: LoadResult);
14026 }
14027
14028 return true;
14029}
14030
14031bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
14032 SDValue &LHS, SDValue &RHS,
14033 SDValue &CC, bool &NeedInvert,
14034 const SDLoc &dl, SDValue &Chain,
14035 bool IsSignaling) const {
14036 MVT OpVT = LHS.getSimpleValueType();
14037 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
14038 NeedInvert = false;
14039 switch (getCondCodeAction(CC: CCCode, VT: OpVT)) {
14040 default:
14041 llvm_unreachable("Unknown condition code action!");
14042 case TargetLowering::Legal:
14043 // Nothing to do.
14044 break;
14045 case TargetLowering::Expand: {
14046 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(Operation: CCCode);
14047 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14048 std::swap(a&: LHS, b&: RHS);
14049 CC = DAG.getCondCode(Cond: InvCC);
14050 return true;
14051 }
14052 // Swapping operands didn't work. Try inverting the condition.
14053 bool NeedSwap = false;
14054 InvCC = getSetCCInverse(Operation: CCCode, Type: OpVT);
14055 if (!isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14056 // If inverting the condition is not enough, try swapping operands
14057 // on top of it.
14058 InvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
14059 NeedSwap = true;
14060 }
14061 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14062 CC = DAG.getCondCode(Cond: InvCC);
14063 NeedInvert = true;
14064 if (NeedSwap)
14065 std::swap(a&: LHS, b&: RHS);
14066 return true;
14067 }
14068
14069 // Special case: expand i1 comparisons using logical operations.
14070 if (OpVT == MVT::i1) {
14071 SDValue Ret;
14072 switch (CCCode) {
14073 default:
14074 llvm_unreachable("Unknown integer setcc!");
14075 case ISD::SETEQ: // X == Y --> ~(X ^ Y)
14076 Ret = DAG.getNOT(DL: dl, Val: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS),
14077 VT: MVT::i1);
14078 break;
14079 case ISD::SETNE: // X != Y --> (X ^ Y)
14080 Ret = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS);
14081 break;
14082 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
14083 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
14084 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: RHS,
14085 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14086 break;
14087 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
14088 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
14089 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: LHS,
14090 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14091 break;
14092 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
14093 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
14094 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: RHS,
14095 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14096 break;
14097 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
14098 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
14099 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: LHS,
14100 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14101 break;
14102 }
14103
14104 LHS = DAG.getZExtOrTrunc(Op: Ret, DL: dl, VT);
14105 RHS = SDValue();
14106 CC = SDValue();
14107 return true;
14108 }
14109
14110 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
14111 unsigned Opc = 0;
14112 switch (CCCode) {
14113 default:
14114 llvm_unreachable("Don't know how to expand this condition!");
14115 case ISD::SETUO:
14116 if (isCondCodeLegal(CC: ISD::SETUNE, VT: OpVT)) {
14117 CC1 = ISD::SETUNE;
14118 CC2 = ISD::SETUNE;
14119 Opc = ISD::OR;
14120 break;
14121 }
14122 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14123 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
14124 NeedInvert = true;
14125 [[fallthrough]];
14126 case ISD::SETO:
14127 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14128 "If SETO is expanded, SETOEQ must be legal!");
14129 CC1 = ISD::SETOEQ;
14130 CC2 = ISD::SETOEQ;
14131 Opc = ISD::AND;
14132 break;
14133 case ISD::SETONE:
14134 case ISD::SETUEQ:
14135 // If the SETUO or SETO CC isn't legal, we might be able to use
14136 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
14137 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
14138 // the operands.
14139 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14140 if (!isCondCodeLegal(CC: CC2, VT: OpVT) && (isCondCodeLegal(CC: ISD::SETOGT, VT: OpVT) ||
14141 isCondCodeLegal(CC: ISD::SETOLT, VT: OpVT))) {
14142 CC1 = ISD::SETOGT;
14143 CC2 = ISD::SETOLT;
14144 Opc = ISD::OR;
14145 NeedInvert = ((unsigned)CCCode & 0x8U);
14146 break;
14147 }
14148 [[fallthrough]];
14149 case ISD::SETOEQ:
14150 case ISD::SETOGT:
14151 case ISD::SETOGE:
14152 case ISD::SETOLT:
14153 case ISD::SETOLE:
14154 case ISD::SETUNE:
14155 case ISD::SETUGT:
14156 case ISD::SETUGE:
14157 case ISD::SETULT:
14158 case ISD::SETULE:
14159 // If we are floating point, assign and break, otherwise fall through.
14160 if (!OpVT.isInteger()) {
14161 // We can use the 4th bit to tell if we are the unordered
14162 // or ordered version of the opcode.
14163 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14164 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
14165 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
14166 break;
14167 }
14168 // Fallthrough if we are unsigned integer.
14169 [[fallthrough]];
14170 case ISD::SETLE:
14171 case ISD::SETGT:
14172 case ISD::SETGE:
14173 case ISD::SETLT:
14174 case ISD::SETNE:
14175 case ISD::SETEQ:
14176 // If all combinations of inverting the condition and swapping operands
14177 // didn't work then we have no means to expand the condition.
14178 llvm_unreachable("Don't know how to expand this condition!");
14179 }
14180
14181 SDValue SetCC1, SetCC2;
14182 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
14183 // If we aren't the ordered or unorder operation,
14184 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
14185 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC1, Chain, IsSignaling);
14186 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC2, Chain, IsSignaling);
14187 } else {
14188 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
14189 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS: LHS, Cond: CC1, Chain, IsSignaling);
14190 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS: RHS, RHS, Cond: CC2, Chain, IsSignaling);
14191 }
14192 if (Chain)
14193 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: SetCC1.getValue(R: 1),
14194 N2: SetCC2.getValue(R: 1));
14195 LHS = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SetCC1, N2: SetCC2);
14196 RHS = SDValue();
14197 CC = SDValue();
14198 return true;
14199 }
14200 }
14201 return false;
14202}
14203
14204SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
14205 SelectionDAG &DAG) const {
14206 EVT VT = Node->getValueType(ResNo: 0);
14207 // Despite its documentation, GetSplitDestVTs will assert if VT cannot be
14208 // split into two equal parts.
14209 if (!VT.isVector() || !VT.getVectorElementCount().isKnownMultipleOf(RHS: 2))
14210 return SDValue();
14211
14212 // Restrict expansion to cases where both parts can be concatenated.
14213 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
14214 if (LoVT != HiVT || !isTypeLegal(VT: LoVT))
14215 return SDValue();
14216
14217 SDLoc DL(Node);
14218 unsigned Opcode = Node->getOpcode();
14219
14220 // Don't expand if the result is likely to be unrolled anyway.
14221 if (!isOperationLegalOrCustomOrPromote(Op: Opcode, VT: LoVT))
14222 return SDValue();
14223
14224 SmallVector<SDValue, 4> LoOps, HiOps;
14225 for (const SDValue &V : Node->op_values()) {
14226 if (!V.getValueType().isVector()) {
14227 // Scalar operands pass through to both halves unchanged.
14228 LoOps.push_back(Elt: V);
14229 HiOps.push_back(Elt: V);
14230 continue;
14231 }
14232 auto [Lo, Hi] = DAG.SplitVector(N: V, DL, LoVT, HiVT);
14233 LoOps.push_back(Elt: Lo);
14234 HiOps.push_back(Elt: Hi);
14235 }
14236
14237 SDValue SplitOpLo = DAG.getNode(Opcode, DL, VT: LoVT, Ops: LoOps, Flags: Node->getFlags());
14238 SDValue SplitOpHi = DAG.getNode(Opcode, DL, VT: HiVT, Ops: HiOps, Flags: Node->getFlags());
14239 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: SplitOpLo, N2: SplitOpHi);
14240}
14241
14242SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
14243 const SDLoc &DL,
14244 EVT InVecVT, SDValue EltNo,
14245 LoadSDNode *OriginalLoad,
14246 SelectionDAG &DAG) const {
14247 assert(OriginalLoad->isSimple());
14248
14249 EVT VecEltVT = InVecVT.getVectorElementType();
14250
14251 // If the vector element type is not a multiple of a byte then we are unable
14252 // to correctly compute an address to load only the extracted element as a
14253 // scalar.
14254 if (!VecEltVT.isByteSized())
14255 return SDValue();
14256
14257 ISD::LoadExtType ExtTy =
14258 ResultVT.bitsGT(VT: VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
14259 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: VecEltVT))
14260 return SDValue();
14261
14262 std::optional<unsigned> ByteOffset;
14263 Align Alignment = OriginalLoad->getAlign();
14264 MachinePointerInfo MPI;
14265 if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo)) {
14266 int Elt = ConstEltNo->getZExtValue();
14267 ByteOffset = VecEltVT.getSizeInBits() * Elt / 8;
14268 MPI = OriginalLoad->getPointerInfo().getWithOffset(O: *ByteOffset);
14269 Alignment = commonAlignment(A: Alignment, Offset: *ByteOffset);
14270 } else {
14271 // Discard the pointer info except the address space because the memory
14272 // operand can't represent this new access since the offset is variable.
14273 MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
14274 Alignment = commonAlignment(A: Alignment, Offset: VecEltVT.getSizeInBits() / 8);
14275 }
14276
14277 if (!shouldReduceLoadWidth(Load: OriginalLoad, ExtTy, NewVT: VecEltVT, ByteOffset))
14278 return SDValue();
14279
14280 unsigned IsFast = 0;
14281 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: VecEltVT,
14282 AddrSpace: OriginalLoad->getAddressSpace(), Alignment,
14283 Flags: OriginalLoad->getMemOperand()->getFlags(), Fast: &IsFast) ||
14284 !IsFast)
14285 return SDValue();
14286
14287 // The original DAG loaded the entire vector from memory, so arithmetic
14288 // within it must be inbounds.
14289 SDValue NewPtr = getInboundsVectorElementPointer(
14290 DAG, VecPtr: OriginalLoad->getBasePtr(), VecVT: InVecVT, Index: EltNo);
14291
14292 // We are replacing a vector load with a scalar load. The new load must have
14293 // identical memory op ordering to the original.
14294 SDValue Load;
14295 if (ResultVT.bitsGT(VT: VecEltVT)) {
14296 // If the result type of vextract is wider than the load, then issue an
14297 // extending load instead.
14298 ISD::LoadExtType ExtType =
14299 isLoadLegal(ValVT: ResultVT, MemVT: VecEltVT, Alignment,
14300 AddrSpace: OriginalLoad->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false)
14301 ? ISD::ZEXTLOAD
14302 : ISD::EXTLOAD;
14303 Load = DAG.getExtLoad(ExtType, dl: DL, VT: ResultVT, Chain: OriginalLoad->getChain(),
14304 Ptr: NewPtr, PtrInfo: MPI, MemVT: VecEltVT, Alignment,
14305 MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14306 Metadata: OriginalLoad->getAAInfo());
14307 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14308 } else {
14309 // The result type is narrower or the same width as the vector element
14310 Load = DAG.getLoad(VT: VecEltVT, dl: DL, Chain: OriginalLoad->getChain(), Ptr: NewPtr, PtrInfo: MPI,
14311 Alignment, MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14312 Metadata: OriginalLoad->getAAInfo());
14313 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14314 if (ResultVT.bitsLT(VT: VecEltVT))
14315 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: Load);
14316 else
14317 Load = DAG.getBitcast(VT: ResultVT, V: Load);
14318 }
14319
14320 return Load;
14321}
14322
14323// Set type id for call site info and metadata 'call_target'.
14324// We are filtering for:
14325// a) The call-graph-section use case that wants to know about indirect
14326// calls, or
14327// b) We want to annotate indirect calls.
14328void TargetLowering::setTypeIdForCallsiteInfo(
14329 const CallBase *CB, MachineFunction &MF,
14330 MachineFunction::CallSiteInfo &CSInfo) const {
14331 if (CB && CB->isIndirectCall() &&
14332 (MF.getTarget().Options.EmitCallGraphSection ||
14333 MF.getTarget().Options.EmitCallSiteInfo))
14334 CSInfo = MachineFunction::CallSiteInfo(*CB);
14335}
14336