1//===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/CodeGen/TargetLowering.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/Analysis/ValueTracking.h"
16#include "llvm/Analysis/VectorUtils.h"
17#include "llvm/CodeGen/Analysis.h"
18#include "llvm/CodeGen/CallingConvLower.h"
19#include "llvm/CodeGen/CodeGenCommonISel.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/SDPatternMatch.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/TargetRegisterInfo.h"
27#include "llvm/IR/DataLayout.h"
28#include "llvm/IR/DerivedTypes.h"
29#include "llvm/IR/GlobalVariable.h"
30#include "llvm/IR/LLVMContext.h"
31#include "llvm/MC/MCAsmInfo.h"
32#include "llvm/MC/MCExpr.h"
33#include "llvm/Support/DivisionByConstantInfo.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Target/TargetMachine.h"
38#include <cctype>
39#include <deque>
40using namespace llvm;
41using namespace llvm::SDPatternMatch;
42
43/// NOTE: The TargetMachine owns TLOF.
44TargetLowering::TargetLowering(const TargetMachine &tm,
45 const TargetSubtargetInfo &STI)
46 : TargetLoweringBase(tm, STI) {}
47
48// Define the virtual destructor out-of-line for build efficiency.
49TargetLowering::~TargetLowering() = default;
50
51const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
52 return nullptr;
53}
54
55bool TargetLowering::isPositionIndependent() const {
56 return getTargetMachine().isPositionIndependent();
57}
58
59/// Check whether a given call node is in tail position within its function. If
60/// so, it sets Chain to the input chain of the tail call.
61bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
62 SDValue &Chain) const {
63 const Function &F = DAG.getMachineFunction().getFunction();
64
65 // First, check if tail calls have been disabled in this function.
66 if (F.getFnAttribute(Kind: "disable-tail-calls").getValueAsBool())
67 return false;
68
69 // Conservatively require the attributes of the call to match those of
70 // the return. Ignore following attributes because they don't affect the
71 // call sequence.
72 AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
73 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
74 Attribute::DereferenceableOrNull, Attribute::NoAlias,
75 Attribute::NonNull, Attribute::NoUndef,
76 Attribute::Range, Attribute::NoFPClass})
77 CallerAttrs.removeAttribute(Val: Attr);
78
79 if (CallerAttrs.hasAttributes())
80 return false;
81
82 // It's not safe to eliminate the sign / zero extension of the return value.
83 if (CallerAttrs.contains(A: Attribute::ZExt) ||
84 CallerAttrs.contains(A: Attribute::SExt))
85 return false;
86
87 // Check if the only use is a function return node.
88 return isUsedByReturnOnly(Node, Chain);
89}
90
91bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
92 const uint32_t *CallerPreservedMask,
93 const SmallVectorImpl<CCValAssign> &ArgLocs,
94 const SmallVectorImpl<SDValue> &OutVals) const {
95 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
96 const CCValAssign &ArgLoc = ArgLocs[I];
97 if (!ArgLoc.isRegLoc())
98 continue;
99 MCRegister Reg = ArgLoc.getLocReg();
100 // Only look at callee saved registers.
101 if (MachineOperand::clobbersPhysReg(RegMask: CallerPreservedMask, PhysReg: Reg))
102 continue;
103 // Check that we pass the value used for the caller.
104 // (We look for a CopyFromReg reading a virtual register that is used
105 // for the function live-in value of register Reg)
106 SDValue Value = OutVals[I];
107 if (Value->getOpcode() == ISD::AssertZext)
108 Value = Value.getOperand(i: 0);
109 if (Value->getOpcode() != ISD::CopyFromReg)
110 return false;
111 Register ArgReg = cast<RegisterSDNode>(Val: Value->getOperand(Num: 1))->getReg();
112 if (MRI.getLiveInPhysReg(VReg: ArgReg) != Reg)
113 return false;
114 }
115 return true;
116}
117
118/// Set CallLoweringInfo attribute flags based on a call instruction
119/// and called function attributes.
120void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
121 unsigned ArgIdx) {
122 IsSExt = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SExt);
123 IsZExt = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::ZExt);
124 IsNoExt = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::NoExt);
125 IsInReg = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::InReg);
126 IsSRet = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::StructRet);
127 IsNest = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::Nest);
128 IsByVal = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::ByVal);
129 IsPreallocated = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::Preallocated);
130 IsInAlloca = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::InAlloca);
131 IsReturned = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::Returned);
132 IsSwiftSelf = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SwiftSelf);
133 IsSwiftAsync = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SwiftAsync);
134 IsSwiftError = Call->paramHasAttr(ArgNo: ArgIdx, Kind: Attribute::SwiftError);
135 Alignment = Call->getParamStackAlign(ArgNo: ArgIdx);
136 IndirectType = nullptr;
137 assert(IsByVal + IsPreallocated + IsInAlloca + IsSRet <= 1 &&
138 "multiple ABI attributes?");
139 if (IsByVal) {
140 IndirectType = Call->getParamByValType(ArgNo: ArgIdx);
141 if (!Alignment)
142 Alignment = Call->getParamAlign(ArgNo: ArgIdx);
143 }
144 if (IsPreallocated)
145 IndirectType = Call->getParamPreallocatedType(ArgNo: ArgIdx);
146 if (IsInAlloca)
147 IndirectType = Call->getParamInAllocaType(ArgNo: ArgIdx);
148 if (IsSRet)
149 IndirectType = Call->getParamStructRetType(ArgNo: ArgIdx);
150}
151
152/// Generate a libcall taking the given operands as arguments and returning a
153/// result of type RetVT.
154std::pair<SDValue, SDValue>
155TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl,
156 EVT RetVT, ArrayRef<SDValue> Ops,
157 MakeLibCallOptions CallOptions, const SDLoc &dl,
158 SDValue InChain) const {
159 if (LibcallImpl == RTLIB::Unsupported)
160 reportFatalInternalError(reason: "unsupported library call operation");
161
162 if (!InChain)
163 InChain = DAG.getEntryNode();
164
165 TargetLowering::ArgListTy Args;
166 Args.reserve(n: Ops.size());
167
168 ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
169 for (unsigned i = 0; i < Ops.size(); ++i) {
170 SDValue NewOp = Ops[i];
171 Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
172 ? OpsTypeOverrides[i]
173 : NewOp.getValueType().getTypeForEVT(Context&: *DAG.getContext());
174 TargetLowering::ArgListEntry Entry(NewOp, Ty);
175 if (CallOptions.IsSoften)
176 Entry.OrigTy =
177 CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(Context&: *DAG.getContext());
178
179 Entry.IsSExt =
180 shouldSignExtendTypeInLibCall(Ty: Entry.Ty, IsSigned: CallOptions.IsSigned);
181 Entry.IsZExt = !Entry.IsSExt;
182
183 if (CallOptions.IsSoften &&
184 !shouldExtendTypeInLibCall(Type: CallOptions.OpsVTBeforeSoften[i])) {
185 Entry.IsSExt = Entry.IsZExt = false;
186 }
187 Args.push_back(x: Entry);
188 }
189
190 SDValue Callee =
191 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
192
193 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
194 Type *OrigRetTy = RetTy;
195 TargetLowering::CallLoweringInfo CLI(DAG);
196 bool signExtend = shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: CallOptions.IsSigned);
197 bool zeroExtend = !signExtend;
198
199 if (CallOptions.IsSoften) {
200 OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(Context&: *DAG.getContext());
201 if (!shouldExtendTypeInLibCall(Type: CallOptions.RetVTBeforeSoften))
202 signExtend = zeroExtend = false;
203 }
204
205 CLI.setDebugLoc(dl)
206 .setChain(InChain)
207 .setLibCallee(CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetTy, OrigResultType: OrigRetTy,
208 Target: Callee, ArgsList: std::move(Args))
209 .setNoReturn(CallOptions.DoesNotReturn)
210 .setDiscardResult(!CallOptions.IsReturnValueUsed)
211 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
212 .setSExtResult(signExtend)
213 .setZExtResult(zeroExtend);
214 return LowerCallTo(CLI);
215}
216
217bool TargetLowering::findOptimalMemOpLowering(
218 LLVMContext &Context, std::vector<EVT> &MemOps, unsigned Limit,
219 const MemOp &Op, unsigned DstAS, unsigned SrcAS,
220 const AttributeList &FuncAttributes, EVT *LargestVT) const {
221 EVT VT = getOptimalMemOpType(Context, Op, FuncAttributes);
222
223 if (VT == MVT::Other) {
224 // Use the largest integer type whose alignment constraints are satisfied.
225 VT = MVT::LAST_INTEGER_VALUETYPE;
226 if (Op.isFixedDstAlign()) {
227 bool LoadsFromSrc = Op.isMemcpyOrMemmove() && !Op.isMemcpyStrSrc();
228 while (VT != MVT::i8) {
229 unsigned VTSize = VT.getSizeInBits() / 8;
230 bool DstOk =
231 Op.getDstAlign() >= VTSize ||
232 allowsMisalignedMemoryAccesses(VT, AddrSpace: DstAS, Alignment: Op.getDstAlign());
233 bool SrcOk =
234 !LoadsFromSrc || Op.getSrcAlign() >= VTSize ||
235 allowsMisalignedMemoryAccesses(VT, AddrSpace: SrcAS, Alignment: Op.getSrcAlign());
236 if (DstOk && SrcOk)
237 break;
238 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
239 }
240 }
241 assert(VT.isInteger());
242
243 // Find the largest legal integer type.
244 MVT LVT = MVT::LAST_INTEGER_VALUETYPE;
245 while (!isTypeLegal(VT: LVT))
246 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
247 assert(LVT.isInteger());
248
249 // If the type we've chosen is larger than the largest legal integer type
250 // then use the largest legal type.
251 if (VT.bitsGT(VT: LVT))
252 VT = LVT;
253 }
254
255 unsigned NumMemOps = 0;
256 uint64_t Size = Op.size();
257 while (Size) {
258 unsigned VTSize = VT.getSizeInBits() / 8;
259 while (VTSize > Size) {
260 // For now, only use non-vector load / store's for the left-over pieces.
261 EVT NewVT = VT;
262 unsigned NewVTSize;
263
264 bool Found = false;
265 if (VT.isVector() || VT.isFloatingPoint()) {
266 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
267 if (isOperationLegalOrCustom(Op: ISD::STORE, VT: NewVT) &&
268 isSafeMemOpType(NewVT.getSimpleVT()))
269 Found = true;
270 else if (NewVT == MVT::i64 &&
271 isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::f64) &&
272 isSafeMemOpType(MVT::f64)) {
273 // i64 is usually not legal on 32-bit targets, but f64 may be.
274 NewVT = MVT::f64;
275 Found = true;
276 }
277 }
278
279 if (!Found) {
280 do {
281 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
282 if (NewVT == MVT::i8)
283 break;
284 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
285 }
286 NewVTSize = NewVT.getSizeInBits() / 8;
287
288 // If the new VT cannot cover all of the remaining bits, then consider
289 // issuing a (or a pair of) unaligned and overlapping load / store.
290 unsigned Fast;
291 if (NumMemOps && !Op.isVolatile() && NewVTSize < Size &&
292 allowsMisalignedMemoryAccesses(
293 VT, AddrSpace: DstAS, Alignment: Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
294 Flags: MachineMemOperand::MONone, &Fast) &&
295 Fast)
296 VTSize = Size;
297 else {
298 VT = NewVT;
299 VTSize = NewVTSize;
300 }
301 }
302
303 if (++NumMemOps > Limit)
304 return false;
305
306 MemOps.push_back(x: VT);
307 Size -= VTSize;
308 }
309
310 return true;
311}
312
313/// Soften the operands of a comparison. This code is shared among BR_CC,
314/// SELECT_CC, and SETCC handlers.
315void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
316 SDValue &NewLHS, SDValue &NewRHS,
317 ISD::CondCode &CCCode,
318 const SDLoc &dl, const SDValue OldLHS,
319 const SDValue OldRHS) const {
320 SDValue Chain;
321 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, DL: dl, OldLHS,
322 OldRHS, Chain);
323}
324
325void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
326 SDValue &NewLHS, SDValue &NewRHS,
327 ISD::CondCode &CCCode,
328 const SDLoc &dl, const SDValue OldLHS,
329 const SDValue OldRHS,
330 SDValue &Chain,
331 bool IsSignaling) const {
332 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
333 // not supporting it. We can update this code when libgcc provides such
334 // functions.
335
336 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
337 && "Unsupported setcc type!");
338
339 // Expand into one or more soft-fp libcall(s).
340 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
341 bool ShouldInvertCC = false;
342 switch (CCCode) {
343 case ISD::SETEQ:
344 case ISD::SETOEQ:
345 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
346 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
347 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
348 break;
349 case ISD::SETNE:
350 case ISD::SETUNE:
351 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
352 (VT == MVT::f64) ? RTLIB::UNE_F64 :
353 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
354 // Some ABIs (e.g. AEABI) only provide an ordered-equal compare; obtain
355 // not-equal (UNE = !OEQ) by inverting the result of that call.
356 if (getLibcallImpl(Call: LC1) == RTLIB::Unsupported) {
357 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32
358 : (VT == MVT::f64) ? RTLIB::OEQ_F64
359 : (VT == MVT::f128) ? RTLIB::OEQ_F128
360 : RTLIB::OEQ_PPCF128;
361 ShouldInvertCC = true;
362 }
363 break;
364 case ISD::SETGE:
365 case ISD::SETOGE:
366 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
367 (VT == MVT::f64) ? RTLIB::OGE_F64 :
368 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
369 break;
370 case ISD::SETLT:
371 case ISD::SETOLT:
372 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
373 (VT == MVT::f64) ? RTLIB::OLT_F64 :
374 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
375 break;
376 case ISD::SETLE:
377 case ISD::SETOLE:
378 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
379 (VT == MVT::f64) ? RTLIB::OLE_F64 :
380 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
381 break;
382 case ISD::SETGT:
383 case ISD::SETOGT:
384 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
385 (VT == MVT::f64) ? RTLIB::OGT_F64 :
386 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
387 break;
388 case ISD::SETO:
389 ShouldInvertCC = true;
390 [[fallthrough]];
391 case ISD::SETUO:
392 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
393 (VT == MVT::f64) ? RTLIB::UO_F64 :
394 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
395 break;
396 case ISD::SETONE:
397 // SETONE = O && UNE
398 ShouldInvertCC = true;
399 [[fallthrough]];
400 case ISD::SETUEQ:
401 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
402 (VT == MVT::f64) ? RTLIB::UO_F64 :
403 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
404 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
405 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
406 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
407 break;
408 default:
409 // Invert CC for unordered comparisons
410 ShouldInvertCC = true;
411 switch (CCCode) {
412 case ISD::SETULT:
413 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
414 (VT == MVT::f64) ? RTLIB::OGE_F64 :
415 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
416 break;
417 case ISD::SETULE:
418 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
419 (VT == MVT::f64) ? RTLIB::OGT_F64 :
420 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
421 break;
422 case ISD::SETUGT:
423 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
424 (VT == MVT::f64) ? RTLIB::OLE_F64 :
425 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
426 break;
427 case ISD::SETUGE:
428 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
429 (VT == MVT::f64) ? RTLIB::OLT_F64 :
430 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
431 break;
432 default: llvm_unreachable("Do not know how to soften this setcc!");
433 }
434 }
435
436 // Use the target specific return value for comparison lib calls.
437 EVT RetVT = getCmpLibcallReturnType();
438 SDValue Ops[2] = {NewLHS, NewRHS};
439 TargetLowering::MakeLibCallOptions CallOptions;
440 EVT OpsVT[2] = { OldLHS.getValueType(),
441 OldRHS.getValueType() };
442 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
443 auto Call = makeLibCall(DAG, LC: LC1, RetVT, Ops, CallOptions, dl, Chain);
444 NewLHS = Call.first;
445 NewRHS = DAG.getConstant(Val: 0, DL: dl, VT: RetVT);
446
447 RTLIB::LibcallImpl LC1Impl = getLibcallImpl(Call: LC1);
448 if (LC1Impl == RTLIB::Unsupported) {
449 reportFatalUsageError(
450 reason: "no libcall available to soften floating-point compare");
451 }
452
453 CCCode = getSoftFloatCmpLibcallPredicate(Call: LC1Impl);
454 if (ShouldInvertCC) {
455 assert(RetVT.isInteger());
456 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
457 }
458
459 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
460 // Update Chain.
461 Chain = Call.second;
462 } else {
463 RTLIB::LibcallImpl LC2Impl = getLibcallImpl(Call: LC2);
464 if (LC2Impl == RTLIB::Unsupported) {
465 reportFatalUsageError(
466 reason: "no libcall available to soften floating-point compare");
467 }
468
469 assert(CCCode == (ShouldInvertCC ? ISD::SETEQ : ISD::SETNE) &&
470 "unordered call should be simple boolean");
471
472 EVT SetCCVT =
473 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: RetVT);
474 if (getBooleanContents(Type: RetVT) == ZeroOrOneBooleanContent) {
475 NewLHS = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: RetVT, N1: Call.first,
476 N2: DAG.getValueType(MVT::i1));
477 }
478
479 SDValue Tmp = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: NewLHS, RHS: NewRHS, Cond: CCCode);
480 auto Call2 = makeLibCall(DAG, LC: LC2, RetVT, Ops, CallOptions, dl, Chain);
481 CCCode = getSoftFloatCmpLibcallPredicate(Call: LC2Impl);
482 if (ShouldInvertCC)
483 CCCode = getSetCCInverse(Operation: CCCode, Type: RetVT);
484 NewLHS = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Call2.first, RHS: NewRHS, Cond: CCCode);
485 if (Chain)
486 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Call.second,
487 N2: Call2.second);
488 NewLHS = DAG.getNode(Opcode: ShouldInvertCC ? ISD::AND : ISD::OR, DL: dl,
489 VT: Tmp.getValueType(), N1: Tmp, N2: NewLHS);
490 NewRHS = SDValue();
491 }
492}
493
494/// Return the entry encoding for a jump table in the current function. The
495/// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
496unsigned TargetLowering::getJumpTableEncoding() const {
497 // In non-pic modes, just use the address of a block.
498 if (!isPositionIndependent())
499 return MachineJumpTableInfo::EK_BlockAddress;
500
501 // Otherwise, use a label difference.
502 return MachineJumpTableInfo::EK_LabelDifference32;
503}
504
505SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
506 SelectionDAG &DAG) const {
507 return Table;
508}
509
510/// This returns the relocation base for the given PIC jumptable, the same as
511/// getPICJumpTableRelocBase, but as an MCExpr.
512const MCExpr *
513TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
514 unsigned JTI,MCContext &Ctx) const{
515 // The normal PIC reloc base is the label at the start of the jump table.
516 return MCSymbolRefExpr::create(Symbol: MF->getJTISymbol(JTI, Ctx), Ctx);
517}
518
519SDValue TargetLowering::expandIndirectJTBranch(const SDLoc &dl, SDValue Value,
520 SDValue Addr, int JTI,
521 SelectionDAG &DAG) const {
522 SDValue Chain = Value;
523 // Jump table debug info is only needed if CodeView is enabled.
524 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF()) {
525 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
526 }
527 return DAG.getNode(Opcode: ISD::BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
528}
529
530bool
531TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
532 const TargetMachine &TM = getTargetMachine();
533 const GlobalValue *GV = GA->getGlobal();
534
535 // If the address is not even local to this DSO we will have to load it from
536 // a got and then add the offset.
537 if (!TM.shouldAssumeDSOLocal(GV))
538 return false;
539
540 // If the code is position independent we will have to add a base register.
541 if (isPositionIndependent())
542 return false;
543
544 // Otherwise we can do it.
545 return true;
546}
547
548//===----------------------------------------------------------------------===//
549// Optimization Methods
550//===----------------------------------------------------------------------===//
551
552/// If the specified instruction has a constant integer operand and there are
553/// bits set in that constant that are not demanded, then clear those bits and
554/// return true.
555bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
556 const APInt &DemandedBits,
557 const APInt &DemandedElts,
558 TargetLoweringOpt &TLO) const {
559 SDLoc DL(Op);
560 unsigned Opcode = Op.getOpcode();
561
562 // Early-out if we've ended up calling an undemanded node, leave this to
563 // constant folding.
564 if (DemandedBits.isZero() || DemandedElts.isZero())
565 return false;
566
567 // Do target-specific constant optimization.
568 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
569 return TLO.New.getNode();
570
571 // FIXME: ISD::SELECT, ISD::SELECT_CC
572 switch (Opcode) {
573 default:
574 break;
575 case ISD::XOR:
576 case ISD::AND:
577 case ISD::OR: {
578 auto *Op1C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
579 if (!Op1C || Op1C->isOpaque())
580 return false;
581
582 // If this is a 'not' op, don't touch it because that's a canonical form.
583 const APInt &C = Op1C->getAPIntValue();
584 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(RHS: C))
585 return false;
586
587 if (!C.isSubsetOf(RHS: DemandedBits)) {
588 EVT VT = Op.getValueType();
589 SDValue NewC = TLO.DAG.getConstant(Val: DemandedBits & C, DL, VT);
590 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, N1: Op.getOperand(i: 0), N2: NewC,
591 Flags: Op->getFlags());
592 return TLO.CombineTo(O: Op, N: NewOp);
593 }
594
595 break;
596 }
597 }
598
599 return false;
600}
601
602bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
603 const APInt &DemandedBits,
604 TargetLoweringOpt &TLO) const {
605 EVT VT = Op.getValueType();
606 APInt DemandedElts = VT.isVector()
607 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
608 : APInt(1, 1);
609 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
610}
611
612/// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
613/// This uses isTruncateFree/isZExtFree and ANY_EXTEND for the widening cast,
614/// but it could be generalized for targets with other types of implicit
615/// widening casts.
616bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
617 const APInt &DemandedBits,
618 TargetLoweringOpt &TLO) const {
619 assert(Op.getNumOperands() == 2 &&
620 "ShrinkDemandedOp only supports binary operators!");
621 assert(Op.getNode()->getNumValues() == 1 &&
622 "ShrinkDemandedOp only supports nodes with one result!");
623
624 EVT VT = Op.getValueType();
625 SelectionDAG &DAG = TLO.DAG;
626 SDLoc dl(Op);
627
628 // Early return, as this function cannot handle vector types.
629 if (VT.isVector())
630 return false;
631
632 assert(Op.getOperand(0).getValueType().getScalarSizeInBits() == BitWidth &&
633 Op.getOperand(1).getValueType().getScalarSizeInBits() == BitWidth &&
634 "ShrinkDemandedOp only supports operands that have the same size!");
635
636 // Don't do this if the node has another user, which may require the
637 // full value.
638 if (!Op.getNode()->hasOneUse())
639 return false;
640
641 // Search for the smallest integer type with free casts to and from
642 // Op's type. For expedience, just check power-of-2 integer types.
643 unsigned DemandedSize = DemandedBits.getActiveBits();
644 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
645 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
646 EVT SmallVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SmallVTBits);
647 if (isTruncateFree(Val: Op, VT2: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT)) {
648 // We found a type with free casts.
649
650 // If the operation has the 'disjoint' flag, then the
651 // operands on the new node are also disjoint.
652 SDNodeFlags Flags(Op->getFlags().hasDisjoint() ? SDNodeFlags::Disjoint
653 : SDNodeFlags::None);
654 unsigned Opcode = Op.getOpcode();
655 if (Opcode == ISD::PTRADD) {
656 // It isn't a ptradd anymore if it doesn't operate on the entire
657 // pointer.
658 Opcode = ISD::ADD;
659 }
660 SDValue X = DAG.getNode(
661 Opcode, DL: dl, VT: SmallVT,
662 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
663 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 1)), Flags);
664 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
665 SDValue Z = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: X);
666 return TLO.CombineTo(O: Op, N: Z);
667 }
668 }
669 return false;
670}
671
672bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
673 DAGCombinerInfo &DCI) const {
674 SelectionDAG &DAG = DCI.DAG;
675 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
676 !DCI.isBeforeLegalizeOps());
677 KnownBits Known;
678
679 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
680 if (Simplified) {
681 DCI.AddToWorklist(N: Op.getNode());
682 DCI.CommitTargetLoweringOpt(TLO);
683 }
684 return Simplified;
685}
686
687bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
688 const APInt &DemandedElts,
689 DAGCombinerInfo &DCI) const {
690 SelectionDAG &DAG = DCI.DAG;
691 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
692 !DCI.isBeforeLegalizeOps());
693 KnownBits Known;
694
695 bool Simplified =
696 SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO);
697 if (Simplified) {
698 DCI.AddToWorklist(N: Op.getNode());
699 DCI.CommitTargetLoweringOpt(TLO);
700 }
701 return Simplified;
702}
703
704bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
705 KnownBits &Known,
706 TargetLoweringOpt &TLO,
707 unsigned Depth,
708 bool AssumeSingleUse) const {
709 EVT VT = Op.getValueType();
710
711 // Since the number of lanes in a scalable vector is unknown at compile time,
712 // we track one bit which is implicitly broadcast to all lanes. This means
713 // that all lanes in a scalable vector are considered demanded.
714 APInt DemandedElts = VT.isFixedLengthVector()
715 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
716 : APInt(1, 1);
717 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
718 AssumeSingleUse);
719}
720
721// TODO: Under what circumstances can we create nodes? Constant folding?
722SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
723 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
724 SelectionDAG &DAG, unsigned Depth) const {
725 EVT VT = Op.getValueType();
726
727 // Limit search depth.
728 if (Depth >= SelectionDAG::MaxRecursionDepth)
729 return SDValue();
730
731 // Ignore UNDEFs.
732 if (Op.isUndef())
733 return SDValue();
734
735 // Not demanding any bits/elts from Op.
736 if (DemandedBits == 0 || DemandedElts == 0)
737 return DAG.getUNDEF(VT);
738
739 bool IsLE = DAG.getDataLayout().isLittleEndian();
740 unsigned NumElts = DemandedElts.getBitWidth();
741 unsigned BitWidth = DemandedBits.getBitWidth();
742 KnownBits LHSKnown, RHSKnown;
743 switch (Op.getOpcode()) {
744 case ISD::BITCAST: {
745 if (VT.isScalableVector())
746 return SDValue();
747
748 SDValue Src = peekThroughBitcasts(V: Op.getOperand(i: 0));
749 EVT SrcVT = Src.getValueType();
750 EVT DstVT = Op.getValueType();
751 if (SrcVT == DstVT)
752 return Src;
753
754 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
755 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
756 if (NumSrcEltBits == NumDstEltBits)
757 if (SDValue V = SimplifyMultipleUseDemandedBits(
758 Op: Src, DemandedBits, DemandedElts, DAG, Depth: Depth + 1))
759 return DAG.getBitcast(VT: DstVT, V);
760
761 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
762 unsigned Scale = NumDstEltBits / NumSrcEltBits;
763 unsigned NumSrcElts = SrcVT.getVectorNumElements();
764 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
765 for (unsigned i = 0; i != Scale; ++i) {
766 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
767 unsigned BitOffset = EltOffset * NumSrcEltBits;
768 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
769 }
770 // Recursive calls below may turn not demanded elements into poison, so we
771 // need to demand all smaller source elements that maps to a demanded
772 // destination element.
773 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
774
775 if (SDValue V = SimplifyMultipleUseDemandedBits(
776 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
777 return DAG.getBitcast(VT: DstVT, V);
778 }
779
780 // TODO - bigendian once we have test coverage.
781 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
782 unsigned Scale = NumSrcEltBits / NumDstEltBits;
783 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
784 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
785 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
786 for (unsigned i = 0; i != NumElts; ++i)
787 if (DemandedElts[i]) {
788 unsigned Offset = (i % Scale) * NumDstEltBits;
789 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
790 DemandedSrcElts.setBit(i / Scale);
791 }
792
793 if (SDValue V = SimplifyMultipleUseDemandedBits(
794 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG, Depth: Depth + 1))
795 return DAG.getBitcast(VT: DstVT, V);
796 }
797
798 break;
799 }
800 case ISD::AND: {
801 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
802 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
803
804 // If all of the demanded bits are known 1 on one side, return the other.
805 // These bits cannot contribute to the result of the 'and' in this
806 // context.
807 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero | RHSKnown.One))
808 return Op.getOperand(i: 0);
809 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero | LHSKnown.One))
810 return Op.getOperand(i: 1);
811 break;
812 }
813 case ISD::OR: {
814 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
815 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
816
817 // If all of the demanded bits are known zero on one side, return the
818 // other. These bits cannot contribute to the result of the 'or' in this
819 // context.
820 if (DemandedBits.isSubsetOf(RHS: LHSKnown.One | RHSKnown.Zero))
821 return Op.getOperand(i: 0);
822 if (DemandedBits.isSubsetOf(RHS: RHSKnown.One | LHSKnown.Zero))
823 return Op.getOperand(i: 1);
824 break;
825 }
826 case ISD::XOR: {
827 LHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
828 RHSKnown = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
829
830 // If all of the demanded bits are known zero on one side, return the
831 // other.
832 if (DemandedBits.isSubsetOf(RHS: RHSKnown.Zero))
833 return Op.getOperand(i: 0);
834 if (DemandedBits.isSubsetOf(RHS: LHSKnown.Zero))
835 return Op.getOperand(i: 1);
836 break;
837 }
838 case ISD::ADD:
839 case ISD::MUL:
840 case ISD::SMIN:
841 case ISD::SMAX:
842 case ISD::UMIN:
843 case ISD::UMAX: {
844 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 1),
845 DemandedElts, OperandNo: 1, Depth: Depth + 1))
846 return Op.getOperand(i: 0);
847
848 if (DAG.isIdentityElement(Opc: Op.getOpcode(), Flags: Op->getFlags(), V: Op.getOperand(i: 0),
849 DemandedElts, OperandNo: 0, Depth: Depth + 1))
850 return Op.getOperand(i: 1);
851 break;
852 }
853 case ISD::SHL: {
854 // If we are only demanding sign bits then we can use the shift source
855 // directly.
856 if (std::optional<unsigned> MaxSA =
857 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
858 SDValue Op0 = Op.getOperand(i: 0);
859 unsigned ShAmt = *MaxSA;
860 unsigned NumSignBits =
861 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
862 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
863 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
864 return Op0;
865 }
866 break;
867 }
868 case ISD::SRL: {
869 // If we are only demanding sign bits then we can use the shift source
870 // directly.
871 if (std::optional<unsigned> MaxSA =
872 DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
873 SDValue Op0 = Op.getOperand(i: 0);
874 unsigned ShAmt = *MaxSA;
875 // Must already be signbits in DemandedBits bounds, and can't demand any
876 // shifted in zeroes.
877 if (DemandedBits.countl_zero() >= ShAmt) {
878 unsigned NumSignBits =
879 DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
880 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
881 return Op0;
882 }
883 }
884 break;
885 }
886 case ISD::SETCC: {
887 SDValue Op0 = Op.getOperand(i: 0);
888 SDValue Op1 = Op.getOperand(i: 1);
889 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
890 // If (1) we only need the sign-bit, (2) the setcc operands are the same
891 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
892 // -1, we may be able to bypass the setcc.
893 if (DemandedBits.isSignMask() &&
894 Op0.getScalarValueSizeInBits() == BitWidth &&
895 getBooleanContents(Type: Op0.getValueType()) ==
896 BooleanContent::ZeroOrNegativeOneBooleanContent) {
897 // If we're testing X < 0, then this compare isn't needed - just use X!
898 // FIXME: We're limiting to integer types here, but this should also work
899 // if we don't care about FP signed-zero. The use of SETLT with FP means
900 // that we don't care about NaNs.
901 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
902 (isNullConstant(V: Op1) || ISD::isBuildVectorAllZeros(N: Op1.getNode())))
903 return Op0;
904 }
905 break;
906 }
907 case ISD::SIGN_EXTEND_INREG: {
908 // If none of the extended bits are demanded, eliminate the sextinreg.
909 SDValue Op0 = Op.getOperand(i: 0);
910 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
911 unsigned ExBits = ExVT.getScalarSizeInBits();
912 if (DemandedBits.getActiveBits() <= ExBits &&
913 shouldRemoveRedundantExtend(Op))
914 return Op0;
915 // If the input is already sign extended, just drop the extension.
916 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
917 if (NumSignBits >= (BitWidth - ExBits + 1))
918 return Op0;
919 break;
920 }
921 case ISD::ANY_EXTEND_VECTOR_INREG:
922 case ISD::SIGN_EXTEND_VECTOR_INREG:
923 case ISD::ZERO_EXTEND_VECTOR_INREG: {
924 if (VT.isScalableVector())
925 return SDValue();
926
927 // If we only want the lowest element and none of extended bits, then we can
928 // return the bitcasted source vector.
929 SDValue Src = Op.getOperand(i: 0);
930 EVT SrcVT = Src.getValueType();
931 EVT DstVT = Op.getValueType();
932 if (IsLE && DemandedElts == 1 &&
933 DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
934 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
935 return DAG.getBitcast(VT: DstVT, V: Src);
936 }
937 break;
938 }
939 case ISD::INSERT_VECTOR_ELT: {
940 if (VT.isScalableVector())
941 return SDValue();
942
943 // If we don't demand the inserted element, return the base vector.
944 SDValue Vec = Op.getOperand(i: 0);
945 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
946 EVT VecVT = Vec.getValueType();
947 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements()) &&
948 !DemandedElts[CIdx->getZExtValue()])
949 return Vec;
950 break;
951 }
952 case ISD::INSERT_SUBVECTOR: {
953 if (VT.isScalableVector())
954 return SDValue();
955
956 SDValue Vec = Op.getOperand(i: 0);
957 SDValue Sub = Op.getOperand(i: 1);
958 uint64_t Idx = Op.getConstantOperandVal(i: 2);
959 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
960 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
961 // If we don't demand the inserted subvector, return the base vector.
962 if (DemandedSubElts == 0)
963 return Vec;
964 break;
965 }
966 case ISD::VECTOR_SHUFFLE: {
967 assert(!VT.isScalableVector());
968 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
969
970 // If all the demanded elts are from one operand and are inline,
971 // then we can use the operand directly.
972 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
973 for (unsigned i = 0; i != NumElts; ++i) {
974 int M = ShuffleMask[i];
975 if (M < 0 || !DemandedElts[i])
976 continue;
977 AllUndef = false;
978 IdentityLHS &= (M == (int)i);
979 IdentityRHS &= ((M - NumElts) == i);
980 }
981
982 if (AllUndef)
983 return DAG.getUNDEF(VT: Op.getValueType());
984 if (IdentityLHS)
985 return Op.getOperand(i: 0);
986 if (IdentityRHS)
987 return Op.getOperand(i: 1);
988 break;
989 }
990 default:
991 // TODO: Probably okay to remove after audit; here to reduce change size
992 // in initial enablement patch for scalable vectors
993 if (VT.isScalableVector())
994 return SDValue();
995
996 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
997 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
998 Op, DemandedBits, DemandedElts, DAG, Depth))
999 return V;
1000 break;
1001 }
1002 return SDValue();
1003}
1004
1005SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
1006 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
1007 unsigned Depth) const {
1008 EVT VT = Op.getValueType();
1009 // Since the number of lanes in a scalable vector is unknown at compile time,
1010 // we track one bit which is implicitly broadcast to all lanes. This means
1011 // that all lanes in a scalable vector are considered demanded.
1012 APInt DemandedElts = VT.isFixedLengthVector()
1013 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
1014 : APInt(1, 1);
1015 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1016 Depth);
1017}
1018
1019SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
1020 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
1021 unsigned Depth) const {
1022 APInt DemandedBits = APInt::getAllOnes(numBits: Op.getScalarValueSizeInBits());
1023 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
1024 Depth);
1025}
1026
1027// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
1028// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
1029static SDValue combineShiftToAVG(SDValue Op,
1030 TargetLowering::TargetLoweringOpt &TLO,
1031 const TargetLowering &TLI,
1032 const APInt &DemandedBits,
1033 const APInt &DemandedElts, unsigned Depth) {
1034 assert((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SRA) &&
1035 "SRL or SRA node is required here!");
1036 // Is the right shift using an immediate value of 1?
1037 ConstantSDNode *N1C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
1038 if (!N1C || !N1C->isOne())
1039 return SDValue();
1040
1041 // We are looking for an avgfloor
1042 // add(ext, ext)
1043 // or one of these as a avgceil
1044 // add(add(ext, ext), 1)
1045 // add(add(ext, 1), ext)
1046 // add(ext, add(ext, 1))
1047 SDValue Add = Op.getOperand(i: 0);
1048 if (Add.getOpcode() != ISD::ADD)
1049 return SDValue();
1050
1051 SDValue ExtOpA = Add.getOperand(i: 0);
1052 SDValue ExtOpB = Add.getOperand(i: 1);
1053 SDValue Add2;
1054 auto MatchOperands = [&](SDValue Op1, SDValue Op2, SDValue Op3, SDValue A) {
1055 ConstantSDNode *ConstOp;
1056 if ((ConstOp = isConstOrConstSplat(N: Op2, DemandedElts)) &&
1057 ConstOp->isOne()) {
1058 ExtOpA = Op1;
1059 ExtOpB = Op3;
1060 Add2 = A;
1061 return true;
1062 }
1063 if ((ConstOp = isConstOrConstSplat(N: Op3, DemandedElts)) &&
1064 ConstOp->isOne()) {
1065 ExtOpA = Op1;
1066 ExtOpB = Op2;
1067 Add2 = A;
1068 return true;
1069 }
1070 return false;
1071 };
1072 bool IsCeil =
1073 (ExtOpA.getOpcode() == ISD::ADD &&
1074 MatchOperands(ExtOpA.getOperand(i: 0), ExtOpA.getOperand(i: 1), ExtOpB, ExtOpA)) ||
1075 (ExtOpB.getOpcode() == ISD::ADD &&
1076 MatchOperands(ExtOpB.getOperand(i: 0), ExtOpB.getOperand(i: 1), ExtOpA, ExtOpB));
1077
1078 // If the shift is signed (sra):
1079 // - Needs >= 2 sign bit for both operands.
1080 // - Needs >= 2 zero bits.
1081 // If the shift is unsigned (srl):
1082 // - Needs >= 1 zero bit for both operands.
1083 // - Needs 1 demanded bit zero and >= 2 sign bits.
1084 SelectionDAG &DAG = TLO.DAG;
1085 unsigned ShiftOpc = Op.getOpcode();
1086 bool IsSigned = false;
1087 unsigned KnownBits;
1088 unsigned NumSignedA = DAG.ComputeNumSignBits(Op: ExtOpA, DemandedElts, Depth);
1089 unsigned NumSignedB = DAG.ComputeNumSignBits(Op: ExtOpB, DemandedElts, Depth);
1090 unsigned NumSigned = std::min(a: NumSignedA, b: NumSignedB) - 1;
1091 unsigned NumZeroA =
1092 DAG.computeKnownBits(Op: ExtOpA, DemandedElts, Depth).countMinLeadingZeros();
1093 unsigned NumZeroB =
1094 DAG.computeKnownBits(Op: ExtOpB, DemandedElts, Depth).countMinLeadingZeros();
1095 unsigned NumZero = std::min(a: NumZeroA, b: NumZeroB);
1096
1097 switch (ShiftOpc) {
1098 default:
1099 llvm_unreachable("Unexpected ShiftOpc in combineShiftToAVG");
1100 case ISD::SRA: {
1101 if (NumZero >= 2 && NumSigned < NumZero) {
1102 IsSigned = false;
1103 KnownBits = NumZero;
1104 break;
1105 }
1106 if (NumSigned >= 1) {
1107 IsSigned = true;
1108 KnownBits = NumSigned;
1109 break;
1110 }
1111 return SDValue();
1112 }
1113 case ISD::SRL: {
1114 if (NumZero >= 1 && NumSigned < NumZero) {
1115 IsSigned = false;
1116 KnownBits = NumZero;
1117 break;
1118 }
1119 if (NumSigned >= 1 && DemandedBits.isSignBitClear()) {
1120 IsSigned = true;
1121 KnownBits = NumSigned;
1122 break;
1123 }
1124 return SDValue();
1125 }
1126 }
1127
1128 unsigned AVGOpc = IsCeil ? (IsSigned ? ISD::AVGCEILS : ISD::AVGCEILU)
1129 : (IsSigned ? ISD::AVGFLOORS : ISD::AVGFLOORU);
1130
1131 // Find the smallest power-2 type that is legal for this vector size and
1132 // operation, given the original type size and the number of known sign/zero
1133 // bits.
1134 EVT VT = Op.getValueType();
1135 unsigned MinWidth =
1136 std::max<unsigned>(a: VT.getScalarSizeInBits() - KnownBits, b: 8);
1137 EVT NVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: llvm::bit_ceil(Value: MinWidth));
1138 if (NVT.getScalarSizeInBits() > VT.getScalarSizeInBits())
1139 return SDValue();
1140 if (VT.isVector())
1141 NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NVT, EC: VT.getVectorElementCount());
1142 if (TLO.LegalTypes() && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT)) {
1143 // If we could not transform, and (both) adds are nuw/nsw, we can use the
1144 // larger type size to do the transform.
1145 if (TLO.LegalOperations() && !TLI.isOperationLegal(Op: AVGOpc, VT))
1146 return SDValue();
1147 if (DAG.willNotOverflowAdd(IsSigned, N0: Add.getOperand(i: 0),
1148 N1: Add.getOperand(i: 1)) &&
1149 (!Add2 || DAG.willNotOverflowAdd(IsSigned, N0: Add2.getOperand(i: 0),
1150 N1: Add2.getOperand(i: 1))))
1151 NVT = VT;
1152 else
1153 return SDValue();
1154 }
1155
1156 // Don't create a AVGFLOOR node with a scalar constant unless its legal as
1157 // this is likely to stop other folds (reassociation, value tracking etc.)
1158 if (!IsCeil && !TLI.isOperationLegal(Op: AVGOpc, VT: NVT) &&
1159 (isa<ConstantSDNode>(Val: ExtOpA) || isa<ConstantSDNode>(Val: ExtOpB)))
1160 return SDValue();
1161
1162 SDLoc DL(Op);
1163 SDValue ResultAVG =
1164 DAG.getNode(Opcode: AVGOpc, DL, VT: NVT, N1: DAG.getExtOrTrunc(IsSigned, Op: ExtOpA, DL, VT: NVT),
1165 N2: DAG.getExtOrTrunc(IsSigned, Op: ExtOpB, DL, VT: NVT));
1166 return DAG.getExtOrTrunc(IsSigned, Op: ResultAVG, DL, VT);
1167}
1168
1169/// Look at Op. At this point, we know that only the OriginalDemandedBits of the
1170/// result of Op are ever used downstream. If we can use this information to
1171/// simplify Op, create a new simplified DAG node and return true, returning the
1172/// original and new nodes in Old and New. Otherwise, analyze the expression and
1173/// return a mask of Known bits for the expression (used to simplify the
1174/// caller). The Known bits may only be accurate for those bits in the
1175/// OriginalDemandedBits and OriginalDemandedElts.
1176bool TargetLowering::SimplifyDemandedBits(
1177 SDValue Op, const APInt &OriginalDemandedBits,
1178 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
1179 unsigned Depth, bool AssumeSingleUse) const {
1180 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
1181 assert(Op.getScalarValueSizeInBits() == BitWidth &&
1182 "Mask size mismatches value type size!");
1183
1184 // Don't know anything.
1185 Known = KnownBits(BitWidth);
1186
1187 EVT VT = Op.getValueType();
1188 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
1189 unsigned NumElts = OriginalDemandedElts.getBitWidth();
1190 assert((!VT.isFixedLengthVector() || NumElts == VT.getVectorNumElements()) &&
1191 "Unexpected vector size");
1192
1193 APInt DemandedBits = OriginalDemandedBits;
1194 APInt DemandedElts = OriginalDemandedElts;
1195 SDLoc dl(Op);
1196
1197 // Undef operand.
1198 if (Op.isUndef())
1199 return false;
1200
1201 // We can't simplify target constants.
1202 if (Op.getOpcode() == ISD::TargetConstant)
1203 return false;
1204
1205 if (Op.getOpcode() == ISD::Constant) {
1206 // We know all of the bits for a constant!
1207 Known = KnownBits::makeConstant(C: Op->getAsAPIntVal());
1208 return false;
1209 }
1210
1211 if (Op.getOpcode() == ISD::ConstantFP) {
1212 // We know all of the bits for a floating point constant!
1213 Known = KnownBits::makeConstant(
1214 C: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF().bitcastToAPInt());
1215 return false;
1216 }
1217
1218 // Other users may use these bits.
1219 bool HasMultiUse = false;
1220 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1221 if (Depth >= SelectionDAG::MaxRecursionDepth) {
1222 // Limit search depth.
1223 return false;
1224 }
1225 // Allow multiple uses, just set the DemandedBits/Elts to all bits.
1226 DemandedBits = APInt::getAllOnes(numBits: BitWidth);
1227 DemandedElts = APInt::getAllOnes(numBits: NumElts);
1228 HasMultiUse = true;
1229 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1230 // Not demanding any bits/elts from Op.
1231 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
1232 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
1233 // Limit search depth.
1234 return false;
1235 }
1236
1237 KnownBits Known2;
1238 switch (Op.getOpcode()) {
1239 case ISD::SCALAR_TO_VECTOR: {
1240 if (VT.isScalableVector())
1241 return false;
1242 if (!DemandedElts[0])
1243 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
1244
1245 KnownBits SrcKnown;
1246 SDValue Src = Op.getOperand(i: 0);
1247 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1248 APInt SrcDemandedBits = DemandedBits.zext(width: SrcBitWidth);
1249 if (SimplifyDemandedBits(Op: Src, DemandedBits: SrcDemandedBits, Known&: SrcKnown, TLO, Depth: Depth + 1))
1250 return true;
1251
1252 // Upper elements are undef, so only get the knownbits if we just demand
1253 // the bottom element.
1254 if (DemandedElts == 1)
1255 Known = SrcKnown.anyextOrTrunc(BitWidth);
1256 break;
1257 }
1258 case ISD::BUILD_VECTOR:
1259 // Collect the known bits that are shared by every demanded element.
1260 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
1261 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1262 return false; // Don't fall through, will infinitely loop.
1263 case ISD::SPLAT_VECTOR: {
1264 SDValue Scl = Op.getOperand(i: 0);
1265 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: Scl.getValueSizeInBits());
1266 KnownBits KnownScl;
1267 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1268 return true;
1269
1270 // Implicitly truncate the bits to match the official semantics of
1271 // SPLAT_VECTOR.
1272 Known = KnownScl.trunc(BitWidth);
1273 break;
1274 }
1275 case ISD::FREEZE: {
1276 SDValue N0 = Op.getOperand(i: 0);
1277 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
1278 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
1279 return TLO.CombineTo(O: Op, N: N0);
1280 break;
1281 }
1282 case ISD::LOAD: {
1283 auto *LD = cast<LoadSDNode>(Val&: Op);
1284 if (getTargetConstantFromLoad(LD)) {
1285 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1286 return false; // Don't fall through, will infinitely loop.
1287 }
1288 if (ISD::isZEXTLoad(N: Op.getNode()) && Op.getResNo() == 0) {
1289 // If this is a ZEXTLoad and we are looking at the loaded value.
1290 EVT MemVT = LD->getMemoryVT();
1291 unsigned MemBits = MemVT.getScalarSizeInBits();
1292 Known.Zero.setBitsFrom(MemBits);
1293 return false; // Don't fall through, will infinitely loop.
1294 }
1295 break;
1296 }
1297 case ISD::INSERT_VECTOR_ELT: {
1298 if (VT.isScalableVector())
1299 return false;
1300 SDValue Vec = Op.getOperand(i: 0);
1301 SDValue Scl = Op.getOperand(i: 1);
1302 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
1303 EVT VecVT = Vec.getValueType();
1304
1305 // If index isn't constant, assume we need all vector elements AND the
1306 // inserted element.
1307 APInt DemandedVecElts(DemandedElts);
1308 if (CIdx && CIdx->getAPIntValue().ult(RHS: VecVT.getVectorNumElements())) {
1309 unsigned Idx = CIdx->getZExtValue();
1310 DemandedVecElts.clearBit(BitPosition: Idx);
1311
1312 // Inserted element is not required.
1313 if (!DemandedElts[Idx])
1314 return TLO.CombineTo(O: Op, N: Vec);
1315 }
1316
1317 KnownBits KnownScl;
1318 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1319 APInt DemandedSclBits = DemandedBits.zextOrTrunc(width: NumSclBits);
1320 if (SimplifyDemandedBits(Op: Scl, DemandedBits: DemandedSclBits, Known&: KnownScl, TLO, Depth: Depth + 1))
1321 return true;
1322
1323 Known = KnownScl.anyextOrTrunc(BitWidth);
1324
1325 KnownBits KnownVec;
1326 if (SimplifyDemandedBits(Op: Vec, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedVecElts, Known&: KnownVec, TLO,
1327 Depth: Depth + 1))
1328 return true;
1329
1330 if (!!DemandedVecElts)
1331 Known = Known.intersectWith(RHS: KnownVec);
1332
1333 return false;
1334 }
1335 case ISD::INSERT_SUBVECTOR: {
1336 if (VT.isScalableVector())
1337 return false;
1338 // Demand any elements from the subvector and the remainder from the src its
1339 // inserted into.
1340 SDValue Src = Op.getOperand(i: 0);
1341 SDValue Sub = Op.getOperand(i: 1);
1342 uint64_t Idx = Op.getConstantOperandVal(i: 2);
1343 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1344 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
1345 APInt DemandedSrcElts = DemandedElts;
1346 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
1347
1348 KnownBits KnownSub, KnownSrc;
1349 if (SimplifyDemandedBits(Op: Sub, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts, Known&: KnownSub, TLO,
1350 Depth: Depth + 1))
1351 return true;
1352 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known&: KnownSrc, TLO,
1353 Depth: Depth + 1))
1354 return true;
1355
1356 Known.setAllConflict();
1357 if (!!DemandedSubElts)
1358 Known = Known.intersectWith(RHS: KnownSub);
1359 if (!!DemandedSrcElts)
1360 Known = Known.intersectWith(RHS: KnownSrc);
1361
1362 // Attempt to avoid multi-use src if we don't need anything from it.
1363 if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1364 !DemandedSrcElts.isAllOnes()) {
1365 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1366 Op: Sub, DemandedBits, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
1367 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1368 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1369 if (NewSub || NewSrc) {
1370 NewSub = NewSub ? NewSub : Sub;
1371 NewSrc = NewSrc ? NewSrc : Src;
1372 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: NewSrc, N2: NewSub,
1373 N3: Op.getOperand(i: 2));
1374 return TLO.CombineTo(O: Op, N: NewOp);
1375 }
1376 }
1377 break;
1378 }
1379 case ISD::EXTRACT_SUBVECTOR: {
1380 if (VT.isScalableVector())
1381 return false;
1382 // Offset the demanded elts by the subvector index.
1383 SDValue Src = Op.getOperand(i: 0);
1384 if (Src.getValueType().isScalableVector())
1385 break;
1386 uint64_t Idx = Op.getConstantOperandVal(i: 1);
1387 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1388 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
1389
1390 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSrcElts, Known, TLO,
1391 Depth: Depth + 1))
1392 return true;
1393
1394 // Attempt to avoid multi-use src if we don't need anything from it.
1395 if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1396 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1397 Op: Src, DemandedBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
1398 if (DemandedSrc) {
1399 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc,
1400 N2: Op.getOperand(i: 1));
1401 return TLO.CombineTo(O: Op, N: NewOp);
1402 }
1403 }
1404 break;
1405 }
1406 case ISD::CONCAT_VECTORS: {
1407 if (VT.isScalableVector())
1408 return false;
1409 Known.setAllConflict();
1410 EVT SubVT = Op.getOperand(i: 0).getValueType();
1411 unsigned NumSubVecs = Op.getNumOperands();
1412 unsigned NumSubElts = SubVT.getVectorNumElements();
1413 for (unsigned i = 0; i != NumSubVecs; ++i) {
1414 APInt DemandedSubElts =
1415 DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
1416 if (SimplifyDemandedBits(Op: Op.getOperand(i), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedSubElts,
1417 Known&: Known2, TLO, Depth: Depth + 1))
1418 return true;
1419 // Known bits are shared by every demanded subvector element.
1420 if (!!DemandedSubElts)
1421 Known = Known.intersectWith(RHS: Known2);
1422 }
1423 break;
1424 }
1425 case ISD::VECTOR_SHUFFLE: {
1426 assert(!VT.isScalableVector());
1427 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
1428
1429 // Collect demanded elements from shuffle operands..
1430 APInt DemandedLHS, DemandedRHS;
1431 if (!getShuffleDemandedElts(SrcWidth: NumElts, Mask: ShuffleMask, DemandedElts, DemandedLHS,
1432 DemandedRHS))
1433 break;
1434
1435 if (!!DemandedLHS || !!DemandedRHS) {
1436 SDValue Op0 = Op.getOperand(i: 0);
1437 SDValue Op1 = Op.getOperand(i: 1);
1438
1439 Known.setAllConflict();
1440 if (!!DemandedLHS) {
1441 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedLHS, Known&: Known2, TLO,
1442 Depth: Depth + 1))
1443 return true;
1444 Known = Known.intersectWith(RHS: Known2);
1445 }
1446 if (!!DemandedRHS) {
1447 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedRHS, Known&: Known2, TLO,
1448 Depth: Depth + 1))
1449 return true;
1450 Known = Known.intersectWith(RHS: Known2);
1451 }
1452
1453 // Attempt to avoid multi-use ops if we don't need anything from them.
1454 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1455 Op: Op0, DemandedBits, DemandedElts: DemandedLHS, DAG&: TLO.DAG, Depth: Depth + 1);
1456 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1457 Op: Op1, DemandedBits, DemandedElts: DemandedRHS, DAG&: TLO.DAG, Depth: Depth + 1);
1458 if (DemandedOp0 || DemandedOp1) {
1459 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1460 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1461 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, N1: Op0, N2: Op1, Mask: ShuffleMask);
1462 return TLO.CombineTo(O: Op, N: NewOp);
1463 }
1464 }
1465 break;
1466 }
1467 case ISD::AND: {
1468 SDValue Op0 = Op.getOperand(i: 0);
1469 SDValue Op1 = Op.getOperand(i: 1);
1470
1471 // If the RHS is a constant, check to see if the LHS would be zero without
1472 // using the bits from the RHS. Below, we use knowledge about the RHS to
1473 // simplify the LHS, here we're using information from the LHS to simplify
1474 // the RHS.
1475 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: Op1, DemandedElts)) {
1476 // Do not increment Depth here; that can cause an infinite loop.
1477 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth);
1478 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1479 if ((LHSKnown.Zero & DemandedBits) ==
1480 (~RHSC->getAPIntValue() & DemandedBits))
1481 return TLO.CombineTo(O: Op, N: Op0);
1482
1483 // If any of the set bits in the RHS are known zero on the LHS, shrink
1484 // the constant.
1485 if (ShrinkDemandedConstant(Op, DemandedBits: ~LHSKnown.Zero & DemandedBits,
1486 DemandedElts, TLO))
1487 return true;
1488
1489 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1490 // constant, but if this 'and' is only clearing bits that were just set by
1491 // the xor, then this 'and' can be eliminated by shrinking the mask of
1492 // the xor. For example, for a 32-bit X:
1493 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1494 if (isBitwiseNot(V: Op0) && Op0.hasOneUse() &&
1495 LHSKnown.One == ~RHSC->getAPIntValue()) {
1496 SDValue Xor = TLO.DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1497 return TLO.CombineTo(O: Op, N: Xor);
1498 }
1499 }
1500
1501 // (X +/- Y) & Y --> ~X & Y when Y is a power of 2 (or zero).
1502 SDValue X, Y;
1503 if (sd_match(N: Op,
1504 P: m_And(L: m_Value(N&: Y),
1505 R: m_OneUse(P: m_AnyOf(preds: m_Add(L: m_Value(N&: X), R: m_Deferred(V&: Y)),
1506 preds: m_Sub(L: m_Value(N&: X), R: m_Deferred(V&: Y)))))) &&
1507 TLO.DAG.isKnownToBeAPowerOfTwo(Val: Y, DemandedElts, /*OrZero=*/true)) {
1508 return TLO.CombineTo(
1509 O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: TLO.DAG.getNOT(DL: dl, Val: X, VT), N2: Y));
1510 }
1511
1512 // AND(INSERT_SUBVECTOR(C,X,I),M) -> INSERT_SUBVECTOR(AND(C,M),X,I)
1513 // iff 'C' is Undef/Constant and AND(X,M) == X (for DemandedBits).
1514 if (Op0.getOpcode() == ISD::INSERT_SUBVECTOR && !VT.isScalableVector() &&
1515 (Op0.getOperand(i: 0).isUndef() ||
1516 ISD::isBuildVectorOfConstantSDNodes(N: Op0.getOperand(i: 0).getNode())) &&
1517 Op0->hasOneUse()) {
1518 unsigned NumSubElts =
1519 Op0.getOperand(i: 1).getValueType().getVectorNumElements();
1520 unsigned SubIdx = Op0.getConstantOperandVal(i: 2);
1521 APInt DemandedSub =
1522 APInt::getBitsSet(numBits: NumElts, loBit: SubIdx, hiBit: SubIdx + NumSubElts);
1523 KnownBits KnownSubMask =
1524 TLO.DAG.computeKnownBits(Op: Op1, DemandedElts: DemandedSub & DemandedElts, Depth: Depth + 1);
1525 if (DemandedBits.isSubsetOf(RHS: KnownSubMask.One)) {
1526 SDValue NewAnd =
1527 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: Op1);
1528 SDValue NewInsert =
1529 TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT, N1: NewAnd,
1530 N2: Op0.getOperand(i: 1), N3: Op0.getOperand(i: 2));
1531 return TLO.CombineTo(O: Op, N: NewInsert);
1532 }
1533 }
1534
1535 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1536 Depth: Depth + 1))
1537 return true;
1538 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.Zero & DemandedBits, OriginalDemandedElts: DemandedElts,
1539 Known&: Known2, TLO, Depth: Depth + 1))
1540 return true;
1541
1542 // If all of the demanded bits are known one on one side, return the other.
1543 // These bits cannot contribute to the result of the 'and'.
1544 if (DemandedBits.isSubsetOf(RHS: Known2.Zero | Known.One))
1545 return TLO.CombineTo(O: Op, N: Op0);
1546 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.One))
1547 return TLO.CombineTo(O: Op, N: Op1);
1548 // If all of the demanded bits in the inputs are known zeros, return zero.
1549 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1550 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: dl, VT));
1551 // If the RHS is a constant, see if we can simplify it.
1552 if (ShrinkDemandedConstant(Op, DemandedBits: ~Known2.Zero & DemandedBits, DemandedElts,
1553 TLO))
1554 return true;
1555 // If the operation can be done in a smaller type, do so.
1556 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1557 return true;
1558
1559 // Attempt to avoid multi-use ops if we don't need anything from them.
1560 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1561 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1562 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1563 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1564 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1565 if (DemandedOp0 || DemandedOp1) {
1566 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1567 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1568 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1569 return TLO.CombineTo(O: Op, N: NewOp);
1570 }
1571 }
1572
1573 Known &= Known2;
1574 break;
1575 }
1576 case ISD::OR: {
1577 SDValue Op0 = Op.getOperand(i: 0);
1578 SDValue Op1 = Op.getOperand(i: 1);
1579 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1580 Depth: Depth + 1)) {
1581 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1582 return true;
1583 }
1584
1585 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~Known.One & DemandedBits, OriginalDemandedElts: DemandedElts,
1586 Known&: Known2, TLO, Depth: Depth + 1)) {
1587 Op->dropFlags(Mask: SDNodeFlags::Disjoint);
1588 return true;
1589 }
1590
1591 // If all of the demanded bits are known zero on one side, return the other.
1592 // These bits cannot contribute to the result of the 'or'.
1593 if (DemandedBits.isSubsetOf(RHS: Known2.One | Known.Zero))
1594 return TLO.CombineTo(O: Op, N: Op0);
1595 if (DemandedBits.isSubsetOf(RHS: Known.One | Known2.Zero))
1596 return TLO.CombineTo(O: Op, N: Op1);
1597 // If the RHS is a constant, see if we can simplify it.
1598 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1599 return true;
1600 // If the operation can be done in a smaller type, do so.
1601 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1602 return true;
1603
1604 // Attempt to avoid multi-use ops if we don't need anything from them.
1605 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1606 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1607 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1608 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1609 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1610 if (DemandedOp0 || DemandedOp1) {
1611 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1612 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1613 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1614 return TLO.CombineTo(O: Op, N: NewOp);
1615 }
1616 }
1617
1618 // (or (and X, C1), (and (or X, Y), C2)) -> (or (and X, C1|C2), (and Y, C2))
1619 // TODO: Use SimplifyMultipleUseDemandedBits to peek through masks.
1620 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::AND &&
1621 Op0->hasOneUse() && Op1->hasOneUse()) {
1622 // Attempt to match all commutations - m_c_Or would've been useful!
1623 for (int I = 0; I != 2; ++I) {
1624 SDValue X = Op.getOperand(i: I).getOperand(i: 0);
1625 SDValue C1 = Op.getOperand(i: I).getOperand(i: 1);
1626 SDValue Alt = Op.getOperand(i: 1 - I).getOperand(i: 0);
1627 SDValue C2 = Op.getOperand(i: 1 - I).getOperand(i: 1);
1628 if (Alt.getOpcode() == ISD::OR) {
1629 for (int J = 0; J != 2; ++J) {
1630 if (X == Alt.getOperand(i: J)) {
1631 SDValue Y = Alt.getOperand(i: 1 - J);
1632 if (SDValue C12 = TLO.DAG.FoldConstantArithmetic(Opcode: ISD::OR, DL: dl, VT,
1633 Ops: {C1, C2})) {
1634 SDValue MaskX = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: X, N2: C12);
1635 SDValue MaskY = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Y, N2: C2);
1636 return TLO.CombineTo(
1637 O: Op, N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: MaskX, N2: MaskY));
1638 }
1639 }
1640 }
1641 }
1642 }
1643 }
1644
1645 Known |= Known2;
1646 break;
1647 }
1648 case ISD::XOR: {
1649 SDValue Op0 = Op.getOperand(i: 0);
1650 SDValue Op1 = Op.getOperand(i: 1);
1651
1652 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
1653 Depth: Depth + 1))
1654 return true;
1655 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
1656 Depth: Depth + 1))
1657 return true;
1658
1659 // If all of the demanded bits are known zero on one side, return the other.
1660 // These bits cannot contribute to the result of the 'xor'.
1661 if (DemandedBits.isSubsetOf(RHS: Known.Zero))
1662 return TLO.CombineTo(O: Op, N: Op0);
1663 if (DemandedBits.isSubsetOf(RHS: Known2.Zero))
1664 return TLO.CombineTo(O: Op, N: Op1);
1665 // If the operation can be done in a smaller type, do so.
1666 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1667 return true;
1668
1669 // If all of the unknown bits are known to be zero on one side or the other
1670 // turn this into an *inclusive* or.
1671 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1672 if (DemandedBits.isSubsetOf(RHS: Known.Zero | Known2.Zero))
1673 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op0, N2: Op1));
1674
1675 ConstantSDNode *C = isConstOrConstSplat(N: Op1, DemandedElts);
1676 if (C) {
1677 // If one side is a constant, and all of the set bits in the constant are
1678 // also known set on the other side, turn this into an AND, as we know
1679 // the bits will be cleared.
1680 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1681 // NB: it is okay if more bits are known than are requested
1682 if (C->getAPIntValue() == Known2.One) {
1683 SDValue ANDC =
1684 TLO.DAG.getConstant(Val: ~C->getAPIntValue() & DemandedBits, DL: dl, VT);
1685 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op0, N2: ANDC));
1686 }
1687
1688 // If the RHS is a constant, see if we can change it. Don't alter a -1
1689 // constant because that's a 'not' op, and that is better for combining
1690 // and codegen.
1691 if (!C->isAllOnes() && DemandedBits.isSubsetOf(RHS: C->getAPIntValue())) {
1692 // We're flipping all demanded bits. Flip the undemanded bits too.
1693 SDValue New = TLO.DAG.getNOT(DL: dl, Val: Op0, VT);
1694 return TLO.CombineTo(O: Op, N: New);
1695 }
1696
1697 unsigned Op0Opcode = Op0.getOpcode();
1698 if ((Op0Opcode == ISD::SRL || Op0Opcode == ISD::SHL) && Op0.hasOneUse()) {
1699 if (ConstantSDNode *ShiftC =
1700 isConstOrConstSplat(N: Op0.getOperand(i: 1), DemandedElts)) {
1701 // Don't crash on an oversized shift. We can not guarantee that a
1702 // bogus shift has been simplified to undef.
1703 if (ShiftC->getAPIntValue().ult(RHS: BitWidth)) {
1704 uint64_t ShiftAmt = ShiftC->getZExtValue();
1705 APInt Ones = APInt::getAllOnes(numBits: BitWidth);
1706 Ones = Op0Opcode == ISD::SHL ? Ones.shl(shiftAmt: ShiftAmt)
1707 : Ones.lshr(shiftAmt: ShiftAmt);
1708 if ((DemandedBits & C->getAPIntValue()) == (DemandedBits & Ones) &&
1709 isDesirableToCommuteXorWithShift(N: Op.getNode())) {
1710 // If the xor constant is a demanded mask, do a 'not' before the
1711 // shift:
1712 // xor (X << ShiftC), XorC --> (not X) << ShiftC
1713 // xor (X >> ShiftC), XorC --> (not X) >> ShiftC
1714 SDValue Not = TLO.DAG.getNOT(DL: dl, Val: Op0.getOperand(i: 0), VT);
1715 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op0Opcode, DL: dl, VT, N1: Not,
1716 N2: Op0.getOperand(i: 1)));
1717 }
1718 }
1719 }
1720 }
1721 }
1722
1723 // If we can't turn this into a 'not', try to shrink the constant.
1724 if (!C || !C->isAllOnes())
1725 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1726 return true;
1727
1728 // Attempt to avoid multi-use ops if we don't need anything from them.
1729 if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1730 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1731 Op: Op0, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1732 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1733 Op: Op1, DemandedBits, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1734 if (DemandedOp0 || DemandedOp1) {
1735 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1736 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1737 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1);
1738 return TLO.CombineTo(O: Op, N: NewOp);
1739 }
1740 }
1741
1742 Known ^= Known2;
1743 break;
1744 }
1745 case ISD::SELECT:
1746 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1747 Known, TLO, Depth: Depth + 1))
1748 return true;
1749 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1750 Known&: Known2, TLO, Depth: Depth + 1))
1751 return true;
1752
1753 // If the operands are constants, see if we can simplify them.
1754 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1755 return true;
1756
1757 // Only known if known in both the LHS and RHS.
1758 Known = Known.intersectWith(RHS: Known2);
1759 break;
1760 case ISD::VSELECT:
1761 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1762 Known, TLO, Depth: Depth + 1))
1763 return true;
1764 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1765 Known&: Known2, TLO, Depth: Depth + 1))
1766 return true;
1767
1768 // Only known if known in both the LHS and RHS.
1769 Known = Known.intersectWith(RHS: Known2);
1770 break;
1771 case ISD::SELECT_CC:
1772 if (SimplifyDemandedBits(Op: Op.getOperand(i: 3), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1773 Known, TLO, Depth: Depth + 1))
1774 return true;
1775 if (SimplifyDemandedBits(Op: Op.getOperand(i: 2), OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
1776 Known&: Known2, TLO, Depth: Depth + 1))
1777 return true;
1778
1779 // If the operands are constants, see if we can simplify them.
1780 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1781 return true;
1782
1783 // Only known if known in both the LHS and RHS.
1784 Known = Known.intersectWith(RHS: Known2);
1785 break;
1786 case ISD::SETCC: {
1787 SDValue Op0 = Op.getOperand(i: 0);
1788 SDValue Op1 = Op.getOperand(i: 1);
1789 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
1790 // If we're testing X < 0, X >= 0, X <= -1 or X > -1
1791 // (X is of integer type) then we only need the sign mask of the previous
1792 // result
1793 if (Op1.getValueType().isInteger() &&
1794 (((CC == ISD::SETLT || CC == ISD::SETGE) && isNullOrNullSplat(V: Op1)) ||
1795 ((CC == ISD::SETLE || CC == ISD::SETGT) &&
1796 isAllOnesOrAllOnesSplat(V: Op1)))) {
1797 KnownBits KnownOp0;
1798 if (SimplifyDemandedBits(
1799 Op: Op0, OriginalDemandedBits: APInt::getSignMask(BitWidth: Op0.getScalarValueSizeInBits()),
1800 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1))
1801 return true;
1802 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1803 // width as the setcc result, and (3) the result of a setcc conforms to 0
1804 // or -1, we may be able to bypass the setcc.
1805 if (DemandedBits.isSignMask() &&
1806 Op0.getScalarValueSizeInBits() == BitWidth &&
1807 getBooleanContents(Type: Op0.getValueType()) ==
1808 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1809 // If we remove a >= 0 or > -1 (for integers), we need to introduce a
1810 // NOT Operation
1811 if (CC == ISD::SETGE || CC == ISD::SETGT) {
1812 SDLoc DL(Op);
1813 EVT VT = Op0.getValueType();
1814 SDValue NotOp0 = TLO.DAG.getNOT(DL, Val: Op0, VT);
1815 return TLO.CombineTo(O: Op, N: NotOp0);
1816 }
1817 return TLO.CombineTo(O: Op, N: Op0);
1818 }
1819 }
1820 if (getBooleanContents(Type: Op0.getValueType()) ==
1821 TargetLowering::ZeroOrOneBooleanContent &&
1822 BitWidth > 1)
1823 Known.Zero.setBitsFrom(1);
1824 break;
1825 }
1826 case ISD::SHL: {
1827 SDValue Op0 = Op.getOperand(i: 0);
1828 SDValue Op1 = Op.getOperand(i: 1);
1829 EVT ShiftVT = Op1.getValueType();
1830
1831 if (std::optional<unsigned> KnownSA =
1832 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
1833 unsigned ShAmt = *KnownSA;
1834 if (ShAmt == 0)
1835 return TLO.CombineTo(O: Op, N: Op0);
1836
1837 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1838 // single shift. We can do this if the bottom bits (which are shifted
1839 // out) are never demanded.
1840 // TODO - support non-uniform vector amounts.
1841 if (Op0.getOpcode() == ISD::SRL) {
1842 if (!DemandedBits.intersects(RHS: APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ShAmt))) {
1843 if (std::optional<unsigned> InnerSA =
1844 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
1845 unsigned C1 = *InnerSA;
1846 unsigned Opc = ISD::SHL;
1847 int Diff = ShAmt - C1;
1848 if (Diff < 0) {
1849 Diff = -Diff;
1850 Opc = ISD::SRL;
1851 }
1852 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
1853 return TLO.CombineTo(
1854 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
1855 }
1856 }
1857 }
1858
1859 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1860 // are not demanded. This will likely allow the anyext to be folded away.
1861 // TODO - support non-uniform vector amounts.
1862 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1863 SDValue InnerOp = Op0.getOperand(i: 0);
1864 EVT InnerVT = InnerOp.getValueType();
1865 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1866 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1867 isTypeDesirableForOp(ISD::SHL, VT: InnerVT)) {
1868 SDValue NarrowShl = TLO.DAG.getNode(
1869 Opcode: ISD::SHL, DL: dl, VT: InnerVT, N1: InnerOp,
1870 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: InnerVT, DL: dl));
1871 return TLO.CombineTo(
1872 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
1873 }
1874
1875 // Repeat the SHL optimization above in cases where an extension
1876 // intervenes: (shl (anyext (shr x, c1)), c2) to
1877 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1878 // aren't demanded (as above) and that the shifted upper c1 bits of
1879 // x aren't demanded.
1880 // TODO - support non-uniform vector amounts.
1881 if (InnerOp.getOpcode() == ISD::SRL && Op0.hasOneUse() &&
1882 InnerOp.hasOneUse()) {
1883 if (std::optional<unsigned> SA2 = TLO.DAG.getValidShiftAmount(
1884 V: InnerOp, DemandedElts, Depth: Depth + 2)) {
1885 unsigned InnerShAmt = *SA2;
1886 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1887 DemandedBits.getActiveBits() <=
1888 (InnerBits - InnerShAmt + ShAmt) &&
1889 DemandedBits.countr_zero() >= ShAmt) {
1890 SDValue NewSA =
1891 TLO.DAG.getConstant(Val: ShAmt - InnerShAmt, DL: dl, VT: ShiftVT);
1892 SDValue NewExt = TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT,
1893 Operand: InnerOp.getOperand(i: 0));
1894 return TLO.CombineTo(
1895 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: NewExt, N2: NewSA));
1896 }
1897 }
1898 }
1899 }
1900
1901 APInt InDemandedMask = DemandedBits.lshr(shiftAmt: ShAmt);
1902 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
1903 Depth: Depth + 1)) {
1904 // Disable the nsw and nuw flags. We can no longer guarantee that we
1905 // won't wrap after simplification.
1906 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
1907 return true;
1908 }
1909 Known <<= ShAmt;
1910 // low bits known zero.
1911 Known.Zero.setLowBits(ShAmt);
1912
1913 // Attempt to avoid multi-use ops if we don't need anything from them.
1914 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1915 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1916 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
1917 if (DemandedOp0) {
1918 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
1919 return TLO.CombineTo(O: Op, N: NewOp);
1920 }
1921 }
1922
1923 // TODO: Can we merge this fold with the one below?
1924 // Try shrinking the operation as long as the shift amount will still be
1925 // in range.
1926 if (ShAmt < DemandedBits.getActiveBits() && !VT.isVector() &&
1927 Op.getNode()->hasOneUse()) {
1928 // Search for the smallest integer type with free casts to and from
1929 // Op's type. For expedience, just check power-of-2 integer types.
1930 unsigned DemandedSize = DemandedBits.getActiveBits();
1931 for (unsigned SmallVTBits = llvm::bit_ceil(Value: DemandedSize);
1932 SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(A: SmallVTBits)) {
1933 EVT SmallVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: SmallVTBits);
1934 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: SmallVT) &&
1935 isTypeDesirableForOp(ISD::SHL, VT: SmallVT) &&
1936 isTruncateFree(FromVT: VT, ToVT: SmallVT) && isZExtFree(FromTy: SmallVT, ToTy: VT) &&
1937 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: SmallVT))) {
1938 assert(DemandedSize <= SmallVTBits &&
1939 "Narrowed below demanded bits?");
1940 // We found a type with free casts.
1941 SDValue NarrowShl = TLO.DAG.getNode(
1942 Opcode: ISD::SHL, DL: dl, VT: SmallVT,
1943 N1: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SmallVT, Operand: Op.getOperand(i: 0)),
1944 N2: TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: SmallVT, DL: dl));
1945 return TLO.CombineTo(
1946 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: NarrowShl));
1947 }
1948 }
1949 }
1950
1951 // Narrow shift to lower half - similar to ShrinkDemandedOp.
1952 // (shl i64:x, K) -> (i64 zero_extend (shl (i32 (trunc i64:x)), K))
1953 // Only do this if we demand the upper half so the knownbits are correct.
1954 unsigned HalfWidth = BitWidth / 2;
1955 if ((BitWidth % 2) == 0 && !VT.isVector() && ShAmt < HalfWidth &&
1956 DemandedBits.countLeadingOnes() >= HalfWidth) {
1957 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: HalfWidth);
1958 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
1959 isTypeDesirableForOp(ISD::SHL, VT: HalfVT) &&
1960 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
1961 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT: HalfVT))) {
1962 // If we're demanding the upper bits at all, we must ensure
1963 // that the upper bits of the shift result are known to be zero,
1964 // which is equivalent to the narrow shift being NUW.
1965 if (bool IsNUW = (Known.countMinLeadingZeros() >= HalfWidth)) {
1966 bool IsNSW = Known.countMinSignBits() > HalfWidth;
1967 SDNodeFlags Flags;
1968 Flags.setNoSignedWrap(IsNSW);
1969 Flags.setNoUnsignedWrap(IsNUW);
1970 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
1971 SDValue NewShiftAmt =
1972 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
1973 SDValue NewShift = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HalfVT, N1: NewOp,
1974 N2: NewShiftAmt, Flags);
1975 SDValue NewExt =
1976 TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift);
1977 return TLO.CombineTo(O: Op, N: NewExt);
1978 }
1979 }
1980 }
1981 } else {
1982 // This is a variable shift, so we can't shift the demand mask by a known
1983 // amount. But if we are not demanding high bits, then we are not
1984 // demanding those bits from the pre-shifted operand either.
1985 if (unsigned CTLZ = DemandedBits.countl_zero()) {
1986 APInt DemandedFromOp(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - CTLZ));
1987 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedFromOp, OriginalDemandedElts: DemandedElts, Known, TLO,
1988 Depth: Depth + 1)) {
1989 // Disable the nsw and nuw flags. We can no longer guarantee that we
1990 // won't wrap after simplification.
1991 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
1992 return true;
1993 }
1994 Known.resetAll();
1995 }
1996 }
1997
1998 // If we are only demanding sign bits then we can use the shift source
1999 // directly.
2000 if (std::optional<unsigned> MaxSA =
2001 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2002 unsigned ShAmt = *MaxSA;
2003 unsigned NumSignBits =
2004 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2005 unsigned UpperDemandedBits = BitWidth - DemandedBits.countr_zero();
2006 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
2007 return TLO.CombineTo(O: Op, N: Op0);
2008 }
2009 break;
2010 }
2011 case ISD::SRL: {
2012 SDValue Op0 = Op.getOperand(i: 0);
2013 SDValue Op1 = Op.getOperand(i: 1);
2014 EVT ShiftVT = Op1.getValueType();
2015
2016 if (std::optional<unsigned> KnownSA =
2017 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2018 unsigned ShAmt = *KnownSA;
2019 if (ShAmt == 0)
2020 return TLO.CombineTo(O: Op, N: Op0);
2021
2022 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
2023 // single shift. We can do this if the top bits (which are shifted out)
2024 // are never demanded.
2025 // TODO - support non-uniform vector amounts.
2026 if (Op0.getOpcode() == ISD::SHL) {
2027 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2028 if (std::optional<unsigned> InnerSA =
2029 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2030 unsigned C1 = *InnerSA;
2031 unsigned Opc = ISD::SRL;
2032 int Diff = ShAmt - C1;
2033 if (Diff < 0) {
2034 Diff = -Diff;
2035 Opc = ISD::SHL;
2036 }
2037 SDValue NewSA = TLO.DAG.getConstant(Val: Diff, DL: dl, VT: ShiftVT);
2038 return TLO.CombineTo(
2039 O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, N1: Op0.getOperand(i: 0), N2: NewSA));
2040 }
2041 }
2042 }
2043
2044 // If this is (srl (sra X, C1), ShAmt), see if we can combine this into a
2045 // single sra. We can do this if the top bits are never demanded.
2046 if (Op0.getOpcode() == ISD::SRA && Op0.hasOneUse()) {
2047 if (!DemandedBits.intersects(RHS: APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: ShAmt))) {
2048 if (std::optional<unsigned> InnerSA =
2049 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2050 unsigned C1 = *InnerSA;
2051 // Clamp the combined shift amount if it exceeds the bit width.
2052 unsigned Combined = std::min(a: C1 + ShAmt, b: BitWidth - 1);
2053 SDValue NewSA = TLO.DAG.getConstant(Val: Combined, DL: dl, VT: ShiftVT);
2054 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT,
2055 N1: Op0.getOperand(i: 0), N2: NewSA));
2056 }
2057 }
2058 }
2059
2060 APInt InDemandedMask = (DemandedBits << ShAmt);
2061
2062 // If the shift is exact, then it does demand the low bits (and knows that
2063 // they are zero).
2064 if (Op->getFlags().hasExact())
2065 InDemandedMask.setLowBits(ShAmt);
2066
2067 // Narrow shift to lower half - similar to ShrinkDemandedOp.
2068 // (srl i64:x, K) -> (i64 zero_extend (srl (i32 (trunc i64:x)), K))
2069 if ((BitWidth % 2) == 0 && !VT.isVector()) {
2070 APInt HiBits = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth / 2);
2071 EVT HalfVT = EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: BitWidth / 2);
2072 if (isNarrowingProfitable(N: Op.getNode(), SrcVT: VT, DestVT: HalfVT) &&
2073 isTypeDesirableForOp(ISD::SRL, VT: HalfVT) &&
2074 isTruncateFree(FromVT: VT, ToVT: HalfVT) && isZExtFree(FromTy: HalfVT, ToTy: VT) &&
2075 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT: HalfVT)) &&
2076 ((InDemandedMask.countLeadingZeros() >= (BitWidth / 2)) ||
2077 TLO.DAG.MaskedValueIsZero(Op: Op0, Mask: HiBits))) {
2078 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HalfVT, Operand: Op0);
2079 SDValue NewShiftAmt =
2080 TLO.DAG.getShiftAmountConstant(Val: ShAmt, VT: HalfVT, DL: dl);
2081 SDValue NewShift =
2082 TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HalfVT, N1: NewOp, N2: NewShiftAmt);
2083 return TLO.CombineTo(
2084 O: Op, N: TLO.DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: NewShift));
2085 }
2086 }
2087
2088 // Compute the new bits that are at the top now.
2089 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2090 Depth: Depth + 1))
2091 return true;
2092 Known >>= ShAmt;
2093 // High bits known zero.
2094 Known.Zero.setHighBits(ShAmt);
2095
2096 // Attempt to avoid multi-use ops if we don't need anything from them.
2097 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2098 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2099 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2100 if (DemandedOp0) {
2101 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2102 return TLO.CombineTo(O: Op, N: NewOp);
2103 }
2104 }
2105 } else {
2106 // Use generic knownbits computation as it has support for non-uniform
2107 // shift amounts.
2108 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2109 }
2110
2111 // If we are only demanding sign bits then we can use the shift source
2112 // directly.
2113 if (std::optional<unsigned> MaxSA =
2114 TLO.DAG.getValidMaximumShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2115 unsigned ShAmt = *MaxSA;
2116 // Must already be signbits in DemandedBits bounds, and can't demand any
2117 // shifted in zeroes.
2118 if (DemandedBits.countl_zero() >= ShAmt) {
2119 unsigned NumSignBits =
2120 TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2121 if (DemandedBits.countr_zero() >= (BitWidth - NumSignBits))
2122 return TLO.CombineTo(O: Op, N: Op0);
2123 }
2124 }
2125
2126 // Try to match AVG patterns (after shift simplification).
2127 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2128 DemandedElts, Depth: Depth + 1))
2129 return TLO.CombineTo(O: Op, N: AVG);
2130
2131 break;
2132 }
2133 case ISD::SRA: {
2134 SDValue Op0 = Op.getOperand(i: 0);
2135 SDValue Op1 = Op.getOperand(i: 1);
2136 EVT ShiftVT = Op1.getValueType();
2137
2138 // If we only want bits that already match the signbit then we don't need
2139 // to shift.
2140 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countr_zero();
2141 if (TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1) >=
2142 NumHiDemandedBits)
2143 return TLO.CombineTo(O: Op, N: Op0);
2144
2145 // If this is an arithmetic shift right and only the low-bit is set, we can
2146 // always convert this into a logical shr, even if the shift amount is
2147 // variable. The low bit of the shift cannot be an input sign bit unless
2148 // the shift amount is >= the size of the datatype, which is undefined.
2149 if (DemandedBits.isOne())
2150 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2151
2152 if (std::optional<unsigned> KnownSA =
2153 TLO.DAG.getValidShiftAmount(V: Op, DemandedElts, Depth: Depth + 1)) {
2154 unsigned ShAmt = *KnownSA;
2155 if (ShAmt == 0)
2156 return TLO.CombineTo(O: Op, N: Op0);
2157
2158 // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target
2159 // supports sext_inreg.
2160 if (Op0.getOpcode() == ISD::SHL) {
2161 if (std::optional<unsigned> InnerSA =
2162 TLO.DAG.getValidShiftAmount(V: Op0, DemandedElts, Depth: Depth + 2)) {
2163 unsigned LowBits = BitWidth - ShAmt;
2164 EVT ExtVT = VT.changeElementType(
2165 Context&: *TLO.DAG.getContext(),
2166 EltVT: EVT::getIntegerVT(Context&: *TLO.DAG.getContext(), BitWidth: LowBits));
2167
2168 if (*InnerSA == ShAmt) {
2169 if (!TLO.LegalOperations() ||
2170 getOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: ExtVT) == Legal)
2171 return TLO.CombineTo(
2172 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT,
2173 N1: Op0.getOperand(i: 0),
2174 N2: TLO.DAG.getValueType(ExtVT)));
2175
2176 // Even if we can't convert to sext_inreg, we might be able to
2177 // remove this shift pair if the input is already sign extended.
2178 unsigned NumSignBits =
2179 TLO.DAG.ComputeNumSignBits(Op: Op0.getOperand(i: 0), DemandedElts);
2180 if (NumSignBits > ShAmt)
2181 return TLO.CombineTo(O: Op, N: Op0.getOperand(i: 0));
2182 }
2183 }
2184 }
2185
2186 APInt InDemandedMask = (DemandedBits << ShAmt);
2187
2188 // If the shift is exact, then it does demand the low bits (and knows that
2189 // they are zero).
2190 if (Op->getFlags().hasExact())
2191 InDemandedMask.setLowBits(ShAmt);
2192
2193 // If any of the demanded bits are produced by the sign extension, we also
2194 // demand the input sign bit.
2195 if (DemandedBits.countl_zero() < ShAmt)
2196 InDemandedMask.setSignBit();
2197
2198 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InDemandedMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2199 Depth: Depth + 1))
2200 return true;
2201 Known >>= ShAmt;
2202
2203 // If the input sign bit is known to be zero, or if none of the top bits
2204 // are demanded, turn this into an unsigned shift right.
2205 if (Known.Zero[BitWidth - ShAmt - 1] ||
2206 DemandedBits.countl_zero() >= ShAmt) {
2207 SDNodeFlags Flags;
2208 Flags.setExact(Op->getFlags().hasExact());
2209 return TLO.CombineTo(
2210 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1, Flags));
2211 }
2212
2213 int Log2 = DemandedBits.exactLogBase2();
2214 if (Log2 >= 0) {
2215 // The bit must come from the sign.
2216 SDValue NewSA = TLO.DAG.getConstant(Val: BitWidth - 1 - Log2, DL: dl, VT: ShiftVT);
2217 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: NewSA));
2218 }
2219
2220 if (Known.One[BitWidth - ShAmt - 1])
2221 // New bits are known one.
2222 Known.One.setHighBits(ShAmt);
2223
2224 // Attempt to avoid multi-use ops if we don't need anything from them.
2225 if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2226 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2227 Op: Op0, DemandedBits: InDemandedMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2228 if (DemandedOp0) {
2229 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: DemandedOp0, N2: Op1);
2230 return TLO.CombineTo(O: Op, N: NewOp);
2231 }
2232 }
2233 }
2234
2235 // Try to match AVG patterns (after shift simplification).
2236 if (SDValue AVG = combineShiftToAVG(Op, TLO, TLI: *this, DemandedBits,
2237 DemandedElts, Depth: Depth + 1))
2238 return TLO.CombineTo(O: Op, N: AVG);
2239
2240 break;
2241 }
2242 case ISD::FSHL:
2243 case ISD::FSHR: {
2244 SDValue Op0 = Op.getOperand(i: 0);
2245 SDValue Op1 = Op.getOperand(i: 1);
2246 SDValue Op2 = Op.getOperand(i: 2);
2247 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
2248
2249 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op2, DemandedElts)) {
2250 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2251
2252 // For fshl, 0-shift returns the 1st arg.
2253 // For fshr, 0-shift returns the 2nd arg.
2254 if (Amt == 0) {
2255 if (SimplifyDemandedBits(Op: IsFSHL ? Op0 : Op1, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts,
2256 Known, TLO, Depth: Depth + 1))
2257 return true;
2258 break;
2259 }
2260
2261 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
2262 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
2263 APInt Demanded0 = DemandedBits.lshr(shiftAmt: IsFSHL ? Amt : (BitWidth - Amt));
2264 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
2265 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2266 Depth: Depth + 1))
2267 return true;
2268 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: Demanded1, OriginalDemandedElts: DemandedElts, Known, TLO,
2269 Depth: Depth + 1))
2270 return true;
2271
2272 Known2 <<= (IsFSHL ? Amt : (BitWidth - Amt));
2273 Known >>= (IsFSHL ? (BitWidth - Amt) : Amt);
2274 Known = Known.unionWith(RHS: Known2);
2275
2276 // Attempt to avoid multi-use ops if we don't need anything from them.
2277 if (!Demanded0.isAllOnes() || !Demanded1.isAllOnes() ||
2278 !DemandedElts.isAllOnes()) {
2279 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2280 Op: Op0, DemandedBits: Demanded0, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2281 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2282 Op: Op1, DemandedBits: Demanded1, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2283 if (DemandedOp0 || DemandedOp1) {
2284 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2285 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2286 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedOp0,
2287 N2: DemandedOp1, N3: Op2);
2288 return TLO.CombineTo(O: Op, N: NewOp);
2289 }
2290 }
2291 }
2292
2293 if (isPowerOf2_32(Value: BitWidth)) {
2294 // Fold FSHR(Op0,Op1,Op2) -> SRL(Op1,Op2)
2295 // iff we're guaranteed not to use Op0.
2296 // TODO: Add FSHL equivalent?
2297 if (!IsFSHL && !DemandedBits.isAllOnes() &&
2298 (!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT))) {
2299 KnownBits KnownAmt =
2300 TLO.DAG.computeKnownBits(Op: Op2, DemandedElts, Depth: Depth + 1);
2301 unsigned MaxShiftAmt =
2302 KnownAmt.getMaxValue().getLimitedValue(Limit: BitWidth - 1);
2303 // Check we don't demand any shifted bits outside Op1.
2304 if (DemandedBits.countl_zero() >= MaxShiftAmt) {
2305 EVT AmtVT = Op2.getValueType();
2306 SDValue NewAmt =
2307 TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Op2,
2308 N2: TLO.DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
2309 SDValue NewOp = TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op1, N2: NewAmt);
2310 return TLO.CombineTo(O: Op, N: NewOp);
2311 }
2312 }
2313
2314 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2315 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
2316 if (SimplifyDemandedBits(Op: Op2, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2317 Depth: Depth + 1))
2318 return true;
2319 }
2320 break;
2321 }
2322 case ISD::ROTL:
2323 case ISD::ROTR: {
2324 SDValue Op0 = Op.getOperand(i: 0);
2325 SDValue Op1 = Op.getOperand(i: 1);
2326 bool IsROTL = (Op.getOpcode() == ISD::ROTL);
2327
2328 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
2329 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1))
2330 return TLO.CombineTo(O: Op, N: Op0);
2331
2332 if (ConstantSDNode *SA = isConstOrConstSplat(N: Op1, DemandedElts)) {
2333 unsigned Amt = SA->getAPIntValue().urem(RHS: BitWidth);
2334 unsigned RevAmt = BitWidth - Amt;
2335
2336 // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
2337 // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
2338 APInt Demanded0 = DemandedBits.rotr(rotateAmt: IsROTL ? Amt : RevAmt);
2339 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: Demanded0, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2340 Depth: Depth + 1))
2341 return true;
2342
2343 // rot*(x, 0) --> x
2344 if (Amt == 0)
2345 return TLO.CombineTo(O: Op, N: Op0);
2346
2347 // See if we don't demand either half of the rotated bits.
2348 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SHL, VT)) &&
2349 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2350 Op1 = TLO.DAG.getConstant(Val: IsROTL ? Amt : RevAmt, DL: dl, VT: Op1.getValueType());
2351 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: Op1));
2352 }
2353 if ((!TLO.LegalOperations() || isOperationLegal(Op: ISD::SRL, VT)) &&
2354 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2355 Op1 = TLO.DAG.getConstant(Val: IsROTL ? RevAmt : Amt, DL: dl, VT: Op1.getValueType());
2356 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op0, N2: Op1));
2357 }
2358 }
2359
2360 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
2361 if (isPowerOf2_32(Value: BitWidth)) {
2362 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
2363 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: DemandedAmtBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2364 Depth: Depth + 1))
2365 return true;
2366 }
2367 break;
2368 }
2369 case ISD::SMIN:
2370 case ISD::SMAX:
2371 case ISD::UMIN:
2372 case ISD::UMAX: {
2373 unsigned Opc = Op.getOpcode();
2374 SDValue Op0 = Op.getOperand(i: 0);
2375 SDValue Op1 = Op.getOperand(i: 1);
2376
2377 // If we're only demanding signbits, then we can simplify to OR/AND node.
2378 unsigned BitOp =
2379 (Opc == ISD::SMIN || Opc == ISD::UMAX) ? ISD::OR : ISD::AND;
2380 unsigned NumSignBits =
2381 std::min(a: TLO.DAG.ComputeNumSignBits(Op: Op0, DemandedElts, Depth: Depth + 1),
2382 b: TLO.DAG.ComputeNumSignBits(Op: Op1, DemandedElts, Depth: Depth + 1));
2383 unsigned NumDemandedUpperBits = BitWidth - DemandedBits.countr_zero();
2384 if (NumSignBits >= NumDemandedUpperBits)
2385 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: BitOp, DL: SDLoc(Op), VT, N1: Op0, N2: Op1));
2386
2387 // Check if one arg is always less/greater than (or equal) to the other arg.
2388 KnownBits Known0 = TLO.DAG.computeKnownBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2389 KnownBits Known1 = TLO.DAG.computeKnownBits(Op: Op1, DemandedElts, Depth: Depth + 1);
2390 switch (Opc) {
2391 case ISD::SMIN:
2392 if (std::optional<bool> IsSLE = KnownBits::sle(LHS: Known0, RHS: Known1))
2393 return TLO.CombineTo(O: Op, N: *IsSLE ? Op0 : Op1);
2394 if (std::optional<bool> IsSLT = KnownBits::slt(LHS: Known0, RHS: Known1))
2395 return TLO.CombineTo(O: Op, N: *IsSLT ? Op0 : Op1);
2396 Known = KnownBits::smin(LHS: Known0, RHS: Known1);
2397 break;
2398 case ISD::SMAX:
2399 if (std::optional<bool> IsSGE = KnownBits::sge(LHS: Known0, RHS: Known1))
2400 return TLO.CombineTo(O: Op, N: *IsSGE ? Op0 : Op1);
2401 if (std::optional<bool> IsSGT = KnownBits::sgt(LHS: Known0, RHS: Known1))
2402 return TLO.CombineTo(O: Op, N: *IsSGT ? Op0 : Op1);
2403 Known = KnownBits::smax(LHS: Known0, RHS: Known1);
2404 break;
2405 case ISD::UMIN:
2406 if (std::optional<bool> IsULE = KnownBits::ule(LHS: Known0, RHS: Known1))
2407 return TLO.CombineTo(O: Op, N: *IsULE ? Op0 : Op1);
2408 if (std::optional<bool> IsULT = KnownBits::ult(LHS: Known0, RHS: Known1))
2409 return TLO.CombineTo(O: Op, N: *IsULT ? Op0 : Op1);
2410 Known = KnownBits::umin(LHS: Known0, RHS: Known1);
2411 break;
2412 case ISD::UMAX:
2413 if (std::optional<bool> IsUGE = KnownBits::uge(LHS: Known0, RHS: Known1))
2414 return TLO.CombineTo(O: Op, N: *IsUGE ? Op0 : Op1);
2415 if (std::optional<bool> IsUGT = KnownBits::ugt(LHS: Known0, RHS: Known1))
2416 return TLO.CombineTo(O: Op, N: *IsUGT ? Op0 : Op1);
2417 Known = KnownBits::umax(LHS: Known0, RHS: Known1);
2418 break;
2419 }
2420 break;
2421 }
2422 case ISD::BITREVERSE: {
2423 SDValue Src = Op.getOperand(i: 0);
2424 APInt DemandedSrcBits = DemandedBits.reverseBits();
2425 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2426 Depth: Depth + 1))
2427 return true;
2428 Known = Known2.reverseBits();
2429 break;
2430 }
2431 case ISD::BSWAP: {
2432 SDValue Src = Op.getOperand(i: 0);
2433
2434 // If the only bits demanded come from one byte of the bswap result,
2435 // just shift the input byte into position to eliminate the bswap.
2436 unsigned NLZ = DemandedBits.countl_zero();
2437 unsigned NTZ = DemandedBits.countr_zero();
2438
2439 // Round NTZ down to the next byte. If we have 11 trailing zeros, then
2440 // we need all the bits down to bit 8. Likewise, round NLZ. If we
2441 // have 14 leading zeros, round to 8.
2442 NLZ = alignDown(Value: NLZ, Align: 8);
2443 NTZ = alignDown(Value: NTZ, Align: 8);
2444 // If we need exactly one byte, we can do this transformation.
2445 if (BitWidth - NLZ - NTZ == 8) {
2446 // Replace this with either a left or right shift to get the byte into
2447 // the right place.
2448 unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
2449 if (!TLO.LegalOperations() || isOperationLegal(Op: ShiftOpcode, VT)) {
2450 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2451 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
2452 SDValue NewOp = TLO.DAG.getNode(Opcode: ShiftOpcode, DL: dl, VT, N1: Src, N2: ShAmt);
2453 return TLO.CombineTo(O: Op, N: NewOp);
2454 }
2455 }
2456
2457 APInt DemandedSrcBits = DemandedBits.byteSwap();
2458 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
2459 Depth: Depth + 1))
2460 return true;
2461 Known = Known2.byteSwap();
2462 break;
2463 }
2464 case ISD::CTPOP: {
2465 // If only 1 bit is demanded, replace with PARITY as long as we're before
2466 // op legalization.
2467 // FIXME: Limit to scalars for now.
2468 if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
2469 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::PARITY, DL: dl, VT,
2470 Operand: Op.getOperand(i: 0)));
2471
2472 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2473 break;
2474 }
2475 case ISD::PDEP: {
2476 SDValue Op0 = Op.getOperand(i: 0);
2477 SDValue Op1 = Op.getOperand(i: 1);
2478
2479 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2480 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
2481
2482 // If the demanded bits has leading zeroes, we don't demand those from the
2483 // mask.
2484 if (SimplifyDemandedBits(Op: Op1, DemandedBits: LoMask, Known, TLO, Depth: Depth + 1))
2485 return true;
2486
2487 // The number of possible 1s in the mask determines the number of LSBs of
2488 // operand 0 used. Undemanded bits from the mask don't matter so filter
2489 // them before counting.
2490 KnownBits Known2;
2491 uint64_t Count = (~Known.Zero & LoMask).popcount();
2492 APInt DemandedMask(APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: Count));
2493 if (SimplifyDemandedBits(Op: Op0, DemandedBits: DemandedMask, Known&: Known2, TLO, Depth: Depth + 1))
2494 return true;
2495
2496 // Zeroes are retained from the mask, but not ones.
2497 Known.One.clearAllBits();
2498 // The result will have at least as many trailing zeros as the non-mask
2499 // operand since bits can only map to the same or higher bit position.
2500 Known.Zero.setLowBits(Known2.countMinTrailingZeros());
2501 break;
2502 }
2503 case ISD::SIGN_EXTEND_INREG: {
2504 SDValue Op0 = Op.getOperand(i: 0);
2505 EVT ExVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2506 unsigned ExVTBits = ExVT.getScalarSizeInBits();
2507
2508 // If we only care about the highest bit, don't bother shifting right.
2509 if (DemandedBits.isSignMask()) {
2510 unsigned MinSignedBits =
2511 TLO.DAG.ComputeMaxSignificantBits(Op: Op0, DemandedElts, Depth: Depth + 1);
2512 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2513 // However if the input is already sign extended we expect the sign
2514 // extension to be dropped altogether later and do not simplify.
2515 if (!AlreadySignExtended) {
2516 // Compute the correct shift amount type, which must be getShiftAmountTy
2517 // for scalar types after legalization.
2518 SDValue ShiftAmt =
2519 TLO.DAG.getShiftAmountConstant(Val: BitWidth - ExVTBits, VT, DL: dl);
2520 return TLO.CombineTo(O: Op,
2521 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op0, N2: ShiftAmt));
2522 }
2523 }
2524
2525 // If none of the extended bits are demanded, eliminate the sextinreg.
2526 if (DemandedBits.getActiveBits() <= ExVTBits)
2527 return TLO.CombineTo(O: Op, N: Op0);
2528
2529 APInt InputDemandedBits = DemandedBits.getLoBits(numBits: ExVTBits);
2530
2531 // Since the sign extended bits are demanded, we know that the sign
2532 // bit is demanded.
2533 InputDemandedBits.setBit(ExVTBits - 1);
2534
2535 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: InputDemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
2536 Depth: Depth + 1))
2537 return true;
2538
2539 // If the sign bit of the input is known set or clear, then we know the
2540 // top bits of the result.
2541
2542 // If the input sign bit is known zero, convert this into a zero extension.
2543 if (Known.Zero[ExVTBits - 1])
2544 return TLO.CombineTo(O: Op, N: TLO.DAG.getZeroExtendInReg(Op: Op0, DL: dl, VT: ExVT));
2545
2546 APInt Mask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ExVTBits);
2547 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
2548 Known.One.setBitsFrom(ExVTBits);
2549 Known.Zero &= Mask;
2550 } else { // Input sign bit unknown
2551 Known.Zero &= Mask;
2552 Known.One &= Mask;
2553 }
2554 break;
2555 }
2556 case ISD::BUILD_PAIR: {
2557 EVT HalfVT = Op.getOperand(i: 0).getValueType();
2558 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
2559
2560 APInt MaskLo = DemandedBits.getLoBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2561 APInt MaskHi = DemandedBits.getHiBits(numBits: HalfBitWidth).trunc(width: HalfBitWidth);
2562
2563 KnownBits KnownLo, KnownHi;
2564
2565 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: MaskLo, Known&: KnownLo, TLO, Depth: Depth + 1))
2566 return true;
2567
2568 if (SimplifyDemandedBits(Op: Op.getOperand(i: 1), DemandedBits: MaskHi, Known&: KnownHi, TLO, Depth: Depth + 1))
2569 return true;
2570
2571 Known = KnownHi.concat(Lo: KnownLo);
2572 break;
2573 }
2574 case ISD::ZERO_EXTEND_VECTOR_INREG:
2575 if (VT.isScalableVector())
2576 return false;
2577 [[fallthrough]];
2578 case ISD::ZERO_EXTEND: {
2579 SDValue Src = Op.getOperand(i: 0);
2580 EVT SrcVT = Src.getValueType();
2581 unsigned InBits = SrcVT.getScalarSizeInBits();
2582 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2583 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
2584
2585 // If none of the top bits are demanded, convert this into an any_extend.
2586 if (DemandedBits.getActiveBits() <= InBits) {
2587 // If we only need the non-extended bits of the bottom element
2588 // then we can just bitcast to the result.
2589 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2590 VT.getSizeInBits() == SrcVT.getSizeInBits())
2591 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2592
2593 unsigned Opc =
2594 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2595 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2596 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2597 }
2598
2599 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2600 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2601 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2602 Depth: Depth + 1)) {
2603 Op->dropFlags(Mask: SDNodeFlags::NonNeg);
2604 return true;
2605 }
2606 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2607 Known = Known.zext(BitWidth);
2608
2609 // Attempt to avoid multi-use ops if we don't need anything from them.
2610 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2611 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2612 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2613 break;
2614 }
2615 case ISD::SIGN_EXTEND_VECTOR_INREG:
2616 if (VT.isScalableVector())
2617 return false;
2618 [[fallthrough]];
2619 case ISD::SIGN_EXTEND: {
2620 SDValue Src = Op.getOperand(i: 0);
2621 EVT SrcVT = Src.getValueType();
2622 unsigned InBits = SrcVT.getScalarSizeInBits();
2623 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2624 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
2625
2626 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2627 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2628
2629 // Since some of the sign extended bits are demanded, we know that the sign
2630 // bit is demanded.
2631 InDemandedBits.setBit(InBits - 1);
2632
2633 // If none of the top bits are demanded, convert this into an any_extend.
2634 if (DemandedBits.getActiveBits() <= InBits) {
2635 // If we only need the non-extended bits of the bottom element
2636 // then we can just bitcast to the result.
2637 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2638 VT.getSizeInBits() == SrcVT.getSizeInBits())
2639 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2640
2641 // Don't lose an all signbits 0/-1 splat on targets with 0/-1 booleans.
2642 if (getBooleanContents(Type: VT) != ZeroOrNegativeOneBooleanContent ||
2643 TLO.DAG.ComputeNumSignBits(Op: Src, DemandedElts: InDemandedElts, Depth: Depth + 1) !=
2644 InBits) {
2645 unsigned Opc =
2646 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
2647 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT))
2648 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src));
2649 }
2650 }
2651
2652 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2653 Depth: Depth + 1))
2654 return true;
2655 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2656
2657 // If the sign bit is known one, the top bits match.
2658 Known = Known.sext(BitWidth);
2659
2660 // If the sign bit is known zero, convert this to a zero extend.
2661 if (Known.isNonNegative()) {
2662 unsigned Opc =
2663 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
2664 if (!TLO.LegalOperations() || isOperationLegal(Op: Opc, VT)) {
2665 SDNodeFlags Flags;
2666 if (!IsVecInReg)
2667 Flags |= SDNodeFlags::NonNeg;
2668 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Opc, DL: dl, VT, Operand: Src, Flags));
2669 }
2670 }
2671
2672 // Attempt to avoid multi-use ops if we don't need anything from them.
2673 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2674 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2675 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2676 break;
2677 }
2678 case ISD::ANY_EXTEND_VECTOR_INREG:
2679 if (VT.isScalableVector())
2680 return false;
2681 [[fallthrough]];
2682 case ISD::ANY_EXTEND: {
2683 SDValue Src = Op.getOperand(i: 0);
2684 EVT SrcVT = Src.getValueType();
2685 unsigned InBits = SrcVT.getScalarSizeInBits();
2686 unsigned InElts = SrcVT.isFixedLengthVector() ? SrcVT.getVectorNumElements() : 1;
2687 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2688
2689 // If we only need the bottom element then we can just bitcast.
2690 // TODO: Handle ANY_EXTEND?
2691 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2692 VT.getSizeInBits() == SrcVT.getSizeInBits())
2693 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
2694
2695 APInt InDemandedBits = DemandedBits.trunc(width: InBits);
2696 APInt InDemandedElts = DemandedElts.zext(width: InElts);
2697 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: InDemandedBits, OriginalDemandedElts: InDemandedElts, Known, TLO,
2698 Depth: Depth + 1))
2699 return true;
2700 assert(Known.getBitWidth() == InBits && "Src width has changed?");
2701 Known = Known.anyext(BitWidth);
2702
2703 // Attempt to avoid multi-use ops if we don't need anything from them.
2704 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2705 Op: Src, DemandedBits: InDemandedBits, DemandedElts: InDemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2706 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, Operand: NewSrc));
2707 break;
2708 }
2709 case ISD::TRUNCATE: {
2710 SDValue Src = Op.getOperand(i: 0);
2711
2712 // Simplify the input, using demanded bit information, and compute the known
2713 // zero/one bits live out.
2714 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2715 APInt TruncMask = DemandedBits.zext(width: OperandBitWidth);
2716 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: TruncMask, OriginalDemandedElts: DemandedElts, Known, TLO,
2717 Depth: Depth + 1)) {
2718 // Disable the nsw and nuw flags. We can no longer guarantee that we
2719 // won't wrap after simplification.
2720 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2721 return true;
2722 }
2723 Known = Known.trunc(BitWidth);
2724
2725 // Attempt to avoid multi-use ops if we don't need anything from them.
2726 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2727 Op: Src, DemandedBits: TruncMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
2728 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: NewSrc));
2729
2730 // If the input is only used by this truncate, see if we can shrink it based
2731 // on the known demanded bits.
2732 switch (Src.getOpcode()) {
2733 default:
2734 break;
2735 case ISD::SRL:
2736 // Shrink SRL by a constant if none of the high bits shifted in are
2737 // demanded.
2738 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2739 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2740 // undesirable.
2741 break;
2742
2743 if (Src.getNode()->hasOneUse()) {
2744 if (isTruncateFree(Val: Src, VT2: VT) &&
2745 !isTruncateFree(FromVT: Src.getValueType(), ToVT: VT)) {
2746 // If truncate is only free at trunc(srl), do not turn it into
2747 // srl(trunc). The check is done by first check the truncate is free
2748 // at Src's opcode(srl), then check the truncate is not done by
2749 // referencing sub-register. In test, if both trunc(srl) and
2750 // srl(trunc)'s trunc are free, srl(trunc) performs better. If only
2751 // trunc(srl)'s trunc is free, trunc(srl) is better.
2752 break;
2753 }
2754
2755 std::optional<unsigned> ShAmtC =
2756 TLO.DAG.getValidShiftAmount(V: Src, DemandedElts, Depth: Depth + 2);
2757 if (!ShAmtC || *ShAmtC >= BitWidth)
2758 break;
2759 unsigned ShVal = *ShAmtC;
2760
2761 APInt HighBits =
2762 APInt::getHighBitsSet(numBits: OperandBitWidth, hiBitsSet: OperandBitWidth - BitWidth);
2763 HighBits.lshrInPlace(ShiftAmt: ShVal);
2764 HighBits = HighBits.trunc(width: BitWidth);
2765 if (!(HighBits & DemandedBits)) {
2766 // None of the shifted in bits are needed. Add a truncate of the
2767 // shift input, then shift it.
2768 SDValue NewShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
2769 SDValue NewTrunc =
2770 TLO.DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Src.getOperand(i: 0));
2771 return TLO.CombineTo(
2772 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NewTrunc, N2: NewShAmt));
2773 }
2774 }
2775 break;
2776 }
2777
2778 break;
2779 }
2780 case ISD::AssertZext: {
2781 // AssertZext demands all of the high bits, plus any of the low bits
2782 // demanded by its users.
2783 EVT ZVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2784 APInt InMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: ZVT.getSizeInBits());
2785 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits: ~InMask | DemandedBits, Known,
2786 TLO, Depth: Depth + 1))
2787 return true;
2788
2789 Known.Zero |= ~InMask;
2790 Known.One &= (~Known.Zero);
2791 break;
2792 }
2793 case ISD::EXTRACT_VECTOR_ELT: {
2794 SDValue Src = Op.getOperand(i: 0);
2795 SDValue Idx = Op.getOperand(i: 1);
2796 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2797 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2798
2799 if (SrcEltCnt.isScalable())
2800 return false;
2801
2802 // Demand the bits from every vector element without a constant index.
2803 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2804 APInt DemandedSrcElts = APInt::getAllOnes(numBits: NumSrcElts);
2805 if (auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx))
2806 if (CIdx->getAPIntValue().ult(RHS: NumSrcElts))
2807 DemandedSrcElts = APInt::getOneBitSet(numBits: NumSrcElts, BitNo: CIdx->getZExtValue());
2808
2809 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2810 // anything about the extended bits.
2811 APInt DemandedSrcBits = DemandedBits;
2812 if (BitWidth > EltBitWidth)
2813 DemandedSrcBits = DemandedSrcBits.trunc(width: EltBitWidth);
2814
2815 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts, Known&: Known2, TLO,
2816 Depth: Depth + 1))
2817 return true;
2818
2819 // Attempt to avoid multi-use ops if we don't need anything from them.
2820 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2821 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2822 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
2823 SDValue NewOp =
2824 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: DemandedSrc, N2: Idx);
2825 return TLO.CombineTo(O: Op, N: NewOp);
2826 }
2827 }
2828
2829 Known = Known2;
2830 if (BitWidth > EltBitWidth)
2831 Known = Known.anyext(BitWidth);
2832 break;
2833 }
2834 case ISD::BITCAST: {
2835 if (VT.isScalableVector())
2836 return false;
2837 SDValue Src = Op.getOperand(i: 0);
2838 EVT SrcVT = Src.getValueType();
2839 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2840
2841 // If this is an FP->Int bitcast and if the sign bit is the only
2842 // thing demanded, turn this into a FGETSIGN.
2843 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2844 DemandedBits == APInt::getSignMask(BitWidth: Op.getValueSizeInBits()) &&
2845 SrcVT.isFloatingPoint()) {
2846 if (isOperationLegalOrCustom(Op: ISD::FGETSIGN, VT)) {
2847 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2848 // place. We expect the SHL to be eliminated by other optimizations.
2849 SDValue Sign = TLO.DAG.getNode(Opcode: ISD::FGETSIGN, DL: dl, VT, Operand: Src);
2850 unsigned ShVal = Op.getValueSizeInBits() - 1;
2851 SDValue ShAmt = TLO.DAG.getShiftAmountConstant(Val: ShVal, VT, DL: dl);
2852 return TLO.CombineTo(O: Op,
2853 N: TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Sign, N2: ShAmt));
2854 }
2855 }
2856
2857 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2858 // Demand the elt/bit if any of the original elts/bits are demanded.
2859 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
2860 unsigned Scale = BitWidth / NumSrcEltBits;
2861 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2862 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
2863 for (unsigned i = 0; i != Scale; ++i) {
2864 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2865 unsigned BitOffset = EltOffset * NumSrcEltBits;
2866 DemandedSrcBits |= DemandedBits.extractBits(numBits: NumSrcEltBits, bitPosition: BitOffset);
2867 }
2868 // Recursive calls below may turn not demanded elements into poison, so we
2869 // need to demand all smaller source elements that maps to a demanded
2870 // destination element.
2871 APInt DemandedSrcElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
2872
2873 APInt KnownSrcUndef, KnownSrcZero;
2874 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
2875 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
2876 return true;
2877
2878 KnownBits KnownSrcBits;
2879 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
2880 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
2881 return true;
2882 } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
2883 // TODO - bigendian once we have test coverage.
2884 unsigned Scale = NumSrcEltBits / BitWidth;
2885 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2886 APInt DemandedSrcBits = APInt::getZero(numBits: NumSrcEltBits);
2887 APInt DemandedSrcElts = APInt::getZero(numBits: NumSrcElts);
2888 for (unsigned i = 0; i != NumElts; ++i)
2889 if (DemandedElts[i]) {
2890 unsigned Offset = (i % Scale) * BitWidth;
2891 DemandedSrcBits.insertBits(SubBits: DemandedBits, bitPosition: Offset);
2892 DemandedSrcElts.setBit(i / Scale);
2893 }
2894
2895 if (SrcVT.isVector()) {
2896 APInt KnownSrcUndef, KnownSrcZero;
2897 if (SimplifyDemandedVectorElts(Op: Src, DemandedEltMask: DemandedSrcElts, KnownUndef&: KnownSrcUndef,
2898 KnownZero&: KnownSrcZero, TLO, Depth: Depth + 1))
2899 return true;
2900 }
2901
2902 KnownBits KnownSrcBits;
2903 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: DemandedSrcBits, OriginalDemandedElts: DemandedSrcElts,
2904 Known&: KnownSrcBits, TLO, Depth: Depth + 1))
2905 return true;
2906
2907 // Attempt to avoid multi-use ops if we don't need anything from them.
2908 if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2909 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2910 Op: Src, DemandedBits: DemandedSrcBits, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1)) {
2911 SDValue NewOp = TLO.DAG.getBitcast(VT, V: DemandedSrc);
2912 return TLO.CombineTo(O: Op, N: NewOp);
2913 }
2914 }
2915 }
2916
2917 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2918 // recursive call where Known may be useful to the caller.
2919 if (Depth > 0) {
2920 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2921 return false;
2922 }
2923 break;
2924 }
2925 case ISD::MUL:
2926 if (DemandedBits.isPowerOf2()) {
2927 // The LSB of X*Y is set only if (X & 1) == 1 and (Y & 1) == 1.
2928 // If we demand exactly one bit N and we have "X * (C' << N)" where C' is
2929 // odd (has LSB set), then the left-shifted low bit of X is the answer.
2930 unsigned CTZ = DemandedBits.countr_zero();
2931 ConstantSDNode *C = isConstOrConstSplat(N: Op.getOperand(i: 1), DemandedElts);
2932 if (C && C->getAPIntValue().countr_zero() == CTZ) {
2933 SDValue AmtC = TLO.DAG.getShiftAmountConstant(Val: CTZ, VT, DL: dl);
2934 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op.getOperand(i: 0), N2: AmtC);
2935 return TLO.CombineTo(O: Op, N: Shl);
2936 }
2937 }
2938 // For a squared value "X * X", the bottom 2 bits are 0 and X[0] because:
2939 // X * X is odd iff X is odd.
2940 // 'Quadratic Reciprocity': X * X -> 0 for bit[1]
2941 if (Op.getOperand(i: 0) == Op.getOperand(i: 1) && DemandedBits.ult(RHS: 4)) {
2942 SDValue One = TLO.DAG.getConstant(Val: 1, DL: dl, VT);
2943 SDValue And1 = TLO.DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op.getOperand(i: 0), N2: One);
2944 return TLO.CombineTo(O: Op, N: And1);
2945 }
2946 [[fallthrough]];
2947 case ISD::PTRADD:
2948 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
2949 break;
2950 // PTRADD behaves like ADD if pointers are represented as integers.
2951 [[fallthrough]];
2952 case ISD::ADD:
2953 case ISD::SUB: {
2954 // Add, Sub, and Mul don't demand any bits in positions beyond that
2955 // of the highest bit demanded of them.
2956 SDValue Op0 = Op.getOperand(i: 0), Op1 = Op.getOperand(i: 1);
2957 SDNodeFlags Flags = Op.getNode()->getFlags();
2958 unsigned DemandedBitsLZ = DemandedBits.countl_zero();
2959 APInt LoMask = APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - DemandedBitsLZ);
2960 KnownBits KnownOp0, KnownOp1;
2961 auto GetDemandedBitsLHSMask = [&](APInt Demanded,
2962 const KnownBits &KnownRHS) {
2963 if (Op.getOpcode() == ISD::MUL)
2964 Demanded.clearHighBits(hiBits: KnownRHS.countMinTrailingZeros());
2965 return Demanded;
2966 };
2967 if (SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: LoMask, OriginalDemandedElts: DemandedElts, Known&: KnownOp1, TLO,
2968 Depth: Depth + 1) ||
2969 SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: GetDemandedBitsLHSMask(LoMask, KnownOp1),
2970 OriginalDemandedElts: DemandedElts, Known&: KnownOp0, TLO, Depth: Depth + 1) ||
2971 // See if the operation should be performed at a smaller bit width.
2972 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2973 // Disable the nsw and nuw flags. We can no longer guarantee that we
2974 // won't wrap after simplification.
2975 Op->dropFlags(Mask: SDNodeFlags::NoWrap);
2976 return true;
2977 }
2978
2979 // neg x with only low bit demanded is simply x.
2980 if (Op.getOpcode() == ISD::SUB && DemandedBits.isOne() &&
2981 isNullConstant(V: Op0))
2982 return TLO.CombineTo(O: Op, N: Op1);
2983
2984 // Attempt to avoid multi-use ops if we don't need anything from them.
2985 if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2986 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2987 Op: Op0, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2988 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2989 Op: Op1, DemandedBits: LoMask, DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1);
2990 if (DemandedOp0 || DemandedOp1) {
2991 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2992 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2993 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Op1,
2994 Flags: Flags & ~SDNodeFlags::NoWrap);
2995 return TLO.CombineTo(O: Op, N: NewOp);
2996 }
2997 }
2998
2999 // If we have a constant operand, we may be able to turn it into -1 if we
3000 // do not demand the high bits. This can make the constant smaller to
3001 // encode, allow more general folding, or match specialized instruction
3002 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
3003 // is probably not useful (and could be detrimental).
3004 ConstantSDNode *C = isConstOrConstSplat(N: Op1);
3005 APInt HighMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: DemandedBitsLZ);
3006 if (C && !C->isAllOnes() && !C->isOne() &&
3007 (C->getAPIntValue() | HighMask).isAllOnes()) {
3008 SDValue Neg1 = TLO.DAG.getAllOnesConstant(DL: dl, VT);
3009 // Disable the nsw and nuw flags. We can no longer guarantee that we
3010 // won't wrap after simplification.
3011 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: dl, VT, N1: Op0, N2: Neg1,
3012 Flags: Flags & ~SDNodeFlags::NoWrap);
3013 return TLO.CombineTo(O: Op, N: NewOp);
3014 }
3015
3016 // Match a multiply with a disguised negated-power-of-2 and convert to a
3017 // an equivalent shift-left amount.
3018 // Example: (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3019 auto getShiftLeftAmt = [&HighMask](SDValue Mul) -> unsigned {
3020 if (Mul.getOpcode() != ISD::MUL || !Mul.hasOneUse())
3021 return 0;
3022
3023 // Don't touch opaque constants. Also, ignore zero and power-of-2
3024 // multiplies. Those will get folded later.
3025 ConstantSDNode *MulC = isConstOrConstSplat(N: Mul.getOperand(i: 1));
3026 if (MulC && !MulC->isOpaque() && !MulC->isZero() &&
3027 !MulC->getAPIntValue().isPowerOf2()) {
3028 APInt UnmaskedC = MulC->getAPIntValue() | HighMask;
3029 if (UnmaskedC.isNegatedPowerOf2())
3030 return (-UnmaskedC).logBase2();
3031 }
3032 return 0;
3033 };
3034
3035 auto foldMul = [&](ISD::NodeType NT, SDValue X, SDValue Y,
3036 unsigned ShlAmt) {
3037 SDValue ShlAmtC = TLO.DAG.getShiftAmountConstant(Val: ShlAmt, VT, DL: dl);
3038 SDValue Shl = TLO.DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: X, N2: ShlAmtC);
3039 SDValue Res = TLO.DAG.getNode(Opcode: NT, DL: dl, VT, N1: Y, N2: Shl);
3040 return TLO.CombineTo(O: Op, N: Res);
3041 };
3042
3043 if (isOperationLegalOrCustom(Op: ISD::SHL, VT)) {
3044 if (Op.getOpcode() == ISD::ADD) {
3045 // (X * MulC) + Op1 --> Op1 - (X << log2(-MulC))
3046 if (unsigned ShAmt = getShiftLeftAmt(Op0))
3047 return foldMul(ISD::SUB, Op0.getOperand(i: 0), Op1, ShAmt);
3048 // Op0 + (X * MulC) --> Op0 - (X << log2(-MulC))
3049 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3050 return foldMul(ISD::SUB, Op1.getOperand(i: 0), Op0, ShAmt);
3051 }
3052 if (Op.getOpcode() == ISD::SUB) {
3053 // Op0 - (X * MulC) --> Op0 + (X << log2(-MulC))
3054 if (unsigned ShAmt = getShiftLeftAmt(Op1))
3055 return foldMul(ISD::ADD, Op1.getOperand(i: 0), Op0, ShAmt);
3056 }
3057 }
3058
3059 if (Op.getOpcode() == ISD::MUL) {
3060 Known = KnownBits::mul(LHS: KnownOp0, RHS: KnownOp1);
3061 } else { // Op.getOpcode() is either ISD::ADD, ISD::PTRADD, or ISD::SUB.
3062 Known = KnownBits::computeForAddSub(
3063 Add: Op.getOpcode() != ISD::SUB, NSW: Flags.hasNoSignedWrap(),
3064 NUW: Flags.hasNoUnsignedWrap(), LHS: KnownOp0, RHS: KnownOp1);
3065 }
3066 break;
3067 }
3068 case ISD::FABS: {
3069 SDValue Op0 = Op.getOperand(i: 0);
3070 APInt SignMask = APInt::getSignMask(BitWidth);
3071
3072 if (!DemandedBits.intersects(RHS: SignMask))
3073 return TLO.CombineTo(O: Op, N: Op0);
3074
3075 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3076 Depth: Depth + 1))
3077 return true;
3078
3079 if (Known.isNonNegative())
3080 return TLO.CombineTo(O: Op, N: Op0);
3081 if (Known.isNegative())
3082 return TLO.CombineTo(
3083 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3084
3085 Known.Zero |= SignMask;
3086 Known.One &= ~SignMask;
3087
3088 break;
3089 }
3090 case ISD::FCOPYSIGN: {
3091 SDValue Op0 = Op.getOperand(i: 0);
3092 SDValue Op1 = Op.getOperand(i: 1);
3093
3094 unsigned BitWidth0 = Op0.getScalarValueSizeInBits();
3095 unsigned BitWidth1 = Op1.getScalarValueSizeInBits();
3096 APInt SignMask0 = APInt::getSignMask(BitWidth: BitWidth0);
3097 APInt SignMask1 = APInt::getSignMask(BitWidth: BitWidth1);
3098
3099 if (!DemandedBits.intersects(RHS: SignMask0))
3100 return TLO.CombineTo(O: Op, N: Op0);
3101
3102 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: ~SignMask0 & DemandedBits, OriginalDemandedElts: DemandedElts,
3103 Known, TLO, Depth: Depth + 1) ||
3104 SimplifyDemandedBits(Op: Op1, OriginalDemandedBits: SignMask1, OriginalDemandedElts: DemandedElts, Known&: Known2, TLO,
3105 Depth: Depth + 1))
3106 return true;
3107
3108 if (Known2.isNonNegative())
3109 return TLO.CombineTo(
3110 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FABS, DL: dl, VT, Operand: Op0, Flags: Op->getFlags()));
3111
3112 if (Known2.isNegative())
3113 return TLO.CombineTo(
3114 O: Op, N: TLO.DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT,
3115 Operand: TLO.DAG.getNode(Opcode: ISD::FABS, DL: SDLoc(Op0), VT, Operand: Op0)));
3116
3117 Known.Zero &= ~SignMask0;
3118 Known.One &= ~SignMask0;
3119 break;
3120 }
3121 case ISD::FNEG: {
3122 SDValue Op0 = Op.getOperand(i: 0);
3123 APInt SignMask = APInt::getSignMask(BitWidth);
3124
3125 if (!DemandedBits.intersects(RHS: SignMask))
3126 return TLO.CombineTo(O: Op, N: Op0);
3127
3128 if (SimplifyDemandedBits(Op: Op0, OriginalDemandedBits: DemandedBits, OriginalDemandedElts: DemandedElts, Known, TLO,
3129 Depth: Depth + 1))
3130 return true;
3131
3132 if (!Known.isSignUnknown()) {
3133 Known.Zero ^= SignMask;
3134 Known.One ^= SignMask;
3135 }
3136
3137 break;
3138 }
3139 default:
3140 // We also ask the target about intrinsics (which could be specific to it).
3141 if (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3142 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
3143 // TODO: Probably okay to remove after audit; here to reduce change size
3144 // in initial enablement patch for scalable vectors
3145 if (Op.getValueType().isScalableVector())
3146 break;
3147 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
3148 Known, TLO, Depth))
3149 return true;
3150 break;
3151 }
3152
3153 // Just use computeKnownBits to compute output bits.
3154 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
3155 break;
3156 }
3157
3158 // If we know the value of all of the demanded bits, return this as a
3159 // constant.
3160 if (!isTargetCanonicalConstantNode(Op) &&
3161 DemandedBits.isSubsetOf(RHS: Known.Zero | Known.One)) {
3162 // Avoid folding to a constant if any OpaqueConstant is involved.
3163 if (llvm::any_of(Range: Op->ops(), P: [](SDValue V) {
3164 auto *C = dyn_cast<ConstantSDNode>(Val&: V);
3165 return C && C->isOpaque();
3166 }))
3167 return false;
3168 if (VT.isInteger())
3169 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: Known.One, DL: dl, VT));
3170 if (VT.isFloatingPoint())
3171 return TLO.CombineTo(
3172 O: Op, N: TLO.DAG.getConstantFP(Val: APFloat(VT.getFltSemantics(), Known.One),
3173 DL: dl, VT));
3174 }
3175
3176 // A multi use 'all demanded elts' simplify failed to find any knownbits.
3177 // Try again just for the original demanded elts.
3178 // Ensure we do this AFTER constant folding above.
3179 if (HasMultiUse && Known.isUnknown() && !OriginalDemandedElts.isAllOnes())
3180 Known = TLO.DAG.computeKnownBits(Op, DemandedElts: OriginalDemandedElts, Depth);
3181
3182 return false;
3183}
3184
3185bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
3186 const APInt &DemandedElts,
3187 DAGCombinerInfo &DCI) const {
3188 SelectionDAG &DAG = DCI.DAG;
3189 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
3190 !DCI.isBeforeLegalizeOps());
3191
3192 APInt KnownUndef, KnownZero;
3193 bool Simplified =
3194 SimplifyDemandedVectorElts(Op, DemandedEltMask: DemandedElts, KnownUndef, KnownZero, TLO);
3195 if (Simplified) {
3196 DCI.AddToWorklist(N: Op.getNode());
3197 DCI.CommitTargetLoweringOpt(TLO);
3198 }
3199
3200 return Simplified;
3201}
3202
3203/// Given a vector binary operation and known undefined elements for each input
3204/// operand, compute whether each element of the output is undefined.
3205static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
3206 const APInt &UndefOp0,
3207 const APInt &UndefOp1) {
3208 EVT VT = BO.getValueType();
3209 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
3210 "Vector binop only");
3211
3212 EVT EltVT = VT.getVectorElementType();
3213 unsigned NumElts = VT.isFixedLengthVector() ? VT.getVectorNumElements() : 1;
3214 assert(UndefOp0.getBitWidth() == NumElts &&
3215 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
3216
3217 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
3218 const APInt &UndefVals) {
3219 if (UndefVals[Index])
3220 return DAG.getUNDEF(VT: EltVT);
3221
3222 if (auto *BV = dyn_cast<BuildVectorSDNode>(Val&: V)) {
3223 // Try hard to make sure that the getNode() call is not creating temporary
3224 // nodes. Ignore opaque integers because they do not constant fold.
3225 SDValue Elt = BV->getOperand(Num: Index);
3226 auto *C = dyn_cast<ConstantSDNode>(Val&: Elt);
3227 if (isa<ConstantFPSDNode>(Val: Elt) || Elt.isUndef() || (C && !C->isOpaque()))
3228 return Elt;
3229 }
3230
3231 return SDValue();
3232 };
3233
3234 APInt KnownUndef = APInt::getZero(numBits: NumElts);
3235 for (unsigned i = 0; i != NumElts; ++i) {
3236 // If both inputs for this element are either constant or undef and match
3237 // the element type, compute the constant/undef result for this element of
3238 // the vector.
3239 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
3240 // not handle FP constants. The code within getNode() should be refactored
3241 // to avoid the danger of creating a bogus temporary node here.
3242 SDValue C0 = getUndefOrConstantElt(BO.getOperand(i: 0), i, UndefOp0);
3243 SDValue C1 = getUndefOrConstantElt(BO.getOperand(i: 1), i, UndefOp1);
3244 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
3245 if (DAG.getNode(Opcode: BO.getOpcode(), DL: SDLoc(BO), VT: EltVT, N1: C0, N2: C1).isUndef())
3246 KnownUndef.setBit(i);
3247 }
3248 return KnownUndef;
3249}
3250
3251bool TargetLowering::SimplifyDemandedVectorElts(
3252 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
3253 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
3254 bool AssumeSingleUse) const {
3255 EVT VT = Op.getValueType();
3256 unsigned Opcode = Op.getOpcode();
3257 APInt DemandedElts = OriginalDemandedElts;
3258 unsigned NumElts = DemandedElts.getBitWidth();
3259 assert(VT.isVector() && "Expected vector op");
3260
3261 KnownUndef = KnownZero = APInt::getZero(numBits: NumElts);
3262
3263 if (!shouldSimplifyDemandedVectorElts(Op, TLO))
3264 return false;
3265
3266 // TODO: For now we assume we know nothing about scalable vectors.
3267 if (VT.isScalableVector())
3268 return false;
3269
3270 assert(VT.getVectorNumElements() == NumElts &&
3271 "Mask size mismatches value type element count!");
3272
3273 // Undef operand.
3274 if (Op.isUndef()) {
3275 KnownUndef.setAllBits();
3276 return false;
3277 }
3278
3279 // If Op has other users, assume that all elements are needed.
3280 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
3281 DemandedElts.setAllBits();
3282
3283 // Not demanding any elements from Op.
3284 if (DemandedElts == 0) {
3285 KnownUndef.setAllBits();
3286 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3287 }
3288
3289 // Limit search depth.
3290 if (Depth >= SelectionDAG::MaxRecursionDepth)
3291 return false;
3292
3293 SDLoc DL(Op);
3294 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3295 bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
3296
3297 auto TryShrinkBinOp = [&](SDValue Op0, SDValue Op1) {
3298 unsigned ShrunkSize = getPreferredShrunkVectorSizeInBits(Op, DemandedElts);
3299 if (!ShrunkSize)
3300 return false;
3301
3302 assert(ShrunkSize % EltSizeInBits == 0 &&
3303 "Shrunk size not a multiple of element size");
3304 assert(ShrunkSize < VT.getSizeInBits() &&
3305 "Shrunk size must be < original vector size");
3306 assert(ShrunkSize >= EltSizeInBits * DemandedElts.getActiveBits() &&
3307 "Shrunk size must be >= demanded size");
3308
3309 EVT ShrunkVT = VT.changeVectorElementCount(
3310 Context&: *TLO.DAG.getContext(),
3311 EC: ElementCount::getFixed(MinVal: ShrunkSize / EltSizeInBits));
3312 Op0 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op0, Idx: 0);
3313 Op1 = TLO.DAG.getExtractSubvector(DL, VT: ShrunkVT, Vec: Op1, Idx: 0);
3314 SDValue NewOp =
3315 TLO.DAG.getNode(Opcode, DL, VT: ShrunkVT, N1: Op0, N2: Op1, Flags: Op->getFlags());
3316 return TLO.CombineTo(
3317 O: Op, N: TLO.DAG.getInsertSubvector(DL, Vec: TLO.DAG.getUNDEF(VT), SubVec: NewOp, Idx: 0));
3318 };
3319
3320 // Helper for demanding the specified elements and all the bits of both binary
3321 // operands.
3322 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
3323 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op: Op0, DemandedElts,
3324 DAG&: TLO.DAG, Depth: Depth + 1);
3325 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op: Op1, DemandedElts,
3326 DAG&: TLO.DAG, Depth: Depth + 1);
3327 if (NewOp0 || NewOp1) {
3328 SDValue NewOp =
3329 TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, N1: NewOp0 ? NewOp0 : Op0,
3330 N2: NewOp1 ? NewOp1 : Op1, Flags: Op->getFlags());
3331 return TLO.CombineTo(O: Op, N: NewOp);
3332 }
3333
3334 if (TryShrinkBinOp(Op0, Op1))
3335 return true;
3336
3337 return false;
3338 };
3339
3340 switch (Opcode) {
3341 case ISD::SCALAR_TO_VECTOR: {
3342 if (!DemandedElts[0]) {
3343 KnownUndef.setAllBits();
3344 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3345 }
3346 KnownUndef.setHighBits(NumElts - 1);
3347 break;
3348 }
3349 case ISD::BITCAST: {
3350 SDValue Src = Op.getOperand(i: 0);
3351 EVT SrcVT = Src.getValueType();
3352
3353 if (!SrcVT.isVector()) {
3354 // TODO - bigendian once we have test coverage.
3355 if (IsLE) {
3356 APInt DemandedSrcBits = APInt::getZero(numBits: SrcVT.getSizeInBits());
3357 unsigned EltSize = VT.getScalarSizeInBits();
3358 for (unsigned I = 0; I != NumElts; ++I) {
3359 if (DemandedElts[I]) {
3360 unsigned Offset = I * EltSize;
3361 DemandedSrcBits.setBits(loBit: Offset, hiBit: Offset + EltSize);
3362 }
3363 }
3364 KnownBits Known;
3365 if (SimplifyDemandedBits(Op: Src, DemandedBits: DemandedSrcBits, Known, TLO, Depth: Depth + 1))
3366 return true;
3367 }
3368 break;
3369 }
3370
3371 // Fast handling of 'identity' bitcasts.
3372 unsigned NumSrcElts = SrcVT.getVectorNumElements();
3373 if (NumSrcElts == NumElts)
3374 return SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedElts, KnownUndef,
3375 KnownZero, TLO, Depth: Depth + 1);
3376
3377 APInt SrcDemandedElts, SrcZero, SrcUndef;
3378
3379 // Bitcast from 'large element' src vector to 'small element' vector, we
3380 // must demand a source element if any DemandedElt maps to it.
3381 if ((NumElts % NumSrcElts) == 0) {
3382 unsigned Scale = NumElts / NumSrcElts;
3383 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3384 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3385 TLO, Depth: Depth + 1))
3386 return true;
3387
3388 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
3389 // of the large element.
3390 // TODO - bigendian once we have test coverage.
3391 if (IsLE) {
3392 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
3393 APInt SrcDemandedBits = APInt::getZero(numBits: SrcEltSizeInBits);
3394 for (unsigned i = 0; i != NumElts; ++i)
3395 if (DemandedElts[i]) {
3396 unsigned Ofs = (i % Scale) * EltSizeInBits;
3397 SrcDemandedBits.setBits(loBit: Ofs, hiBit: Ofs + EltSizeInBits);
3398 }
3399
3400 KnownBits Known;
3401 if (SimplifyDemandedBits(Op: Src, OriginalDemandedBits: SrcDemandedBits, OriginalDemandedElts: SrcDemandedElts, Known,
3402 TLO, Depth: Depth + 1))
3403 return true;
3404
3405 // The bitcast has split each wide element into a number of
3406 // narrow subelements. We have just computed the Known bits
3407 // for wide elements. See if element splitting results in
3408 // some subelements being zero. Only for demanded elements!
3409 for (unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
3410 if (!Known.Zero.extractBits(numBits: EltSizeInBits, bitPosition: SubElt * EltSizeInBits)
3411 .isAllOnes())
3412 continue;
3413 for (unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
3414 unsigned Elt = Scale * SrcElt + SubElt;
3415 if (DemandedElts[Elt])
3416 KnownZero.setBit(Elt);
3417 }
3418 }
3419 }
3420
3421 // If the src element is zero/undef then all the output elements will be -
3422 // only demanded elements are guaranteed to be correct.
3423 for (unsigned i = 0; i != NumSrcElts; ++i) {
3424 if (SrcDemandedElts[i]) {
3425 if (SrcZero[i])
3426 KnownZero.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3427 if (SrcUndef[i])
3428 KnownUndef.setBits(loBit: i * Scale, hiBit: (i + 1) * Scale);
3429 }
3430 }
3431 }
3432
3433 // Bitcast from 'small element' src vector to 'large element' vector, we
3434 // demand all smaller source elements covered by the larger demanded element
3435 // of this vector.
3436 if ((NumSrcElts % NumElts) == 0) {
3437 unsigned Scale = NumSrcElts / NumElts;
3438 SrcDemandedElts = APIntOps::ScaleBitMask(A: DemandedElts, NewBitWidth: NumSrcElts);
3439 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: SrcDemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero,
3440 TLO, Depth: Depth + 1))
3441 return true;
3442
3443 // If all the src elements covering an output element are zero/undef, then
3444 // the output element will be as well, assuming it was demanded.
3445 for (unsigned i = 0; i != NumElts; ++i) {
3446 if (DemandedElts[i]) {
3447 if (SrcZero.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3448 KnownZero.setBit(i);
3449 if (SrcUndef.extractBits(numBits: Scale, bitPosition: i * Scale).isAllOnes())
3450 KnownUndef.setBit(i);
3451 }
3452 }
3453 }
3454 break;
3455 }
3456 case ISD::FREEZE: {
3457 SDValue N0 = Op.getOperand(i: 0);
3458 if (TLO.DAG.isGuaranteedNotToBeUndefOrPoison(
3459 Op: N0, DemandedElts, Kind: UndefPoisonKind::UndefOrPoison, Depth: Depth + 1))
3460 return TLO.CombineTo(O: Op, N: N0);
3461
3462 // TODO: Replace this with the general fold from DAGCombiner::visitFREEZE
3463 // freeze(op(x, ...)) -> op(freeze(x), ...).
3464 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && DemandedElts == 1)
3465 return TLO.CombineTo(
3466 O: Op, N: TLO.DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT,
3467 Operand: TLO.DAG.getFreeze(V: N0.getOperand(i: 0))));
3468 break;
3469 }
3470 case ISD::BUILD_VECTOR: {
3471 // Check all elements and simplify any unused elements with UNDEF.
3472 if (!DemandedElts.isAllOnes()) {
3473 // Don't simplify BROADCASTS.
3474 if (llvm::any_of(Range: Op->op_values(),
3475 P: [&](SDValue Elt) { return Op.getOperand(i: 0) != Elt; })) {
3476 SmallVector<SDValue, 32> Ops(Op->ops());
3477 bool Updated = false;
3478 for (unsigned i = 0; i != NumElts; ++i) {
3479 if (!DemandedElts[i] && !Ops[i].isUndef()) {
3480 Ops[i] = TLO.DAG.getUNDEF(VT: Ops[0].getValueType());
3481 KnownUndef.setBit(i);
3482 Updated = true;
3483 }
3484 }
3485 if (Updated)
3486 return TLO.CombineTo(O: Op, N: TLO.DAG.getBuildVector(VT, DL, Ops));
3487 }
3488 }
3489 for (unsigned i = 0; i != NumElts; ++i) {
3490 SDValue SrcOp = Op.getOperand(i);
3491 if (SrcOp.isUndef()) {
3492 KnownUndef.setBit(i);
3493 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
3494 (isNullConstant(V: SrcOp) || isNullFPConstant(V: SrcOp))) {
3495 KnownZero.setBit(i);
3496 }
3497 }
3498 break;
3499 }
3500 case ISD::CONCAT_VECTORS: {
3501 EVT SubVT = Op.getOperand(i: 0).getValueType();
3502 unsigned NumSubVecs = Op.getNumOperands();
3503 unsigned NumSubElts = SubVT.getVectorNumElements();
3504 for (unsigned i = 0; i != NumSubVecs; ++i) {
3505 SDValue SubOp = Op.getOperand(i);
3506 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3507 APInt SubUndef, SubZero;
3508 if (SimplifyDemandedVectorElts(Op: SubOp, OriginalDemandedElts: SubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3509 Depth: Depth + 1))
3510 return true;
3511 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: i * NumSubElts);
3512 KnownZero.insertBits(SubBits: SubZero, bitPosition: i * NumSubElts);
3513 }
3514
3515 // Attempt to avoid multi-use ops if we don't need anything from them.
3516 if (!DemandedElts.isAllOnes()) {
3517 bool FoundNewSub = false;
3518 SmallVector<SDValue, 2> DemandedSubOps;
3519 for (unsigned i = 0; i != NumSubVecs; ++i) {
3520 SDValue SubOp = Op.getOperand(i);
3521 APInt SubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: i * NumSubElts);
3522 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3523 Op: SubOp, DemandedElts: SubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3524 DemandedSubOps.push_back(Elt: NewSubOp ? NewSubOp : SubOp);
3525 FoundNewSub = NewSubOp ? true : FoundNewSub;
3526 }
3527 if (FoundNewSub) {
3528 SDValue NewOp =
3529 TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, Ops: DemandedSubOps);
3530 return TLO.CombineTo(O: Op, N: NewOp);
3531 }
3532 }
3533 break;
3534 }
3535 case ISD::INSERT_SUBVECTOR: {
3536 // Demand any elements from the subvector and the remainder from the src it
3537 // is inserted into.
3538 SDValue Src = Op.getOperand(i: 0);
3539 SDValue Sub = Op.getOperand(i: 1);
3540 uint64_t Idx = Op.getConstantOperandVal(i: 2);
3541 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3542 APInt DemandedSubElts = DemandedElts.extractBits(numBits: NumSubElts, bitPosition: Idx);
3543 APInt DemandedSrcElts = DemandedElts;
3544 DemandedSrcElts.clearBits(LoBit: Idx, HiBit: Idx + NumSubElts);
3545
3546 // If none of the sub operand elements are demanded, bypass the insert.
3547 if (!DemandedSubElts)
3548 return TLO.CombineTo(O: Op, N: Src);
3549
3550 APInt SubUndef, SubZero;
3551 if (SimplifyDemandedVectorElts(Op: Sub, OriginalDemandedElts: DemandedSubElts, KnownUndef&: SubUndef, KnownZero&: SubZero, TLO,
3552 Depth: Depth + 1))
3553 return true;
3554
3555 // If none of the src operand elements are demanded, replace it with undef.
3556 if (!DemandedSrcElts && !Src.isUndef())
3557 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT,
3558 N1: TLO.DAG.getUNDEF(VT), N2: Sub,
3559 N3: Op.getOperand(i: 2)));
3560
3561 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef, KnownZero,
3562 TLO, Depth: Depth + 1))
3563 return true;
3564 KnownUndef.insertBits(SubBits: SubUndef, bitPosition: Idx);
3565 KnownZero.insertBits(SubBits: SubZero, bitPosition: Idx);
3566
3567 // Attempt to avoid multi-use ops if we don't need anything from them.
3568 if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
3569 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3570 Op: Src, DemandedElts: DemandedSrcElts, DAG&: TLO.DAG, Depth: Depth + 1);
3571 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3572 Op: Sub, DemandedElts: DemandedSubElts, DAG&: TLO.DAG, Depth: Depth + 1);
3573 if (NewSrc || NewSub) {
3574 NewSrc = NewSrc ? NewSrc : Src;
3575 NewSub = NewSub ? NewSub : Sub;
3576 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3577 N2: NewSub, N3: Op.getOperand(i: 2));
3578 return TLO.CombineTo(O: Op, N: NewOp);
3579 }
3580 }
3581 break;
3582 }
3583 case ISD::EXTRACT_SUBVECTOR: {
3584 // Offset the demanded elts by the subvector index.
3585 SDValue Src = Op.getOperand(i: 0);
3586 if (Src.getValueType().isScalableVector())
3587 break;
3588 uint64_t Idx = Op.getConstantOperandVal(i: 1);
3589 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3590 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts).shl(shiftAmt: Idx);
3591
3592 APInt SrcUndef, SrcZero;
3593 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3594 Depth: Depth + 1))
3595 return true;
3596 KnownUndef = SrcUndef.extractBits(numBits: NumElts, bitPosition: Idx);
3597 KnownZero = SrcZero.extractBits(numBits: NumElts, bitPosition: Idx);
3598
3599 // Attempt to avoid multi-use ops if we don't need anything from them.
3600 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(Op: Src, DemandedElts: DemandedSrcElts,
3601 DAG&: TLO.DAG, Depth: Depth + 1);
3602 if (NewSrc) {
3603 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, N1: NewSrc,
3604 N2: Op.getOperand(i: 1));
3605 return TLO.CombineTo(O: Op, N: NewOp);
3606 }
3607 break;
3608 }
3609 case ISD::INSERT_VECTOR_ELT: {
3610 SDValue Vec = Op.getOperand(i: 0);
3611 SDValue Scl = Op.getOperand(i: 1);
3612 auto *CIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
3613
3614 // For a legal, constant insertion index, if we don't need this insertion
3615 // then strip it, else remove it from the demanded elts.
3616 if (CIdx && CIdx->getAPIntValue().ult(RHS: NumElts)) {
3617 unsigned Idx = CIdx->getZExtValue();
3618 if (!DemandedElts[Idx])
3619 return TLO.CombineTo(O: Op, N: Vec);
3620
3621 APInt DemandedVecElts(DemandedElts);
3622 DemandedVecElts.clearBit(BitPosition: Idx);
3623 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedVecElts, KnownUndef,
3624 KnownZero, TLO, Depth: Depth + 1))
3625 return true;
3626
3627 KnownUndef.setBitVal(BitPosition: Idx, BitValue: Scl.isUndef());
3628
3629 KnownZero.setBitVal(BitPosition: Idx, BitValue: isNullConstant(V: Scl) || isNullFPConstant(V: Scl));
3630 break;
3631 }
3632
3633 APInt VecUndef, VecZero;
3634 if (SimplifyDemandedVectorElts(Op: Vec, OriginalDemandedElts: DemandedElts, KnownUndef&: VecUndef, KnownZero&: VecZero, TLO,
3635 Depth: Depth + 1))
3636 return true;
3637 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
3638 break;
3639 }
3640 case ISD::VSELECT: {
3641 SDValue Sel = Op.getOperand(i: 0);
3642 SDValue LHS = Op.getOperand(i: 1);
3643 SDValue RHS = Op.getOperand(i: 2);
3644
3645 // Try to transform the select condition based on the current demanded
3646 // elements.
3647 APInt UndefSel, ZeroSel;
3648 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3649 Depth: Depth + 1))
3650 return true;
3651
3652 // See if we can simplify either vselect operand.
3653 APInt DemandedLHS(DemandedElts);
3654 APInt DemandedRHS(DemandedElts);
3655 APInt UndefLHS, ZeroLHS;
3656 APInt UndefRHS, ZeroRHS;
3657 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3658 Depth: Depth + 1))
3659 return true;
3660 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3661 Depth: Depth + 1))
3662 return true;
3663
3664 KnownUndef = UndefLHS & UndefRHS;
3665 KnownZero = ZeroLHS & ZeroRHS;
3666
3667 // If we know that the selected element is always zero, we don't need the
3668 // select value element.
3669 APInt DemandedSel = DemandedElts & ~KnownZero;
3670 if (DemandedSel != DemandedElts)
3671 if (SimplifyDemandedVectorElts(Op: Sel, OriginalDemandedElts: DemandedSel, KnownUndef&: UndefSel, KnownZero&: ZeroSel, TLO,
3672 Depth: Depth + 1))
3673 return true;
3674
3675 break;
3676 }
3677 case ISD::VECTOR_SHUFFLE: {
3678 SDValue LHS = Op.getOperand(i: 0);
3679 SDValue RHS = Op.getOperand(i: 1);
3680 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
3681
3682 // Collect demanded elements from shuffle operands..
3683 APInt DemandedLHS(NumElts, 0);
3684 APInt DemandedRHS(NumElts, 0);
3685 for (unsigned i = 0; i != NumElts; ++i) {
3686 int M = ShuffleMask[i];
3687 if (M < 0 || !DemandedElts[i])
3688 continue;
3689 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
3690 if (M < (int)NumElts)
3691 DemandedLHS.setBit(M);
3692 else
3693 DemandedRHS.setBit(M - NumElts);
3694 }
3695
3696 // If either side isn't demanded, replace it by UNDEF. We handle this
3697 // explicitly here to also simplify in case of multiple uses (on the
3698 // contrary to the SimplifyDemandedVectorElts calls below).
3699 bool FoldLHS = !DemandedLHS && !LHS.isUndef();
3700 bool FoldRHS = !DemandedRHS && !RHS.isUndef();
3701 if (FoldLHS || FoldRHS) {
3702 LHS = FoldLHS ? TLO.DAG.getUNDEF(VT: LHS.getValueType()) : LHS;
3703 RHS = FoldRHS ? TLO.DAG.getUNDEF(VT: RHS.getValueType()) : RHS;
3704 SDValue NewOp =
3705 TLO.DAG.getVectorShuffle(VT, dl: SDLoc(Op), N1: LHS, N2: RHS, Mask: ShuffleMask);
3706 return TLO.CombineTo(O: Op, N: NewOp);
3707 }
3708
3709 // See if we can simplify either shuffle operand.
3710 APInt UndefLHS, ZeroLHS;
3711 APInt UndefRHS, ZeroRHS;
3712 if (SimplifyDemandedVectorElts(Op: LHS, OriginalDemandedElts: DemandedLHS, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3713 Depth: Depth + 1))
3714 return true;
3715 if (SimplifyDemandedVectorElts(Op: RHS, OriginalDemandedElts: DemandedRHS, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3716 Depth: Depth + 1))
3717 return true;
3718
3719 // Simplify mask using undef elements from LHS/RHS.
3720 bool Updated = false;
3721 bool IdentityLHS = true, IdentityRHS = true;
3722 SmallVector<int, 32> NewMask(ShuffleMask);
3723 for (unsigned i = 0; i != NumElts; ++i) {
3724 int &M = NewMask[i];
3725 if (M < 0)
3726 continue;
3727 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
3728 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
3729 Updated = true;
3730 M = -1;
3731 }
3732 IdentityLHS &= (M < 0) || (M == (int)i);
3733 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3734 }
3735
3736 // Update legal shuffle masks based on demanded elements if it won't reduce
3737 // to Identity which can cause premature removal of the shuffle mask.
3738 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
3739 SDValue LegalShuffle =
3740 buildLegalVectorShuffle(VT, DL, N0: LHS, N1: RHS, Mask: NewMask, DAG&: TLO.DAG);
3741 if (LegalShuffle)
3742 return TLO.CombineTo(O: Op, N: LegalShuffle);
3743 }
3744
3745 // Propagate undef/zero elements from LHS/RHS.
3746 for (unsigned i = 0; i != NumElts; ++i) {
3747 int M = ShuffleMask[i];
3748 if (M < 0) {
3749 KnownUndef.setBit(i);
3750 } else if (M < (int)NumElts) {
3751 if (UndefLHS[M])
3752 KnownUndef.setBit(i);
3753 if (ZeroLHS[M])
3754 KnownZero.setBit(i);
3755 } else {
3756 if (UndefRHS[M - NumElts])
3757 KnownUndef.setBit(i);
3758 if (ZeroRHS[M - NumElts])
3759 KnownZero.setBit(i);
3760 }
3761 }
3762 break;
3763 }
3764 case ISD::ANY_EXTEND_VECTOR_INREG:
3765 case ISD::SIGN_EXTEND_VECTOR_INREG:
3766 case ISD::ZERO_EXTEND_VECTOR_INREG: {
3767 APInt SrcUndef, SrcZero;
3768 SDValue Src = Op.getOperand(i: 0);
3769 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3770 APInt DemandedSrcElts = DemandedElts.zext(width: NumSrcElts);
3771 if (SimplifyDemandedVectorElts(Op: Src, OriginalDemandedElts: DemandedSrcElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3772 Depth: Depth + 1))
3773 return true;
3774 KnownZero = SrcZero.zextOrTrunc(width: NumElts);
3775 KnownUndef = SrcUndef.zextOrTrunc(width: NumElts);
3776
3777 if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
3778 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3779 DemandedSrcElts == 1) {
3780 // aext - if we just need the bottom element then we can bitcast.
3781 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Src));
3782 }
3783
3784 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
3785 // zext(undef) upper bits are guaranteed to be zero.
3786 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
3787 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3788 KnownUndef.clearAllBits();
3789
3790 // zext - if we just need the bottom element then we can mask:
3791 // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
3792 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
3793 Op->isOnlyUserOf(N: Src.getNode()) &&
3794 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3795 SDLoc DL(Op);
3796 EVT SrcVT = Src.getValueType();
3797 EVT SrcSVT = SrcVT.getScalarType();
3798
3799 // If we're after type legalization and SrcSVT is not legal, use the
3800 // promoted type for creating constants to avoid creating nodes with
3801 // illegal types.
3802 if (TLO.LegalTypes())
3803 SrcSVT = getLegalTypeToTransformTo(Context&: *TLO.DAG.getContext(), VT: SrcSVT);
3804
3805 SmallVector<SDValue> MaskElts;
3806 MaskElts.push_back(Elt: TLO.DAG.getAllOnesConstant(DL, VT: SrcSVT));
3807 MaskElts.append(NumInputs: NumSrcElts - 1, Elt: TLO.DAG.getConstant(Val: 0, DL, VT: SrcSVT));
3808 SDValue Mask = TLO.DAG.getBuildVector(VT: SrcVT, DL, Ops: MaskElts);
3809 if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
3810 Opcode: ISD::AND, DL, VT: SrcVT, Ops: {Src.getOperand(i: 1), Mask})) {
3811 Fold = TLO.DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: Src.getOperand(i: 0), N2: Fold);
3812 return TLO.CombineTo(O: Op, N: TLO.DAG.getBitcast(VT, V: Fold));
3813 }
3814 }
3815 }
3816 break;
3817 }
3818
3819 // TODO: There are more binop opcodes that could be handled here - MIN,
3820 // MAX, saturated math, etc.
3821 case ISD::ADD: {
3822 SDValue Op0 = Op.getOperand(i: 0);
3823 SDValue Op1 = Op.getOperand(i: 1);
3824 if (Op0 == Op1 && Op->isOnlyUserOf(N: Op0.getNode())) {
3825 APInt UndefLHS, ZeroLHS;
3826 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3827 Depth: Depth + 1, /*AssumeSingleUse*/ true))
3828 return true;
3829 }
3830 [[fallthrough]];
3831 }
3832 case ISD::AVGCEILS:
3833 case ISD::AVGCEILU:
3834 case ISD::AVGFLOORS:
3835 case ISD::AVGFLOORU:
3836 case ISD::OR:
3837 case ISD::XOR:
3838 case ISD::SUB:
3839 case ISD::FADD:
3840 case ISD::FSUB:
3841 case ISD::FMUL:
3842 case ISD::FDIV:
3843 case ISD::FREM:
3844 case ISD::PSEUDO_FMIN:
3845 case ISD::PSEUDO_FMAX: {
3846 SDValue Op0 = Op.getOperand(i: 0);
3847 SDValue Op1 = Op.getOperand(i: 1);
3848
3849 APInt UndefRHS, ZeroRHS;
3850 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3851 Depth: Depth + 1))
3852 return true;
3853 APInt UndefLHS, ZeroLHS;
3854 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3855 Depth: Depth + 1))
3856 return true;
3857
3858 KnownZero = ZeroLHS & ZeroRHS;
3859 KnownUndef = getKnownUndefForVectorBinop(BO: Op, DAG&: TLO.DAG, UndefOp0: UndefLHS, UndefOp1: UndefRHS);
3860
3861 // Attempt to avoid multi-use ops if we don't need anything from them.
3862 // TODO - use KnownUndef to relax the demandedelts?
3863 if (!DemandedElts.isAllOnes())
3864 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3865 return true;
3866 break;
3867 }
3868 case ISD::SHL:
3869 case ISD::SRL:
3870 case ISD::SRA:
3871 case ISD::ROTL:
3872 case ISD::ROTR: {
3873 SDValue Op0 = Op.getOperand(i: 0);
3874 SDValue Op1 = Op.getOperand(i: 1);
3875
3876 APInt UndefRHS, ZeroRHS;
3877 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefRHS, KnownZero&: ZeroRHS, TLO,
3878 Depth: Depth + 1))
3879 return true;
3880 APInt UndefLHS, ZeroLHS;
3881 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef&: UndefLHS, KnownZero&: ZeroLHS, TLO,
3882 Depth: Depth + 1))
3883 return true;
3884
3885 KnownZero = ZeroLHS;
3886 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
3887
3888 // Attempt to avoid multi-use ops if we don't need anything from them.
3889 // TODO - use KnownUndef to relax the demandedelts?
3890 if (!DemandedElts.isAllOnes())
3891 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3892 return true;
3893 break;
3894 }
3895 case ISD::MUL:
3896 case ISD::MULHU:
3897 case ISD::MULHS:
3898 case ISD::AND: {
3899 SDValue Op0 = Op.getOperand(i: 0);
3900 SDValue Op1 = Op.getOperand(i: 1);
3901
3902 APInt SrcUndef, SrcZero;
3903 if (SimplifyDemandedVectorElts(Op: Op1, OriginalDemandedElts: DemandedElts, KnownUndef&: SrcUndef, KnownZero&: SrcZero, TLO,
3904 Depth: Depth + 1))
3905 return true;
3906 // FIXME: If we know that a demanded element was zero in Op1 we don't need
3907 // to demand it in Op0 - its guaranteed to be zero. There is however a
3908 // restriction, as we must not make any of the originally demanded elements
3909 // more poisonous. We could reduce amount of elements demanded, but then we
3910 // also need a to inform SimplifyDemandedVectorElts that some elements must
3911 // not be made more poisonous.
3912 if (SimplifyDemandedVectorElts(Op: Op0, OriginalDemandedElts: DemandedElts, KnownUndef, KnownZero,
3913 TLO, Depth: Depth + 1))
3914 return true;
3915
3916 KnownUndef &= DemandedElts;
3917 KnownZero &= DemandedElts;
3918
3919 // If every element pair has a zero/undef/poison then just fold to zero.
3920 // fold (and x, undef/poison) -> 0 / (and x, 0) -> 0
3921 // fold (mul x, undef/poison) -> 0 / (mul x, 0) -> 0
3922 if (DemandedElts.isSubsetOf(RHS: SrcZero | KnownZero | SrcUndef | KnownUndef))
3923 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3924
3925 // If either side has a zero element, then the result element is zero, even
3926 // if the other is an UNDEF.
3927 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
3928 // and then handle 'and' nodes with the rest of the binop opcodes.
3929 KnownZero |= SrcZero;
3930 KnownUndef &= SrcUndef;
3931 KnownUndef &= ~KnownZero;
3932
3933 // Attempt to avoid multi-use ops if we don't need anything from them.
3934 if (!DemandedElts.isAllOnes())
3935 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3936 return true;
3937 break;
3938 }
3939 case ISD::TRUNCATE:
3940 case ISD::SIGN_EXTEND:
3941 case ISD::ZERO_EXTEND:
3942 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
3943 KnownZero, TLO, Depth: Depth + 1))
3944 return true;
3945
3946 if (!DemandedElts.isAllOnes())
3947 if (SDValue NewOp = SimplifyMultipleUseDemandedVectorElts(
3948 Op: Op.getOperand(i: 0), DemandedElts, DAG&: TLO.DAG, Depth: Depth + 1))
3949 return TLO.CombineTo(O: Op, N: TLO.DAG.getNode(Opcode, DL: SDLoc(Op), VT, Operand: NewOp));
3950
3951 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
3952 // zext(undef) upper bits are guaranteed to be zero.
3953 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
3954 return TLO.CombineTo(O: Op, N: TLO.DAG.getConstant(Val: 0, DL: SDLoc(Op), VT));
3955 KnownUndef.clearAllBits();
3956 }
3957 break;
3958 case ISD::SINT_TO_FP:
3959 case ISD::UINT_TO_FP:
3960 case ISD::FP_TO_SINT:
3961 case ISD::FP_TO_UINT:
3962 if (SimplifyDemandedVectorElts(Op: Op.getOperand(i: 0), OriginalDemandedElts: DemandedElts, KnownUndef,
3963 KnownZero, TLO, Depth: Depth + 1))
3964 return true;
3965 // Don't fall through to generic undef -> undef handling.
3966 return false;
3967 default: {
3968 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
3969 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3970 KnownZero, TLO, Depth))
3971 return true;
3972 } else {
3973 KnownBits Known;
3974 APInt DemandedBits = APInt::getAllOnes(numBits: EltSizeInBits);
3975 if (SimplifyDemandedBits(Op, OriginalDemandedBits: DemandedBits, OriginalDemandedElts, Known,
3976 TLO, Depth, AssumeSingleUse))
3977 return true;
3978 }
3979 break;
3980 }
3981 }
3982 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
3983
3984 // Constant fold all undef cases.
3985 // TODO: Handle zero cases as well.
3986 if (DemandedElts.isSubsetOf(RHS: KnownUndef))
3987 return TLO.CombineTo(O: Op, N: TLO.DAG.getUNDEF(VT));
3988
3989 return false;
3990}
3991
3992/// Determine which of the bits specified in Mask are known to be either zero or
3993/// one and return them in the Known.
3994void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
3995 KnownBits &Known,
3996 const APInt &DemandedElts,
3997 const SelectionDAG &DAG,
3998 unsigned Depth) const {
3999 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4000 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4001 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4002 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4003 "Should use MaskedValueIsZero if you don't know whether Op"
4004 " is a target node!");
4005 Known.resetAll();
4006}
4007
4008void TargetLowering::computeKnownBitsForTargetInstr(
4009 GISelValueTracking &Analysis, Register R, KnownBits &Known,
4010 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4011 unsigned Depth) const {
4012 Known.resetAll();
4013}
4014
4015void TargetLowering::computeKnownFPClassForTargetInstr(
4016 GISelValueTracking &Analysis, Register R, KnownFPClass &Known,
4017 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
4018 unsigned Depth) const {
4019 Known.resetAll();
4020}
4021
4022void TargetLowering::computeKnownBitsForStackObjectPointer(
4023 KnownBits &Known, const MachineFunction &, Align Alignment) const {
4024 // The low bits are known zero if the pointer is aligned.
4025 Known.Zero.setLowBits(Log2(A: Alignment));
4026}
4027
4028SDValue TargetLowering::annotateStackObjectPointer(SDValue Ptr,
4029 SelectionDAG &DAG,
4030 const SDLoc &DL,
4031 Align Alignment) const {
4032 // Materialize leading-zero stack object pointer facts as AssertZext.
4033 // Alignment-derived low zero bits are not represented on the returned DAG
4034 // value here.
4035 EVT PtrVT = Ptr.getValueType();
4036
4037 unsigned RegSize = PtrVT.getScalarSizeInBits();
4038 KnownBits Known(RegSize);
4039 computeKnownBitsForStackObjectPointer(Known, DAG.getMachineFunction(),
4040 Alignment);
4041
4042 unsigned NumZeroBits = Known.countMinLeadingZeros();
4043 if (!NumZeroBits)
4044 return Ptr;
4045
4046 EVT FromVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: RegSize - NumZeroBits);
4047 return DAG.getNode(Opcode: ISD::AssertZext, DL, VT: PtrVT, N1: Ptr, N2: DAG.getValueType(FromVT));
4048}
4049
4050Align TargetLowering::computeKnownAlignForTargetInstr(
4051 GISelValueTracking &Analysis, Register R, const MachineRegisterInfo &MRI,
4052 unsigned Depth) const {
4053 return Align(1);
4054}
4055
4056/// This method can be implemented by targets that want to expose additional
4057/// information about sign bits to the DAG Combiner.
4058unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
4059 const APInt &,
4060 const SelectionDAG &,
4061 unsigned Depth) const {
4062 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4063 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4064 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4065 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4066 "Should use ComputeNumSignBits if you don't know whether Op"
4067 " is a target node!");
4068 return 1;
4069}
4070
4071unsigned TargetLowering::computeNumSignBitsForTargetInstr(
4072 GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
4073 const MachineRegisterInfo &MRI, unsigned Depth) const {
4074 return 1;
4075}
4076
4077bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
4078 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
4079 TargetLoweringOpt &TLO, unsigned Depth) const {
4080 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4081 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4082 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4083 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4084 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
4085 " is a target node!");
4086 return false;
4087}
4088
4089bool TargetLowering::SimplifyDemandedBitsForTargetNode(
4090 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4091 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
4092 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4093 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4094 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4095 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4096 "Should use SimplifyDemandedBits if you don't know whether Op"
4097 " is a target node!");
4098 computeKnownBitsForTargetNode(Op, Known, DemandedElts, DAG: TLO.DAG, Depth);
4099 return false;
4100}
4101
4102SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
4103 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
4104 SelectionDAG &DAG, unsigned Depth) const {
4105 assert(
4106 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4107 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4108 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4109 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4110 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
4111 " is a target node!");
4112 return SDValue();
4113}
4114
4115SDValue
4116TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
4117 SDValue N1, MutableArrayRef<int> Mask,
4118 SelectionDAG &DAG) const {
4119 bool LegalMask = isShuffleMaskLegal(Mask, VT);
4120 if (!LegalMask) {
4121 std::swap(a&: N0, b&: N1);
4122 ShuffleVectorSDNode::commuteMask(Mask);
4123 LegalMask = isShuffleMaskLegal(Mask, VT);
4124 }
4125
4126 if (!LegalMask)
4127 return SDValue();
4128
4129 return DAG.getVectorShuffle(VT, dl: DL, N1: N0, N2: N1, Mask);
4130}
4131
4132const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
4133 return nullptr;
4134}
4135
4136bool TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode(
4137 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4138 UndefPoisonKind Kind, unsigned Depth) const {
4139 assert(
4140 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4141 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4142 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4143 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4144 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
4145 " is a target node!");
4146
4147 // If Op can't create undef/poison and none of its operands are undef/poison
4148 // then Op is never undef/poison.
4149 return !canCreateUndefOrPoisonForTargetNode(Op, DemandedElts, DAG, Kind,
4150 /*ConsiderFlags*/ true, Depth) &&
4151 all_of(Range: Op->ops(), P: [&](SDValue V) {
4152 return DAG.isGuaranteedNotToBeUndefOrPoison(Op: V, Kind, Depth: Depth + 1);
4153 });
4154}
4155
4156bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
4157 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
4158 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
4159 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4160 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4161 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4162 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4163 "Should use canCreateUndefOrPoison if you don't know whether Op"
4164 " is a target node!");
4165 // Be conservative and return true.
4166 return true;
4167}
4168
4169void TargetLowering::computeKnownFPClassForTargetNode(const SDValue Op,
4170 KnownFPClass &Known,
4171 const APInt &DemandedElts,
4172 const SelectionDAG &DAG,
4173 unsigned Depth) const {
4174 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4175 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4176 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4177 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4178 "Should use computeKnownFPClass if you don't know whether Op"
4179 " is a target node!");
4180}
4181
4182bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
4183 const APInt &DemandedElts,
4184 const SelectionDAG &DAG,
4185 bool SNaN,
4186 unsigned Depth) const {
4187 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4188 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4189 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4190 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4191 "Should use isKnownNeverNaN if you don't know whether Op"
4192 " is a target node!");
4193 return false;
4194}
4195
4196bool TargetLowering::isSplatValueForTargetNode(SDValue Op,
4197 const APInt &DemandedElts,
4198 APInt &UndefElts,
4199 const SelectionDAG &DAG,
4200 unsigned Depth) const {
4201 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
4202 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
4203 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
4204 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
4205 "Should use isSplatValue if you don't know whether Op"
4206 " is a target node!");
4207 return false;
4208}
4209
4210// FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
4211// work with truncating build vectors and vectors with elements of less than
4212// 8 bits.
4213bool TargetLowering::isConstTrueVal(SDValue N) const {
4214 if (!N)
4215 return false;
4216
4217 unsigned EltWidth;
4218 APInt CVal;
4219 if (ConstantSDNode *CN = isConstOrConstSplat(N, /*AllowUndefs=*/false,
4220 /*AllowTruncation=*/true)) {
4221 CVal = CN->getAPIntValue();
4222 EltWidth = N.getValueType().getScalarSizeInBits();
4223 } else
4224 return false;
4225
4226 // If this is a truncating splat, truncate the splat value.
4227 // Otherwise, we may fail to match the expected values below.
4228 if (EltWidth < CVal.getBitWidth())
4229 CVal = CVal.trunc(width: EltWidth);
4230
4231 switch (getBooleanContents(Type: N.getValueType())) {
4232 case UndefinedBooleanContent:
4233 return CVal[0];
4234 case ZeroOrOneBooleanContent:
4235 return CVal.isOne();
4236 case ZeroOrNegativeOneBooleanContent:
4237 return CVal.isAllOnes();
4238 }
4239
4240 llvm_unreachable("Invalid boolean contents");
4241}
4242
4243bool TargetLowering::isConstFalseVal(SDValue N) const {
4244 if (!N)
4245 return false;
4246
4247 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N);
4248 if (!CN) {
4249 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val&: N);
4250 if (!BV)
4251 return false;
4252
4253 // Only interested in constant splats, we don't care about undef
4254 // elements in identifying boolean constants and getConstantSplatNode
4255 // returns NULL if all ops are undef;
4256 CN = BV->getConstantSplatNode();
4257 if (!CN)
4258 return false;
4259 }
4260
4261 if (getBooleanContents(Type: N->getValueType(ResNo: 0)) == UndefinedBooleanContent)
4262 return !CN->getAPIntValue()[0];
4263
4264 return CN->isZero();
4265}
4266
4267bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
4268 bool SExt) const {
4269 if (VT == MVT::i1)
4270 return N->isOne();
4271
4272 TargetLowering::BooleanContent Cnt = getBooleanContents(Type: VT);
4273 switch (Cnt) {
4274 case TargetLowering::ZeroOrOneBooleanContent:
4275 // An extended value of 1 is always true, unless its original type is i1,
4276 // in which case it will be sign extended to -1.
4277 return (N->isOne() && !SExt) || (SExt && (N->getValueType(ResNo: 0) != MVT::i1));
4278 case TargetLowering::UndefinedBooleanContent:
4279 case TargetLowering::ZeroOrNegativeOneBooleanContent:
4280 return N->isAllOnes() && SExt;
4281 }
4282 llvm_unreachable("Unexpected enumeration.");
4283}
4284
4285/// This helper function of SimplifySetCC tries to optimize the comparison when
4286/// either operand of the SetCC node is a bitwise-and instruction.
4287SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
4288 ISD::CondCode Cond, const SDLoc &DL,
4289 DAGCombinerInfo &DCI) const {
4290 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
4291 std::swap(a&: N0, b&: N1);
4292
4293 SelectionDAG &DAG = DCI.DAG;
4294 EVT OpVT = N0.getValueType();
4295 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
4296 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4297 return SDValue();
4298
4299 // (X & Y) != 0 --> zextOrTrunc(X & Y)
4300 // iff everything but LSB is known zero:
4301 if (Cond == ISD::SETNE && isNullConstant(V: N1) &&
4302 (getBooleanContents(Type: OpVT) == TargetLowering::UndefinedBooleanContent ||
4303 getBooleanContents(Type: OpVT) == TargetLowering::ZeroOrOneBooleanContent)) {
4304 unsigned NumEltBits = OpVT.getScalarSizeInBits();
4305 APInt UpperBits = APInt::getHighBitsSet(numBits: NumEltBits, hiBitsSet: NumEltBits - 1);
4306 if (DAG.MaskedValueIsZero(Op: N0, Mask: UpperBits))
4307 return DAG.getBoolExtOrTrunc(Op: N0, SL: DL, VT, OpVT);
4308 }
4309
4310 // Try to eliminate a power-of-2 mask constant by converting to a signbit
4311 // test in a narrow type that we can truncate to with no cost. Examples:
4312 // (i32 X & 32768) == 0 --> (trunc X to i16) >= 0
4313 // (i32 X & 32768) != 0 --> (trunc X to i16) < 0
4314 // TODO: This conservatively checks for type legality on the source and
4315 // destination types. That may inhibit optimizations, but it also
4316 // allows setcc->shift transforms that may be more beneficial.
4317 auto *AndC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
4318 if (AndC && isNullConstant(V: N1) && AndC->getAPIntValue().isPowerOf2() &&
4319 isTypeLegal(VT: OpVT) && N0.hasOneUse()) {
4320 EVT NarrowVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
4321 BitWidth: AndC->getAPIntValue().getActiveBits());
4322 if (isTruncateFree(FromVT: OpVT, ToVT: NarrowVT) && isTypeLegal(VT: NarrowVT)) {
4323 SDValue Trunc = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 0), DL, VT: NarrowVT);
4324 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: NarrowVT);
4325 return DAG.getSetCC(DL, VT, LHS: Trunc, RHS: Zero,
4326 Cond: Cond == ISD::SETEQ ? ISD::SETGE : ISD::SETLT);
4327 }
4328 }
4329
4330 // Match these patterns in any of their permutations:
4331 // (X & Y) == Y
4332 // (X & Y) != Y
4333 SDValue X, Y;
4334 if (N0.getOperand(i: 0) == N1) {
4335 X = N0.getOperand(i: 1);
4336 Y = N0.getOperand(i: 0);
4337 } else if (N0.getOperand(i: 1) == N1) {
4338 X = N0.getOperand(i: 0);
4339 Y = N0.getOperand(i: 1);
4340 } else {
4341 return SDValue();
4342 }
4343
4344 // TODO: We should invert (X & Y) eq/ne 0 -> (X & Y) ne/eq Y if
4345 // `isXAndYEqZeroPreferableToXAndYEqY` is false. This is a bit difficult as
4346 // its liable to create and infinite loop.
4347 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: OpVT);
4348 if (isXAndYEqZeroPreferableToXAndYEqY(Cond, OpVT) &&
4349 DAG.isKnownToBeAPowerOfTwo(Val: Y)) {
4350 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
4351 // Note that where Y is variable and is known to have at most one bit set
4352 // (for example, if it is Z & 1) we cannot do this; the expressions are not
4353 // equivalent when Y == 0.
4354 assert(OpVT.isInteger());
4355 Cond = ISD::getSetCCInverse(Operation: Cond, Type: OpVT);
4356 if (DCI.isBeforeLegalizeOps() ||
4357 isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType()))
4358 return DAG.getSetCC(DL, VT, LHS: N0, RHS: Zero, Cond);
4359 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
4360 // If the target supports an 'and-not' or 'and-complement' logic operation,
4361 // try to use that to make a comparison operation more efficient.
4362 // But don't do this transform if the mask is a single bit because there are
4363 // more efficient ways to deal with that case (for example, 'bt' on x86 or
4364 // 'rlwinm' on PPC).
4365
4366 // Bail out if the compare operand that we want to turn into a zero is
4367 // already a zero (otherwise, infinite loop).
4368 if (isNullConstant(V: Y))
4369 return SDValue();
4370
4371 // Transform this into: ~X & Y == 0.
4372 SDValue NotX = DAG.getNOT(DL: SDLoc(X), Val: X, VT: OpVT);
4373 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: NotX, N2: Y);
4374 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: Zero, Cond);
4375 }
4376
4377 return SDValue();
4378}
4379
4380/// This helper function of SimplifySetCC tries to optimize the comparison when
4381/// either operand of the SetCC node is a bitwise-or instruction.
4382/// For now, this just transforms (X | Y) ==/!= Y into X & ~Y ==/!= 0.
4383SDValue TargetLowering::foldSetCCWithOr(EVT VT, SDValue N0, SDValue N1,
4384 ISD::CondCode Cond, const SDLoc &DL,
4385 DAGCombinerInfo &DCI) const {
4386 if (N1.getOpcode() == ISD::OR && N0.getOpcode() != ISD::OR)
4387 std::swap(a&: N0, b&: N1);
4388
4389 SelectionDAG &DAG = DCI.DAG;
4390 EVT OpVT = N0.getValueType();
4391 if (!N0.hasOneUse() || !OpVT.isInteger() ||
4392 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
4393 return SDValue();
4394
4395 // (X | Y) == Y
4396 // (X | Y) != Y
4397 SDValue X;
4398 if (sd_match(N: N0, P: m_Or(L: m_Value(N&: X), R: m_Specific(N: N1))) && hasAndNotCompare(Y: X)) {
4399 // If the target supports an 'and-not' or 'and-complement' logic operation,
4400 // try to use that to make a comparison operation more efficient.
4401
4402 // Bail out if the compare operand that we want to turn into a zero is
4403 // already a zero (otherwise, infinite loop).
4404 if (isNullConstant(V: N1))
4405 return SDValue();
4406
4407 // Transform this into: X & ~Y ==/!= 0.
4408 SDValue NotY = DAG.getNOT(DL: SDLoc(N1), Val: N1, VT: OpVT);
4409 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: X, N2: NotY);
4410 return DAG.getSetCC(DL, VT, LHS: NewAnd, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4411 }
4412
4413 return SDValue();
4414}
4415
4416/// There are multiple IR patterns that could be checking whether certain
4417/// truncation of a signed number would be lossy or not. The pattern which is
4418/// best at IR level, may not lower optimally. Thus, we want to unfold it.
4419/// We are looking for the following pattern: (KeptBits is a constant)
4420/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
4421/// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
4422/// KeptBits also can't be 1, that would have been folded to %x dstcond 0
4423/// We will unfold it into the natural trunc+sext pattern:
4424/// ((%x << C) a>> C) dstcond %x
4425/// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
4426SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
4427 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
4428 const SDLoc &DL) const {
4429 // We must be comparing with a constant.
4430 ConstantSDNode *C1;
4431 if (!(C1 = dyn_cast<ConstantSDNode>(Val&: N1)))
4432 return SDValue();
4433
4434 // N0 should be: add %x, (1 << (KeptBits-1))
4435 if (N0->getOpcode() != ISD::ADD)
4436 return SDValue();
4437
4438 // And we must be 'add'ing a constant.
4439 ConstantSDNode *C01;
4440 if (!(C01 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1))))
4441 return SDValue();
4442
4443 SDValue X = N0->getOperand(Num: 0);
4444 EVT XVT = X.getValueType();
4445
4446 // Validate constants ...
4447
4448 APInt I1 = C1->getAPIntValue();
4449
4450 ISD::CondCode NewCond;
4451 if (Cond == ISD::CondCode::SETULT) {
4452 NewCond = ISD::CondCode::SETEQ;
4453 } else if (Cond == ISD::CondCode::SETULE) {
4454 NewCond = ISD::CondCode::SETEQ;
4455 // But need to 'canonicalize' the constant.
4456 I1 += 1;
4457 } else if (Cond == ISD::CondCode::SETUGT) {
4458 NewCond = ISD::CondCode::SETNE;
4459 // But need to 'canonicalize' the constant.
4460 I1 += 1;
4461 } else if (Cond == ISD::CondCode::SETUGE) {
4462 NewCond = ISD::CondCode::SETNE;
4463 } else
4464 return SDValue();
4465
4466 APInt I01 = C01->getAPIntValue();
4467
4468 auto checkConstants = [&I1, &I01]() -> bool {
4469 // Both of them must be power-of-two, and the constant from setcc is bigger.
4470 return I1.ugt(RHS: I01) && I1.isPowerOf2() && I01.isPowerOf2();
4471 };
4472
4473 if (checkConstants()) {
4474 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
4475 } else {
4476 // What if we invert constants? (and the target predicate)
4477 I1.negate();
4478 I01.negate();
4479 assert(XVT.isInteger());
4480 NewCond = getSetCCInverse(Operation: NewCond, Type: XVT);
4481 if (!checkConstants())
4482 return SDValue();
4483 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
4484 }
4485
4486 // They are power-of-two, so which bit is set?
4487 const unsigned KeptBits = I1.logBase2();
4488 const unsigned KeptBitsMinusOne = I01.logBase2();
4489
4490 // Magic!
4491 if (KeptBits != (KeptBitsMinusOne + 1))
4492 return SDValue();
4493 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
4494
4495 // We don't want to do this in every single case.
4496 SelectionDAG &DAG = DCI.DAG;
4497 if (!shouldTransformSignedTruncationCheck(XVT, KeptBits))
4498 return SDValue();
4499
4500 // Unfold into: sext_inreg(%x) cond %x
4501 // Where 'cond' will be either 'eq' or 'ne'.
4502 SDValue SExtInReg = DAG.getNode(
4503 Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: XVT, N1: X,
4504 N2: DAG.getValueType(EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: KeptBits)));
4505 return DAG.getSetCC(DL, VT: SCCVT, LHS: SExtInReg, RHS: X, Cond: NewCond);
4506}
4507
4508// (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4509SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
4510 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
4511 DAGCombinerInfo &DCI, const SDLoc &DL) const {
4512 assert(isConstOrConstSplat(N1C) && isConstOrConstSplat(N1C)->isZero() &&
4513 "Should be a comparison with 0.");
4514 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4515 "Valid only for [in]equality comparisons.");
4516
4517 unsigned NewShiftOpcode;
4518 SDValue X, C, Y;
4519
4520 SelectionDAG &DAG = DCI.DAG;
4521
4522 // Look for '(C l>>/<< Y)'.
4523 auto Match = [&NewShiftOpcode, &X, &C, &Y, &DAG, this](SDValue V) {
4524 // The shift should be one-use.
4525 if (!V.hasOneUse())
4526 return false;
4527 unsigned OldShiftOpcode = V.getOpcode();
4528 switch (OldShiftOpcode) {
4529 case ISD::SHL:
4530 NewShiftOpcode = ISD::SRL;
4531 break;
4532 case ISD::SRL:
4533 NewShiftOpcode = ISD::SHL;
4534 break;
4535 default:
4536 return false; // must be a logical shift.
4537 }
4538 // We should be shifting a constant.
4539 // FIXME: best to use isConstantOrConstantVector().
4540 C = V.getOperand(i: 0);
4541 ConstantSDNode *CC =
4542 isConstOrConstSplat(N: C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4543 if (!CC)
4544 return false;
4545 Y = V.getOperand(i: 1);
4546
4547 ConstantSDNode *XC =
4548 isConstOrConstSplat(N: X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
4549 return shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
4550 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
4551 };
4552
4553 // LHS of comparison should be an one-use 'and'.
4554 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
4555 return SDValue();
4556
4557 X = N0.getOperand(i: 0);
4558 SDValue Mask = N0.getOperand(i: 1);
4559
4560 // 'and' is commutative!
4561 if (!Match(Mask)) {
4562 std::swap(a&: X, b&: Mask);
4563 if (!Match(Mask))
4564 return SDValue();
4565 }
4566
4567 EVT VT = X.getValueType();
4568
4569 // Produce:
4570 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
4571 SDValue T0 = DAG.getNode(Opcode: NewShiftOpcode, DL, VT, N1: X, N2: Y);
4572 SDValue T1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T0, N2: C);
4573 SDValue T2 = DAG.getSetCC(DL, VT: SCCVT, LHS: T1, RHS: N1C, Cond);
4574 return T2;
4575}
4576
4577/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
4578/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
4579/// handle the commuted versions of these patterns.
4580SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
4581 ISD::CondCode Cond, const SDLoc &DL,
4582 DAGCombinerInfo &DCI) const {
4583 unsigned BOpcode = N0.getOpcode();
4584 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
4585 "Unexpected binop");
4586 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
4587
4588 // (X + Y) == X --> Y == 0
4589 // (X - Y) == X --> Y == 0
4590 // (X ^ Y) == X --> Y == 0
4591 SelectionDAG &DAG = DCI.DAG;
4592 EVT OpVT = N0.getValueType();
4593 SDValue X = N0.getOperand(i: 0);
4594 SDValue Y = N0.getOperand(i: 1);
4595 if (X == N1)
4596 return DAG.getSetCC(DL, VT, LHS: Y, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4597
4598 if (Y != N1)
4599 return SDValue();
4600
4601 // (X + Y) == Y --> X == 0
4602 // (X ^ Y) == Y --> X == 0
4603 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
4604 return DAG.getSetCC(DL, VT, LHS: X, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond);
4605
4606 // The shift would not be valid if the operands are boolean (i1).
4607 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
4608 return SDValue();
4609
4610 // (X - Y) == Y --> X == Y << 1
4611 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: OpVT, DL);
4612 SDValue YShl1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: N1.getValueType(), N1: Y, N2: One);
4613 if (!DCI.isCalledByLegalizer())
4614 DCI.AddToWorklist(N: YShl1.getNode());
4615 return DAG.getSetCC(DL, VT, LHS: X, RHS: YShl1, Cond);
4616}
4617
4618static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
4619 SDValue N0, const APInt &C1,
4620 ISD::CondCode Cond, const SDLoc &dl,
4621 SelectionDAG &DAG) {
4622 // Look through truncs that don't change the value of a ctpop.
4623 // FIXME: Add vector support? Need to be careful with setcc result type below.
4624 SDValue CTPOP = N0;
4625 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
4626 N0.getScalarValueSizeInBits() > Log2_32(Value: N0.getOperand(i: 0).getScalarValueSizeInBits()))
4627 CTPOP = N0.getOperand(i: 0);
4628
4629 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
4630 return SDValue();
4631
4632 EVT CTVT = CTPOP.getValueType();
4633 SDValue CTOp = CTPOP.getOperand(i: 0);
4634
4635 // Expand a power-of-2-or-zero comparison based on ctpop:
4636 // (ctpop x) u< 2 -> (x & x-1) == 0
4637 // (ctpop x) u> 1 -> (x & x-1) != 0
4638 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
4639 // Keep the CTPOP if it is a cheap vector op.
4640 if (CTVT.isVector() && TLI.isCtpopFast(VT: CTVT))
4641 return SDValue();
4642
4643 unsigned CostLimit = TLI.getCustomCtpopCost(VT: CTVT, Cond);
4644 if (C1.ugt(RHS: CostLimit + (Cond == ISD::SETULT)))
4645 return SDValue();
4646 if (C1 == 0 && (Cond == ISD::SETULT))
4647 return SDValue(); // This is handled elsewhere.
4648
4649 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
4650
4651 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4652 SDValue Result = CTOp;
4653 for (unsigned i = 0; i < Passes; i++) {
4654 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: Result, N2: NegOne);
4655 Result = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: Result, N2: Add);
4656 }
4657 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
4658 return DAG.getSetCC(DL: dl, VT, LHS: Result, RHS: DAG.getConstant(Val: 0, DL: dl, VT: CTVT), Cond: CC);
4659 }
4660
4661 // Expand a power-of-2 comparison based on ctpop
4662 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
4663 // Keep the CTPOP if it is cheap.
4664 if (TLI.isCtpopFast(VT: CTVT))
4665 return SDValue();
4666
4667 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: CTVT);
4668 SDValue NegOne = DAG.getAllOnesConstant(DL: dl, VT: CTVT);
4669 assert(CTVT.isInteger());
4670 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: CTVT, N1: CTOp, N2: NegOne);
4671
4672 // Its not uncommon for known-never-zero X to exist in (ctpop X) eq/ne 1, so
4673 // check before emitting a potentially unnecessary op.
4674 if (DAG.isKnownNeverZero(Op: CTOp)) {
4675 // (ctpop x) == 1 --> (x & x-1) == 0
4676 // (ctpop x) != 1 --> (x & x-1) != 0
4677 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4678 SDValue RHS = DAG.getSetCC(DL: dl, VT, LHS: And, RHS: Zero, Cond);
4679 return RHS;
4680 }
4681
4682 // (ctpop x) == 1 --> (x ^ x-1) > x-1
4683 // (ctpop x) != 1 --> (x ^ x-1) <= x-1
4684 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: CTVT, N1: CTOp, N2: Add);
4685 ISD::CondCode CmpCond = Cond == ISD::SETEQ ? ISD::SETUGT : ISD::SETULE;
4686 return DAG.getSetCC(DL: dl, VT, LHS: Xor, RHS: Add, Cond: CmpCond);
4687 }
4688
4689 return SDValue();
4690}
4691
4692static SDValue foldSetCCWithRotate(EVT VT, SDValue N0, SDValue N1,
4693 ISD::CondCode Cond, const SDLoc &dl,
4694 SelectionDAG &DAG) {
4695 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4696 return SDValue();
4697
4698 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4699 if (!C1 || !(C1->isZero() || C1->isAllOnes()))
4700 return SDValue();
4701
4702 auto getRotateSource = [](SDValue X) {
4703 if (X.getOpcode() == ISD::ROTL || X.getOpcode() == ISD::ROTR)
4704 return X.getOperand(i: 0);
4705 return SDValue();
4706 };
4707
4708 // Peek through a rotated value compared against 0 or -1:
4709 // (rot X, Y) == 0/-1 --> X == 0/-1
4710 // (rot X, Y) != 0/-1 --> X != 0/-1
4711 if (SDValue R = getRotateSource(N0))
4712 return DAG.getSetCC(DL: dl, VT, LHS: R, RHS: N1, Cond);
4713
4714 // Peek through an 'or' of a rotated value compared against 0:
4715 // or (rot X, Y), Z ==/!= 0 --> (or X, Z) ==/!= 0
4716 // or Z, (rot X, Y) ==/!= 0 --> (or X, Z) ==/!= 0
4717 //
4718 // TODO: Add the 'and' with -1 sibling.
4719 // TODO: Recurse through a series of 'or' ops to find the rotate.
4720 EVT OpVT = N0.getValueType();
4721 if (N0.hasOneUse() && N0.getOpcode() == ISD::OR && C1->isZero()) {
4722 if (SDValue R = getRotateSource(N0.getOperand(i: 0))) {
4723 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 1));
4724 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4725 }
4726 if (SDValue R = getRotateSource(N0.getOperand(i: 1))) {
4727 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: R, N2: N0.getOperand(i: 0));
4728 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4729 }
4730 }
4731
4732 return SDValue();
4733}
4734
4735static SDValue foldSetCCWithFunnelShift(EVT VT, SDValue N0, SDValue N1,
4736 ISD::CondCode Cond, const SDLoc &dl,
4737 SelectionDAG &DAG) {
4738 // If we are testing for all-bits-clear, we might be able to do that with
4739 // less shifting since bit-order does not matter.
4740 if (Cond != ISD::SETEQ && Cond != ISD::SETNE)
4741 return SDValue();
4742
4743 auto *C1 = isConstOrConstSplat(N: N1, /* AllowUndefs */ true);
4744 if (!C1 || !C1->isZero())
4745 return SDValue();
4746
4747 if (!N0.hasOneUse() ||
4748 (N0.getOpcode() != ISD::FSHL && N0.getOpcode() != ISD::FSHR))
4749 return SDValue();
4750
4751 unsigned BitWidth = N0.getScalarValueSizeInBits();
4752 auto *ShAmtC = isConstOrConstSplat(N: N0.getOperand(i: 2));
4753 if (!ShAmtC)
4754 return SDValue();
4755
4756 uint64_t ShAmt = ShAmtC->getAPIntValue().urem(RHS: BitWidth);
4757 if (ShAmt == 0)
4758 return SDValue();
4759
4760 // Canonicalize fshr as fshl to reduce pattern-matching.
4761 if (N0.getOpcode() == ISD::FSHR)
4762 ShAmt = BitWidth - ShAmt;
4763
4764 // Match an 'or' with a specific operand 'Other' in either commuted variant.
4765 SDValue X, Y;
4766 auto matchOr = [&X, &Y](SDValue Or, SDValue Other) {
4767 if (Or.getOpcode() != ISD::OR || !Or.hasOneUse())
4768 return false;
4769 if (Or.getOperand(i: 0) == Other) {
4770 X = Or.getOperand(i: 0);
4771 Y = Or.getOperand(i: 1);
4772 return true;
4773 }
4774 if (Or.getOperand(i: 1) == Other) {
4775 X = Or.getOperand(i: 1);
4776 Y = Or.getOperand(i: 0);
4777 return true;
4778 }
4779 return false;
4780 };
4781
4782 EVT OpVT = N0.getValueType();
4783 EVT ShAmtVT = N0.getOperand(i: 2).getValueType();
4784 SDValue F0 = N0.getOperand(i: 0);
4785 SDValue F1 = N0.getOperand(i: 1);
4786 if (matchOr(F0, F1)) {
4787 // fshl (or X, Y), X, C ==/!= 0 --> or (shl Y, C), X ==/!= 0
4788 SDValue NewShAmt = DAG.getConstant(Val: ShAmt, DL: dl, VT: ShAmtVT);
4789 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4790 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4791 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4792 }
4793 if (matchOr(F1, F0)) {
4794 // fshl X, (or X, Y), C ==/!= 0 --> or (srl Y, BW-C), X ==/!= 0
4795 SDValue NewShAmt = DAG.getConstant(Val: BitWidth - ShAmt, DL: dl, VT: ShAmtVT);
4796 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: Y, N2: NewShAmt);
4797 SDValue NewOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: Shift, N2: X);
4798 return DAG.getSetCC(DL: dl, VT, LHS: NewOr, RHS: N1, Cond);
4799 }
4800
4801 return SDValue();
4802}
4803
4804/// Try to simplify a setcc built with the specified operands and cc. If it is
4805/// unable to simplify it, return a null SDValue.
4806SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
4807 ISD::CondCode Cond, bool foldBooleans,
4808 DAGCombinerInfo &DCI,
4809 const SDLoc &dl) const {
4810 SelectionDAG &DAG = DCI.DAG;
4811 const DataLayout &Layout = DAG.getDataLayout();
4812 EVT OpVT = N0.getValueType();
4813 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4814
4815 // Constant fold or commute setcc.
4816 if (SDValue Fold = DAG.FoldSetCC(VT, N1: N0, N2: N1, Cond, dl))
4817 return Fold;
4818
4819 bool N0ConstOrSplat =
4820 isConstOrConstSplat(N: N0, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
4821 bool N1ConstOrSplat =
4822 isConstOrConstSplat(N: N1, /*AllowUndefs*/ false, /*AllowTruncate*/ AllowTruncation: true);
4823
4824 // Canonicalize toward having the constant on the RHS.
4825 // TODO: Handle non-splat vector constants. All undef causes trouble.
4826 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
4827 // infinite loop here when we encounter one.
4828 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Operation: Cond);
4829 if (N0ConstOrSplat && !N1ConstOrSplat &&
4830 (DCI.isBeforeLegalizeOps() ||
4831 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())))
4832 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
4833
4834 // If we have a subtract with the same 2 non-constant operands as this setcc
4835 // -- but in reverse order -- then try to commute the operands of this setcc
4836 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
4837 // instruction on some targets.
4838 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4839 (DCI.isBeforeLegalizeOps() ||
4840 isCondCodeLegal(CC: SwappedCC, VT: N0.getSimpleValueType())) &&
4841 DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N1, N0}) &&
4842 !DAG.doesNodeExist(Opcode: ISD::SUB, VTList: DAG.getVTList(VT: OpVT), Ops: {N0, N1}))
4843 return DAG.getSetCC(DL: dl, VT, LHS: N1, RHS: N0, Cond: SwappedCC);
4844
4845 if (SDValue V = foldSetCCWithRotate(VT, N0, N1, Cond, dl, DAG))
4846 return V;
4847
4848 if (SDValue V = foldSetCCWithFunnelShift(VT, N0, N1, Cond, dl, DAG))
4849 return V;
4850
4851 if (auto *N1C = isConstOrConstSplat(N: N1)) {
4852 const APInt &C1 = N1C->getAPIntValue();
4853
4854 // Optimize some CTPOP cases.
4855 if (SDValue V = simplifySetCCWithCTPOP(TLI: *this, VT, N0, C1, Cond, dl, DAG))
4856 return V;
4857
4858 // For equality to 0 of a no-wrap multiply, decompose and test each op:
4859 // X * Y == 0 --> (X == 0) || (Y == 0)
4860 // X * Y != 0 --> (X != 0) && (Y != 0)
4861 // TODO: This bails out if minsize is set, but if the target doesn't have a
4862 // single instruction multiply for this type, it would likely be
4863 // smaller to decompose.
4864 if (C1.isZero() && (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4865 N0.getOpcode() == ISD::MUL && N0.hasOneUse() &&
4866 (N0->getFlags().hasNoUnsignedWrap() ||
4867 N0->getFlags().hasNoSignedWrap()) &&
4868 !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
4869 SDValue IsXZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
4870 SDValue IsYZero = DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1, Cond);
4871 unsigned LogicOp = Cond == ISD::SETEQ ? ISD::OR : ISD::AND;
4872 return DAG.getNode(Opcode: LogicOp, DL: dl, VT, N1: IsXZero, N2: IsYZero);
4873 }
4874
4875 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
4876 // equality comparison, then we're just comparing whether X itself is
4877 // zero.
4878 if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
4879 N0.getOperand(i: 0).getOpcode() == ISD::CTLZ &&
4880 llvm::has_single_bit<uint32_t>(Value: N0.getScalarValueSizeInBits())) {
4881 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
4882 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4883 ShAmt->getAPIntValue() == Log2_32(Value: N0.getScalarValueSizeInBits())) {
4884 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
4885 // (srl (ctlz x), 5) == 0 -> X != 0
4886 // (srl (ctlz x), 5) != 1 -> X != 0
4887 Cond = ISD::SETNE;
4888 } else {
4889 // (srl (ctlz x), 5) != 0 -> X == 0
4890 // (srl (ctlz x), 5) == 1 -> X == 0
4891 Cond = ISD::SETEQ;
4892 }
4893 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: N0.getValueType());
4894 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0).getOperand(i: 0), RHS: Zero,
4895 Cond);
4896 }
4897 }
4898 }
4899 }
4900
4901 // setcc X, 0, setlt --> X (when X is all sign bits)
4902 // setcc X, 0, setne --> X (when X is all sign bits)
4903 //
4904 // When we know that X has 0 or -1 in each element (or scalar), this
4905 // comparison will produce X. This is only true when boolean contents are
4906 // represented via 0s and -1s.
4907 if (VT == OpVT &&
4908 // Check that the result of setcc is 0 and -1.
4909 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent &&
4910 // Match only for checks X < 0 and X != 0
4911 (Cond == ISD::SETLT || Cond == ISD::SETNE) && isNullOrNullSplat(V: N1) &&
4912 // The identity holds iff we know all sign bits for all lanes.
4913 DAG.ComputeNumSignBits(Op: N0) == N0.getScalarValueSizeInBits())
4914 return N0;
4915
4916 // FIXME: Support vectors.
4917 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
4918 const APInt &C1 = N1C->getAPIntValue();
4919
4920 // (zext x) == C --> x == (trunc C)
4921 // (sext x) == C --> x == (trunc C)
4922 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4923 DCI.isBeforeLegalize() && N0->hasOneUse()) {
4924 unsigned MinBits = N0.getValueSizeInBits();
4925 SDValue PreExt;
4926 bool Signed = false;
4927 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
4928 // ZExt
4929 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
4930 PreExt = N0->getOperand(Num: 0);
4931 } else if (N0->getOpcode() == ISD::AND) {
4932 // DAGCombine turns costly ZExts into ANDs
4933 if (auto *C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1)))
4934 if ((C->getAPIntValue()+1).isPowerOf2()) {
4935 MinBits = C->getAPIntValue().countr_one();
4936 PreExt = N0->getOperand(Num: 0);
4937 }
4938 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
4939 // SExt
4940 MinBits = N0->getOperand(Num: 0).getValueSizeInBits();
4941 PreExt = N0->getOperand(Num: 0);
4942 Signed = true;
4943 } else if (auto *LN0 = dyn_cast<LoadSDNode>(Val&: N0)) {
4944 // ZEXTLOAD / SEXTLOAD
4945 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
4946 MinBits = LN0->getMemoryVT().getSizeInBits();
4947 PreExt = N0;
4948 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
4949 Signed = true;
4950 MinBits = LN0->getMemoryVT().getSizeInBits();
4951 PreExt = N0;
4952 }
4953 }
4954
4955 // Figure out how many bits we need to preserve this constant.
4956 unsigned ReqdBits = Signed ? C1.getSignificantBits() : C1.getActiveBits();
4957
4958 // Make sure we're not losing bits from the constant.
4959 if (MinBits > 0 &&
4960 MinBits < C1.getBitWidth() &&
4961 MinBits >= ReqdBits) {
4962 EVT MinVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MinBits);
4963 if (isTypeDesirableForOp(ISD::SETCC, VT: MinVT)) {
4964 // Will get folded away.
4965 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MinVT, Operand: PreExt);
4966 if (MinBits == 1 && C1 == 1)
4967 // Invert the condition.
4968 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i1),
4969 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
4970 SDValue C = DAG.getConstant(Val: C1.trunc(width: MinBits), DL: dl, VT: MinVT);
4971 return DAG.getSetCC(DL: dl, VT, LHS: Trunc, RHS: C, Cond);
4972 }
4973
4974 // If truncating the setcc operands is not desirable, we can still
4975 // simplify the expression in some cases:
4976 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
4977 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
4978 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
4979 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
4980 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
4981 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
4982 SDValue TopSetCC = N0->getOperand(Num: 0);
4983 unsigned N0Opc = N0->getOpcode();
4984 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
4985 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
4986 TopSetCC.getOpcode() == ISD::SETCC &&
4987 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
4988 (isConstFalseVal(N: N1) ||
4989 isExtendedTrueVal(N: N1C, VT: N0->getValueType(ResNo: 0), SExt))) {
4990
4991 bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
4992 (!N1C->isZero() && Cond == ISD::SETNE);
4993
4994 if (!Inverse)
4995 return TopSetCC;
4996
4997 ISD::CondCode InvCond = ISD::getSetCCInverse(
4998 Operation: cast<CondCodeSDNode>(Val: TopSetCC.getOperand(i: 2))->get(),
4999 Type: TopSetCC.getOperand(i: 0).getValueType());
5000 return DAG.getSetCC(DL: dl, VT, LHS: TopSetCC.getOperand(i: 0),
5001 RHS: TopSetCC.getOperand(i: 1),
5002 Cond: InvCond);
5003 }
5004 }
5005 }
5006
5007 // If the LHS is '(and load, const)', the RHS is 0, the test is for
5008 // equality or unsigned, and all 1 bits of the const are in the same
5009 // partial word, see if we can shorten the load.
5010 if (DCI.isBeforeLegalize() &&
5011 !ISD::isSignedIntSetCC(Code: Cond) &&
5012 N0.getOpcode() == ISD::AND && C1 == 0 &&
5013 N0.getNode()->hasOneUse() &&
5014 isa<LoadSDNode>(Val: N0.getOperand(i: 0)) &&
5015 N0.getOperand(i: 0).getNode()->hasOneUse() &&
5016 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5017 auto *Lod = cast<LoadSDNode>(Val: N0.getOperand(i: 0));
5018 APInt bestMask;
5019 unsigned bestWidth = 0, bestOffset = 0;
5020 if (Lod->isSimple() && Lod->isUnindexed() &&
5021 (Lod->getMemoryVT().isByteSized() ||
5022 isPaddedAtMostSignificantBitsWhenStored(VT: Lod->getMemoryVT()))) {
5023 unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
5024 unsigned origWidth = N0.getValueSizeInBits();
5025 unsigned maskWidth = origWidth;
5026 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
5027 // 8 bits, but have to be careful...
5028 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
5029 origWidth = Lod->getMemoryVT().getSizeInBits();
5030 const APInt &Mask = N0.getConstantOperandAPInt(i: 1);
5031 // Only consider power-of-2 widths (and at least one byte) as candiates
5032 // for the narrowed load.
5033 for (unsigned width = 8; width < origWidth; width *= 2) {
5034 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: width);
5035 APInt newMask = APInt::getLowBitsSet(numBits: maskWidth, loBitsSet: width);
5036 // Avoid accessing any padding here for now (we could use memWidth
5037 // instead of origWidth here otherwise).
5038 unsigned maxOffset = origWidth - width;
5039 for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
5040 if (Mask.isSubsetOf(RHS: newMask)) {
5041 unsigned ptrOffset =
5042 Layout.isLittleEndian() ? offset : memWidth - width - offset;
5043 unsigned IsFast = 0;
5044 assert((ptrOffset % 8) == 0 && "Non-Bytealigned pointer offset");
5045 Align NewAlign = commonAlignment(A: Lod->getAlign(), Offset: ptrOffset / 8);
5046 if (shouldReduceLoadWidth(Load: Lod, ExtTy: ISD::NON_EXTLOAD, NewVT: newVT,
5047 ByteOffset: ptrOffset / 8) &&
5048 allowsMemoryAccess(
5049 Context&: *DAG.getContext(), DL: Layout, VT: newVT, AddrSpace: Lod->getAddressSpace(),
5050 Alignment: NewAlign, Flags: Lod->getMemOperand()->getFlags(), Fast: &IsFast) &&
5051 IsFast) {
5052 bestOffset = ptrOffset / 8;
5053 bestMask = Mask.lshr(shiftAmt: offset);
5054 bestWidth = width;
5055 break;
5056 }
5057 }
5058 newMask <<= 8;
5059 }
5060 if (bestWidth)
5061 break;
5062 }
5063 }
5064 if (bestWidth) {
5065 EVT newVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: bestWidth);
5066 SDValue Ptr = Lod->getBasePtr();
5067 if (bestOffset != 0)
5068 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: bestOffset));
5069 SDValue NewLoad =
5070 DAG.getLoad(VT: newVT, dl, Chain: Lod->getChain(), Ptr,
5071 PtrInfo: Lod->getPointerInfo().getWithOffset(O: bestOffset),
5072 Alignment: Lod->getBaseAlign());
5073 SDValue And =
5074 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: newVT, N1: NewLoad,
5075 N2: DAG.getConstant(Val: bestMask.trunc(width: bestWidth), DL: dl, VT: newVT));
5076 return DAG.getSetCC(DL: dl, VT, LHS: And, RHS: DAG.getConstant(Val: 0LL, DL: dl, VT: newVT), Cond);
5077 }
5078 }
5079
5080 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
5081 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
5082 unsigned InSize = N0.getOperand(i: 0).getValueSizeInBits();
5083
5084 // If the comparison constant has bits in the upper part, the
5085 // zero-extended value could never match.
5086 if (C1.intersects(RHS: APInt::getHighBitsSet(numBits: C1.getBitWidth(),
5087 hiBitsSet: C1.getBitWidth() - InSize))) {
5088 switch (Cond) {
5089 case ISD::SETUGT:
5090 case ISD::SETUGE:
5091 case ISD::SETEQ:
5092 return DAG.getConstant(Val: 0, DL: dl, VT);
5093 case ISD::SETULT:
5094 case ISD::SETULE:
5095 case ISD::SETNE:
5096 return DAG.getConstant(Val: 1, DL: dl, VT);
5097 case ISD::SETGT:
5098 case ISD::SETGE:
5099 // True if the sign bit of C1 is set.
5100 return DAG.getConstant(Val: C1.isNegative(), DL: dl, VT);
5101 case ISD::SETLT:
5102 case ISD::SETLE:
5103 // True if the sign bit of C1 isn't set.
5104 return DAG.getConstant(Val: C1.isNonNegative(), DL: dl, VT);
5105 default:
5106 break;
5107 }
5108 }
5109
5110 // Otherwise, we can perform the comparison with the low bits.
5111 switch (Cond) {
5112 case ISD::SETEQ:
5113 case ISD::SETNE:
5114 case ISD::SETUGT:
5115 case ISD::SETUGE:
5116 case ISD::SETULT:
5117 case ISD::SETULE: {
5118 EVT newVT = N0.getOperand(i: 0).getValueType();
5119 // FIXME: Should use isNarrowingProfitable.
5120 if (DCI.isBeforeLegalizeOps() ||
5121 (isOperationLegal(Op: ISD::SETCC, VT: newVT) &&
5122 isCondCodeLegal(CC: Cond, VT: newVT.getSimpleVT()) &&
5123 isTypeDesirableForOp(ISD::SETCC, VT: newVT))) {
5124 EVT NewSetCCVT = getSetCCResultType(DL: Layout, Context&: *DAG.getContext(), VT: newVT);
5125 SDValue NewConst = DAG.getConstant(Val: C1.trunc(width: InSize), DL: dl, VT: newVT);
5126
5127 SDValue NewSetCC = DAG.getSetCC(DL: dl, VT: NewSetCCVT, LHS: N0.getOperand(i: 0),
5128 RHS: NewConst, Cond);
5129 return DAG.getBoolExtOrTrunc(Op: NewSetCC, SL: dl, VT, OpVT: N0.getValueType());
5130 }
5131 break;
5132 }
5133 default:
5134 break; // todo, be more careful with signed comparisons
5135 }
5136 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
5137 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5138 !isSExtCheaperThanZExt(FromTy: cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT(),
5139 ToTy: OpVT)) {
5140 EVT ExtSrcTy = cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT();
5141 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
5142 EVT ExtDstTy = N0.getValueType();
5143 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
5144
5145 // If the constant doesn't fit into the number of bits for the source of
5146 // the sign extension, it is impossible for both sides to be equal.
5147 if (C1.getSignificantBits() > ExtSrcTyBits)
5148 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
5149
5150 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
5151 ExtDstTy != ExtSrcTy && "Unexpected types!");
5152 APInt Imm = APInt::getLowBitsSet(numBits: ExtDstTyBits, loBitsSet: ExtSrcTyBits);
5153 SDValue ZextOp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ExtDstTy, N1: N0.getOperand(i: 0),
5154 N2: DAG.getConstant(Val: Imm, DL: dl, VT: ExtDstTy));
5155 if (!DCI.isCalledByLegalizer())
5156 DCI.AddToWorklist(N: ZextOp.getNode());
5157 // Otherwise, make this a use of a zext.
5158 return DAG.getSetCC(DL: dl, VT, LHS: ZextOp,
5159 RHS: DAG.getConstant(Val: C1 & Imm, DL: dl, VT: ExtDstTy), Cond);
5160 } else if ((N1C->isZero() || N1C->isOne()) &&
5161 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5162 // SETCC (X), [0|1], [EQ|NE] -> X if X is known 0/1. i1 types are
5163 // excluded as they are handled below whilst checking for foldBooleans.
5164 if ((N0.getOpcode() == ISD::SETCC || VT.getScalarType() != MVT::i1) &&
5165 isTypeLegal(VT) && VT.bitsLE(VT: N0.getValueType()) &&
5166 (N0.getValueType() == MVT::i1 ||
5167 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5168 DAG.MaskedValueIsZero(
5169 Op: N0, Mask: APInt::getBitsSetFrom(numBits: N0.getValueSizeInBits(), loBit: 1))) {
5170 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
5171 if (TrueWhenTrue)
5172 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: N0);
5173 // Invert the condition.
5174 if (N0.getOpcode() == ISD::SETCC) {
5175 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
5176 CC = ISD::getSetCCInverse(Operation: CC, Type: N0.getOperand(i: 0).getValueType());
5177 if (DCI.isBeforeLegalizeOps() ||
5178 isCondCodeLegal(CC, VT: N0.getOperand(i: 0).getSimpleValueType()))
5179 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond: CC);
5180 }
5181 }
5182
5183 if ((N0.getOpcode() == ISD::XOR ||
5184 (N0.getOpcode() == ISD::AND &&
5185 N0.getOperand(i: 0).getOpcode() == ISD::XOR &&
5186 N0.getOperand(i: 1) == N0.getOperand(i: 0).getOperand(i: 1))) &&
5187 isOneConstant(V: N0.getOperand(i: 1))) {
5188 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
5189 // can only do this if the top bits are known zero.
5190 unsigned BitWidth = N0.getValueSizeInBits();
5191 if (DAG.MaskedValueIsZero(Op: N0,
5192 Mask: APInt::getHighBitsSet(numBits: BitWidth,
5193 hiBitsSet: BitWidth-1))) {
5194 // Okay, get the un-inverted input value.
5195 SDValue Val;
5196 if (N0.getOpcode() == ISD::XOR) {
5197 Val = N0.getOperand(i: 0);
5198 } else {
5199 assert(N0.getOpcode() == ISD::AND &&
5200 N0.getOperand(0).getOpcode() == ISD::XOR);
5201 // ((X^1)&1)^1 -> X & 1
5202 Val = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N0.getValueType(),
5203 N1: N0.getOperand(i: 0).getOperand(i: 0),
5204 N2: N0.getOperand(i: 1));
5205 }
5206
5207 return DAG.getSetCC(DL: dl, VT, LHS: Val, RHS: N1,
5208 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5209 }
5210 } else if (N1C->isOne()) {
5211 SDValue Op0 = N0;
5212 if (Op0.getOpcode() == ISD::TRUNCATE)
5213 Op0 = Op0.getOperand(i: 0);
5214
5215 if ((Op0.getOpcode() == ISD::XOR) &&
5216 Op0.getOperand(i: 0).getOpcode() == ISD::SETCC &&
5217 Op0.getOperand(i: 1).getOpcode() == ISD::SETCC) {
5218 SDValue XorLHS = Op0.getOperand(i: 0);
5219 SDValue XorRHS = Op0.getOperand(i: 1);
5220 // Ensure that the input setccs return an i1 type or 0/1 value.
5221 if (Op0.getValueType() == MVT::i1 ||
5222 (getBooleanContents(Type: XorLHS.getOperand(i: 0).getValueType()) ==
5223 ZeroOrOneBooleanContent &&
5224 getBooleanContents(Type: XorRHS.getOperand(i: 0).getValueType()) ==
5225 ZeroOrOneBooleanContent)) {
5226 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
5227 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
5228 return DAG.getSetCC(DL: dl, VT, LHS: XorLHS, RHS: XorRHS, Cond);
5229 }
5230 }
5231 if (Op0.getOpcode() == ISD::AND && isOneConstant(V: Op0.getOperand(i: 1))) {
5232 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
5233 if (Op0.getValueType().bitsGT(VT))
5234 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5235 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5236 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5237 else if (Op0.getValueType().bitsLT(VT))
5238 Op0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
5239 N1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Op0.getOperand(i: 0)),
5240 N2: DAG.getConstant(Val: 1, DL: dl, VT));
5241
5242 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5243 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5244 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5245 }
5246 if (Op0.getOpcode() == ISD::AssertZext &&
5247 cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT() == MVT::i1)
5248 return DAG.getSetCC(DL: dl, VT, LHS: Op0,
5249 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Op0.getValueType()),
5250 Cond: Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
5251 }
5252 }
5253
5254 // Given:
5255 // icmp eq/ne (urem %x, %y), 0
5256 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
5257 // icmp eq/ne %x, 0
5258 if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
5259 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5260 KnownBits XKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 0));
5261 KnownBits YKnown = DAG.computeKnownBits(Op: N0.getOperand(i: 1));
5262 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
5263 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1, Cond);
5264 }
5265
5266 // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
5267 // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
5268 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5269 N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
5270 N0.getConstantOperandAPInt(i: 1) == OpVT.getScalarSizeInBits() - 1 &&
5271 N1C->isAllOnes()) {
5272 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
5273 RHS: DAG.getConstant(Val: 0, DL: dl, VT: OpVT),
5274 Cond: Cond == ISD::SETEQ ? ISD::SETLT : ISD::SETGE);
5275 }
5276
5277 // fold (setcc (trunc x) c) -> (setcc x c)
5278 if (N0.getOpcode() == ISD::TRUNCATE &&
5279 ((N0->getFlags().hasNoUnsignedWrap() && !ISD::isSignedIntSetCC(Code: Cond)) ||
5280 (N0->getFlags().hasNoSignedWrap() &&
5281 !ISD::isUnsignedIntSetCC(Code: Cond))) &&
5282 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5283 EVT NewVT = N0.getOperand(i: 0).getValueType();
5284 SDValue NewConst = DAG.getConstant(
5285 Val: (N0->getFlags().hasNoSignedWrap() && !ISD::isUnsignedIntSetCC(Code: Cond))
5286 ? C1.sext(width: NewVT.getSizeInBits())
5287 : C1.zext(width: NewVT.getSizeInBits()),
5288 DL: dl, VT: NewVT);
5289 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NewConst, Cond);
5290 }
5291
5292 if (SDValue V =
5293 optimizeSetCCOfSignedTruncationCheck(SCCVT: VT, N0, N1, Cond, DCI, DL: dl))
5294 return V;
5295 }
5296
5297 // These simplifications apply to splat vectors as well.
5298 // TODO: Handle more splat vector cases.
5299 if (auto *N1C = isConstOrConstSplat(N: N1)) {
5300 const APInt &C1 = N1C->getAPIntValue();
5301
5302 APInt MinVal, MaxVal;
5303 unsigned OperandBitSize = N1C->getValueType(ResNo: 0).getScalarSizeInBits();
5304 if (ISD::isSignedIntSetCC(Code: Cond)) {
5305 MinVal = APInt::getSignedMinValue(numBits: OperandBitSize);
5306 MaxVal = APInt::getSignedMaxValue(numBits: OperandBitSize);
5307 } else {
5308 MinVal = APInt::getMinValue(numBits: OperandBitSize);
5309 MaxVal = APInt::getMaxValue(numBits: OperandBitSize);
5310 }
5311
5312 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
5313 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
5314 // X >= MIN --> true
5315 if (C1 == MinVal)
5316 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5317
5318 if (!VT.isVector()) { // TODO: Support this for vectors.
5319 // X >= C0 --> X > (C0 - 1)
5320 APInt C = C1 - 1;
5321 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
5322 if ((DCI.isBeforeLegalizeOps() ||
5323 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5324 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5325 isLegalICmpImmediate(C.getSExtValue())))) {
5326 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5327 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5328 Cond: NewCC);
5329 }
5330 }
5331 }
5332
5333 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
5334 // X <= MAX --> true
5335 if (C1 == MaxVal)
5336 return DAG.getBoolConstant(V: true, DL: dl, VT, OpVT);
5337
5338 // X <= C0 --> X < (C0 + 1)
5339 if (!VT.isVector()) { // TODO: Support this for vectors.
5340 APInt C = C1 + 1;
5341 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
5342 if ((DCI.isBeforeLegalizeOps() ||
5343 isCondCodeLegal(CC: NewCC, VT: OpVT.getSimpleVT())) &&
5344 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
5345 isLegalICmpImmediate(C.getSExtValue())))) {
5346 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5347 RHS: DAG.getConstant(Val: C, DL: dl, VT: N1.getValueType()),
5348 Cond: NewCC);
5349 }
5350 }
5351 }
5352
5353 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
5354 if (C1 == MinVal)
5355 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X < MIN --> false
5356
5357 // TODO: Support this for vectors after legalize ops.
5358 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5359 // Canonicalize setlt X, Max --> setne X, Max
5360 if (C1 == MaxVal)
5361 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5362
5363 // If we have setult X, 1, turn it into seteq X, 0
5364 if (C1 == MinVal+1)
5365 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5366 RHS: DAG.getConstant(Val: MinVal, DL: dl, VT: N0.getValueType()),
5367 Cond: ISD::SETEQ);
5368 }
5369 }
5370
5371 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
5372 if (C1 == MaxVal)
5373 return DAG.getBoolConstant(V: false, DL: dl, VT, OpVT); // X > MAX --> false
5374
5375 // TODO: Support this for vectors after legalize ops.
5376 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5377 // Canonicalize setgt X, Min --> setne X, Min
5378 if (C1 == MinVal)
5379 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: ISD::SETNE);
5380
5381 // If we have setugt X, Max-1, turn it into seteq X, Max
5382 if (C1 == MaxVal-1)
5383 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5384 RHS: DAG.getConstant(Val: MaxVal, DL: dl, VT: N0.getValueType()),
5385 Cond: ISD::SETEQ);
5386 }
5387 }
5388
5389 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
5390 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
5391 if (C1.isZero())
5392 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
5393 SCCVT: VT, N0, N1C: N1, Cond, DCI, DL: dl))
5394 return CC;
5395
5396 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
5397 // For example, when high 32-bits of i64 X are known clear:
5398 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
5399 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
5400 bool CmpZero = N1C->isZero();
5401 bool CmpNegOne = N1C->isAllOnes();
5402 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
5403 // Match or(lo,shl(hi,bw/2)) pattern.
5404 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
5405 unsigned EltBits = V.getScalarValueSizeInBits();
5406 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
5407 return false;
5408 SDValue LHS = V.getOperand(i: 0);
5409 SDValue RHS = V.getOperand(i: 1);
5410 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: EltBits / 2);
5411 // Unshifted element must have zero upperbits.
5412 if (RHS.getOpcode() == ISD::SHL &&
5413 isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)) &&
5414 RHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5415 DAG.MaskedValueIsZero(Op: LHS, Mask: HiBits)) {
5416 Lo = LHS;
5417 Hi = RHS.getOperand(i: 0);
5418 return true;
5419 }
5420 if (LHS.getOpcode() == ISD::SHL &&
5421 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
5422 LHS.getConstantOperandAPInt(i: 1) == (EltBits / 2) &&
5423 DAG.MaskedValueIsZero(Op: RHS, Mask: HiBits)) {
5424 Lo = RHS;
5425 Hi = LHS.getOperand(i: 0);
5426 return true;
5427 }
5428 return false;
5429 };
5430
5431 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
5432 unsigned EltBits = N0.getScalarValueSizeInBits();
5433 unsigned HalfBits = EltBits / 2;
5434 APInt HiBits = APInt::getHighBitsSet(numBits: EltBits, hiBitsSet: HalfBits);
5435 SDValue LoBits = DAG.getConstant(Val: ~HiBits, DL: dl, VT: OpVT);
5436 SDValue HiMask = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: Hi, N2: LoBits);
5437 SDValue NewN0 =
5438 DAG.getNode(Opcode: CmpZero ? ISD::OR : ISD::AND, DL: dl, VT: OpVT, N1: Lo, N2: HiMask);
5439 SDValue NewN1 = CmpZero ? DAG.getConstant(Val: 0, DL: dl, VT: OpVT) : LoBits;
5440 return DAG.getSetCC(DL: dl, VT, LHS: NewN0, RHS: NewN1, Cond);
5441 };
5442
5443 SDValue Lo, Hi;
5444 if (IsConcat(N0, Lo, Hi))
5445 return MergeConcat(Lo, Hi);
5446
5447 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
5448 SDValue Lo0, Lo1, Hi0, Hi1;
5449 if (IsConcat(N0.getOperand(i: 0), Lo0, Hi0) &&
5450 IsConcat(N0.getOperand(i: 1), Lo1, Hi1)) {
5451 return MergeConcat(DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Lo0, N2: Lo1),
5452 DAG.getNode(Opcode: N0.getOpcode(), DL: dl, VT: OpVT, N1: Hi0, N2: Hi1));
5453 }
5454 }
5455 }
5456 }
5457
5458 // If we have "setcc X, C0", check to see if we can shrink the immediate
5459 // by changing cc.
5460 // TODO: Support this for vectors after legalize ops.
5461 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
5462 // SETUGT X, SINTMAX -> SETLT X, 0
5463 // SETUGE X, SINTMIN -> SETLT X, 0
5464 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
5465 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
5466 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5467 RHS: DAG.getConstant(Val: 0, DL: dl, VT: N1.getValueType()),
5468 Cond: ISD::SETLT);
5469
5470 // SETULT X, SINTMIN -> SETGT X, -1
5471 // SETULE X, SINTMAX -> SETGT X, -1
5472 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
5473 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
5474 return DAG.getSetCC(DL: dl, VT, LHS: N0,
5475 RHS: DAG.getAllOnesConstant(DL: dl, VT: N1.getValueType()),
5476 Cond: ISD::SETGT);
5477 }
5478 }
5479
5480 // Back to non-vector simplifications.
5481 // TODO: Can we do these for vector splats?
5482 if (auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode())) {
5483 const APInt &C1 = N1C->getAPIntValue();
5484 EVT ShValTy = N0.getValueType();
5485
5486 // Fold bit comparisons when we can. This will result in an
5487 // incorrect value when boolean false is negative one, unless
5488 // the bitsize is 1 in which case the false value is the same
5489 // in practice regardless of the representation.
5490 if ((VT.getSizeInBits() == 1 ||
5491 getBooleanContents(Type: N0.getValueType()) == ZeroOrOneBooleanContent) &&
5492 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5493 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(VT: ShValTy))) &&
5494 N0.getOpcode() == ISD::AND) {
5495 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5496 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
5497 // Perform the xform if the AND RHS is a single bit.
5498 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
5499 if (AndRHS->getAPIntValue().isPowerOf2() &&
5500 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5501 return DAG.getNode(
5502 Opcode: ISD::TRUNCATE, DL: dl, VT,
5503 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5504 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5505 }
5506 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
5507 // (X & 8) == 8 --> (X & 8) >> 3
5508 // Perform the xform if C1 is a single bit.
5509 unsigned ShCt = C1.logBase2();
5510 if (C1.isPowerOf2() && !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShCt)) {
5511 return DAG.getNode(
5512 Opcode: ISD::TRUNCATE, DL: dl, VT,
5513 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5514 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: ShValTy, DL: dl)));
5515 }
5516 }
5517 }
5518 }
5519
5520 if (C1.getSignificantBits() <= 64 &&
5521 !isLegalICmpImmediate(C1.getSExtValue())) {
5522 // (X & -256) == 256 -> (X >> 8) == 1
5523 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5524 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
5525 if (auto *AndRHS = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5526 const APInt &AndRHSC = AndRHS->getAPIntValue();
5527 if (AndRHSC.isNegatedPowerOf2() && C1.isSubsetOf(RHS: AndRHSC)) {
5528 unsigned ShiftBits = AndRHSC.countr_zero();
5529 if (!shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5530 // If using an unsigned shift doesn't yield a legal compare
5531 // immediate, try using sra instead.
5532 APInt NewC = C1.lshr(shiftAmt: ShiftBits);
5533 if (NewC.getSignificantBits() <= 64 &&
5534 !isLegalICmpImmediate(NewC.getSExtValue())) {
5535 APInt SignedC = C1.ashr(ShiftAmt: ShiftBits);
5536 if (SignedC.getSignificantBits() <= 64 &&
5537 isLegalICmpImmediate(SignedC.getSExtValue())) {
5538 SDValue Shift = DAG.getNode(
5539 Opcode: ISD::SRA, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5540 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5541 SDValue CmpRHS = DAG.getConstant(Val: SignedC, DL: dl, VT: ShValTy);
5542 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5543 }
5544 }
5545 SDValue Shift = DAG.getNode(
5546 Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0.getOperand(i: 0),
5547 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5548 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5549 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond);
5550 }
5551 }
5552 }
5553 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
5554 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
5555 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
5556 // X < 0x100000000 -> (X >> 32) < 1
5557 // X >= 0x100000000 -> (X >> 32) >= 1
5558 // X <= 0x0ffffffff -> (X >> 32) < 1
5559 // X > 0x0ffffffff -> (X >> 32) >= 1
5560 unsigned ShiftBits;
5561 APInt NewC = C1;
5562 ISD::CondCode NewCond = Cond;
5563 if (AdjOne) {
5564 ShiftBits = C1.countr_one();
5565 NewC = NewC + 1;
5566 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
5567 } else {
5568 ShiftBits = C1.countr_zero();
5569 }
5570 NewC.lshrInPlace(ShiftAmt: ShiftBits);
5571 if (ShiftBits && NewC.getSignificantBits() <= 64 &&
5572 isLegalICmpImmediate(NewC.getSExtValue()) &&
5573 !shouldAvoidTransformToShift(VT: ShValTy, Amount: ShiftBits)) {
5574 SDValue Shift =
5575 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ShValTy, N1: N0,
5576 N2: DAG.getShiftAmountConstant(Val: ShiftBits, VT: ShValTy, DL: dl));
5577 SDValue CmpRHS = DAG.getConstant(Val: NewC, DL: dl, VT: ShValTy);
5578 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: CmpRHS, Cond: NewCond);
5579 }
5580 }
5581 }
5582 }
5583
5584 if (!isa<ConstantFPSDNode>(Val: N0) && isa<ConstantFPSDNode>(Val: N1)) {
5585 auto *CFP = cast<ConstantFPSDNode>(Val&: N1);
5586 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
5587
5588 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
5589 // constant if knowing that the operand is non-nan is enough. We prefer to
5590 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
5591 // materialize 0.0.
5592 if (Cond == ISD::SETO || Cond == ISD::SETUO)
5593 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N0, Cond);
5594
5595 // setcc (fneg x), C -> setcc swap(pred) x, -C
5596 if (N0.getOpcode() == ISD::FNEG) {
5597 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Operation: Cond);
5598 if (DCI.isBeforeLegalizeOps() ||
5599 isCondCodeLegal(CC: SwapCond, VT: N0.getSimpleValueType())) {
5600 SDValue NegN1 = DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: N0.getValueType(), Operand: N1);
5601 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: NegN1, Cond: SwapCond);
5602 }
5603 }
5604
5605 // setueq/setoeq X, (fabs Inf) -> is_fpclass X, fcInf
5606 if (isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT: N0.getValueType()) &&
5607 !isFPImmLegal(CFP->getValueAPF(), CFP->getValueType(ResNo: 0))) {
5608 bool IsFabs = N0.getOpcode() == ISD::FABS;
5609 SDValue Op = IsFabs ? N0.getOperand(i: 0) : N0;
5610 if ((Cond == ISD::SETOEQ || Cond == ISD::SETUEQ) && CFP->isInfinity()) {
5611 FPClassTest Flag = CFP->isNegative() ? (IsFabs ? fcNone : fcNegInf)
5612 : (IsFabs ? fcInf : fcPosInf);
5613 if (Cond == ISD::SETUEQ)
5614 Flag |= fcNan;
5615 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL: dl, VT, N1: Op,
5616 N2: DAG.getTargetConstant(Val: Flag, DL: dl, VT: MVT::i32));
5617 }
5618 }
5619
5620 // If the condition is not legal, see if we can find an equivalent one
5621 // which is legal.
5622 if (!isCondCodeLegal(CC: Cond, VT: N0.getSimpleValueType())) {
5623 // If the comparison was an awkward floating-point == or != and one of
5624 // the comparison operands is infinity or negative infinity, convert the
5625 // condition to a less-awkward <= or >=.
5626 if (CFP->getValueAPF().isInfinity()) {
5627 bool IsNegInf = CFP->getValueAPF().isNegative();
5628 ISD::CondCode NewCond = ISD::SETCC_INVALID;
5629 switch (Cond) {
5630 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
5631 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
5632 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
5633 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
5634 default: break;
5635 }
5636 if (NewCond != ISD::SETCC_INVALID &&
5637 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType()))
5638 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5639 }
5640 }
5641 }
5642
5643 if (N0 == N1) {
5644 // The sext(setcc()) => setcc() optimization relies on the appropriate
5645 // constant being emitted.
5646 assert(!N0.getValueType().isInteger() &&
5647 "Integer types should be handled by FoldSetCC");
5648
5649 bool EqTrue = ISD::isTrueWhenEqual(Cond);
5650 unsigned UOF = ISD::getUnorderedFlavor(Cond);
5651 if (UOF == 2) // FP operators that are undefined on NaNs.
5652 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5653 if (UOF == unsigned(EqTrue))
5654 return DAG.getBoolConstant(V: EqTrue, DL: dl, VT, OpVT);
5655 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
5656 // if it is not already.
5657 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
5658 if (NewCond != Cond &&
5659 (DCI.isBeforeLegalizeOps() ||
5660 isCondCodeLegal(CC: NewCond, VT: N0.getSimpleValueType())))
5661 return DAG.getSetCC(DL: dl, VT, LHS: N0, RHS: N1, Cond: NewCond);
5662 }
5663
5664 // ~X > ~Y --> Y > X
5665 // ~X < ~Y --> Y < X
5666 // ~X < C --> X > ~C
5667 // ~X > C --> X < ~C
5668 if ((isSignedIntSetCC(Code: Cond) || isUnsignedIntSetCC(Code: Cond)) &&
5669 N0.getValueType().isInteger()) {
5670 if (isBitwiseNot(V: N0)) {
5671 if (isBitwiseNot(V: N1))
5672 return DAG.getSetCC(DL: dl, VT, LHS: N1.getOperand(i: 0), RHS: N0.getOperand(i: 0), Cond);
5673
5674 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N1) &&
5675 !DAG.isConstantIntBuildVectorOrConstantInt(N: N0.getOperand(i: 0))) {
5676 SDValue Not = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5677 return DAG.getSetCC(DL: dl, VT, LHS: Not, RHS: N0.getOperand(i: 0), Cond);
5678 }
5679 }
5680 }
5681
5682 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5683 N0.getValueType().isInteger()) {
5684 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
5685 N0.getOpcode() == ISD::XOR) {
5686 // Simplify (X+Y) == (X+Z) --> Y == Z
5687 if (N0.getOpcode() == N1.getOpcode()) {
5688 if (N0.getOperand(i: 0) == N1.getOperand(i: 0))
5689 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 1), Cond);
5690 if (N0.getOperand(i: 1) == N1.getOperand(i: 1))
5691 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
5692 if (isCommutativeBinOp(Opcode: N0.getOpcode())) {
5693 // If X op Y == Y op X, try other combinations.
5694 if (N0.getOperand(i: 0) == N1.getOperand(i: 1))
5695 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 1), RHS: N1.getOperand(i: 0),
5696 Cond);
5697 if (N0.getOperand(i: 1) == N1.getOperand(i: 0))
5698 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 1),
5699 Cond);
5700 }
5701 }
5702
5703 // If RHS is a legal immediate value for a compare instruction, we need
5704 // to be careful about increasing register pressure needlessly.
5705 bool LegalRHSImm = false;
5706
5707 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: N1)) {
5708 if (auto *LHSR = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
5709 // Turn (X+C1) == C2 --> X == C2-C1
5710 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse())
5711 return DAG.getSetCC(
5712 DL: dl, VT, LHS: N0.getOperand(i: 0),
5713 RHS: DAG.getConstant(Val: RHSC->getAPIntValue() - LHSR->getAPIntValue(),
5714 DL: dl, VT: N0.getValueType()),
5715 Cond);
5716
5717 // Turn (X^C1) == C2 --> X == C1^C2
5718 if (N0.getOpcode() == ISD::XOR && N0.getNode()->hasOneUse())
5719 return DAG.getSetCC(
5720 DL: dl, VT, LHS: N0.getOperand(i: 0),
5721 RHS: DAG.getConstant(Val: LHSR->getAPIntValue() ^ RHSC->getAPIntValue(),
5722 DL: dl, VT: N0.getValueType()),
5723 Cond);
5724 }
5725
5726 // Turn (C1-X) == C2 --> X == C1-C2
5727 if (auto *SUBC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0)))
5728 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse())
5729 return DAG.getSetCC(
5730 DL: dl, VT, LHS: N0.getOperand(i: 1),
5731 RHS: DAG.getConstant(Val: SUBC->getAPIntValue() - RHSC->getAPIntValue(),
5732 DL: dl, VT: N0.getValueType()),
5733 Cond);
5734
5735 // Could RHSC fold directly into a compare?
5736 if (RHSC->getValueType(ResNo: 0).getSizeInBits() <= 64)
5737 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
5738 }
5739
5740 // (X+Y) == X --> Y == 0 and similar folds.
5741 // Don't do this if X is an immediate that can fold into a cmp
5742 // instruction and X+Y has other uses. It could be an induction variable
5743 // chain, and the transform would increase register pressure.
5744 if (!LegalRHSImm || N0.hasOneUse())
5745 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, DL: dl, DCI))
5746 return V;
5747 }
5748
5749 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
5750 N1.getOpcode() == ISD::XOR)
5751 if (SDValue V = foldSetCCWithBinOp(VT, N0: N1, N1: N0, Cond, DL: dl, DCI))
5752 return V;
5753
5754 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, DL: dl, DCI))
5755 return V;
5756
5757 if (SDValue V = foldSetCCWithOr(VT, N0, N1, Cond, DL: dl, DCI))
5758 return V;
5759 }
5760
5761 // Fold remainder of division by a constant.
5762 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
5763 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
5764 // When division is cheap or optimizing for minimum size,
5765 // fall through to DIVREM creation by skipping this fold.
5766 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Kind: Attribute::MinSize)) {
5767 if (N0.getOpcode() == ISD::UREM) {
5768 if (SDValue Folded = buildUREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5769 return Folded;
5770 } else if (N0.getOpcode() == ISD::SREM) {
5771 if (SDValue Folded = buildSREMEqFold(SETCCVT: VT, REMNode: N0, CompTargetNode: N1, Cond, DCI, DL: dl))
5772 return Folded;
5773 }
5774 }
5775 }
5776
5777 // Fold away ALL boolean setcc's.
5778 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
5779 SDValue Temp;
5780 switch (Cond) {
5781 default: llvm_unreachable("Unknown integer setcc!");
5782 case ISD::SETEQ: // X == Y -> ~(X^Y)
5783 Temp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
5784 N0 = DAG.getNOT(DL: dl, Val: Temp, VT: OpVT);
5785 if (!DCI.isCalledByLegalizer())
5786 DCI.AddToWorklist(N: Temp.getNode());
5787 break;
5788 case ISD::SETNE: // X != Y --> (X^Y)
5789 N0 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OpVT, N1: N0, N2: N1);
5790 break;
5791 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
5792 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
5793 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
5794 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1, N2: Temp);
5795 if (!DCI.isCalledByLegalizer())
5796 DCI.AddToWorklist(N: Temp.getNode());
5797 break;
5798 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
5799 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
5800 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5801 N0 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: OpVT, N1: N0, N2: Temp);
5802 if (!DCI.isCalledByLegalizer())
5803 DCI.AddToWorklist(N: Temp.getNode());
5804 break;
5805 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
5806 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
5807 Temp = DAG.getNOT(DL: dl, Val: N0, VT: OpVT);
5808 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1, N2: Temp);
5809 if (!DCI.isCalledByLegalizer())
5810 DCI.AddToWorklist(N: Temp.getNode());
5811 break;
5812 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
5813 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
5814 Temp = DAG.getNOT(DL: dl, Val: N1, VT: OpVT);
5815 N0 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: OpVT, N1: N0, N2: Temp);
5816 break;
5817 }
5818 if (VT.getScalarType() != MVT::i1) {
5819 if (!DCI.isCalledByLegalizer())
5820 DCI.AddToWorklist(N: N0.getNode());
5821 // FIXME: If running after legalize, we probably can't do this.
5822 ISD::NodeType ExtendCode = getExtendForContent(Content: getBooleanContents(Type: OpVT));
5823 N0 = DAG.getNode(Opcode: ExtendCode, DL: dl, VT, Operand: N0);
5824 }
5825 return N0;
5826 }
5827
5828 // Fold (setcc (trunc x) (trunc y)) -> (setcc x y)
5829 if (N0.getOpcode() == ISD::TRUNCATE && N1.getOpcode() == ISD::TRUNCATE &&
5830 N0.getOperand(i: 0).getValueType() == N1.getOperand(i: 0).getValueType() &&
5831 ((!ISD::isSignedIntSetCC(Code: Cond) && N0->getFlags().hasNoUnsignedWrap() &&
5832 N1->getFlags().hasNoUnsignedWrap()) ||
5833 (!ISD::isUnsignedIntSetCC(Code: Cond) && N0->getFlags().hasNoSignedWrap() &&
5834 N1->getFlags().hasNoSignedWrap())) &&
5835 isTypeDesirableForOp(ISD::SETCC, VT: N0.getOperand(i: 0).getValueType())) {
5836 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N1.getOperand(i: 0), Cond);
5837 }
5838
5839 // Fold (setcc (sub nsw a, b), zero, s??) -> (setcc a, b, s??)
5840 // TODO: Remove that .isVector() check
5841 if (VT.isVector() && isZeroOrZeroSplat(N: N1) && N0.getOpcode() == ISD::SUB &&
5842 N0->getFlags().hasNoSignedWrap() && ISD::isSignedIntSetCC(Code: Cond)) {
5843 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1), Cond);
5844 }
5845
5846 // Could not fold it.
5847 return SDValue();
5848}
5849
5850/// Returns true (and the GlobalValue and the offset) if the node is a
5851/// GlobalAddress + offset.
5852bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
5853 int64_t &Offset) const {
5854
5855 SDNode *N = unwrapAddress(N: SDValue(WN, 0)).getNode();
5856
5857 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(Val: N)) {
5858 GA = GASD->getGlobal();
5859 Offset += GASD->getOffset();
5860 return true;
5861 }
5862
5863 if (N->isAnyAdd()) {
5864 SDValue N1 = N->getOperand(Num: 0);
5865 SDValue N2 = N->getOperand(Num: 1);
5866 if (isGAPlusOffset(WN: N1.getNode(), GA, Offset)) {
5867 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N2)) {
5868 Offset += V->getSExtValue();
5869 return true;
5870 }
5871 } else if (isGAPlusOffset(WN: N2.getNode(), GA, Offset)) {
5872 if (auto *V = dyn_cast<ConstantSDNode>(Val&: N1)) {
5873 Offset += V->getSExtValue();
5874 return true;
5875 }
5876 }
5877 }
5878
5879 return false;
5880}
5881
5882SDValue TargetLowering::PerformDAGCombine(SDNode *N,
5883 DAGCombinerInfo &DCI) const {
5884 // Default implementation: no optimization.
5885 return SDValue();
5886}
5887
5888//===----------------------------------------------------------------------===//
5889// Inline Assembler Implementation Methods
5890//===----------------------------------------------------------------------===//
5891
5892TargetLowering::ConstraintType
5893TargetLowering::getConstraintType(StringRef Constraint) const {
5894 unsigned S = Constraint.size();
5895
5896 if (S == 1) {
5897 switch (Constraint[0]) {
5898 default: break;
5899 case 'r':
5900 return C_RegisterClass;
5901 case 'm': // memory
5902 case 'o': // offsetable
5903 case 'V': // not offsetable
5904 return C_Memory;
5905 case 'p': // Address.
5906 return C_Address;
5907 case 'n': // Simple Integer
5908 case 'E': // Floating Point Constant
5909 case 'F': // Floating Point Constant
5910 return C_Immediate;
5911 case 'i': // Simple Integer or Relocatable Constant
5912 case 's': // Relocatable Constant
5913 case 'X': // Allow ANY value.
5914 case 'I': // Target registers.
5915 case 'J':
5916 case 'K':
5917 case 'L':
5918 case 'M':
5919 case 'N':
5920 case 'O':
5921 case 'P':
5922 case '<':
5923 case '>':
5924 return C_Other;
5925 }
5926 }
5927
5928 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
5929 if (S == 8 && Constraint.substr(Start: 1, N: 6) == "memory") // "{memory}"
5930 return C_Memory;
5931 return C_Register;
5932 }
5933 return C_Unknown;
5934}
5935
5936/// Try to replace an X constraint, which matches anything, with another that
5937/// has more specific requirements based on the type of the corresponding
5938/// operand.
5939const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
5940 if (ConstraintVT.isInteger())
5941 return "r";
5942 if (ConstraintVT.isFloatingPoint())
5943 return "f"; // works for many targets
5944 return nullptr;
5945}
5946
5947SDValue TargetLowering::LowerAsmOutputForConstraint(
5948 SDValue &Chain, SDValue &Glue, const SDLoc &DL,
5949 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
5950 return SDValue();
5951}
5952
5953/// Lower the specified operand into the Ops vector.
5954/// If it is invalid, don't add anything to Ops.
5955void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
5956 StringRef Constraint,
5957 std::vector<SDValue> &Ops,
5958 SelectionDAG &DAG) const {
5959
5960 if (Constraint.size() > 1)
5961 return;
5962
5963 char ConstraintLetter = Constraint[0];
5964 switch (ConstraintLetter) {
5965 default: break;
5966 case 'X': // Allows any operand
5967 case 'i': // Simple Integer or Relocatable Constant
5968 case 'n': // Simple Integer
5969 case 's': { // Relocatable Constant
5970
5971 ConstantSDNode *C;
5972 uint64_t Offset = 0;
5973
5974 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
5975 // etc., since getelementpointer is variadic. We can't use
5976 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
5977 // while in this case the GA may be furthest from the root node which is
5978 // likely an ISD::ADD.
5979 while (true) {
5980 if ((C = dyn_cast<ConstantSDNode>(Val&: Op)) && ConstraintLetter != 's') {
5981 // gcc prints these as sign extended. Sign extend value to 64 bits
5982 // now; without this it would get ZExt'd later in
5983 // ScheduleDAGSDNodes::EmitNode, which is very generic.
5984 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
5985 BooleanContent BCont = getBooleanContents(Type: MVT::i64);
5986 ISD::NodeType ExtOpc =
5987 IsBool ? getExtendForContent(Content: BCont) : ISD::SIGN_EXTEND;
5988 int64_t ExtVal =
5989 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
5990 Ops.push_back(
5991 x: DAG.getTargetConstant(Val: Offset + ExtVal, DL: SDLoc(C), VT: MVT::i64));
5992 return;
5993 }
5994 if (ConstraintLetter != 'n') {
5995 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
5996 Ops.push_back(x: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL: SDLoc(Op),
5997 VT: GA->getValueType(ResNo: 0),
5998 offset: Offset + GA->getOffset()));
5999 return;
6000 }
6001 if (const auto *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
6002 Ops.push_back(x: DAG.getTargetBlockAddress(
6003 BA: BA->getBlockAddress(), VT: BA->getValueType(ResNo: 0),
6004 Offset: Offset + BA->getOffset(), TargetFlags: BA->getTargetFlags()));
6005 return;
6006 }
6007 if (isa<BasicBlockSDNode>(Val: Op)) {
6008 Ops.push_back(x: Op);
6009 return;
6010 }
6011 }
6012 const unsigned OpCode = Op.getOpcode();
6013 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
6014 if ((C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 0))))
6015 Op = Op.getOperand(i: 1);
6016 // Subtraction is not commutative.
6017 else if (OpCode == ISD::ADD &&
6018 (C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))))
6019 Op = Op.getOperand(i: 0);
6020 else
6021 return;
6022 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
6023 continue;
6024 }
6025 return;
6026 }
6027 break;
6028 }
6029 }
6030}
6031
6032void TargetLowering::CollectTargetIntrinsicOperands(
6033 const CallInst &I, SmallVectorImpl<SDValue> &Ops, SelectionDAG &DAG) const {
6034}
6035
6036std::pair<unsigned, const TargetRegisterClass *>
6037TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
6038 StringRef Constraint,
6039 MVT VT) const {
6040 if (!Constraint.starts_with(Prefix: "{"))
6041 return std::make_pair(x: 0u, y: static_cast<TargetRegisterClass *>(nullptr));
6042 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
6043
6044 // Remove the braces from around the name.
6045 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
6046
6047 std::pair<unsigned, const TargetRegisterClass *> R =
6048 std::make_pair(x: 0u, y: static_cast<const TargetRegisterClass *>(nullptr));
6049
6050 // Figure out which register class contains this reg.
6051 for (const TargetRegisterClass &RC : RI->regclasses()) {
6052 // If none of the value types for this register class are valid, we
6053 // can't use it. For example, 64-bit reg classes on 32-bit targets.
6054 if (!isLegalRC(TRI: *RI, RC))
6055 continue;
6056
6057 for (const MCPhysReg &PR : RC) {
6058 if (RegName.equals_insensitive(RHS: RI->getRegAsmName(Reg: PR))) {
6059 std::pair<unsigned, const TargetRegisterClass *> S =
6060 std::make_pair(x: PR, y: &RC);
6061
6062 // If this register class has the requested value type, return it,
6063 // otherwise keep searching and return the first class found
6064 // if no other is found which explicitly has the requested type.
6065 if (RI->isTypeLegalForClass(RC, T: VT))
6066 return S;
6067 if (!R.second)
6068 R = S;
6069 }
6070 }
6071 }
6072
6073 return R;
6074}
6075
6076//===----------------------------------------------------------------------===//
6077// Constraint Selection.
6078
6079/// Return true of this is an input operand that is a matching constraint like
6080/// "4".
6081bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
6082 assert(!ConstraintCode.empty() && "No known constraint!");
6083 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
6084}
6085
6086/// If this is an input matching constraint, this method returns the output
6087/// operand it matches.
6088unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
6089 assert(!ConstraintCode.empty() && "No known constraint!");
6090 return atoi(nptr: ConstraintCode.c_str());
6091}
6092
6093/// Split up the constraint string from the inline assembly value into the
6094/// specific constraints and their prefixes, and also tie in the associated
6095/// operand values.
6096/// If this returns an empty vector, and if the constraint string itself
6097/// isn't empty, there was an error parsing.
6098TargetLowering::AsmOperandInfoVector
6099TargetLowering::ParseConstraints(const DataLayout &DL,
6100 const TargetRegisterInfo *TRI,
6101 const CallBase &Call) const {
6102 /// Information about all of the constraints.
6103 AsmOperandInfoVector ConstraintOperands;
6104 const InlineAsm *IA = cast<InlineAsm>(Val: Call.getCalledOperand());
6105 unsigned maCount = 0; // Largest number of multiple alternative constraints.
6106
6107 // Do a prepass over the constraints, canonicalizing them, and building up the
6108 // ConstraintOperands list.
6109 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
6110 unsigned ResNo = 0; // ResNo - The result number of the next output.
6111 unsigned LabelNo = 0; // LabelNo - CallBr indirect dest number.
6112
6113 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
6114 ConstraintOperands.emplace_back(args: std::move(CI));
6115 AsmOperandInfo &OpInfo = ConstraintOperands.back();
6116
6117 // Update multiple alternative constraint count.
6118 if (OpInfo.multipleAlternatives.size() > maCount)
6119 maCount = OpInfo.multipleAlternatives.size();
6120
6121 OpInfo.ConstraintVT = MVT::Other;
6122
6123 // Compute the value type for each operand.
6124 switch (OpInfo.Type) {
6125 case InlineAsm::isOutput: {
6126 // Indirect outputs just consume an argument.
6127 if (OpInfo.isIndirect) {
6128 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6129 break;
6130 }
6131
6132 // The return value of the call is this value. As such, there is no
6133 // corresponding argument.
6134 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
6135 EVT VT;
6136 if (auto *STy = dyn_cast<StructType>(Val: Call.getType())) {
6137 VT = getAsmOperandValueType(DL, Ty: STy->getElementType(N: ResNo));
6138 } else {
6139 assert(ResNo == 0 && "Asm only has one result!");
6140 VT = getAsmOperandValueType(DL, Ty: Call.getType());
6141 }
6142 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6143 ++ResNo;
6144 break;
6145 }
6146 case InlineAsm::isInput:
6147 OpInfo.CallOperandVal = Call.getArgOperand(i: ArgNo);
6148 break;
6149 case InlineAsm::isLabel:
6150 OpInfo.CallOperandVal = cast<CallBrInst>(Val: &Call)->getIndirectDest(i: LabelNo);
6151 ++LabelNo;
6152 continue;
6153 case InlineAsm::isClobber:
6154 // Nothing to do.
6155 break;
6156 }
6157
6158 if (OpInfo.CallOperandVal) {
6159 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
6160 if (OpInfo.isIndirect) {
6161 OpTy = Call.getParamElementType(ArgNo);
6162 assert(OpTy && "Indirect operand must have elementtype attribute");
6163 }
6164
6165 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
6166 if (StructType *STy = dyn_cast<StructType>(Val: OpTy))
6167 if (STy->getNumElements() == 1)
6168 OpTy = STy->getElementType(N: 0);
6169
6170 // If OpTy is not a single value, it may be a struct/union that we
6171 // can tile with integers.
6172 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
6173 unsigned BitSize = DL.getTypeSizeInBits(Ty: OpTy);
6174 switch (BitSize) {
6175 default: break;
6176 case 1:
6177 case 8:
6178 case 16:
6179 case 32:
6180 case 64:
6181 case 128:
6182 OpTy = IntegerType::get(C&: OpTy->getContext(), NumBits: BitSize);
6183 break;
6184 }
6185 }
6186
6187 EVT VT = getAsmOperandValueType(DL, Ty: OpTy, AllowUnknown: true);
6188 OpInfo.ConstraintVT = VT.isSimple() ? VT.getSimpleVT() : MVT::Other;
6189 ArgNo++;
6190 }
6191 }
6192
6193 // If we have multiple alternative constraints, select the best alternative.
6194 if (!ConstraintOperands.empty()) {
6195 if (maCount) {
6196 unsigned bestMAIndex = 0;
6197 int bestWeight = -1;
6198 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
6199 int weight = -1;
6200 unsigned maIndex;
6201 // Compute the sums of the weights for each alternative, keeping track
6202 // of the best (highest weight) one so far.
6203 for (maIndex = 0; maIndex < maCount; ++maIndex) {
6204 int weightSum = 0;
6205 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6206 cIndex != eIndex; ++cIndex) {
6207 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6208 if (OpInfo.Type == InlineAsm::isClobber)
6209 continue;
6210
6211 // If this is an output operand with a matching input operand,
6212 // look up the matching input. If their types mismatch, e.g. one
6213 // is an integer, the other is floating point, or their sizes are
6214 // different, flag it as an maCantMatch.
6215 if (OpInfo.hasMatchingInput()) {
6216 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6217 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6218 if ((OpInfo.ConstraintVT.isInteger() !=
6219 Input.ConstraintVT.isInteger()) ||
6220 (OpInfo.ConstraintVT.getSizeInBits() !=
6221 Input.ConstraintVT.getSizeInBits())) {
6222 weightSum = -1; // Can't match.
6223 break;
6224 }
6225 }
6226 }
6227 weight = getMultipleConstraintMatchWeight(info&: OpInfo, maIndex);
6228 if (weight == -1) {
6229 weightSum = -1;
6230 break;
6231 }
6232 weightSum += weight;
6233 }
6234 // Update best.
6235 if (weightSum > bestWeight) {
6236 bestWeight = weightSum;
6237 bestMAIndex = maIndex;
6238 }
6239 }
6240
6241 // Now select chosen alternative in each constraint.
6242 for (AsmOperandInfo &cInfo : ConstraintOperands)
6243 if (cInfo.Type != InlineAsm::isClobber)
6244 cInfo.selectAlternative(index: bestMAIndex);
6245 }
6246 }
6247
6248 // Check and hook up tied operands, choose constraint code to use.
6249 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
6250 cIndex != eIndex; ++cIndex) {
6251 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
6252
6253 // If this is an output operand with a matching input operand, look up the
6254 // matching input. If their types mismatch, e.g. one is an integer, the
6255 // other is floating point, or their sizes are different, flag it as an
6256 // error.
6257 if (OpInfo.hasMatchingInput()) {
6258 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
6259
6260 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
6261 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
6262 getRegForInlineAsmConstraint(RI: TRI, Constraint: OpInfo.ConstraintCode,
6263 VT: OpInfo.ConstraintVT);
6264 std::pair<unsigned, const TargetRegisterClass *> InputRC =
6265 getRegForInlineAsmConstraint(RI: TRI, Constraint: Input.ConstraintCode,
6266 VT: Input.ConstraintVT);
6267 const bool OutOpIsIntOrFP = OpInfo.ConstraintVT.isInteger() ||
6268 OpInfo.ConstraintVT.isFloatingPoint();
6269 const bool InOpIsIntOrFP = Input.ConstraintVT.isInteger() ||
6270 Input.ConstraintVT.isFloatingPoint();
6271 if ((OutOpIsIntOrFP != InOpIsIntOrFP) ||
6272 (MatchRC.second != InputRC.second)) {
6273 report_fatal_error(reason: "Unsupported asm: input constraint"
6274 " with a matching output constraint of"
6275 " incompatible type!");
6276 }
6277 }
6278 }
6279 }
6280
6281 return ConstraintOperands;
6282}
6283
6284/// Return a number indicating our preference for chosing a type of constraint
6285/// over another, for the purpose of sorting them. Immediates are almost always
6286/// preferrable (when they can be emitted). A higher return value means a
6287/// stronger preference for one constraint type relative to another.
6288/// FIXME: We should prefer registers over memory but doing so may lead to
6289/// unrecoverable register exhaustion later.
6290/// https://github.com/llvm/llvm-project/issues/20571
6291static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) {
6292 switch (CT) {
6293 case TargetLowering::C_Immediate:
6294 case TargetLowering::C_Other:
6295 return 4;
6296 case TargetLowering::C_Memory:
6297 case TargetLowering::C_Address:
6298 return 3;
6299 case TargetLowering::C_RegisterClass:
6300 return 2;
6301 case TargetLowering::C_Register:
6302 return 1;
6303 case TargetLowering::C_Unknown:
6304 return 0;
6305 }
6306 llvm_unreachable("Invalid constraint type");
6307}
6308
6309/// Examine constraint type and operand type and determine a weight value.
6310/// This object must already have been set up with the operand type
6311/// and the current alternative constraint selected.
6312TargetLowering::ConstraintWeight
6313 TargetLowering::getMultipleConstraintMatchWeight(
6314 AsmOperandInfo &info, int maIndex) const {
6315 InlineAsm::ConstraintCodeVector *rCodes;
6316 if (maIndex >= (int)info.multipleAlternatives.size())
6317 rCodes = &info.Codes;
6318 else
6319 rCodes = &info.multipleAlternatives[maIndex].Codes;
6320 ConstraintWeight BestWeight = CW_Invalid;
6321
6322 // Loop over the options, keeping track of the most general one.
6323 for (const std::string &rCode : *rCodes) {
6324 ConstraintWeight weight =
6325 getSingleConstraintMatchWeight(info, constraint: rCode.c_str());
6326 if (weight > BestWeight)
6327 BestWeight = weight;
6328 }
6329
6330 return BestWeight;
6331}
6332
6333/// Examine constraint type and operand type and determine a weight value.
6334/// This object must already have been set up with the operand type
6335/// and the current alternative constraint selected.
6336TargetLowering::ConstraintWeight
6337 TargetLowering::getSingleConstraintMatchWeight(
6338 AsmOperandInfo &info, const char *constraint) const {
6339 ConstraintWeight weight = CW_Invalid;
6340 Value *CallOperandVal = info.CallOperandVal;
6341 // If we don't have a value, we can't do a match,
6342 // but allow it at the lowest weight.
6343 if (!CallOperandVal)
6344 return CW_Default;
6345 // Look at the constraint type.
6346 switch (*constraint) {
6347 case 'i': // immediate integer.
6348 case 'n': // immediate integer with a known value.
6349 if (isa<ConstantInt>(Val: CallOperandVal))
6350 weight = CW_Constant;
6351 break;
6352 case 's': // non-explicit intregal immediate.
6353 if (isa<GlobalValue>(Val: CallOperandVal))
6354 weight = CW_Constant;
6355 break;
6356 case 'E': // immediate float if host format.
6357 case 'F': // immediate float.
6358 if (isa<ConstantFP>(Val: CallOperandVal))
6359 weight = CW_Constant;
6360 break;
6361 case '<': // memory operand with autodecrement.
6362 case '>': // memory operand with autoincrement.
6363 case 'm': // memory operand.
6364 case 'o': // offsettable memory operand
6365 case 'V': // non-offsettable memory operand
6366 weight = CW_Memory;
6367 break;
6368 case 'r': // general register.
6369 case 'g': // general register, memory operand or immediate integer.
6370 // note: Clang converts "g" to "imr".
6371 if (CallOperandVal->getType()->isIntegerTy())
6372 weight = CW_Register;
6373 break;
6374 case 'X': // any operand.
6375 default:
6376 weight = CW_Default;
6377 break;
6378 }
6379 return weight;
6380}
6381
6382/// If there are multiple different constraints that we could pick for this
6383/// operand (e.g. "imr") try to pick the 'best' one.
6384/// This is somewhat tricky: constraints (TargetLowering::ConstraintType) fall
6385/// into seven classes:
6386/// Register -> one specific register
6387/// RegisterClass -> a group of regs
6388/// Memory -> memory
6389/// Address -> a symbolic memory reference
6390/// Immediate -> immediate values
6391/// Other -> magic values (such as "Flag Output Operands")
6392/// Unknown -> something we don't recognize yet and can't handle
6393/// Ideally, we would pick the most specific constraint possible: if we have
6394/// something that fits into a register, we would pick it. The problem here
6395/// is that if we have something that could either be in a register or in
6396/// memory that use of the register could cause selection of *other*
6397/// operands to fail: they might only succeed if we pick memory. Because of
6398/// this the heuristic we use is:
6399///
6400/// 1) If there is an 'other' constraint, and if the operand is valid for
6401/// that constraint, use it. This makes us take advantage of 'i'
6402/// constraints when available.
6403/// 2) Otherwise, pick the most general constraint present. This prefers
6404/// 'm' over 'r', for example.
6405///
6406TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
6407 TargetLowering::AsmOperandInfo &OpInfo) const {
6408 ConstraintGroup Ret;
6409
6410 Ret.reserve(N: OpInfo.Codes.size());
6411 for (StringRef Code : OpInfo.Codes) {
6412 TargetLowering::ConstraintType CType = getConstraintType(Constraint: Code);
6413
6414 // Indirect 'other' or 'immediate' constraints are not allowed.
6415 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
6416 CType == TargetLowering::C_Register ||
6417 CType == TargetLowering::C_RegisterClass))
6418 continue;
6419
6420 // Things with matching constraints can only be registers, per gcc
6421 // documentation. This mainly affects "g" constraints.
6422 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
6423 continue;
6424
6425 Ret.emplace_back(Args&: Code, Args&: CType);
6426 }
6427
6428 llvm::stable_sort(Range&: Ret, C: [](ConstraintPair a, ConstraintPair b) {
6429 return getConstraintPiority(CT: a.second) > getConstraintPiority(CT: b.second);
6430 });
6431
6432 return Ret;
6433}
6434
6435/// If we have an immediate, see if we can lower it. Return true if we can,
6436/// false otherwise.
6437static bool lowerImmediateIfPossible(TargetLowering::ConstraintPair &P,
6438 SDValue Op, SelectionDAG *DAG,
6439 const TargetLowering &TLI) {
6440
6441 assert((P.second == TargetLowering::C_Other ||
6442 P.second == TargetLowering::C_Immediate) &&
6443 "need immediate or other");
6444
6445 if (!Op.getNode())
6446 return false;
6447
6448 std::vector<SDValue> ResultOps;
6449 TLI.LowerAsmOperandForConstraint(Op, Constraint: P.first, Ops&: ResultOps, DAG&: *DAG);
6450 return !ResultOps.empty();
6451}
6452
6453/// Determines the constraint code and constraint type to use for the specific
6454/// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
6455void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
6456 SDValue Op,
6457 SelectionDAG *DAG) const {
6458 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
6459
6460 // Single-letter constraints ('r') are very common.
6461 if (OpInfo.Codes.size() == 1) {
6462 OpInfo.ConstraintCode = OpInfo.Codes[0];
6463 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6464 } else {
6465 ConstraintGroup G = getConstraintPreferences(OpInfo);
6466 if (G.empty())
6467 return;
6468
6469 unsigned BestIdx = 0;
6470 for (const unsigned E = G.size();
6471 BestIdx < E && (G[BestIdx].second == TargetLowering::C_Other ||
6472 G[BestIdx].second == TargetLowering::C_Immediate);
6473 ++BestIdx) {
6474 if (lowerImmediateIfPossible(P&: G[BestIdx], Op, DAG, TLI: *this))
6475 break;
6476 // If we're out of constraints, just pick the first one.
6477 if (BestIdx + 1 == E) {
6478 BestIdx = 0;
6479 break;
6480 }
6481 }
6482
6483 OpInfo.ConstraintCode = G[BestIdx].first;
6484 OpInfo.ConstraintType = G[BestIdx].second;
6485 }
6486
6487 // 'X' matches anything.
6488 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
6489 // Constants are handled elsewhere. For Functions, the type here is the
6490 // type of the result, which is not what we want to look at; leave them
6491 // alone.
6492 Value *v = OpInfo.CallOperandVal;
6493 if (isa<ConstantInt>(Val: v) || isa<Function>(Val: v)) {
6494 return;
6495 }
6496
6497 if (isa<BasicBlock>(Val: v) || isa<BlockAddress>(Val: v)) {
6498 OpInfo.ConstraintCode = "i";
6499 return;
6500 }
6501
6502 // Otherwise, try to resolve it to something we know about by looking at
6503 // the actual operand type.
6504 if (const char *Repl = LowerXConstraint(ConstraintVT: OpInfo.ConstraintVT)) {
6505 OpInfo.ConstraintCode = Repl;
6506 OpInfo.ConstraintType = getConstraintType(Constraint: OpInfo.ConstraintCode);
6507 }
6508 }
6509}
6510
6511/// Given an exact SDIV by a constant, create a multiplication
6512/// with the multiplicative inverse of the constant.
6513/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6514static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
6515 const SDLoc &dl, SelectionDAG &DAG,
6516 SmallVectorImpl<SDNode *> &Created) {
6517 SDValue Op0 = N->getOperand(Num: 0);
6518 SDValue Op1 = N->getOperand(Num: 1);
6519 EVT VT = N->getValueType(ResNo: 0);
6520 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6521 EVT ShSVT = ShVT.getScalarType();
6522
6523 bool UseSRA = false;
6524 SmallVector<SDValue, 16> Shifts, Factors;
6525
6526 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6527 if (C->isZero())
6528 return false;
6529
6530 EVT CT = C->getValueType(ResNo: 0);
6531 APInt Divisor = C->getAPIntValue();
6532 unsigned Shift = Divisor.countr_zero();
6533 if (Shift) {
6534 Divisor.ashrInPlace(ShiftAmt: Shift);
6535 UseSRA = true;
6536 }
6537 APInt Factor = Divisor.multiplicativeInverse();
6538 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6539 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6540 return true;
6541 };
6542
6543 // Collect all magic values from the build vector.
6544 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildSDIVPattern))
6545 return SDValue();
6546
6547 SDValue Shift, Factor;
6548 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6549 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6550 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6551 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6552 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6553 "Expected matchUnaryPredicate to return one element for scalable "
6554 "vectors");
6555 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6556 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6557 } else {
6558 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6559 Shift = Shifts[0];
6560 Factor = Factors[0];
6561 }
6562
6563 SDValue Res = Op0;
6564 if (UseSRA) {
6565 Res = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6566 Created.push_back(Elt: Res.getNode());
6567 }
6568
6569 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6570}
6571
6572/// Given an exact UDIV by a constant, create a multiplication
6573/// with the multiplicative inverse of the constant.
6574/// Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242
6575static SDValue BuildExactUDIV(const TargetLowering &TLI, SDNode *N,
6576 const SDLoc &dl, SelectionDAG &DAG,
6577 SmallVectorImpl<SDNode *> &Created) {
6578 EVT VT = N->getValueType(ResNo: 0);
6579 EVT ShVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6580 EVT ShSVT = ShVT.getScalarType();
6581
6582 bool UseSRL = false;
6583 SmallVector<SDValue, 16> Shifts, Factors;
6584
6585 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6586 if (C->isZero())
6587 return false;
6588
6589 EVT CT = C->getValueType(ResNo: 0);
6590 APInt Divisor = C->getAPIntValue();
6591 unsigned Shift = Divisor.countr_zero();
6592 if (Shift) {
6593 Divisor.lshrInPlace(ShiftAmt: Shift);
6594 UseSRL = true;
6595 }
6596 // Calculate the multiplicative inverse modulo BW.
6597 APInt Factor = Divisor.multiplicativeInverse();
6598 Shifts.push_back(Elt: DAG.getConstant(Val: Shift, DL: dl, VT: ShSVT));
6599 Factors.push_back(Elt: DAG.getConstant(Val: Factor, DL: dl, VT: CT));
6600 return true;
6601 };
6602
6603 SDValue Op1 = N->getOperand(Num: 1);
6604
6605 // Collect all magic values from the build vector.
6606 if (!ISD::matchUnaryPredicate(Op: Op1, Match: BuildUDIVPattern))
6607 return SDValue();
6608
6609 SDValue Shift, Factor;
6610 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
6611 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6612 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6613 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
6614 assert(Shifts.size() == 1 && Factors.size() == 1 &&
6615 "Expected matchUnaryPredicate to return one element for scalable "
6616 "vectors");
6617 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6618 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6619 } else {
6620 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
6621 Shift = Shifts[0];
6622 Factor = Factors[0];
6623 }
6624
6625 SDValue Res = N->getOperand(Num: 0);
6626 if (UseSRL) {
6627 Res = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Res, N2: Shift, Flags: SDNodeFlags::Exact);
6628 Created.push_back(Elt: Res.getNode());
6629 }
6630
6631 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Res, N2: Factor);
6632}
6633
6634SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
6635 SelectionDAG &DAG,
6636 SmallVectorImpl<SDNode *> &Created) const {
6637 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6638 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6639 return SDValue(N, 0); // Lower SDIV as SDIV
6640 return SDValue();
6641}
6642
6643SDValue
6644TargetLowering::BuildSREMPow2(SDNode *N, const APInt &Divisor,
6645 SelectionDAG &DAG,
6646 SmallVectorImpl<SDNode *> &Created) const {
6647 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
6648 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
6649 return SDValue(N, 0); // Lower SREM as SREM
6650 return SDValue();
6651}
6652
6653/// Build sdiv by power-of-2 with conditional move instructions
6654/// Ref: "Hacker's Delight" by Henry Warren 10-1
6655/// If conditional move/branch is preferred, we lower sdiv x, +/-2**k into:
6656/// bgez x, label
6657/// add x, x, 2**k-1
6658/// label:
6659/// sra res, x, k
6660/// neg res, res (when the divisor is negative)
6661SDValue TargetLowering::buildSDIVPow2WithCMov(
6662 SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
6663 SmallVectorImpl<SDNode *> &Created) const {
6664 unsigned Lg2 = Divisor.countr_zero();
6665 EVT VT = N->getValueType(ResNo: 0);
6666
6667 SDLoc DL(N);
6668 SDValue N0 = N->getOperand(Num: 0);
6669 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
6670 APInt Lg2Mask = APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: Lg2);
6671 SDValue Pow2MinusOne = DAG.getConstant(Val: Lg2Mask, DL, VT);
6672
6673 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
6674 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
6675 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: N0, RHS: Zero, Cond: ISD::SETLT);
6676 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: Pow2MinusOne);
6677 SDValue CMov = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cmp, N2: Add, N3: N0);
6678
6679 Created.push_back(Elt: Cmp.getNode());
6680 Created.push_back(Elt: Add.getNode());
6681 Created.push_back(Elt: CMov.getNode());
6682
6683 // Divide by pow2.
6684 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CMov,
6685 N2: DAG.getShiftAmountConstant(Val: Lg2, VT, DL));
6686
6687 // If we're dividing by a positive value, we're done. Otherwise, we must
6688 // negate the result.
6689 if (Divisor.isNonNegative())
6690 return SRA;
6691
6692 Created.push_back(Elt: SRA.getNode());
6693 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: SRA);
6694}
6695
6696/// Given an ISD::SDIV node expressing a divide by constant,
6697/// return a DAG expression to select that will generate the same value by
6698/// multiplying by a magic number.
6699/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6700SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
6701 bool IsAfterLegalization,
6702 bool IsAfterLegalTypes,
6703 SmallVectorImpl<SDNode *> &Created) const {
6704 SDLoc dl(N);
6705
6706 // If the sdiv has an 'exact' bit we can use a simpler lowering.
6707 if (N->getFlags().hasExact())
6708 return BuildExactSDIV(TLI: *this, N, dl, DAG, Created);
6709
6710 EVT VT = N->getValueType(ResNo: 0);
6711 EVT SVT = VT.getScalarType();
6712 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6713 EVT ShSVT = ShVT.getScalarType();
6714 unsigned EltBits = VT.getScalarSizeInBits();
6715 EVT MulVT;
6716
6717 // Check to see if we can do this.
6718 // FIXME: We should be more aggressive here.
6719 EVT QueryVT = VT;
6720 if (VT.isVector()) {
6721 // If the vector type will be legalized to a vector type with the same
6722 // element type, allow the transform before type legalization if MULHS or
6723 // SMUL_LOHI are supported.
6724 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
6725 if (!QueryVT.isVector() ||
6726 QueryVT.getVectorElementType() != VT.getVectorElementType())
6727 return SDValue();
6728 } else if (!isTypeLegal(VT)) {
6729 // Limit this to simple scalars for now.
6730 if (!VT.isSimple())
6731 return SDValue();
6732
6733 // If this type will be promoted to a large enough type with a legal
6734 // multiply operation, we can go ahead and do this transform.
6735 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
6736 return SDValue();
6737
6738 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6739 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6740 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
6741 return SDValue();
6742 }
6743
6744 bool HasMULHS =
6745 isOperationLegalOrCustom(Op: ISD::MULHS, VT: QueryVT, LegalOnly: IsAfterLegalization);
6746 bool HasSMUL_LOHI =
6747 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
6748
6749 if (isTypeLegal(VT) && !HasMULHS && !HasSMUL_LOHI && MulVT == EVT()) {
6750 // If type twice as wide legal, widen and use a mul plus a shift.
6751 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
6752 // Some targets like AMDGPU try to go from SDIV to SDIVREM which is then
6753 // custom lowered. This is very expensive so avoid it at all costs for
6754 // constant divisors.
6755 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::SDIV, VT) &&
6756 isOperationCustom(Op: ISD::SDIVREM, VT: VT.getScalarType())) ||
6757 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
6758 MulVT = WideVT;
6759 }
6760
6761 if (!HasMULHS && !HasSMUL_LOHI && MulVT == EVT())
6762 return SDValue();
6763
6764 // If we're after type legalization and SVT is not legal, use the
6765 // promoted type for creating constants to avoid creating nodes with
6766 // illegal types.
6767 if (IsAfterLegalTypes && VT.isVector()) {
6768 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
6769 if (SVT.bitsLT(VT: VT.getScalarType()))
6770 return SDValue();
6771 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
6772 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
6773 return SDValue();
6774 }
6775 const unsigned SVTBits = SVT.getSizeInBits();
6776
6777 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
6778
6779 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
6780 if (C->isZero())
6781 return false;
6782 // Truncate the divisor to the target scalar type in case it was promoted
6783 // during type legalization.
6784 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
6785 SignedDivisionByConstantInfo magics = SignedDivisionByConstantInfo::get(D: Divisor);
6786 int NumeratorFactor = 0;
6787 int ShiftMask = -1;
6788
6789 if (Divisor.isOne() || Divisor.isAllOnes()) {
6790 // If d is +1/-1, we just multiply the numerator by +1/-1.
6791 NumeratorFactor = Divisor.getSExtValue();
6792 magics.Magic = 0;
6793 magics.ShiftAmount = 0;
6794 ShiftMask = 0;
6795 } else if (Divisor.isStrictlyPositive() && magics.Magic.isNegative()) {
6796 // If d > 0 and m < 0, add the numerator.
6797 NumeratorFactor = 1;
6798 } else if (Divisor.isNegative() && magics.Magic.isStrictlyPositive()) {
6799 // If d < 0 and m > 0, subtract the numerator.
6800 NumeratorFactor = -1;
6801 }
6802
6803 MagicFactors.push_back(
6804 Elt: DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT));
6805 Factors.push_back(Elt: DAG.getSignedConstant(Val: NumeratorFactor, DL: dl, VT: SVT));
6806 Shifts.push_back(Elt: DAG.getConstant(Val: magics.ShiftAmount, DL: dl, VT: ShSVT));
6807 ShiftMasks.push_back(Elt: DAG.getSignedConstant(Val: ShiftMask, DL: dl, VT: SVT));
6808 return true;
6809 };
6810
6811 SDValue N0 = N->getOperand(Num: 0);
6812 SDValue N1 = N->getOperand(Num: 1);
6813
6814 // Collect the shifts / magic values from each element.
6815 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildSDIVPattern, /*AllowUndefs=*/false,
6816 /*AllowTruncation=*/true))
6817 return SDValue();
6818
6819 SDValue MagicFactor, Factor, Shift, ShiftMask;
6820 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
6821 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
6822 Factor = DAG.getBuildVector(VT, DL: dl, Ops: Factors);
6823 Shift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: Shifts);
6824 ShiftMask = DAG.getBuildVector(VT, DL: dl, Ops: ShiftMasks);
6825 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
6826 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
6827 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
6828 "Expected matchUnaryPredicate to return one element for scalable "
6829 "vectors");
6830 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
6831 Factor = DAG.getSplatVector(VT, DL: dl, Op: Factors[0]);
6832 Shift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: Shifts[0]);
6833 ShiftMask = DAG.getSplatVector(VT, DL: dl, Op: ShiftMasks[0]);
6834 } else {
6835 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
6836 MagicFactor = MagicFactors[0];
6837 Factor = Factors[0];
6838 Shift = Shifts[0];
6839 ShiftMask = ShiftMasks[0];
6840 }
6841
6842 // Multiply the numerator (operand 0) by the magic value.
6843 auto GetMULHS = [&](SDValue X, SDValue Y) {
6844 if (HasMULHS)
6845 return DAG.getNode(Opcode: ISD::MULHS, DL: dl, VT, N1: X, N2: Y);
6846 if (HasSMUL_LOHI) {
6847 SDValue LoHi =
6848 DAG.getNode(Opcode: ISD::SMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
6849 return LoHi.getValue(R: 1);
6850 }
6851
6852 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: X);
6853 Y = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MulVT, Operand: Y);
6854 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
6855 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
6856 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
6857 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
6858 };
6859
6860 SDValue Q = GetMULHS(N0, MagicFactor);
6861 if (!Q)
6862 return SDValue();
6863
6864 Created.push_back(Elt: Q.getNode());
6865
6866 // (Optionally) Add/subtract the numerator using Factor.
6867 Factor = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: N0, N2: Factor);
6868 Created.push_back(Elt: Factor.getNode());
6869 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: Factor);
6870 Created.push_back(Elt: Q.getNode());
6871
6872 // Shift right algebraic by shift value.
6873 Q = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Q, N2: Shift);
6874 Created.push_back(Elt: Q.getNode());
6875
6876 // Extract the sign bit, mask it and add it to the quotient.
6877 SDValue SignShift = DAG.getConstant(Val: EltBits - 1, DL: dl, VT: ShVT);
6878 SDValue T = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: SignShift);
6879 Created.push_back(Elt: T.getNode());
6880 T = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: ShiftMask);
6881 Created.push_back(Elt: T.getNode());
6882 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Q, N2: T);
6883}
6884
6885/// Given an ISD::UDIV node expressing a divide by constant,
6886/// return a DAG expression to select that will generate the same value by
6887/// multiplying by a magic number.
6888/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
6889SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
6890 bool IsAfterLegalization,
6891 bool IsAfterLegalTypes,
6892 SmallVectorImpl<SDNode *> &Created) const {
6893 SDLoc dl(N);
6894
6895 // If the udiv has an 'exact' bit we can use a simpler lowering.
6896 if (N->getFlags().hasExact())
6897 return BuildExactUDIV(TLI: *this, N, dl, DAG, Created);
6898
6899 EVT VT = N->getValueType(ResNo: 0);
6900 EVT SVT = VT.getScalarType();
6901 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
6902 EVT ShSVT = ShVT.getScalarType();
6903 unsigned EltBits = VT.getScalarSizeInBits();
6904 EVT MulVT;
6905
6906 // Check to see if we can do this.
6907 // FIXME: We should be more aggressive here.
6908 EVT QueryVT = VT;
6909 if (VT.isVector()) {
6910 // If the vector type will be legalized to a vector type with the same
6911 // element type, allow the transform before type legalization if MULHU or
6912 // UMUL_LOHI are supported.
6913 QueryVT = getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
6914 if (!QueryVT.isVector() ||
6915 QueryVT.getVectorElementType() != VT.getVectorElementType())
6916 return SDValue();
6917 } else if (!isTypeLegal(VT)) {
6918 // Limit this to simple scalars for now.
6919 if (!VT.isSimple())
6920 return SDValue();
6921
6922 // If this type will be promoted to a large enough type with a legal
6923 // multiply operation, we can go ahead and do this transform.
6924 if (getTypeAction(VT: VT.getSimpleVT()) != TypePromoteInteger)
6925 return SDValue();
6926
6927 MulVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
6928 if (MulVT.getSizeInBits() < (2 * EltBits) ||
6929 !isOperationLegal(Op: ISD::MUL, VT: MulVT))
6930 return SDValue();
6931 }
6932
6933 bool HasMULHU =
6934 isOperationLegalOrCustom(Op: ISD::MULHU, VT: QueryVT, LegalOnly: IsAfterLegalization);
6935 bool HasUMUL_LOHI =
6936 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: QueryVT, LegalOnly: IsAfterLegalization);
6937
6938 if (isTypeLegal(VT) && !HasMULHU && !HasUMUL_LOHI && MulVT == EVT()) {
6939 // If type twice as wide legal, widen and use a mul plus a shift.
6940 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
6941 // Some targets like AMDGPU try to go from UDIV to UDIVREM which is then
6942 // custom lowered. This is very expensive so avoid it at all costs for
6943 // constant divisors.
6944 if ((!IsAfterLegalTypes && isOperationExpand(Op: ISD::UDIV, VT) &&
6945 isOperationCustom(Op: ISD::UDIVREM, VT: VT.getScalarType())) ||
6946 isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT))
6947 MulVT = WideVT;
6948 }
6949
6950 if (!HasMULHU && !HasUMUL_LOHI && MulVT == EVT())
6951 return SDValue();
6952
6953 SDValue N0 = N->getOperand(Num: 0);
6954 SDValue N1 = N->getOperand(Num: 1);
6955
6956 // Try to use leading zeros of the dividend to reduce the multiplier and
6957 // avoid expensive fixups.
6958 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
6959
6960 // If we're after type legalization and SVT is not legal, use the
6961 // promoted type for creating constants to avoid creating nodes with
6962 // illegal types.
6963 if (IsAfterLegalTypes && VT.isVector()) {
6964 SVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: SVT);
6965 if (SVT.bitsLT(VT: VT.getScalarType()))
6966 return SDValue();
6967 ShSVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: ShSVT);
6968 if (ShSVT.bitsLT(VT: ShVT.getScalarType()))
6969 return SDValue();
6970 }
6971 const unsigned SVTBits = SVT.getSizeInBits();
6972
6973 // Allow i32 to be widened to i64 for uncooperative divisors if i64 MULHU or
6974 // UMUL_LOHI is supported.
6975 const EVT WideSVT = MVT::i64;
6976 const bool HasWideMULHU =
6977 VT == MVT::i32 &&
6978 isOperationLegalOrCustom(Op: ISD::MULHU, VT: WideSVT, LegalOnly: IsAfterLegalization);
6979 const bool HasWideUMUL_LOHI =
6980 VT == MVT::i32 &&
6981 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: WideSVT, LegalOnly: IsAfterLegalization);
6982 const bool AllowWiden = (HasWideMULHU || HasWideUMUL_LOHI);
6983
6984 // For even divisors with a 33-bit magic number, the widened high-multiply
6985 // path is only worthwhile over the even-divisor rewrite on targets that
6986 // zero-extend i32 to i64 for free (e.g. x86-64 and AArch64). Elsewhere (e.g.
6987 // RISC-V) keep the even-divisor rewrite, which avoids the explicit extension.
6988 const bool AllowEvenToWiden = AllowWiden && isZExtFree(FromTy: VT, ToTy: WideSVT);
6989
6990 bool UseNPQ = false, UsePreShift = false, UsePostShift = false;
6991 bool UseWiden = false;
6992 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
6993
6994 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
6995 if (C->isZero())
6996 return false;
6997 // Truncate the divisor to the target scalar type in case it was promoted
6998 // during type legalization.
6999 APInt Divisor = C->getAPIntValue().trunc(width: EltBits);
7000
7001 SDValue PreShift, MagicFactor, NPQFactor, PostShift;
7002
7003 // Magic algorithm doesn't work for division by 1. We need to emit a select
7004 // at the end.
7005 if (Divisor.isOne()) {
7006 PreShift = PostShift = DAG.getUNDEF(VT: ShSVT);
7007 MagicFactor = NPQFactor = DAG.getUNDEF(VT: SVT);
7008 } else {
7009 UnsignedDivisionByConstantInfo magics =
7010 UnsignedDivisionByConstantInfo::get(
7011 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()),
7012 /*AllowEvenDivisorOptimization=*/!AllowEvenToWiden,
7013 /*AllowWidenOptimization=*/AllowWiden);
7014
7015 if (magics.Widen) {
7016 UseWiden = true;
7017 MagicFactor = DAG.getConstant(Val: magics.Magic, DL: dl, VT: WideSVT);
7018 } else {
7019 MagicFactor = DAG.getConstant(Val: magics.Magic.zext(width: SVTBits), DL: dl, VT: SVT);
7020 }
7021
7022 assert(magics.PreShift < Divisor.getBitWidth() &&
7023 "We shouldn't generate an undefined shift!");
7024 assert(magics.PostShift < Divisor.getBitWidth() &&
7025 "We shouldn't generate an undefined shift!");
7026 assert((!magics.IsAdd || magics.PreShift == 0) &&
7027 "Unexpected pre-shift");
7028 PreShift = DAG.getConstant(Val: magics.PreShift, DL: dl, VT: ShSVT);
7029 PostShift = DAG.getConstant(Val: magics.PostShift, DL: dl, VT: ShSVT);
7030 NPQFactor = DAG.getConstant(
7031 Val: magics.IsAdd ? APInt::getOneBitSet(numBits: SVTBits, BitNo: EltBits - 1)
7032 : APInt::getZero(numBits: SVTBits),
7033 DL: dl, VT: SVT);
7034 UseNPQ |= magics.IsAdd;
7035 UsePreShift |= magics.PreShift != 0;
7036 UsePostShift |= magics.PostShift != 0;
7037 }
7038
7039 PreShifts.push_back(Elt: PreShift);
7040 MagicFactors.push_back(Elt: MagicFactor);
7041 NPQFactors.push_back(Elt: NPQFactor);
7042 PostShifts.push_back(Elt: PostShift);
7043 return true;
7044 };
7045
7046 // Collect the shifts/magic values from each element.
7047 if (!ISD::matchUnaryPredicate(Op: N1, Match: BuildUDIVPattern, /*AllowUndefs=*/false,
7048 /*AllowTruncation=*/true))
7049 return SDValue();
7050
7051 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
7052 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
7053 PreShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PreShifts);
7054 MagicFactor = DAG.getBuildVector(VT, DL: dl, Ops: MagicFactors);
7055 NPQFactor = DAG.getBuildVector(VT, DL: dl, Ops: NPQFactors);
7056 PostShift = DAG.getBuildVector(VT: ShVT, DL: dl, Ops: PostShifts);
7057 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
7058 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
7059 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
7060 "Expected matchUnaryPredicate to return one for scalable vectors");
7061 PreShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PreShifts[0]);
7062 MagicFactor = DAG.getSplatVector(VT, DL: dl, Op: MagicFactors[0]);
7063 NPQFactor = DAG.getSplatVector(VT, DL: dl, Op: NPQFactors[0]);
7064 PostShift = DAG.getSplatVector(VT: ShVT, DL: dl, Op: PostShifts[0]);
7065 } else {
7066 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
7067 PreShift = PreShifts[0];
7068 MagicFactor = MagicFactors[0];
7069 PostShift = PostShifts[0];
7070 }
7071
7072 if (UseWiden) {
7073 // Compute: (WideSVT(x) * MagicFactor) >> WideSVTBits.
7074 SDValue WideN0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: WideSVT, Operand: N0);
7075
7076 // Perform WideSVTxWideSVT -> 2*WideSVT multiplication and extract high
7077 // WideSVT bits
7078 SDValue High;
7079 if (HasWideMULHU) {
7080 High = DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT: WideSVT, N1: WideN0, N2: MagicFactor);
7081 } else {
7082 assert(HasWideUMUL_LOHI);
7083 SDValue LoHi =
7084 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: WideSVT, VT2: WideSVT),
7085 N1: WideN0, N2: MagicFactor);
7086 High = LoHi.getValue(R: 1);
7087 }
7088
7089 Created.push_back(Elt: High.getNode());
7090 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: High);
7091 }
7092
7093 SDValue Q = N0;
7094 if (UsePreShift) {
7095 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PreShift);
7096 Created.push_back(Elt: Q.getNode());
7097 }
7098
7099 auto GetMULHU = [&](SDValue X, SDValue Y) {
7100 if (HasMULHU)
7101 return DAG.getNode(Opcode: ISD::MULHU, DL: dl, VT, N1: X, N2: Y);
7102 if (HasUMUL_LOHI) {
7103 SDValue LoHi =
7104 DAG.getNode(Opcode: ISD::UMUL_LOHI, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: X, N2: Y);
7105 return LoHi.getValue(R: 1);
7106 }
7107
7108 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: X);
7109 Y = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MulVT, Operand: Y);
7110 Y = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MulVT, N1: X, N2: Y);
7111 Y = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MulVT, N1: Y,
7112 N2: DAG.getShiftAmountConstant(Val: EltBits, VT: MulVT, DL: dl));
7113 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Y);
7114 };
7115
7116 // Multiply the numerator (operand 0) by the magic value.
7117 Q = GetMULHU(Q, MagicFactor);
7118 if (!Q)
7119 return SDValue();
7120
7121 Created.push_back(Elt: Q.getNode());
7122
7123 if (UseNPQ) {
7124 SDValue NPQ = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: N0, N2: Q);
7125 Created.push_back(Elt: NPQ.getNode());
7126
7127 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
7128 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
7129 if (VT.isVector())
7130 NPQ = GetMULHU(NPQ, NPQFactor);
7131 else
7132 NPQ = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: NPQ, N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT));
7133
7134 Created.push_back(Elt: NPQ.getNode());
7135
7136 Q = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: NPQ, N2: Q);
7137 Created.push_back(Elt: Q.getNode());
7138 }
7139
7140 if (UsePostShift) {
7141 Q = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Q, N2: PostShift);
7142 Created.push_back(Elt: Q.getNode());
7143 }
7144
7145 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7146
7147 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT);
7148 SDValue IsOne = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: N1, RHS: One, Cond: ISD::SETEQ);
7149 return DAG.getSelect(DL: dl, VT, Cond: IsOne, LHS: N0, RHS: Q);
7150}
7151
7152/// If all values in Values that *don't* match the predicate are same 'splat'
7153/// value, then replace all values with that splat value.
7154/// Else, if AlternativeReplacement was provided, then replace all values that
7155/// do match predicate with AlternativeReplacement value.
7156static void
7157turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
7158 std::function<bool(SDValue)> Predicate,
7159 SDValue AlternativeReplacement = SDValue()) {
7160 SDValue Replacement;
7161 // Is there a value for which the Predicate does *NOT* match? What is it?
7162 auto SplatValue = llvm::find_if_not(Range&: Values, P: Predicate);
7163 if (SplatValue != Values.end()) {
7164 // Does Values consist only of SplatValue's and values matching Predicate?
7165 if (llvm::all_of(Range&: Values, P: [Predicate, SplatValue](SDValue Value) {
7166 return Value == *SplatValue || Predicate(Value);
7167 })) // Then we shall replace values matching predicate with SplatValue.
7168 Replacement = *SplatValue;
7169 }
7170 if (!Replacement) {
7171 // Oops, we did not find the "baseline" splat value.
7172 if (!AlternativeReplacement)
7173 return; // Nothing to do.
7174 // Let's replace with provided value then.
7175 Replacement = AlternativeReplacement;
7176 }
7177 std::replace_if(first: Values.begin(), last: Values.end(), pred: Predicate, new_value: Replacement);
7178}
7179
7180/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
7181/// where the divisor and comparison target are constants,
7182/// return a DAG expression that will generate the same comparison result
7183/// using only multiplications, additions and shifts/rotations.
7184/// Ref: "Hacker's Delight" 10-17.
7185SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
7186 SDValue CompTargetNode,
7187 ISD::CondCode Cond,
7188 DAGCombinerInfo &DCI,
7189 const SDLoc &DL) const {
7190 SmallVector<SDNode *, 5> Built;
7191 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7192 DCI, DL, Created&: Built)) {
7193 for (SDNode *N : Built)
7194 DCI.AddToWorklist(N);
7195 return Folded;
7196 }
7197
7198 return SDValue();
7199}
7200
7201SDValue
7202TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
7203 SDValue CompTargetNode, ISD::CondCode Cond,
7204 DAGCombinerInfo &DCI, const SDLoc &DL,
7205 SmallVectorImpl<SDNode *> &Created) const {
7206 // fold (seteq/ne (urem N, D), C) ->
7207 // (setule/ugt (rotr (mul (sub N, C), P), K), Q)
7208 // - D must be constant, with D = D0 * 2^K where D0 is odd
7209 // - P is the multiplicative inverse of D0 modulo 2^W
7210 // - Q = floor(((2^W) - 1) / D)
7211 // where W is the width of the common type of N and D.
7212 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7213 "Only applicable for (in)equality comparisons.");
7214
7215 SelectionDAG &DAG = DCI.DAG;
7216
7217 EVT VT = REMNode.getValueType();
7218 EVT SVT = VT.getScalarType();
7219 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7220 EVT ShSVT = ShVT.getScalarType();
7221
7222 // If MUL is unavailable, we cannot proceed in any case.
7223 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7224 return SDValue();
7225
7226 bool ComparingWithAllZeros = true;
7227 bool AllComparisonsWithNonZerosAreTautological = true;
7228 bool HadTautologicalLanes = false;
7229 bool AllLanesAreTautological = true;
7230 bool HadEvenDivisor = false;
7231 bool AllDivisorsArePowerOfTwo = true;
7232 bool HadTautologicalInvertedLanes = false;
7233 SmallVector<SDValue, 16> PAmts, KAmts, QAmts;
7234
7235 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
7236 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7237 if (CDiv->isZero())
7238 return false;
7239
7240 const APInt &D = CDiv->getAPIntValue();
7241 const APInt &Cmp = CCmp->getAPIntValue();
7242
7243 ComparingWithAllZeros &= Cmp.isZero();
7244
7245 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7246 // if C2 is not less than C1, the comparison is always false.
7247 // But we will only be able to produce the comparison that will give the
7248 // opposive tautological answer. So this lane would need to be fixed up.
7249 bool TautologicalInvertedLane = D.ule(RHS: Cmp);
7250 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
7251
7252 // If all lanes are tautological (either all divisors are ones, or divisor
7253 // is not greater than the constant we are comparing with),
7254 // we will prefer to avoid the fold.
7255 bool TautologicalLane = D.isOne() || TautologicalInvertedLane;
7256 HadTautologicalLanes |= TautologicalLane;
7257 AllLanesAreTautological &= TautologicalLane;
7258
7259 // If we are comparing with non-zero, we need'll need to subtract said
7260 // comparison value from the LHS. But there is no point in doing that if
7261 // every lane where we are comparing with non-zero is tautological..
7262 if (!Cmp.isZero())
7263 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
7264
7265 // Decompose D into D0 * 2^K
7266 unsigned K = D.countr_zero();
7267 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7268 APInt D0 = D.lshr(shiftAmt: K);
7269
7270 // D is even if it has trailing zeros.
7271 HadEvenDivisor |= (K != 0);
7272 // D is a power-of-two if D0 is one.
7273 // If all divisors are power-of-two, we will prefer to avoid the fold.
7274 AllDivisorsArePowerOfTwo &= D0.isOne();
7275
7276 // P = inv(D0, 2^W)
7277 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7278 unsigned W = D.getBitWidth();
7279 APInt P = D0.multiplicativeInverse();
7280 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7281
7282 // Q = floor((2^W - 1) u/ D)
7283 // R = ((2^W - 1) u% D)
7284 APInt Q, R;
7285 APInt::udivrem(LHS: APInt::getAllOnes(numBits: W), RHS: D, Quotient&: Q, Remainder&: R);
7286
7287 // If we are comparing with zero, then that comparison constant is okay,
7288 // else it may need to be one less than that.
7289 if (Cmp.ugt(RHS: R))
7290 Q -= 1;
7291
7292 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7293 "We are expecting that K is always less than all-ones for ShSVT");
7294
7295 // If the lane is tautological the result can be constant-folded.
7296 if (TautologicalLane) {
7297 // Set P and K amount to a bogus values so we can try to splat them.
7298 P = 0;
7299 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7300 // And ensure that comparison constant is tautological,
7301 // it will always compare true/false.
7302 Q.setAllBits();
7303 } else {
7304 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7305 }
7306
7307 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7308 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7309 return true;
7310 };
7311
7312 SDValue N = REMNode.getOperand(i: 0);
7313 SDValue D = REMNode.getOperand(i: 1);
7314
7315 // Collect the values from each element.
7316 if (!ISD::matchBinaryPredicate(LHS: D, RHS: CompTargetNode, Match: BuildUREMPattern))
7317 return SDValue();
7318
7319 // If all lanes are tautological, the result can be constant-folded.
7320 if (AllLanesAreTautological)
7321 return SDValue();
7322
7323 // If this is a urem by a powers-of-two, avoid the fold since it can be
7324 // best implemented as a bit test.
7325 if (AllDivisorsArePowerOfTwo)
7326 return SDValue();
7327
7328 SDValue PVal, KVal, QVal;
7329 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7330 if (HadTautologicalLanes) {
7331 // Try to turn PAmts into a splat, since we don't care about the values
7332 // that are currently '0'. If we can't, just keep '0'`s.
7333 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7334 // Try to turn KAmts into a splat, since we don't care about the values
7335 // that are currently '-1'. If we can't, change them to '0'`s.
7336 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7337 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7338 }
7339
7340 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7341 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7342 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7343 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7344 assert(PAmts.size() == 1 && KAmts.size() == 1 && QAmts.size() == 1 &&
7345 "Expected matchBinaryPredicate to return one element for "
7346 "SPLAT_VECTORs");
7347 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7348 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7349 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7350 } else {
7351 PVal = PAmts[0];
7352 KVal = KAmts[0];
7353 QVal = QAmts[0];
7354 }
7355
7356 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
7357 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::SUB, VT))
7358 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
7359 assert(CompTargetNode.getValueType() == N.getValueType() &&
7360 "Expecting that the types on LHS and RHS of comparisons match.");
7361 N = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N, N2: CompTargetNode);
7362 }
7363
7364 // (mul N, P)
7365 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7366 Created.push_back(Elt: Op0.getNode());
7367
7368 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7369 // divisors as a performance improvement, since rotating by 0 is a no-op.
7370 if (HadEvenDivisor) {
7371 // We need ROTR to do this.
7372 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7373 return SDValue();
7374 // UREM: (rotr (mul N, P), K)
7375 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7376 Created.push_back(Elt: Op0.getNode());
7377 }
7378
7379 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
7380 SDValue NewCC =
7381 DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7382 Cond: ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
7383 if (!HadTautologicalInvertedLanes)
7384 return NewCC;
7385
7386 // If any lanes previously compared always-false, the NewCC will give
7387 // always-true result for them, so we need to fixup those lanes.
7388 // Or the other way around for inequality predicate.
7389 assert(VT.isVector() && "Can/should only get here for vectors.");
7390 Created.push_back(Elt: NewCC.getNode());
7391
7392 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
7393 // if C2 is not less than C1, the comparison is always false.
7394 // But we have produced the comparison that will give the
7395 // opposive tautological answer. So these lanes would need to be fixed up.
7396 SDValue TautologicalInvertedChannels =
7397 DAG.getSetCC(DL, VT: SETCCVT, LHS: D, RHS: CompTargetNode, Cond: ISD::SETULE);
7398 Created.push_back(Elt: TautologicalInvertedChannels.getNode());
7399
7400 // NOTE: we avoid letting illegal types through even if we're before legalize
7401 // ops – legalization has a hard time producing good code for this.
7402 if (isOperationLegalOrCustom(Op: ISD::VSELECT, VT: SETCCVT)) {
7403 // If we have a vector select, let's replace the comparison results in the
7404 // affected lanes with the correct tautological result.
7405 SDValue Replacement = DAG.getBoolConstant(V: Cond == ISD::SETEQ ? false : true,
7406 DL, VT: SETCCVT, OpVT: SETCCVT);
7407 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: SETCCVT, N1: TautologicalInvertedChannels,
7408 N2: Replacement, N3: NewCC);
7409 }
7410
7411 // Else, we can just invert the comparison result in the appropriate lanes.
7412 //
7413 // NOTE: see the note above VSELECT above.
7414 if (isOperationLegalOrCustom(Op: ISD::XOR, VT: SETCCVT))
7415 return DAG.getNode(Opcode: ISD::XOR, DL, VT: SETCCVT, N1: NewCC,
7416 N2: TautologicalInvertedChannels);
7417
7418 return SDValue(); // Don't know how to lower.
7419}
7420
7421/// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
7422/// where the divisor is constant and the comparison target is zero,
7423/// return a DAG expression that will generate the same comparison result
7424/// using only multiplications, additions and shifts/rotations.
7425/// Ref: "Hacker's Delight" 10-17.
7426SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
7427 SDValue CompTargetNode,
7428 ISD::CondCode Cond,
7429 DAGCombinerInfo &DCI,
7430 const SDLoc &DL) const {
7431 SmallVector<SDNode *, 7> Built;
7432 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
7433 DCI, DL, Created&: Built)) {
7434 assert(Built.size() <= 7 && "Max size prediction failed.");
7435 for (SDNode *N : Built)
7436 DCI.AddToWorklist(N);
7437 return Folded;
7438 }
7439
7440 return SDValue();
7441}
7442
7443SDValue
7444TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
7445 SDValue CompTargetNode, ISD::CondCode Cond,
7446 DAGCombinerInfo &DCI, const SDLoc &DL,
7447 SmallVectorImpl<SDNode *> &Created) const {
7448 // Derived from Hacker's Delight, 2nd Edition, by Hank Warren. Section 10-17.
7449 // Fold:
7450 // (seteq/ne (srem N, D), 0)
7451 // To:
7452 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
7453 //
7454 // - D must be constant, with D = D0 * 2^K where D0 is odd
7455 // - P is the multiplicative inverse of D0 modulo 2^W
7456 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
7457 // - Q = floor((2 * A) / (2^K))
7458 // where W is the width of the common type of N and D.
7459 //
7460 // When D is a power of two (and thus D0 is 1), the normal
7461 // formula for A and Q don't apply, because the derivation
7462 // depends on D not dividing 2^(W-1), and thus theorem ZRS
7463 // does not apply. This specifically fails when N = INT_MIN.
7464 //
7465 // Instead, for power-of-two D, we use:
7466 // - A = 0
7467 // | -> No offset needed. We're effectively treating it the same as urem.
7468 // - Q = 2^(W-K) - 1
7469 // |-> Test that the top K bits are zero after rotation
7470 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
7471 "Only applicable for (in)equality comparisons.");
7472
7473 SelectionDAG &DAG = DCI.DAG;
7474
7475 EVT VT = REMNode.getValueType();
7476 EVT SVT = VT.getScalarType();
7477 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
7478 EVT ShSVT = ShVT.getScalarType();
7479
7480 // If we are after ops legalization, and MUL is unavailable, we can not
7481 // proceed.
7482 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::MUL, VT))
7483 return SDValue();
7484
7485 // TODO: Could support comparing with non-zero too.
7486 ConstantSDNode *CompTarget = isConstOrConstSplat(N: CompTargetNode);
7487 if (!CompTarget || !CompTarget->isZero())
7488 return SDValue();
7489
7490 bool HadOneDivisor = false;
7491 bool AllDivisorsAreOnes = true;
7492 bool HadEvenDivisor = false;
7493 bool AllDivisorsArePowerOfTwo = true;
7494 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
7495
7496 auto BuildSREMPattern = [&](ConstantSDNode *C) {
7497 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
7498 if (C->isZero())
7499 return false;
7500
7501 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
7502
7503 // WARNING: this fold is only valid for positive divisors!
7504 // `rem %X, -C` is equivalent to `rem %X, C`
7505 APInt D = C->getAPIntValue().abs();
7506
7507 // If all divisors are ones, we will prefer to avoid the fold.
7508 HadOneDivisor |= D.isOne();
7509 AllDivisorsAreOnes &= D.isOne();
7510
7511 // Decompose D into D0 * 2^K
7512 unsigned K = D.countr_zero();
7513 assert((!D.isOne() || (K == 0)) && "For divisor '1' we won't rotate.");
7514 APInt D0 = D.lshr(shiftAmt: K);
7515
7516 // D is even if it has trailing zeros.
7517 HadEvenDivisor |= (K != 0);
7518
7519 // D is a power-of-two if D0 is one. This includes INT_MIN.
7520 // If all divisors are power-of-two, we will prefer to avoid the fold.
7521 AllDivisorsArePowerOfTwo &= D0.isOne();
7522
7523 // P = inv(D0, 2^W)
7524 // 2^W requires W + 1 bits, so we have to extend and then truncate.
7525 unsigned W = D.getBitWidth();
7526 APInt P = D0.multiplicativeInverse();
7527 assert((D0 * P).isOne() && "Multiplicative inverse basic check failed.");
7528
7529 // A = floor((2^(W - 1) - 1) / D0) & -2^K
7530 APInt A = APInt::getSignedMaxValue(numBits: W).udiv(RHS: D0);
7531 A.clearLowBits(loBits: K);
7532
7533 // Q = floor((2 * A) / (2^K))
7534 APInt Q = (2 * A).udiv(RHS: APInt::getOneBitSet(numBits: W, BitNo: K));
7535
7536 assert(APInt::getAllOnes(SVT.getSizeInBits()).ugt(A) &&
7537 "We are expecting that A is always less than all-ones for SVT");
7538 assert(APInt::getAllOnes(ShSVT.getSizeInBits()).ugt(K) &&
7539 "We are expecting that K is always less than all-ones for ShSVT");
7540
7541 // If D was a power of two, apply the alternate constant derivation.
7542 if (D0.isOne()) {
7543 // A = 0
7544 A = APInt(W, 0);
7545 // - Q = 2^(W-K) - 1
7546 Q = APInt::getLowBitsSet(numBits: W, loBitsSet: W - K);
7547 }
7548
7549 // If the divisor is 1 the result can be constant-folded.
7550 if (D.isOne()) {
7551 // Set P, A and K to a bogus values so we can try to splat them.
7552 P = 0;
7553 A.setAllBits();
7554 KAmts.push_back(Elt: DAG.getAllOnesConstant(DL, VT: ShSVT));
7555
7556 // x ?% 1 == 0 <--> true <--> x u<= -1
7557 Q.setAllBits();
7558 } else {
7559 KAmts.push_back(Elt: DAG.getConstant(Val: K, DL, VT: ShSVT));
7560 }
7561
7562 PAmts.push_back(Elt: DAG.getConstant(Val: P, DL, VT: SVT));
7563 AAmts.push_back(Elt: DAG.getConstant(Val: A, DL, VT: SVT));
7564 QAmts.push_back(Elt: DAG.getConstant(Val: Q, DL, VT: SVT));
7565 return true;
7566 };
7567
7568 SDValue N = REMNode.getOperand(i: 0);
7569 SDValue D = REMNode.getOperand(i: 1);
7570
7571 // Collect the values from each element.
7572 if (!ISD::matchUnaryPredicate(Op: D, Match: BuildSREMPattern))
7573 return SDValue();
7574
7575 // If this is a srem by a one, avoid the fold since it can be constant-folded.
7576 if (AllDivisorsAreOnes)
7577 return SDValue();
7578
7579 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
7580 // since it can be best implemented as a bit test.
7581 if (AllDivisorsArePowerOfTwo)
7582 return SDValue();
7583
7584 SDValue PVal, AVal, KVal, QVal;
7585 if (D.getOpcode() == ISD::BUILD_VECTOR) {
7586 if (HadOneDivisor) {
7587 // Try to turn PAmts into a splat, since we don't care about the values
7588 // that are currently '0'. If we can't, just keep '0'`s.
7589 turnVectorIntoSplatVector(Values: PAmts, Predicate: isNullConstant);
7590 // Try to turn AAmts into a splat, since we don't care about the
7591 // values that are currently '-1'. If we can't, change them to '0'`s.
7592 turnVectorIntoSplatVector(Values: AAmts, Predicate: isAllOnesConstant,
7593 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: SVT));
7594 // Try to turn KAmts into a splat, since we don't care about the values
7595 // that are currently '-1'. If we can't, change them to '0'`s.
7596 turnVectorIntoSplatVector(Values: KAmts, Predicate: isAllOnesConstant,
7597 AlternativeReplacement: DAG.getConstant(Val: 0, DL, VT: ShSVT));
7598 }
7599
7600 PVal = DAG.getBuildVector(VT, DL, Ops: PAmts);
7601 AVal = DAG.getBuildVector(VT, DL, Ops: AAmts);
7602 KVal = DAG.getBuildVector(VT: ShVT, DL, Ops: KAmts);
7603 QVal = DAG.getBuildVector(VT, DL, Ops: QAmts);
7604 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
7605 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
7606 QAmts.size() == 1 &&
7607 "Expected matchUnaryPredicate to return one element for scalable "
7608 "vectors");
7609 PVal = DAG.getSplatVector(VT, DL, Op: PAmts[0]);
7610 AVal = DAG.getSplatVector(VT, DL, Op: AAmts[0]);
7611 KVal = DAG.getSplatVector(VT: ShVT, DL, Op: KAmts[0]);
7612 QVal = DAG.getSplatVector(VT, DL, Op: QAmts[0]);
7613 } else {
7614 assert(isa<ConstantSDNode>(D) && "Expected a constant");
7615 PVal = PAmts[0];
7616 AVal = AAmts[0];
7617 KVal = KAmts[0];
7618 QVal = QAmts[0];
7619 }
7620
7621 // (mul N, P)
7622 SDValue Op0 = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N, N2: PVal);
7623 Created.push_back(Elt: Op0.getNode());
7624
7625 // We need ADD to do this.
7626 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ADD, VT))
7627 return SDValue();
7628
7629 // (add (mul N, P), A)
7630 Op0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: AVal);
7631 Created.push_back(Elt: Op0.getNode());
7632
7633 // Rotate right only if any divisor was even. We avoid rotates for all-odd
7634 // divisors as a performance improvement, since rotating by 0 is a no-op.
7635 if (HadEvenDivisor) {
7636 // We need ROTR to do this.
7637 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(Op: ISD::ROTR, VT))
7638 return SDValue();
7639 // SREM: (rotr (add (mul N, P), A), K)
7640 Op0 = DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: Op0, N2: KVal);
7641 Created.push_back(Elt: Op0.getNode());
7642 }
7643
7644 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
7645 return DAG.getSetCC(DL, VT: SETCCVT, LHS: Op0, RHS: QVal,
7646 Cond: (Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT);
7647}
7648
7649SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
7650 const DenormalMode &Mode,
7651 SDNodeFlags Flags) const {
7652 SDLoc DL(Op);
7653 EVT VT = Op.getValueType();
7654 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
7655 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL, VT);
7656
7657 // This is specifically a check for the handling of denormal inputs, not the
7658 // result.
7659 if (Mode.Input == DenormalMode::PreserveSign ||
7660 Mode.Input == DenormalMode::PositiveZero) {
7661 // Test = X == 0.0
7662 return DAG.getSetCC(DL, VT: CCVT, LHS: Op, RHS: FPZero, Cond: ISD::SETEQ, /*Chain=*/{},
7663 /*Signaling=*/IsSignaling: false, Flags);
7664 }
7665
7666 // Testing it with denormal inputs to avoid wrong estimate.
7667 //
7668 // Test = fabs(X) < SmallestNormal
7669 const fltSemantics &FltSem = VT.getFltSemantics();
7670 APFloat SmallestNorm = APFloat::getSmallestNormalized(Sem: FltSem);
7671 SDValue NormC = DAG.getConstantFP(Val: SmallestNorm, DL, VT);
7672 SDValue Fabs = DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: Op, Flags);
7673 return DAG.getSetCC(DL, VT: CCVT, LHS: Fabs, RHS: NormC, Cond: ISD::SETLT, /*Chain=*/{},
7674 /*Signaling=*/IsSignaling: false, Flags);
7675}
7676
7677SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
7678 bool LegalOps, bool OptForSize,
7679 NegatibleCost &Cost,
7680 unsigned Depth) const {
7681 // fneg is removable even if it has multiple uses.
7682 if (Op.getOpcode() == ISD::FNEG) {
7683 Cost = NegatibleCost::Cheaper;
7684 return Op.getOperand(i: 0);
7685 }
7686
7687 // Don't recurse exponentially.
7688 if (Depth > SelectionDAG::MaxRecursionDepth)
7689 return SDValue();
7690
7691 // Pre-increment recursion depth for use in recursive calls.
7692 ++Depth;
7693 const SDNodeFlags Flags = Op->getFlags();
7694 EVT VT = Op.getValueType();
7695 unsigned Opcode = Op.getOpcode();
7696
7697 // Don't allow anything with multiple uses unless we know it is free.
7698 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
7699 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
7700 isFPExtFree(DestVT: VT, SrcVT: Op.getOperand(i: 0).getValueType());
7701 if (!IsFreeExtend)
7702 return SDValue();
7703 }
7704
7705 auto RemoveDeadNode = [&](SDValue N) {
7706 if (N && N.getNode()->use_empty())
7707 DAG.RemoveDeadNode(N: N.getNode());
7708 };
7709
7710 SDLoc DL(Op);
7711
7712 // Because getNegatedExpression can delete nodes we need a handle to keep
7713 // temporary nodes alive in case the recursion manages to create an identical
7714 // node.
7715 std::list<HandleSDNode> Handles;
7716
7717 switch (Opcode) {
7718 case ISD::ConstantFP: {
7719 // Don't invert constant FP values after legalization unless the target says
7720 // the negated constant is legal.
7721 bool IsOpLegal =
7722 isOperationLegal(Op: ISD::ConstantFP, VT) ||
7723 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: Op)->getValueAPF()), VT,
7724 ForCodeSize: OptForSize);
7725
7726 if (LegalOps && !IsOpLegal)
7727 break;
7728
7729 APFloat V = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
7730 V.changeSign();
7731 SDValue CFP = DAG.getConstantFP(Val: V, DL, VT);
7732
7733 // If we already have the use of the negated floating constant, it is free
7734 // to negate it even it has multiple uses.
7735 if (!Op.hasOneUse() && CFP.use_empty())
7736 break;
7737 Cost = NegatibleCost::Neutral;
7738 return CFP;
7739 }
7740 case ISD::SPLAT_VECTOR: {
7741 // fold splat_vector(fneg(X)) -> splat_vector(-X)
7742 SDValue X = Op.getOperand(i: 0);
7743 if (!isOperationLegal(Op: ISD::SPLAT_VECTOR, VT))
7744 break;
7745
7746 SDValue NegX = getCheaperNegatedExpression(Op: X, DAG, LegalOps, OptForSize);
7747 if (!NegX)
7748 break;
7749 Cost = NegatibleCost::Cheaper;
7750 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: NegX);
7751 }
7752 case ISD::BUILD_VECTOR: {
7753 // Only permit BUILD_VECTOR of constants.
7754 if (llvm::any_of(Range: Op->op_values(), P: [&](SDValue N) {
7755 return !N.isUndef() && !isa<ConstantFPSDNode>(Val: N);
7756 }))
7757 break;
7758
7759 bool IsOpLegal =
7760 (isOperationLegal(Op: ISD::ConstantFP, VT) &&
7761 isOperationLegal(Op: ISD::BUILD_VECTOR, VT)) ||
7762 llvm::all_of(Range: Op->op_values(), P: [&](SDValue N) {
7763 return N.isUndef() ||
7764 isFPImmLegal(neg(X: cast<ConstantFPSDNode>(Val&: N)->getValueAPF()), VT,
7765 ForCodeSize: OptForSize);
7766 });
7767
7768 if (LegalOps && !IsOpLegal)
7769 break;
7770
7771 SmallVector<SDValue, 4> Ops;
7772 for (SDValue C : Op->op_values()) {
7773 if (C.isUndef()) {
7774 Ops.push_back(Elt: C);
7775 continue;
7776 }
7777 APFloat V = cast<ConstantFPSDNode>(Val&: C)->getValueAPF();
7778 V.changeSign();
7779 Ops.push_back(Elt: DAG.getConstantFP(Val: V, DL, VT: C.getValueType()));
7780 }
7781 Cost = NegatibleCost::Neutral;
7782 return DAG.getBuildVector(VT, DL, Ops);
7783 }
7784 case ISD::FADD: {
7785 if (!Flags.hasNoSignedZeros())
7786 break;
7787
7788 // After operation legalization, it might not be legal to create new FSUBs.
7789 if (LegalOps && !isOperationLegalOrCustom(Op: ISD::FSUB, VT))
7790 break;
7791 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7792
7793 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
7794 NegatibleCost CostX = NegatibleCost::Expensive;
7795 SDValue NegX =
7796 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
7797 // Prevent this node from being deleted by the next call.
7798 if (NegX)
7799 Handles.emplace_back(args&: NegX);
7800
7801 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
7802 NegatibleCost CostY = NegatibleCost::Expensive;
7803 SDValue NegY =
7804 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
7805
7806 // We're done with the handles.
7807 Handles.clear();
7808
7809 // Negate the X if its cost is less or equal than Y.
7810 if (NegX && (CostX <= CostY)) {
7811 Cost = CostX;
7812 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegX, N2: Y, Flags);
7813 if (NegY != N)
7814 RemoveDeadNode(NegY);
7815 return N;
7816 }
7817
7818 // Negate the Y if it is not expensive.
7819 if (NegY) {
7820 Cost = CostY;
7821 SDValue N = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: NegY, N2: X, Flags);
7822 if (NegX != N)
7823 RemoveDeadNode(NegX);
7824 return N;
7825 }
7826 break;
7827 }
7828 case ISD::FSUB: {
7829 // We can't turn -(A-B) into B-A when we honor signed zeros.
7830 if (!Flags.hasNoSignedZeros())
7831 break;
7832
7833 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7834 // fold (fneg (fsub 0, Y)) -> Y
7835 if (ConstantFPSDNode *C = isConstOrConstSplatFP(N: X, /*AllowUndefs*/ true))
7836 if (C->isZero()) {
7837 Cost = NegatibleCost::Cheaper;
7838 return Y;
7839 }
7840
7841 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
7842 Cost = NegatibleCost::Neutral;
7843 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: Y, N2: X, Flags);
7844 }
7845 case ISD::FMUL:
7846 case ISD::FDIV: {
7847 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1);
7848
7849 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
7850 NegatibleCost CostX = NegatibleCost::Expensive;
7851 SDValue NegX =
7852 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
7853 // Prevent this node from being deleted by the next call.
7854 if (NegX)
7855 Handles.emplace_back(args&: NegX);
7856
7857 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
7858 NegatibleCost CostY = NegatibleCost::Expensive;
7859 SDValue NegY =
7860 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
7861
7862 // We're done with the handles.
7863 Handles.clear();
7864
7865 // Negate the X if its cost is less or equal than Y.
7866 if (NegX && (CostX <= CostY)) {
7867 Cost = CostX;
7868 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, Flags);
7869 if (NegY != N)
7870 RemoveDeadNode(NegY);
7871 return N;
7872 }
7873
7874 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
7875 if (auto *C = isConstOrConstSplatFP(N: Op.getOperand(i: 1)))
7876 if (C->isExactlyValue(V: 2.0) && Op.getOpcode() == ISD::FMUL)
7877 break;
7878
7879 // Negate the Y if it is not expensive.
7880 if (NegY) {
7881 Cost = CostY;
7882 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, Flags);
7883 if (NegX != N)
7884 RemoveDeadNode(NegX);
7885 return N;
7886 }
7887 break;
7888 }
7889 case ISD::FMA:
7890 case ISD::FMULADD:
7891 case ISD::FMAD: {
7892 if (!Flags.hasNoSignedZeros())
7893 break;
7894
7895 SDValue X = Op.getOperand(i: 0), Y = Op.getOperand(i: 1), Z = Op.getOperand(i: 2);
7896 NegatibleCost CostZ = NegatibleCost::Expensive;
7897 SDValue NegZ =
7898 getNegatedExpression(Op: Z, DAG, LegalOps, OptForSize, Cost&: CostZ, Depth);
7899 // Give up if fail to negate the Z.
7900 if (!NegZ)
7901 break;
7902
7903 // Prevent this node from being deleted by the next two calls.
7904 Handles.emplace_back(args&: NegZ);
7905
7906 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
7907 NegatibleCost CostX = NegatibleCost::Expensive;
7908 SDValue NegX =
7909 getNegatedExpression(Op: X, DAG, LegalOps, OptForSize, Cost&: CostX, Depth);
7910 // Prevent this node from being deleted by the next call.
7911 if (NegX)
7912 Handles.emplace_back(args&: NegX);
7913
7914 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
7915 NegatibleCost CostY = NegatibleCost::Expensive;
7916 SDValue NegY =
7917 getNegatedExpression(Op: Y, DAG, LegalOps, OptForSize, Cost&: CostY, Depth);
7918
7919 // We're done with the handles.
7920 Handles.clear();
7921
7922 // Negate the X if its cost is less or equal than Y.
7923 if (NegX && (CostX <= CostY)) {
7924 Cost = std::min(a: CostX, b: CostZ);
7925 SDValue N = DAG.getNode(Opcode, DL, VT, N1: NegX, N2: Y, N3: NegZ, Flags);
7926 if (NegY != N)
7927 RemoveDeadNode(NegY);
7928 return N;
7929 }
7930
7931 // Negate the Y if it is not expensive.
7932 if (NegY) {
7933 Cost = std::min(a: CostY, b: CostZ);
7934 SDValue N = DAG.getNode(Opcode, DL, VT, N1: X, N2: NegY, N3: NegZ, Flags);
7935 if (NegX != N)
7936 RemoveDeadNode(NegX);
7937 return N;
7938 }
7939 break;
7940 }
7941
7942 case ISD::FP_EXTEND:
7943 case ISD::FSIN:
7944 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
7945 OptForSize, Cost, Depth))
7946 return DAG.getNode(Opcode, DL, VT, Operand: NegV);
7947 break;
7948 case ISD::FP_ROUND:
7949 if (SDValue NegV = getNegatedExpression(Op: Op.getOperand(i: 0), DAG, LegalOps,
7950 OptForSize, Cost, Depth))
7951 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: NegV, N2: Op.getOperand(i: 1));
7952 break;
7953 case ISD::SELECT:
7954 case ISD::VSELECT: {
7955 // fold (fneg (select C, LHS, RHS)) -> (select C, (fneg LHS), (fneg RHS))
7956 // iff at least one cost is cheaper and the other is neutral/cheaper
7957 SDValue LHS = Op.getOperand(i: 1);
7958 NegatibleCost CostLHS = NegatibleCost::Expensive;
7959 SDValue NegLHS =
7960 getNegatedExpression(Op: LHS, DAG, LegalOps, OptForSize, Cost&: CostLHS, Depth);
7961 if (!NegLHS || CostLHS > NegatibleCost::Neutral) {
7962 RemoveDeadNode(NegLHS);
7963 break;
7964 }
7965
7966 // Prevent this node from being deleted by the next call.
7967 Handles.emplace_back(args&: NegLHS);
7968
7969 SDValue RHS = Op.getOperand(i: 2);
7970 NegatibleCost CostRHS = NegatibleCost::Expensive;
7971 SDValue NegRHS =
7972 getNegatedExpression(Op: RHS, DAG, LegalOps, OptForSize, Cost&: CostRHS, Depth);
7973
7974 // We're done with the handles.
7975 Handles.clear();
7976
7977 if (!NegRHS || CostRHS > NegatibleCost::Neutral ||
7978 (CostLHS != NegatibleCost::Cheaper &&
7979 CostRHS != NegatibleCost::Cheaper)) {
7980 RemoveDeadNode(NegLHS);
7981 RemoveDeadNode(NegRHS);
7982 break;
7983 }
7984
7985 Cost = std::min(a: CostLHS, b: CostRHS);
7986 return DAG.getSelect(DL, VT, Cond: Op.getOperand(i: 0), LHS: NegLHS, RHS: NegRHS);
7987 }
7988 }
7989
7990 return SDValue();
7991}
7992
7993//===----------------------------------------------------------------------===//
7994// Legalization Utilities
7995//===----------------------------------------------------------------------===//
7996
7997bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
7998 SDValue LHS, SDValue RHS,
7999 SmallVectorImpl<SDValue> &Result,
8000 EVT HiLoVT, SelectionDAG &DAG,
8001 MulExpansionKind Kind, SDValue LL,
8002 SDValue LH, SDValue RL, SDValue RH) const {
8003 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
8004 Opcode == ISD::SMUL_LOHI);
8005
8006 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
8007 isOperationLegalOrCustom(Op: ISD::MULHS, VT: HiLoVT);
8008 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
8009 isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT);
8010 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8011 isOperationLegalOrCustom(Op: ISD::SMUL_LOHI, VT: HiLoVT);
8012 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
8013 isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT);
8014
8015 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
8016 return false;
8017
8018 unsigned OuterBitSize = VT.getScalarSizeInBits();
8019 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
8020
8021 // LL, LH, RL, and RH must be either all NULL or all set to a value.
8022 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
8023 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
8024
8025 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
8026 bool Signed) -> bool {
8027 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
8028 SDVTList VTs = DAG.getVTList(VT1: HiLoVT, VT2: HiLoVT);
8029 Lo = DAG.getNode(Opcode: Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, DL: dl, VTList: VTs, N1: L, N2: R);
8030 Hi = Lo.getValue(R: 1);
8031 return true;
8032 }
8033 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
8034 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: L, N2: R);
8035 Hi = DAG.getNode(Opcode: Signed ? ISD::MULHS : ISD::MULHU, DL: dl, VT: HiLoVT, N1: L, N2: R);
8036 return true;
8037 }
8038 return false;
8039 };
8040
8041 SDValue Lo, Hi;
8042
8043 if (!LL.getNode() && !RL.getNode() &&
8044 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8045 LL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LHS);
8046 RL = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RHS);
8047 }
8048
8049 if (!LL.getNode())
8050 return false;
8051
8052 APInt HighMask = APInt::getHighBitsSet(numBits: OuterBitSize, hiBitsSet: InnerBitSize);
8053 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask) &&
8054 DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask)) {
8055 // The inputs are both zero-extended.
8056 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
8057 Result.push_back(Elt: Lo);
8058 Result.push_back(Elt: Hi);
8059 if (Opcode != ISD::MUL) {
8060 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8061 Result.push_back(Elt: Zero);
8062 Result.push_back(Elt: Zero);
8063 }
8064 return true;
8065 }
8066 }
8067
8068 if (!VT.isVector() && Opcode == ISD::MUL &&
8069 DAG.ComputeMaxSignificantBits(Op: LHS) <= InnerBitSize &&
8070 DAG.ComputeMaxSignificantBits(Op: RHS) <= InnerBitSize) {
8071 // The input values are both sign-extended.
8072 // TODO non-MUL case?
8073 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
8074 Result.push_back(Elt: Lo);
8075 Result.push_back(Elt: Hi);
8076 return true;
8077 }
8078 }
8079
8080 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
8081 SDValue Shift = DAG.getShiftAmountConstant(Val: ShiftAmount, VT, DL: dl);
8082
8083 if (!LH.getNode() && !RH.getNode() &&
8084 isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
8085 isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: HiLoVT)) {
8086 LH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: LHS, N2: Shift);
8087 LH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: LH);
8088 RH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: RHS, N2: Shift);
8089 RH = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: RH);
8090 }
8091
8092 if (!LH.getNode())
8093 return false;
8094
8095 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
8096 return false;
8097
8098 Result.push_back(Elt: Lo);
8099
8100 if (Opcode == ISD::MUL) {
8101 RH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LL, N2: RH);
8102 LH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: HiLoVT, N1: LH, N2: RL);
8103 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: RH);
8104 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Hi, N2: LH);
8105 Result.push_back(Elt: Hi);
8106 return true;
8107 }
8108
8109 // Compute the full width result.
8110 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
8111 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Lo);
8112 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8113 Hi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: Shift);
8114 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Lo, N2: Hi);
8115 };
8116
8117 SDValue Next = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Hi);
8118 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
8119 return false;
8120
8121 // This is effectively the add part of a multiply-add of half-sized operands,
8122 // so it cannot overflow.
8123 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8124
8125 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
8126 return false;
8127
8128 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8129 EVT BoolType = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
8130
8131 bool UseGlue = (isOperationLegalOrCustom(Op: ISD::ADDC, VT) &&
8132 isOperationLegalOrCustom(Op: ISD::ADDE, VT));
8133 if (UseGlue)
8134 Next = DAG.getNode(Opcode: ISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Glue), N1: Next,
8135 N2: Merge(Lo, Hi));
8136 else
8137 Next = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolType), N1: Next,
8138 N2: Merge(Lo, Hi), N3: DAG.getConstant(Val: 0, DL: dl, VT: BoolType));
8139
8140 SDValue Carry = Next.getValue(R: 1);
8141 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8142 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8143
8144 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
8145 return false;
8146
8147 if (UseGlue)
8148 Hi = DAG.getNode(Opcode: ISD::ADDE, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::Glue), N1: Hi, N2: Zero,
8149 N3: Carry);
8150 else
8151 Hi = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList: DAG.getVTList(VT1: HiLoVT, VT2: BoolType), N1: Hi,
8152 N2: Zero, N3: Carry);
8153
8154 Next = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Next, N2: Merge(Lo, Hi));
8155
8156 if (Opcode == ISD::SMUL_LOHI) {
8157 SDValue NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8158 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: RL));
8159 Next = DAG.getSelectCC(DL: dl, LHS: LH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8160
8161 NextSub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Next,
8162 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: LL));
8163 Next = DAG.getSelectCC(DL: dl, LHS: RH, RHS: Zero, True: NextSub, False: Next, Cond: ISD::SETLT);
8164 }
8165
8166 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8167 Next = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Next, N2: Shift);
8168 Result.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: HiLoVT, Operand: Next));
8169 return true;
8170}
8171
8172bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
8173 SelectionDAG &DAG, MulExpansionKind Kind,
8174 SDValue LL, SDValue LH, SDValue RL,
8175 SDValue RH) const {
8176 SmallVector<SDValue, 2> Result;
8177 bool Ok = expandMUL_LOHI(Opcode: N->getOpcode(), VT: N->getValueType(ResNo: 0), dl: SDLoc(N),
8178 LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), Result, HiLoVT,
8179 DAG, Kind, LL, LH, RL, RH);
8180 if (Ok) {
8181 assert(Result.size() == 2);
8182 Lo = Result[0];
8183 Hi = Result[1];
8184 }
8185 return Ok;
8186}
8187
8188// Optimize unsigned division or remainder by constants for types twice as large
8189// as a legal VT.
8190//
8191// If (1 << (BitWidth / 2)) % Constant == 1, then the remainder
8192// can be computed
8193// as:
8194// Sum = __builtin_uadd_overflow(Lo, High, &Sum);
8195// Remainder = Sum % Constant;
8196//
8197// If (1 << (BitWidth / 2)) % Constant != 1, we can search for a smaller value
8198// W such that W != (BitWidth / 2) and (1 << W) % Constant == 1. We can break
8199// High:Low into 3 chunks of W bits and compute remainder as
8200// Sum = Chunk0 + Chunk1 + Chunk2;
8201// Remainder = Sum % Constant;
8202//
8203// This is based on "Remainder by Summing Digits" from Hacker's Delight.
8204//
8205// For division, we can compute the remainder using the algorithm described
8206// above, subtract it from the dividend to get an exact multiple of Constant.
8207// Then multiply that exact multiply by the multiplicative inverse modulo
8208// (1 << (BitWidth / 2)) to get the quotient.
8209
8210// If Constant is even, we can shift right the dividend and the divisor by the
8211// number of trailing zeros in Constant before applying the remainder algorithm.
8212// If we're after the quotient, we can subtract this value from the shifted
8213// dividend and multiply by the multiplicative inverse of the shifted divisor.
8214// If we want the remainder, we shift the value left by the number of trailing
8215// zeros and add the bits that were shifted out of the dividend.
8216bool TargetLowering::expandUDIVREMByConstantViaUREMDecomposition(
8217 SDNode *N, APInt Divisor, SmallVectorImpl<SDValue> &Result, EVT HiLoVT,
8218 SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8219 unsigned Opcode = N->getOpcode();
8220 EVT VT = N->getValueType(ResNo: 0);
8221
8222 unsigned BitWidth = Divisor.getBitWidth();
8223 unsigned HBitWidth = BitWidth / 2;
8224 assert(VT.getScalarSizeInBits() == BitWidth &&
8225 HiLoVT.getScalarSizeInBits() == HBitWidth && "Unexpected VTs");
8226
8227 // If the divisor is even, shift it until it becomes odd.
8228 unsigned TrailingZeros = 0;
8229 if (!Divisor[0]) {
8230 TrailingZeros = Divisor.countr_zero();
8231 Divisor.lshrInPlace(ShiftAmt: TrailingZeros);
8232 }
8233
8234 // After removing trailing zeros, the divisor needs to be less than
8235 // (1 << HBitWidth).
8236 APInt HalfMaxPlus1 = APInt::getOneBitSet(numBits: BitWidth, BitNo: HBitWidth);
8237 if (Divisor.uge(RHS: HalfMaxPlus1))
8238 return false;
8239
8240 // Look for the largest chunk width W such that (1 << W) % Divisor == 1 or
8241 // (1 << W) % Divisor == -1.
8242 unsigned BestChunkWidth = 0, AltChunkWidth = 0;
8243 for (unsigned I = HBitWidth, E = HBitWidth / 2; I > E; --I) {
8244 // Skip HBitWidth-1, it doesn't have enough bits for carries.
8245 if (I == HBitWidth - 1)
8246 continue;
8247
8248 APInt Mod = APInt::getOneBitSet(numBits: Divisor.getBitWidth(), BitNo: I).urem(RHS: Divisor);
8249
8250 if (Mod.isOne()) {
8251 BestChunkWidth = I;
8252 break;
8253 }
8254
8255 // We have an alternate strategy for Remainder == Divisor - 1.
8256 // FIXME: Support HBitWidth.
8257 if (I != HBitWidth && Mod == Divisor - 1)
8258 AltChunkWidth = I;
8259 }
8260
8261 bool Alternate = false;
8262 if (!BestChunkWidth) {
8263 if (!AltChunkWidth)
8264 return false;
8265 Alternate = true;
8266 BestChunkWidth = AltChunkWidth;
8267 }
8268
8269 SDLoc dl(N);
8270
8271 assert(!LL == !LH && "Expected both input halves or no input halves!");
8272 if (!LL)
8273 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N->getOperand(Num: 0), DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8274
8275 bool HasFSHR = isOperationLegal(Op: ISD::FSHR, VT: HiLoVT);
8276
8277 auto GetFSHR = [&](SDValue Lo, SDValue Hi, unsigned ShiftAmt) {
8278 assert(ShiftAmt > 0 && ShiftAmt < HBitWidth);
8279 if (HasFSHR)
8280 return DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT: HiLoVT, N1: Hi, N2: Lo,
8281 N3: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8282 return DAG.getNode(
8283 Opcode: ISD::OR, DL: dl, VT: HiLoVT,
8284 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Lo,
8285 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl)),
8286 N2: DAG.getNode(
8287 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: Hi,
8288 N2: DAG.getShiftAmountConstant(Val: HBitWidth - ShiftAmt, VT: HiLoVT, DL: dl)));
8289 };
8290
8291 // Helper to perform a right shift on a 128-bit value split into two halves.
8292 // Handles shifts >= HBitWidth by moving Hi to Lo and shifting Hi.
8293 auto ShiftRight = [&](SDValue &Lo, SDValue &Hi, unsigned ShiftAmt) {
8294 if (ShiftAmt == 0)
8295 return;
8296 if (ShiftAmt < HBitWidth) {
8297 Lo = GetFSHR(Lo, Hi, ShiftAmt);
8298 Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8299 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: HiLoVT, DL: dl));
8300 } else if (ShiftAmt == HBitWidth) {
8301 Lo = Hi;
8302 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8303 } else {
8304 Lo = DAG.getNode(
8305 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: Hi,
8306 N2: DAG.getShiftAmountConstant(Val: ShiftAmt - HBitWidth, VT: HiLoVT, DL: dl));
8307 Hi = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8308 }
8309 };
8310
8311 // Shift the input by the number of TrailingZeros in the divisor. The
8312 // shifted out bits will be added to the remainder later.
8313 SDValue PartialRemL, PartialRemH;
8314 if (TrailingZeros && Opcode != ISD::UDIV) {
8315 // Save the shifted off bits if we need the remainder.
8316 if (TrailingZeros < HBitWidth) {
8317 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros);
8318 PartialRemL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LL,
8319 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8320 } else if (TrailingZeros == HBitWidth) {
8321 // All of LL is part of the remainder.
8322 PartialRemL = LL;
8323 } else {
8324 // TrailingZeros > HBitWidth: LL and part of LH are the remainder.
8325 PartialRemL = LL;
8326 APInt Mask = APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: TrailingZeros - HBitWidth);
8327 PartialRemH = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: LH,
8328 N2: DAG.getConstant(Val: Mask, DL: dl, VT: HiLoVT));
8329 }
8330 }
8331
8332 SDValue Sum;
8333 // If BestChunkWidth is HBitWidth add low and high half. If there is a carry
8334 // out, add that to the final sum.
8335 if (BestChunkWidth == HBitWidth) {
8336 assert(!Alternate);
8337 // Shift LH:LL right if there were trailing zeros in the divisor.
8338 ShiftRight(LL, LH, TrailingZeros);
8339
8340 // Use uaddo_carry if we can, otherwise use a compare to detect overflow.
8341 EVT SetCCType =
8342 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: HiLoVT);
8343 if (isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT: HiLoVT)) {
8344 SDVTList VTList = DAG.getVTList(VT1: HiLoVT, VT2: SetCCType);
8345 Sum = DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList, N1: LL, N2: LH);
8346 Sum = DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: dl, VTList, N1: Sum,
8347 N2: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT), N3: Sum.getValue(R: 1));
8348 } else {
8349 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: LL, N2: LH);
8350 SDValue Carry = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Sum, RHS: LL, Cond: ISD::SETULT);
8351 // If the boolean for the target is 0 or 1, we can add the setcc result
8352 // directly.
8353 if (getBooleanContents(Type: HiLoVT) ==
8354 TargetLoweringBase::ZeroOrOneBooleanContent)
8355 Carry = DAG.getZExtOrTrunc(Op: Carry, DL: dl, VT: HiLoVT);
8356 else
8357 Carry = DAG.getSelect(DL: dl, VT: HiLoVT, Cond: Carry, LHS: DAG.getConstant(Val: 1, DL: dl, VT: HiLoVT),
8358 RHS: DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT));
8359 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum, N2: Carry);
8360 }
8361 } else {
8362 // Otherwise split into multple chunks and add them together. We chose
8363 // BestChunkWidth so that the sum will not overflow.
8364 SDValue Mask = DAG.getConstant(
8365 Val: APInt::getLowBitsSet(numBits: HBitWidth, loBitsSet: BestChunkWidth), DL: dl, VT: HiLoVT);
8366
8367 for (unsigned I = 0; I < BitWidth - TrailingZeros; I += BestChunkWidth) {
8368 // If there were trailing zeros in the divisor, increase the shift amount.
8369 unsigned Shift = I + TrailingZeros;
8370 SDValue Chunk;
8371 if (Shift == 0)
8372 Chunk = LL;
8373 else if (Shift >= HBitWidth)
8374 Chunk = DAG.getNode(
8375 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: LH,
8376 N2: DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL: dl));
8377 else
8378 Chunk = GetFSHR(LL, LH, Shift);
8379 // If we're on the last chunk, we don't need an AND.
8380 if (I + BestChunkWidth < BitWidth - TrailingZeros)
8381 Chunk = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: HiLoVT, N1: Chunk, N2: Mask);
8382 if (!Sum) {
8383 Sum = Chunk;
8384 } else {
8385 // For Alternate, we need to subtract odd chunks.
8386 unsigned ChunkNum = I / BestChunkWidth;
8387 unsigned Opc = (Alternate && (ChunkNum % 2) != 0) ? ISD::SUB : ISD::ADD;
8388 Sum = DAG.getNode(Opcode: Opc, DL: dl, VT: HiLoVT, N1: Sum, N2: Chunk);
8389 }
8390 }
8391
8392 // For Alternate, the sum may be negative, but we need a positive sum. We
8393 // can increase it by a multiple of the divisor to make it positive. For 3
8394 // chunks the largest negative value is -(2^BestChunkWidth - 1). For 4
8395 // chunks, it's 2*-(2^BestChunkWidth - 1). We know that 2^BestChunkWidth + 1
8396 // is a multiple of the divisor. Add that 1 or 2 times to make the sum
8397 // positive.
8398 if (Alternate) {
8399 unsigned NumChunks = divideCeil(Numerator: BitWidth - TrailingZeros, Denominator: BestChunkWidth);
8400 assert(NumChunks <= 4);
8401
8402 APInt Adjust = APInt::getOneBitSet(numBits: HBitWidth, BitNo: BestChunkWidth);
8403 Adjust.setBit(0);
8404 // If there are 4 chunks, we need to adjust twice.
8405 if (NumChunks == 4)
8406 Adjust <<= 1;
8407 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: HiLoVT, N1: Sum,
8408 N2: DAG.getConstant(Val: Adjust, DL: dl, VT: HiLoVT));
8409 }
8410 }
8411
8412 // Perform a HiLoVT urem on the Sum using truncated divisor.
8413 SDValue RemL =
8414 DAG.getNode(Opcode: ISD::UREM, DL: dl, VT: HiLoVT, N1: Sum,
8415 N2: DAG.getConstant(Val: Divisor.trunc(width: HBitWidth), DL: dl, VT: HiLoVT));
8416 SDValue RemH = DAG.getConstant(Val: 0, DL: dl, VT: HiLoVT);
8417
8418 if (Opcode != ISD::UREM) {
8419 // If we didn't shift LH/LR earlier, do it now.
8420 if (BestChunkWidth != HBitWidth)
8421 ShiftRight(LL, LH, TrailingZeros);
8422
8423 // Subtract the remainder from the shifted dividend.
8424 SDValue Dividend = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: LL, N2: LH);
8425 SDValue Rem = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT, N1: RemL, N2: RemH);
8426
8427 Dividend = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Rem);
8428
8429 // Multiply by the multiplicative inverse of the divisor modulo
8430 // (1 << BitWidth).
8431 APInt MulFactor = Divisor.multiplicativeInverse();
8432
8433 SDValue Quotient = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Dividend,
8434 N2: DAG.getConstant(Val: MulFactor, DL: dl, VT));
8435
8436 // Split the quotient into low and high parts.
8437 SDValue QuotL, QuotH;
8438 std::tie(args&: QuotL, args&: QuotH) = DAG.SplitScalar(N: Quotient, DL: dl, LoVT: HiLoVT, HiVT: HiLoVT);
8439 Result.push_back(Elt: QuotL);
8440 Result.push_back(Elt: QuotH);
8441 }
8442
8443 if (Opcode != ISD::UDIV) {
8444 // If we shifted the input, shift the remainder left and add the bits we
8445 // shifted off the input.
8446 if (TrailingZeros) {
8447 if (TrailingZeros < HBitWidth) {
8448 // Shift RemH:RemL left by TrailingZeros.
8449 // RemH gets the high bits shifted out of RemL.
8450 RemH = DAG.getNode(
8451 Opcode: ISD::SRL, DL: dl, VT: HiLoVT, N1: RemL,
8452 N2: DAG.getShiftAmountConstant(Val: HBitWidth - TrailingZeros, VT: HiLoVT, DL: dl));
8453 RemL =
8454 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8455 N2: DAG.getShiftAmountConstant(Val: TrailingZeros, VT: HiLoVT, DL: dl));
8456 // OR in the partial remainder.
8457 RemL = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemL, N2: PartialRemL,
8458 Flags: SDNodeFlags::Disjoint);
8459 } else if (TrailingZeros == HBitWidth) {
8460 // Shift left by exactly HBitWidth: RemH becomes RemL, RemL becomes
8461 // PartialRemL.
8462 RemH = RemL;
8463 RemL = PartialRemL;
8464 } else {
8465 // Shift left by more than HBitWidth.
8466 RemH = DAG.getNode(
8467 Opcode: ISD::SHL, DL: dl, VT: HiLoVT, N1: RemL,
8468 N2: DAG.getShiftAmountConstant(Val: TrailingZeros - HBitWidth, VT: HiLoVT, DL: dl));
8469 RemH = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: HiLoVT, N1: RemH, N2: PartialRemH,
8470 Flags: SDNodeFlags::Disjoint);
8471 RemL = PartialRemL;
8472 }
8473 }
8474 Result.push_back(Elt: RemL);
8475 Result.push_back(Elt: RemH);
8476 }
8477
8478 return true;
8479}
8480
8481bool TargetLowering::expandUDIVREMByConstantViaUMulHiMagic(
8482 SDNode *N, const APInt &Divisor, SmallVectorImpl<SDValue> &Result,
8483 EVT HiLoVT, SelectionDAG &DAG, SDValue LL, SDValue LH) const {
8484
8485 SDValue N0 = N->getOperand(Num: 0);
8486 EVT VT = N0->getValueType(ResNo: 0);
8487 SDLoc DL{N};
8488
8489 assert(!Divisor.isOne() && "Magic algorithm does not work for division by 1");
8490
8491 // This helper creates a MUL_LOHI of the pair (LL, LH) by a constant.
8492 auto MakeMUL_LOHIByConst = [&](unsigned Opc, SDValue LL, SDValue LH,
8493 const APInt &Const,
8494 SmallVectorImpl<SDValue> &Result) {
8495 SDValue LHS = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT, N1: LL, N2: LH);
8496 SDValue RHS = DAG.getConstant(Val: Const, DL, VT);
8497 auto [RL, RH] = DAG.SplitScalar(N: RHS, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8498 return expandMUL_LOHI(Opcode: Opc, VT, dl: DL, LHS, RHS, Result, HiLoVT, DAG,
8499 Kind: TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
8500 LL, LH, RL, RH);
8501 };
8502
8503 // This helper creates an ADD/SUB of the pairs (LL, LH) and (RL, RH).
8504 auto MakeAddSubLong = [&](unsigned Opc, SDValue LL, SDValue LH, SDValue RL,
8505 SDValue RH) {
8506 SDValue AddSubNode =
8507 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO : ISD::USUBO, DL,
8508 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LL, N2: RL);
8509 SDValue OutL = AddSubNode.getValue(R: 0);
8510 SDValue Overflow = AddSubNode.getValue(R: 1);
8511 SDValue AddSubWithOverflow =
8512 DAG.getNode(Opcode: Opc == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY, DL,
8513 VTList: DAG.getVTList(VT1: HiLoVT, VT2: MVT::i1), N1: LH, N2: RH, N3: Overflow);
8514 SDValue OutH = AddSubWithOverflow.getValue(R: 0);
8515 return std::make_pair(x&: OutL, y&: OutH);
8516 };
8517
8518 // This helper creates a SRL of the pair (LL, LH) by Shift.
8519 auto MakeSRLLong = [&](SDValue LL, SDValue LH, unsigned Shift) {
8520 unsigned HBitWidth = HiLoVT.getScalarSizeInBits();
8521 if (Shift < HBitWidth) {
8522 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift, VT: HiLoVT, DL);
8523 SDValue ResL = DAG.getNode(Opcode: ISD::FSHR, DL, VT: HiLoVT, N1: LH, N2: LL, N3: ShAmt);
8524 SDValue ResH = DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt);
8525 return std::make_pair(x&: ResL, y&: ResH);
8526 }
8527 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: HiLoVT);
8528 if (Shift == HBitWidth)
8529 return std::make_pair(x&: LH, y&: Zero);
8530 assert(Shift - HBitWidth < HBitWidth &&
8531 "We shouldn't generate an undefined shift");
8532 SDValue ShAmt = DAG.getShiftAmountConstant(Val: Shift - HBitWidth, VT: HiLoVT, DL);
8533 return std::make_pair(x: DAG.getNode(Opcode: ISD::SRL, DL, VT: HiLoVT, N1: LH, N2: ShAmt), y&: Zero);
8534 };
8535
8536 // Knowledge of leading zeros may help to reduce the multiplier.
8537 unsigned KnownLeadingZeros = DAG.computeKnownBits(Op: N0).countMinLeadingZeros();
8538
8539 UnsignedDivisionByConstantInfo Magics = UnsignedDivisionByConstantInfo::get(
8540 D: Divisor, LeadingZeros: std::min(a: KnownLeadingZeros, b: Divisor.countl_zero()));
8541
8542 assert(!LL == !LH && "Expected both input halves or no input halves!");
8543 if (!LL)
8544 std::tie(args&: LL, args&: LH) = DAG.SplitScalar(N: N0, DL, LoVT: HiLoVT, HiVT: HiLoVT);
8545 SDValue QL = LL;
8546 SDValue QH = LH;
8547 if (Magics.PreShift != 0)
8548 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PreShift);
8549
8550 SmallVector<SDValue, 4> UMulResult;
8551 if (!MakeMUL_LOHIByConst(ISD::UMUL_LOHI, QL, QH, Magics.Magic, UMulResult))
8552 return false;
8553
8554 QL = UMulResult[2];
8555 QH = UMulResult[3];
8556
8557 if (Magics.IsAdd) {
8558 auto [NPQL, NPQH] = MakeAddSubLong(ISD::SUB, LL, LH, QL, QH);
8559 std::tie(args&: NPQL, args&: NPQH) = MakeSRLLong(NPQL, NPQH, 1);
8560 std::tie(args&: QL, args&: QH) = MakeAddSubLong(ISD::ADD, NPQL, NPQH, QL, QH);
8561 }
8562
8563 if (Magics.PostShift != 0)
8564 std::tie(args&: QL, args&: QH) = MakeSRLLong(QL, QH, Magics.PostShift);
8565
8566 unsigned Opcode = N->getOpcode();
8567 if (Opcode != ISD::UREM) {
8568 Result.push_back(Elt: QL);
8569 Result.push_back(Elt: QH);
8570 }
8571
8572 if (Opcode != ISD::UDIV) {
8573 SmallVector<SDValue, 2> MulResult;
8574 if (!MakeMUL_LOHIByConst(ISD::MUL, QL, QH, Divisor, MulResult))
8575 return false;
8576
8577 assert(MulResult.size() == 2);
8578
8579 auto [RemL, RemH] =
8580 MakeAddSubLong(ISD::SUB, LL, LH, MulResult[0], MulResult[1]);
8581
8582 Result.push_back(Elt: RemL);
8583 Result.push_back(Elt: RemH);
8584 }
8585
8586 return true;
8587}
8588
8589bool TargetLowering::expandDIVREMByConstant(SDNode *N,
8590 SmallVectorImpl<SDValue> &Result,
8591 EVT HiLoVT, SelectionDAG &DAG,
8592 SDValue LL, SDValue LH) const {
8593 unsigned Opcode = N->getOpcode();
8594
8595 // TODO: Support signed division/remainder.
8596 if (Opcode == ISD::SREM || Opcode == ISD::SDIV || Opcode == ISD::SDIVREM)
8597 return false;
8598 assert(
8599 (Opcode == ISD::UREM || Opcode == ISD::UDIV || Opcode == ISD::UDIVREM) &&
8600 "Unexpected opcode");
8601
8602 auto *CN = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
8603 if (!CN)
8604 return false;
8605
8606 APInt Divisor = CN->getAPIntValue();
8607
8608 // The generated half-width UREM is normally optimized using high multiply.
8609 // If the wide UREM libcall is unavailable, a legal or custom half-width
8610 // UDIVREM can lower it instead.
8611 bool CanDecomposeUREMWithoutMulHi =
8612 Opcode == ISD::UREM &&
8613 getLibcallImpl(Call: RTLIB::getUREM(VT: N->getValueType(ResNo: 0))) ==
8614 RTLIB::Unsupported &&
8615 isOperationLegalOrCustom(Op: ISD::UDIVREM, VT: HiLoVT);
8616 if (!CanDecomposeUREMWithoutMulHi &&
8617 !isOperationLegalOrCustom(Op: ISD::MULHU, VT: HiLoVT) &&
8618 !isOperationLegalOrCustom(Op: ISD::UMUL_LOHI, VT: HiLoVT))
8619 return false;
8620
8621 // Prefer the smaller libcall when one is available.
8622 if (DAG.shouldOptForSize() && !CanDecomposeUREMWithoutMulHi)
8623 return false;
8624
8625 // Early out for 0 or 1 divisors.
8626 if (Divisor.ule(RHS: 1))
8627 return false;
8628
8629 if (expandUDIVREMByConstantViaUREMDecomposition(N, Divisor, Result, HiLoVT,
8630 DAG, LL, LH))
8631 return true;
8632
8633 if (expandUDIVREMByConstantViaUMulHiMagic(N, Divisor, Result, HiLoVT, DAG, LL,
8634 LH))
8635 return true;
8636
8637 return false;
8638}
8639
8640// Check that (every element of) Z is undef or not an exact multiple of BW.
8641static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
8642 return ISD::matchUnaryPredicate(
8643 Op: Z,
8644 Match: [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(RHS: BW) != 0; },
8645 /*AllowUndefs=*/true, /*AllowTruncation=*/true);
8646}
8647
8648static SDValue expandVPFunnelShift(SDNode *Node, SelectionDAG &DAG) {
8649 EVT VT = Node->getValueType(ResNo: 0);
8650 SDValue ShX, ShY;
8651 SDValue ShAmt, InvShAmt;
8652 SDValue X = Node->getOperand(Num: 0);
8653 SDValue Y = Node->getOperand(Num: 1);
8654 SDValue Z = Node->getOperand(Num: 2);
8655 SDValue Mask = Node->getOperand(Num: 3);
8656 SDValue VL = Node->getOperand(Num: 4);
8657
8658 unsigned BW = VT.getScalarSizeInBits();
8659 bool IsFSHL = Node->getOpcode() == ISD::VP_FSHL;
8660 SDLoc DL(SDValue(Node, 0));
8661
8662 EVT ShVT = Z.getValueType();
8663 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8664 // fshl: X << C | Y >> (BW - C)
8665 // fshr: X << (BW - C) | Y >> C
8666 // where C = Z % BW is not zero
8667 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8668 ShAmt = DAG.getNode(Opcode: ISD::VP_UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC, N3: Mask, N4: VL);
8669 InvShAmt = DAG.getNode(Opcode: ISD::VP_SUB, DL, VT: ShVT, N1: BitWidthC, N2: ShAmt, N3: Mask, N4: VL);
8670 ShX = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: X, N2: IsFSHL ? ShAmt : InvShAmt, N3: Mask,
8671 N4: VL);
8672 ShY = DAG.getNode(Opcode: ISD::VP_SRL, DL, VT, N1: Y, N2: IsFSHL ? InvShAmt : ShAmt, N3: Mask,
8673 N4: VL);
8674 } else {
8675 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8676 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8677 SDValue BitMask = DAG.getConstant(Val: BW - 1, DL, VT: ShVT);
8678 if (isPowerOf2_32(Value: BW)) {
8679 // Z % BW -> Z & (BW - 1)
8680 ShAmt = DAG.getNode(Opcode: ISD::VP_AND, DL, VT: ShVT, N1: Z, N2: BitMask, N3: Mask, N4: VL);
8681 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8682 SDValue NotZ = DAG.getNode(Opcode: ISD::VP_XOR, DL, VT: ShVT, N1: Z,
8683 N2: DAG.getAllOnesConstant(DL, VT: ShVT), N3: Mask, N4: VL);
8684 InvShAmt = DAG.getNode(Opcode: ISD::VP_AND, DL, VT: ShVT, N1: NotZ, N2: BitMask, N3: Mask, N4: VL);
8685 } else {
8686 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8687 ShAmt = DAG.getNode(Opcode: ISD::VP_UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC, N3: Mask, N4: VL);
8688 InvShAmt = DAG.getNode(Opcode: ISD::VP_SUB, DL, VT: ShVT, N1: BitMask, N2: ShAmt, N3: Mask, N4: VL);
8689 }
8690
8691 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8692 if (IsFSHL) {
8693 ShX = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: X, N2: ShAmt, N3: Mask, N4: VL);
8694 SDValue ShY1 = DAG.getNode(Opcode: ISD::VP_SRL, DL, VT, N1: Y, N2: One, N3: Mask, N4: VL);
8695 ShY = DAG.getNode(Opcode: ISD::VP_SRL, DL, VT, N1: ShY1, N2: InvShAmt, N3: Mask, N4: VL);
8696 } else {
8697 SDValue ShX1 = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: X, N2: One, N3: Mask, N4: VL);
8698 ShX = DAG.getNode(Opcode: ISD::VP_SHL, DL, VT, N1: ShX1, N2: InvShAmt, N3: Mask, N4: VL);
8699 ShY = DAG.getNode(Opcode: ISD::VP_SRL, DL, VT, N1: Y, N2: ShAmt, N3: Mask, N4: VL);
8700 }
8701 }
8702 return DAG.getNode(Opcode: ISD::VP_OR, DL, VT, N1: ShX, N2: ShY, N3: Mask, N4: VL);
8703}
8704
8705SDValue TargetLowering::expandFunnelShift(SDNode *Node,
8706 SelectionDAG &DAG) const {
8707 if (Node->isVPOpcode())
8708 return expandVPFunnelShift(Node, DAG);
8709
8710 EVT VT = Node->getValueType(ResNo: 0);
8711
8712 if (VT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8713 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8714 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8715 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
8716 return SDValue();
8717
8718 SDValue X = Node->getOperand(Num: 0);
8719 SDValue Y = Node->getOperand(Num: 1);
8720 SDValue Z = Node->getOperand(Num: 2);
8721
8722 unsigned BW = VT.getScalarSizeInBits();
8723 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
8724 SDLoc DL(SDValue(Node, 0));
8725
8726 EVT ShVT = Z.getValueType();
8727
8728 // If a funnel shift in the other direction is more supported, use it.
8729 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
8730 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8731 isOperationLegalOrCustom(Op: RevOpcode, VT) && isPowerOf2_32(Value: BW)) {
8732 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8733 // fshl X, Y, Z -> fshr X, Y, -Z
8734 // fshr X, Y, Z -> fshl X, Y, -Z
8735 Z = DAG.getNegative(Val: Z, DL, VT: ShVT);
8736 } else {
8737 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
8738 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
8739 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8740 if (IsFSHL) {
8741 Y = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8742 X = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: One);
8743 } else {
8744 X = DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: One);
8745 Y = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Y, N2: One);
8746 }
8747 Z = DAG.getNOT(DL, Val: Z, VT: ShVT);
8748 }
8749 return DAG.getNode(Opcode: RevOpcode, DL, VT, N1: X, N2: Y, N3: Z);
8750 }
8751
8752 SDValue ShX, ShY;
8753 SDValue ShAmt, InvShAmt;
8754 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
8755 // fshl: X << C | Y >> (BW - C)
8756 // fshr: X << (BW - C) | Y >> C
8757 // where C = Z % BW is not zero
8758 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8759 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8760 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthC, N2: ShAmt);
8761 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: IsFSHL ? ShAmt : InvShAmt);
8762 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: IsFSHL ? InvShAmt : ShAmt);
8763 } else {
8764 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
8765 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
8766 SDValue Mask = DAG.getConstant(Val: BW - 1, DL, VT: ShVT);
8767 if (isPowerOf2_32(Value: BW)) {
8768 // Z % BW -> Z & (BW - 1)
8769 ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Z, N2: Mask);
8770 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
8771 InvShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: DAG.getNOT(DL, Val: Z, VT: ShVT), N2: Mask);
8772 } else {
8773 SDValue BitWidthC = DAG.getConstant(Val: BW, DL, VT: ShVT);
8774 ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Z, N2: BitWidthC);
8775 InvShAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Mask, N2: ShAmt);
8776 }
8777
8778 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8779 if (IsFSHL) {
8780 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShAmt);
8781 SDValue ShY1 = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: One);
8782 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShY1, N2: InvShAmt);
8783 } else {
8784 SDValue ShX1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: One);
8785 ShX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShX1, N2: InvShAmt);
8786 ShY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt);
8787 }
8788 }
8789 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShX, N2: ShY);
8790}
8791
8792// TODO: Merge with expandFunnelShift.
8793SDValue TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
8794 SelectionDAG &DAG) const {
8795 EVT VT = Node->getValueType(ResNo: 0);
8796 unsigned EltSizeInBits = VT.getScalarSizeInBits();
8797 bool IsLeft = Node->getOpcode() == ISD::ROTL;
8798 SDValue Op0 = Node->getOperand(Num: 0);
8799 SDValue Op1 = Node->getOperand(Num: 1);
8800 SDLoc DL(SDValue(Node, 0));
8801
8802 EVT ShVT = Op1.getValueType();
8803 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: ShVT);
8804
8805 // If a rotate in the other direction is more supported, use it.
8806 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
8807 if (!isOperationLegalOrCustom(Op: Node->getOpcode(), VT) &&
8808 isOperationLegalOrCustom(Op: RevRot, VT) && isPowerOf2_32(Value: EltSizeInBits)) {
8809 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8810 return DAG.getNode(Opcode: RevRot, DL, VT, N1: Op0, N2: Sub);
8811 }
8812
8813 if (!AllowVectorOps && VT.isVector() &&
8814 (!isOperationLegalOrCustom(Op: ISD::SHL, VT) ||
8815 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
8816 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
8817 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT) ||
8818 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT)))
8819 return SDValue();
8820
8821 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
8822 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
8823 SDValue BitWidthMinusOneC = DAG.getConstant(Val: EltSizeInBits - 1, DL, VT: ShVT);
8824 SDValue ShVal;
8825 SDValue HsVal;
8826 if (isPowerOf2_32(Value: EltSizeInBits)) {
8827 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
8828 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
8829 SDValue NegOp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: Zero, N2: Op1);
8830 SDValue ShAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: Op1, N2: BitWidthMinusOneC);
8831 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
8832 SDValue HsAmt = DAG.getNode(Opcode: ISD::AND, DL, VT: ShVT, N1: NegOp1, N2: BitWidthMinusOneC);
8833 HsVal = DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: HsAmt);
8834 } else {
8835 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
8836 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
8837 SDValue BitWidthC = DAG.getConstant(Val: EltSizeInBits, DL, VT: ShVT);
8838 SDValue ShAmt = DAG.getNode(Opcode: ISD::UREM, DL, VT: ShVT, N1: Op1, N2: BitWidthC);
8839 ShVal = DAG.getNode(Opcode: ShOpc, DL, VT, N1: Op0, N2: ShAmt);
8840 SDValue HsAmt = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShVT, N1: BitWidthMinusOneC, N2: ShAmt);
8841 SDValue One = DAG.getConstant(Val: 1, DL, VT: ShVT);
8842 HsVal =
8843 DAG.getNode(Opcode: HsOpc, DL, VT, N1: DAG.getNode(Opcode: HsOpc, DL, VT, N1: Op0, N2: One), N2: HsAmt);
8844 }
8845 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShVal, N2: HsVal);
8846}
8847
8848/// Check if CLMUL on VT can eventually reach a type with legal CLMUL through
8849/// a chain of halving decompositions (halving element width) and/or vector
8850/// widening (doubling element count). This guides expansion strategy selection:
8851/// if true, the halving/widening path produces better code than bit-by-bit.
8852///
8853/// HalveDepth tracks halving steps only (each creates ~4x more operations).
8854/// Widening steps are cheap (O(1) pad/extract) and don't count.
8855/// Limiting halvings to 2 prevents exponential blowup:
8856/// 1 halving: ~4 sub-CLMULs (good, e.g. v8i16 -> v8i8)
8857/// 2 halvings: ~16 sub-CLMULs (acceptable, e.g. v4i32 -> v4i16 -> v8i8)
8858/// 3 halvings: ~64 sub-CLMULs (worse than bit-by-bit expansion)
8859static bool canNarrowCLMULToLegal(const TargetLowering &TLI, LLVMContext &Ctx,
8860 EVT VT, unsigned HalveDepth = 0,
8861 unsigned TotalDepth = 0) {
8862 if (HalveDepth > 2 || TotalDepth > 8 || !VT.isFixedLengthVector())
8863 return false;
8864 if (TLI.isOperationLegalOrCustom(Op: ISD::CLMUL, VT))
8865 return true;
8866 if (!TLI.isTypeLegal(VT))
8867 return false;
8868
8869 unsigned BW = VT.getScalarSizeInBits();
8870
8871 // Halve: halve element width, same element count.
8872 // This is the expensive step -- each halving creates ~4x more operations.
8873 if (BW % 2 == 0) {
8874 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: BW / 2);
8875 EVT HalfVT = VT.changeVectorElementType(Context&: Ctx, EltVT: HalfEltVT);
8876 if (TLI.isTypeLegal(VT: HalfVT) &&
8877 canNarrowCLMULToLegal(TLI, Ctx, VT: HalfVT, HalveDepth: HalveDepth + 1, TotalDepth: TotalDepth + 1))
8878 return true;
8879 }
8880
8881 // Widen: double element count (fixed-width vectors only).
8882 // This is cheap -- just INSERT_SUBVECTOR + EXTRACT_SUBVECTOR.
8883 EVT WideVT = VT.getDoubleNumVectorElementsVT(Context&: Ctx);
8884 if (TLI.isTypeLegal(VT: WideVT) &&
8885 canNarrowCLMULToLegal(TLI, Ctx, VT: WideVT, HalveDepth, TotalDepth: TotalDepth + 1))
8886 return true;
8887
8888 return false;
8889}
8890
8891SDValue TargetLowering::expandCLMUL(SDNode *Node, SelectionDAG &DAG) const {
8892 SDLoc DL(Node);
8893 EVT VT = Node->getValueType(ResNo: 0);
8894 SDValue X = Node->getOperand(Num: 0);
8895 SDValue Y = Node->getOperand(Num: 1);
8896 unsigned BW = VT.getScalarSizeInBits();
8897 unsigned Opcode = Node->getOpcode();
8898 LLVMContext &Ctx = *DAG.getContext();
8899
8900 switch (Opcode) {
8901 case ISD::CLMUL: {
8902 // For vector types, try decomposition strategies that leverage legal
8903 // CLMUL on narrower or wider element types, avoiding the expensive
8904 // bit-by-bit expansion.
8905 if (VT.isVector()) {
8906 // Strategy 1: Halving decomposition to half-element-width CLMUL.
8907 // Applies ExpandIntRes_CLMUL's identity element-wise:
8908 // CLMUL(X, Y) = (Hi << HalfBW) | Lo
8909 // where:
8910 // Lo = CLMUL(XLo, YLo)
8911 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
8912 unsigned HalfBW = BW / 2;
8913 if (BW % 2 == 0) {
8914 EVT HalfEltVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: HalfBW);
8915 EVT HalfVT =
8916 EVT::getVectorVT(Context&: Ctx, VT: HalfEltVT, EC: VT.getVectorElementCount());
8917 if (isTypeLegal(VT: HalfVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: HalfVT,
8918 /*HalveDepth=*/1)) {
8919 SDValue ShAmt = DAG.getShiftAmountConstant(Val: HalfBW, VT, DL);
8920
8921 // Extract low and high halves of each element.
8922 SDValue XLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: X);
8923 SDValue XHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
8924 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: ShAmt));
8925 SDValue YLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT, Operand: Y);
8926 SDValue YHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: HalfVT,
8927 Operand: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: ShAmt));
8928
8929 // Lo = CLMUL(XLo, YLo)
8930 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YLo);
8931
8932 // Hi = CLMULH(XLo, YLo) ^ CLMUL(XLo, YHi) ^ CLMUL(XHi, YLo)
8933 SDValue LoH = DAG.getNode(Opcode: ISD::CLMULH, DL, VT: HalfVT, N1: XLo, N2: YLo);
8934 SDValue Cross1 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XLo, N2: YHi);
8935 SDValue Cross2 = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: HalfVT, N1: XHi, N2: YLo);
8936 SDValue Cross = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: Cross1, N2: Cross2);
8937 SDValue Hi = DAG.getNode(Opcode: ISD::XOR, DL, VT: HalfVT, N1: LoH, N2: Cross);
8938
8939 // Reassemble: Result = ZExt(Lo) | (AnyExt(Hi) << HalfBW)
8940 SDValue LoExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Lo);
8941 SDValue HiExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Hi);
8942 SDValue HiShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: HiExt, N2: ShAmt);
8943 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoExt, N2: HiShifted);
8944 }
8945 }
8946
8947 // Strategy 2: Promote to double-element-width CLMUL.
8948 // CLMUL(X, Y) = Trunc(CLMUL(AnyExt(X), AnyExt(Y)))
8949 {
8950 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
8951 if (isTypeLegal(VT: ExtVT) && isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT)) {
8952 // If CLMUL on ExtVT is Custom (not Legal), the target may
8953 // scalarize it, costing O(NumElements) scalar ops. The bit-by-bit
8954 // fallback costs O(BW) vectorized iterations. Only widen when
8955 // element count is small enough that scalarization is cheaper.
8956 unsigned NumElts = VT.getVectorMinNumElements();
8957 if (isOperationLegal(Op: ISD::CLMUL, VT: ExtVT) || NumElts < BW) {
8958 SDValue XExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: X);
8959 SDValue YExt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ExtVT, Operand: Y);
8960 SDValue Mul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
8961 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Mul);
8962 }
8963 }
8964 }
8965
8966 // Strategy 3: Widen element count (pad with undef, do CLMUL on wider
8967 // vector, extract lower result). CLMUL is element-wise, so upper
8968 // (undef) lanes don't affect the lower results.
8969 // e.g. v4i16 => pad to v8i16 => halve to v8i8 PMUL => extract v4i16.
8970 if (auto EC = VT.getVectorElementCount(); EC.isFixed()) {
8971 EVT WideVT = EVT::getVectorVT(Context&: Ctx, VT: VT.getVectorElementType(), EC: EC * 2);
8972 if (isTypeLegal(VT: WideVT) && canNarrowCLMULToLegal(TLI: *this, Ctx, VT: WideVT)) {
8973 SDValue Undef = DAG.getUNDEF(VT: WideVT);
8974 SDValue XWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
8975 N2: X, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8976 SDValue YWide = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: WideVT, N1: Undef,
8977 N2: Y, N3: DAG.getVectorIdxConstant(Val: 0, DL));
8978 SDValue WideRes = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: WideVT, N1: XWide, N2: YWide);
8979 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: WideRes,
8980 N2: DAG.getVectorIdxConstant(Val: 0, DL));
8981 }
8982 }
8983 }
8984
8985 // Special case: clmul(X, ~0) is equivalent to a "parallel prefix XOR" or
8986 // "bitwise parity" operation.
8987 if (isAllOnesOrAllOnesSplat(V: Y)) {
8988 SDValue R = X;
8989 for (unsigned I = 1; I < BW; I <<= 1) {
8990 SDValue ShAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
8991 SDValue Shifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: R, N2: ShAmt);
8992 R = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: R, N2: Shifted);
8993 }
8994 return R;
8995 }
8996
8997 // NOTE: If you change this expansion, please update the cost model
8998 // calculation in BasicTTIImpl::getTypeBasedIntrinsicInstrCost for
8999 // Intrinsic::clmul.
9000
9001 // Strategy 4: multiplication with holes.
9002 //
9003 // Uses "holes" (sequences of zeroes) to avoid carry spilling. When carries
9004 // do occur, they wind up in a "hole" and are subsequently masked out of the
9005 // result.
9006 //
9007 // A hole of 3 bits is optimal for 32-bit and 64-bit inputs. 128-bit
9008 // integers need a larger hole, and for smaller integers the fallback below
9009 // is more efficient.
9010 //
9011 // Based on bmul64 in bearssl and bmul in the rust polyval crate.
9012 if (BW >= 32 && BW <= 64 &&
9013 isOperationLegalOrCustom(Op: ISD::MUL, VT: getTypeToTransformTo(Context&: Ctx, VT))) {
9014
9015 // Set every fourth bit of each nibble, equivalent to 0b00010001...0001.
9016 APInt MaskVal = APInt::getSplat(NewLen: BW, V: APInt(4, 0b0001));
9017
9018 // Create versions of X and Y that keep only the I-th bit of
9019 // each nibble.
9020 SDValue M[4], Xp[4], Yp[4];
9021 for (unsigned I = 0; I < 4; ++I) {
9022 M[I] = DAG.getConstant(Val: MaskVal.shl(shiftAmt: I), DL, VT);
9023 Xp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: M[I]);
9024 Yp[I] = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: M[I]);
9025 }
9026
9027 // Codegens these expressions (16 multiplications):
9028 //
9029 // z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
9030 // z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
9031 // z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
9032 // z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
9033 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9034 for (unsigned I = 0; I < 4; ++I) {
9035 SDValue Zi = DAG.getConstant(Val: 0, DL, VT);
9036 for (unsigned J = 0; J < 4; ++J) {
9037 unsigned K = (I + 4 - J) % 4;
9038 SDValue P = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: Xp[J], N2: Yp[K]);
9039 Zi = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Zi, N2: P);
9040 }
9041
9042 // Keep only the bits belonging to this iteration, and bitwise or it all
9043 // together.
9044 Zi = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Zi, N2: M[I]);
9045 Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Res, N2: Zi, Flags: SDNodeFlags::Disjoint);
9046 }
9047 return Res;
9048 }
9049
9050 // Strategy 5: the naive fallback.
9051 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
9052
9053 SDValue Res = DAG.getConstant(Val: 0, DL, VT);
9054 for (unsigned I = 0; I < BW; ++I) {
9055 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: I, VT, DL);
9056 SDValue Mask = DAG.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: I), DL, VT);
9057 SDValue YMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: Mask);
9058
9059 // For targets with a fast bit test instruction (e.g., x86 BT) or without
9060 // multiply, use a shift-based expansion to avoid expensive MUL
9061 // instructions.
9062 SDValue Part;
9063 if (!hasBitTest(X: Y, Y: ShiftAmt) &&
9064 isOperationLegalOrCustom(
9065 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
9066 Part = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: YMasked);
9067 } else {
9068 // Canonical bit test: (Y & (1 << I)) != 0
9069 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
9070 SDValue Cond = DAG.getSetCC(DL, VT: SetCCVT, LHS: YMasked, RHS: Zero, Cond: ISD::SETEQ);
9071 SDValue XShifted = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftAmt);
9072 Part = DAG.getSelect(DL, VT, Cond, LHS: Zero, RHS: XShifted);
9073 }
9074 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Res, N2: Part);
9075 }
9076 return Res;
9077 }
9078 case ISD::CLMULR:
9079 // If we have CLMUL/CLMULH, merge the shifted results to form CLMULR.
9080 if (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) &&
9081 isOperationLegalOrCustom(Op: ISD::CLMULH, VT)) {
9082 SDValue Lo = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: X, N2: Y);
9083 SDValue Hi = DAG.getNode(Opcode: ISD::CLMULH, DL, VT, N1: X, N2: Y);
9084 Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo,
9085 N2: DAG.getShiftAmountConstant(Val: BW - 1, VT, DL));
9086 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi,
9087 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9088 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Lo, N2: Hi);
9089 }
9090 [[fallthrough]];
9091 case ISD::CLMULH: {
9092 EVT ExtVT = VT.widenIntegerElementType(Context&: Ctx);
9093 // Use bitreverse-based lowering (CLMULR/H = rev(CLMUL(rev,rev)) >> S)
9094 // when any of these hold:
9095 // (a) ZERO_EXTEND to ExtVT or SRL on ExtVT isn't legal.
9096 // (b) CLMUL is legal on VT but not on ExtVT (e.g. v8i8 on AArch64).
9097 // (c) CLMUL on ExtVT isn't legal, but CLMUL on VT can be efficiently
9098 // expanded via halving/widening to reach legal CLMUL. The bitreverse
9099 // path creates CLMUL(VT) which will be expanded efficiently. The
9100 // promote path would create CLMUL(ExtVT) => halving => CLMULH(VT),
9101 // causing a cycle.
9102 // Note: when CLMUL is legal on ExtVT, the zext => CLMUL(ExtVT) => shift
9103 // => trunc path is preferred over the bitreverse path, as it avoids the
9104 // cost of 3 bitreverse operations.
9105 if (!isOperationLegalOrCustom(Op: ISD::ZERO_EXTEND, VT: ExtVT) ||
9106 !isOperationLegalOrCustom(Op: ISD::SRL, VT: ExtVT) ||
9107 (!isOperationLegalOrCustom(Op: ISD::CLMUL, VT: ExtVT) &&
9108 (isOperationLegalOrCustom(Op: ISD::CLMUL, VT) ||
9109 canNarrowCLMULToLegal(TLI: *this, Ctx, VT)))) {
9110 SDValue XRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: X);
9111 SDValue YRev = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: Y);
9112 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: XRev, N2: YRev);
9113 SDValue Res = DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: ClMul);
9114 if (Opcode == ISD::CLMULH)
9115 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res,
9116 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
9117 return Res;
9118 }
9119 SDValue XExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: X);
9120 SDValue YExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: Y);
9121 SDValue ClMul = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: ExtVT, N1: XExt, N2: YExt);
9122 unsigned ShAmt = Opcode == ISD::CLMULR ? BW - 1 : BW;
9123 SDValue HiBits = DAG.getNode(Opcode: ISD::SRL, DL, VT: ExtVT, N1: ClMul,
9124 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: ExtVT, DL));
9125 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: HiBits);
9126 }
9127 }
9128 llvm_unreachable("Expected CLMUL, CLMULR, or CLMULH");
9129}
9130
9131SDValue TargetLowering::expandPEXT(SDNode *Node, SelectionDAG &DAG) const {
9132 SDLoc DL(Node);
9133 EVT VT = Node->getValueType(ResNo: 0);
9134 SDValue Val = Node->getOperand(Num: 0);
9135 SDValue Msk = Node->getOperand(Num: 1);
9136 unsigned BW = VT.getScalarSizeInBits();
9137
9138 // Hacker's Delight §7-4: Compress, or Generalized Extract
9139 SDValue X = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Val, N2: Msk);
9140 SDValue M = Msk;
9141 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9142 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: M, VT), N2: One);
9143
9144 // Repeatedly compute which bits would shift to the right by an odd amount,
9145 // shift all such bits in parallel using a mask, and double the shift amount.
9146 for (unsigned I = 1; I < BW; I *= 2) {
9147 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9148 SDValue Mp =
9149 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9150 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: M);
9151 SDValue ShiftI = DAG.getShiftAmountConstant(Val: I, VT, DL);
9152 SDValue MvS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mv, N2: ShiftI);
9153 M = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: M, N2: Mv), N2: MvS,
9154 Flags: SDNodeFlags::Disjoint);
9155 SDValue T = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Mv);
9156 SDValue TS = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: T, N2: ShiftI);
9157 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: X, N2: T), N2: TS,
9158 Flags: SDNodeFlags::Disjoint);
9159 if (I * 2 < BW)
9160 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9161 }
9162
9163 return X;
9164}
9165
9166SDValue TargetLowering::expandPDEP(SDNode *Node, SelectionDAG &DAG) const {
9167 SDLoc DL(Node);
9168 EVT VT = Node->getValueType(ResNo: 0);
9169 SDValue Val = Node->getOperand(Num: 0);
9170 SDValue Msk = Node->getOperand(Num: 1);
9171 unsigned BW = VT.getScalarSizeInBits();
9172
9173 // Hacker's Delight §7-5: Expand, or Generalized Insert.
9174 unsigned LogBW = Log2_32_Ceil(Value: BW);
9175 SmallVector<SDValue, 8> MvArray(LogBW);
9176 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT, DL);
9177 SDValue Mc = Msk;
9178 SDValue Mk = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: DAG.getNOT(DL, Val: Msk, VT), N2: One);
9179
9180 // First pass: compute move masks for each power of two that a bit moves by.
9181 for (unsigned S = 0; S < LogBW; ++S) {
9182 unsigned ShiftS = 1u << S;
9183 // This expands the "parallel prefix" operation to clmul(Mk, ~0).
9184 SDValue Mp =
9185 DAG.getNode(Opcode: ISD::CLMUL, DL, VT, N1: Mk, N2: DAG.getAllOnesConstant(DL, VT));
9186 SDValue Mv = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mp, N2: Mc);
9187 MvArray[S] = Mv;
9188 if (S + 1 < LogBW) {
9189 SDValue McXorMv = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Mc, N2: Mv);
9190 SDValue MvShifted = DAG.getNode(
9191 Opcode: ISD::SRL, DL, VT, N1: Mv, N2: DAG.getShiftAmountConstant(Val: ShiftS, VT, DL));
9192 Mc = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: McXorMv, N2: MvShifted,
9193 Flags: SDNodeFlags::Disjoint);
9194 Mk = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mk, N2: DAG.getNOT(DL, Val: Mp, VT));
9195 }
9196 }
9197
9198 // Second pass: move bits by 32, 16, 8, 4, 2, 1, using masks, in parallel.
9199 // Each pass handles half the shift amount of the previous pass.
9200 SDValue X = Val;
9201 for (int S = (int)LogBW - 1; S >= 0; --S) {
9202 SDValue ShiftSv = DAG.getShiftAmountConstant(Val: 1ull << S, VT, DL);
9203 SDValue T = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: ShiftSv);
9204 SDValue UnshiftedBits =
9205 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: DAG.getNOT(DL, Val: MvArray[S], VT));
9206 SDValue ShiftedBits = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: T, N2: MvArray[S]);
9207 X = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: UnshiftedBits, N2: ShiftedBits,
9208 Flags: SDNodeFlags::Disjoint);
9209 }
9210
9211 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: Msk);
9212}
9213
9214void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
9215 SelectionDAG &DAG) const {
9216 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
9217 EVT VT = Node->getValueType(ResNo: 0);
9218 unsigned VTBits = VT.getScalarSizeInBits();
9219 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
9220
9221 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
9222 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
9223 SDValue ShOpLo = Node->getOperand(Num: 0);
9224 SDValue ShOpHi = Node->getOperand(Num: 1);
9225 SDValue ShAmt = Node->getOperand(Num: 2);
9226 EVT ShAmtVT = ShAmt.getValueType();
9227 EVT ShAmtCCVT =
9228 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ShAmtVT);
9229 SDLoc dl(Node);
9230
9231 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
9232 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
9233 // away during isel.
9234 SDValue SafeShAmt = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9235 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT));
9236 SDValue Tmp1 = IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: ShOpHi,
9237 N2: DAG.getConstant(Val: VTBits - 1, DL: dl, VT: ShAmtVT))
9238 : DAG.getConstant(Val: 0, DL: dl, VT);
9239
9240 SDValue Tmp2, Tmp3;
9241 if (IsSHL) {
9242 Tmp2 = DAG.getNode(Opcode: ISD::FSHL, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9243 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: ShOpLo, N2: SafeShAmt);
9244 } else {
9245 Tmp2 = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: ShOpHi, N2: ShOpLo, N3: ShAmt);
9246 Tmp3 = DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: ShOpHi, N2: SafeShAmt);
9247 }
9248
9249 // If the shift amount is larger or equal than the width of a part we don't
9250 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
9251 // values for large shift amounts.
9252 SDValue AndNode = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ShAmtVT, N1: ShAmt,
9253 N2: DAG.getConstant(Val: VTBits, DL: dl, VT: ShAmtVT));
9254 SDValue Cond = DAG.getSetCC(DL: dl, VT: ShAmtCCVT, LHS: AndNode,
9255 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ShAmtVT), Cond: ISD::SETNE);
9256
9257 if (IsSHL) {
9258 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9259 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9260 } else {
9261 Lo = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp3, N3: Tmp2);
9262 Hi = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT, N1: Cond, N2: Tmp1, N3: Tmp3);
9263 }
9264}
9265
9266SDValue TargetLowering::expandFCANONICALIZE(SDNode *Node,
9267 SelectionDAG &DAG) const {
9268 // This implements llvm.canonicalize.f* by multiplication with 1.0, as
9269 // suggested in
9270 // https://llvm.org/docs/LangRef.html#llvm-canonicalize-intrinsic.
9271 // It uses strict_fp operations even outside a strict_fp context in order
9272 // to guarantee that the canonicalization is not optimized away by later
9273 // passes. The result chain introduced by that is intentionally ignored
9274 // since no ordering requirement is intended here.
9275 EVT VT = Node->getValueType(ResNo: 0);
9276 SDLoc DL(Node);
9277 SDNodeFlags Flags = Node->getFlags();
9278 Flags.setNoFPExcept(true);
9279 SDValue One = DAG.getConstantFP(Val: 1.0, DL, VT);
9280 SDValue Mul =
9281 DAG.getNode(Opcode: ISD::STRICT_FMUL, DL, ResultTys: {VT, MVT::Other},
9282 Ops: {DAG.getEntryNode(), Node->getOperand(Num: 0), One}, Flags);
9283 return Mul;
9284}
9285
9286SDValue TargetLowering::expandCONVERT_TO_ARBITRARY_FP(SDNode *Node,
9287 SelectionDAG &DAG) const {
9288 // Expand conversion from a native IEEE float type to an arbitrary FP format
9289 // returning the result as an integer using bit manipulation.
9290 EVT ResVT = Node->getValueType(ResNo: 0);
9291 SDLoc dl(Node);
9292
9293 SDValue FloatVal = Node->getOperand(Num: 0);
9294 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9295 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9296 const auto RoundMode =
9297 static_cast<RoundingMode>(Node->getConstantOperandVal(Num: 2));
9298 const bool Saturate = Node->getConstantOperandVal(Num: 3) != 0;
9299
9300 // Supported destination formats.
9301 switch (Sem) {
9302 case APFloatBase::S_Float8E5M2:
9303 case APFloatBase::S_Float8E4M3FN:
9304 case APFloatBase::S_Float6E3M2FN:
9305 case APFloatBase::S_Float6E2M3FN:
9306 case APFloatBase::S_Float4E2M1FN:
9307 break;
9308 default:
9309 DAG.getContext()->emitError(ErrorStr: "CONVERT_TO_ARBITRARY_FP: not implemented "
9310 "destination format (semantics enum " +
9311 Twine(SemEnum) + ")");
9312 return SDValue();
9313 }
9314
9315 // Supported rounding modes.
9316 switch (RoundMode) {
9317 case RoundingMode::NearestTiesToEven:
9318 case RoundingMode::TowardZero:
9319 case RoundingMode::TowardPositive:
9320 case RoundingMode::TowardNegative:
9321 case RoundingMode::NearestTiesToAway:
9322 break;
9323 default:
9324 DAG.getContext()->emitError(
9325 ErrorStr: "CONVERT_TO_ARBITRARY_FP: unsupported rounding mode (enum " +
9326 Twine(static_cast<int>(RoundMode)) + ")");
9327 return SDValue();
9328 }
9329
9330 // Destination format parameters.
9331 const fltSemantics &DstSem = APFloatBase::EnumToSemantics(S: Sem);
9332 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9333 const unsigned DstPrecision = APFloat::semanticsPrecision(DstSem);
9334 const unsigned DstMant = DstPrecision - 1;
9335 const unsigned DstExpBits = DstBits - DstMant - 1;
9336 const int DstBias = 1 - APFloat::semanticsMinExponent(DstSem);
9337 const unsigned DstExpMax = (1U << DstExpBits) - 1;
9338 const uint64_t DstMantMask = (DstMant > 0) ? ((1ULL << DstMant) - 1) : 0;
9339 const fltNonfiniteBehavior DstNFBehavior = DstSem.nonFiniteBehavior;
9340 const fltNanEncoding DstNanEnc = DstSem.nanEncoding;
9341
9342 // Compute the maximum normal exponent for the destination format.
9343 const unsigned DstExpMaxNormal =
9344 DstNFBehavior == fltNonfiniteBehavior::IEEE754 ? DstExpMax - 1
9345 : DstExpMax;
9346
9347 // For NanOnly formats the max exponent field for finite values
9348 // is DstExpMax, but the encoding with exp = DstExpMax and
9349 // mant = all-ones is NaN. So DstExpMaxNormal = DstExpMax, but max
9350 // mantissa at that exponent is DstMantMask - 1 (if NanEnc == AllOnes) to
9351 // avoid the NaN encoding.
9352 uint64_t DstMaxMantAtMaxExp = DstMantMask;
9353 if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9354 DstNanEnc == fltNanEncoding::AllOnes)
9355 DstMaxMantAtMaxExp = DstMantMask - 1;
9356
9357 // Source format parameters.
9358 EVT SrcVT = FloatVal.getValueType();
9359 const fltSemantics &SrcSem = SrcVT.getScalarType().getFltSemantics();
9360 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9361 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9362 const unsigned SrcMant = SrcPrecision - 1;
9363 const uint64_t SrcMantMask = (1ULL << SrcMant) - 1;
9364
9365 // Work in the source integer type. Match the destination shape so the
9366 // expansion stays vector when ResVT is a vector.
9367 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcBits);
9368 EVT IntVT = ResVT.changeElementType(Context&: *DAG.getContext(), EltVT: IntScalarVT);
9369 EVT SetCCVT =
9370 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9371 EVT FPSetCCVT =
9372 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
9373
9374 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9375 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9376
9377 // Bitcast source float to integer to extract the sign bit.
9378 SDValue Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FloatVal);
9379 SDValue SignBit =
9380 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9381 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
9382
9383 // Classify the input.
9384 SDValue FPZero = DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT);
9385 SDValue FPInf = DAG.getConstantFP(Val: APFloat::getInf(Sem: SrcSem), DL: dl, VT: SrcVT);
9386 SDValue AbsVal = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: SrcVT, Operand: FloatVal);
9387 SDValue IsNaN = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETUO);
9388 SDValue IsInf = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: AbsVal, RHS: FPInf, Cond: ISD::SETOEQ);
9389 SDValue IsZero = DAG.getSetCC(DL: dl, VT: FPSetCCVT, LHS: FloatVal, RHS: FPZero, Cond: ISD::SETOEQ);
9390
9391 // Split into a normalized fraction and unbiased exponent. FFREXP normalizes
9392 // source denormals automatically. The result is unspecified for Inf/NaN, but
9393 // those inputs are detected above and override the final result.
9394 EVT FrexpExpScalarVT =
9395 getValueType(DL: DAG.getDataLayout(), Ty: Type::getInt32Ty(C&: *DAG.getContext()));
9396 EVT FrexpExpVT = SrcVT.changeElementType(Context&: *DAG.getContext(), EltVT: FrexpExpScalarVT);
9397 SDValue Frexp =
9398 DAG.getNode(Opcode: ISD::FFREXP, DL: dl, VTList: DAG.getVTList(VT1: SrcVT, VT2: FrexpExpVT), N: FloatVal);
9399 SDValue FrexpFrac = Frexp.getValue(R: 0);
9400 SDValue FrexpExp = Frexp.getValue(R: 1);
9401
9402 SDValue FrexpFracInt = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: FrexpFrac);
9403 SDValue EffSrcMant = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FrexpFracInt,
9404 N2: DAG.getConstant(Val: SrcMantMask, DL: dl, VT: IntVT));
9405
9406 SDValue FrexpExpExt = DAG.getSExtOrTrunc(Op: FrexpExp, DL: dl, VT: IntVT);
9407 SDValue NewExp = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: FrexpExpExt,
9408 N2: DAG.getConstant(Val: DstBias - 1, DL: dl, VT: IntVT));
9409
9410 // Compute rounding increment given the round bit, sticky bits, and LSB
9411 // of the truncated mantissa.
9412 auto ComputeRoundUp = [&](SDValue RoundBit, SDValue StickyBits,
9413 SDValue LSB) -> SDValue {
9414 switch (RoundMode) {
9415 case RoundingMode::NearestTiesToEven: {
9416 // Round up if round_bit && (sticky || lsb)
9417 SDValue StickyOrLSB = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: StickyBits, N2: LSB);
9418 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyOrLSB);
9419 }
9420 case RoundingMode::TowardZero:
9421 return Zero;
9422 case RoundingMode::TowardPositive: {
9423 // Round up if positive and any truncated bits are set.
9424 SDValue AnyTruncBits =
9425 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9426 SDValue HasTruncBits =
9427 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9428 SDValue IsPositive = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETEQ);
9429 SDValue DoRound =
9430 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsPositive);
9431 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9432 }
9433 case RoundingMode::TowardNegative: {
9434 // Round up if negative and any truncated bits are set (to -Inf).
9435 SDValue AnyTruncBits =
9436 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: RoundBit, N2: StickyBits);
9437 SDValue HasTruncBits =
9438 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AnyTruncBits, RHS: Zero, Cond: ISD::SETNE);
9439 SDValue IsNegative = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: SignBit, RHS: Zero, Cond: ISD::SETNE);
9440 SDValue DoRound =
9441 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: HasTruncBits, N2: IsNegative);
9442 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: DoRound);
9443 }
9444 case RoundingMode::NearestTiesToAway:
9445 return RoundBit;
9446 default:
9447 llvm_unreachable("unsupported rounding mode");
9448 }
9449 };
9450
9451 // Round mantissa from SrcMant bits to DstMant bits.
9452 SDValue TruncMant;
9453 SDValue RoundUp;
9454 if (SrcMant > DstMant) {
9455 const unsigned Shift = SrcMant - DstMant;
9456 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: Shift, VT: IntVT, DL: dl);
9457 TruncMant = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ShiftConst);
9458
9459 // Check bit at position Shift - 1 aka the round bit.
9460 SDValue RoundBit;
9461 if (Shift >= 1) {
9462 SDValue RoundBitShift = DAG.getShiftAmountConstant(Val: Shift - 1, VT: IntVT, DL: dl);
9463 SDValue ShiftedMant =
9464 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: RoundBitShift);
9465 RoundBit = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: ShiftedMant, N2: One);
9466 } else {
9467 RoundBit = Zero;
9468 }
9469
9470 // OR of all bits below the round bit to get sticky bits.
9471 SDValue StickyBits;
9472 if (Shift >= 2) {
9473 uint64_t StickyMask = maskTrailingOnes<uint64_t>(N: Shift - 1);
9474 StickyBits = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: EffSrcMant,
9475 N2: DAG.getConstant(Val: StickyMask, DL: dl, VT: IntVT));
9476 StickyBits = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: StickyBits, RHS: Zero, Cond: ISD::SETNE);
9477 StickyBits = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: StickyBits);
9478 } else {
9479 StickyBits = Zero;
9480 }
9481
9482 // LSB of truncated mantissa.
9483 SDValue LSB = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: TruncMant, N2: One);
9484
9485 RoundUp = ComputeRoundUp(RoundBit, StickyBits, LSB);
9486 } else {
9487 // If DstMant >= SrcMant, then no rounding needed, just shift left.
9488 SDValue MantShift =
9489 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
9490 TruncMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: EffSrcMant, N2: MantShift);
9491 RoundUp = Zero;
9492 }
9493
9494 // Apply rounding.
9495 SDValue RoundedMant = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: TruncMant, N2: RoundUp);
9496
9497 // Handle mantissa overflow from rounding.
9498 // If rounded_mant > DstMantMask, carry into exponent.
9499 SDValue MantOverflow =
9500 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: RoundedMant,
9501 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9502 // On overflow: mant = 0, exp += 1.
9503 SDValue AdjMant = DAG.getSelect(DL: dl, VT: IntVT, Cond: MantOverflow, LHS: Zero, RHS: RoundedMant);
9504 SDValue AdjExp =
9505 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: NewExp,
9506 N2: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT, Operand: MantOverflow));
9507
9508 // Precompute sign shifted to MSB of destination.
9509 SDValue SignShifted =
9510 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
9511 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl));
9512
9513 // Destination denormal conversion (when new_exp <= 0).
9514 // Shift the mantissa right by 1 - new_exp additional bits and set the
9515 // exponent field to 0.
9516 SDValue ExpIsNeg = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9517 RHS: DAG.getConstant(Val: 1, DL: dl, VT: IntVT), Cond: ISD::SETLT);
9518
9519 SDValue DenormResult;
9520 {
9521 // denorm_shift = 1 - NewExp.
9522 SDValue DenormShift = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: One, N2: NewExp);
9523
9524 // full_src_mant = (1 << SrcMant) | EffSrcMant.
9525 SDValue ImplicitOne =
9526 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One,
9527 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl));
9528 SDValue FullSrcMant =
9529 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: EffSrcMant, N2: ImplicitOne);
9530
9531 // Total right shift = DenormShift + (SrcMant - DstMant).
9532 int64_t MantDelta = static_cast<int64_t>(SrcMant) - DstMant;
9533 SDValue TotalShift =
9534 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormShift,
9535 N2: DAG.getSignedConstant(Val: MantDelta, DL: dl, VT: IntVT));
9536
9537 // Clamp total shift to avoid UB, then truncate denorm mantissa.
9538 EVT ShiftVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
9539 SDValue MaxShift = DAG.getConstant(Val: SrcBits - 1, DL: dl, VT: IntVT);
9540 SDValue ClampedShift =
9541 DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IntVT, N1: TotalShift, N2: MaxShift);
9542 SDValue DenormTruncMant =
9543 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant,
9544 N2: DAG.getZExtOrTrunc(Op: ClampedShift, DL: dl, VT: ShiftVT));
9545
9546 // Rounding for denorm path.
9547 SDValue DenormRoundUp;
9548 {
9549 // Round bit is at position TotalShift - 1 of FullSrcMant.
9550 // Clamp to at least 1 so the subtraction doesn't underflow and create
9551 // shift nodes with invalid shift amounts.
9552 SDValue SafeShift = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT: IntVT, N1: ClampedShift, N2: One);
9553 SDValue RoundBitPos = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: SafeShift, N2: One);
9554 SDValue RoundBitPosAmt = DAG.getZExtOrTrunc(Op: RoundBitPos, DL: dl, VT: ShiftVT);
9555 SDValue DenormRoundBit = DAG.getNode(
9556 Opcode: ISD::AND, DL: dl, VT: IntVT,
9557 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: FullSrcMant, N2: RoundBitPosAmt), N2: One);
9558
9559 // Sticky: all bits below round bit.
9560 // sticky_mask = (1 << RoundBitPos) - 1
9561 SDValue StickyMask = DAG.getNode(
9562 Opcode: ISD::SUB, DL: dl, VT: IntVT,
9563 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: RoundBitPosAmt), N2: One);
9564 SDValue DenormStickyBits =
9565 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: FullSrcMant, N2: StickyMask);
9566 SDValue HasSticky = DAG.getNode(
9567 Opcode: ISD::ZERO_EXTEND, DL: dl, VT: IntVT,
9568 Operand: DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormStickyBits, RHS: Zero, Cond: ISD::SETNE));
9569
9570 SDValue DenormLSB =
9571 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: One);
9572
9573 DenormRoundUp = ComputeRoundUp(DenormRoundBit, HasSticky, DenormLSB);
9574
9575 // Only apply rounding if TotalShift >= 1 (i.e., there are bits to round).
9576 SDValue ShiftGEOne =
9577 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ClampedShift, RHS: One, Cond: ISD::SETUGE);
9578 DenormRoundUp = DAG.getSelect(DL: dl, VT: IntVT, Cond: ShiftGEOne, LHS: DenormRoundUp, RHS: Zero);
9579 }
9580
9581 SDValue DenormRoundedMant =
9582 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: DenormTruncMant, N2: DenormRoundUp);
9583
9584 // If rounding caused overflow into the normal range, then we get the
9585 // smallest normal number.
9586 SDValue DenormMantOF =
9587 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: DenormRoundedMant,
9588 RHS: DAG.getConstant(Val: DstMantMask, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9589 SDValue DenormFinalMant =
9590 DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: Zero, RHS: DenormRoundedMant);
9591 SDValue DenormFinalExp = DAG.getSelect(DL: dl, VT: IntVT, Cond: DenormMantOF, LHS: One, RHS: Zero);
9592
9593 // Assemble: sign | (exp << DstMant) | mant
9594 SDValue DenormExpShifted =
9595 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormFinalExp,
9596 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9597 DenormResult = DAG.getNode(
9598 Opcode: ISD::OR, DL: dl, VT: IntVT,
9599 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
9600 N2: DenormFinalMant);
9601 }
9602
9603 // Exponent overflow detection.
9604 SDValue ExpOF =
9605 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9606 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9607
9608 // Also check if AdjExp == DstExpMaxNormal and mantissa overflow into
9609 // a value that exceeds the max allowed mantissa at that exponent.
9610 SDValue ExpAtMax =
9611 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjExp,
9612 RHS: DAG.getConstant(Val: DstExpMaxNormal, DL: dl, VT: IntVT), Cond: ISD::SETEQ);
9613 SDValue MantExceedsMax =
9614 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: AdjMant,
9615 RHS: DAG.getConstant(Val: DstMaxMantAtMaxExp, DL: dl, VT: IntVT), Cond: ISD::SETGT);
9616 SDValue ExpMantOF =
9617 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: ExpAtMax, N2: MantExceedsMax);
9618 SDValue IsOverflow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SetCCVT, N1: ExpOF, N2: ExpMantOF);
9619
9620 // Build overflow result.
9621 SDValue OverflowResult;
9622
9623 if (Saturate) {
9624 // Clamp to max finite value:
9625 // sign | (DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp
9626 uint64_t MaxFinite =
9627 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9628 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9629 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9630 } else if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9631 // Produce infinity.
9632 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9633 OverflowResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9634 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9635 } else {
9636 // Emit poison if no Inf in format and not saturating.
9637 OverflowResult = DAG.getPOISON(VT: IntVT);
9638 }
9639
9640 // Assemble normal result: sign | (AdjExp << DstMant) | AdjMant
9641 SDValue NormExpShifted =
9642 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: AdjExp,
9643 N2: DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl));
9644 SDValue NormResult = DAG.getNode(
9645 Opcode: ISD::OR, DL: dl, VT: IntVT,
9646 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted), N2: AdjMant);
9647
9648 // Build special-value results.
9649 SDValue NaNResult;
9650 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9651 // Produce canonical NaN.
9652 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
9653 NaNResult =
9654 DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | QNaNBit, DL: dl, VT: IntVT);
9655 } else if (DstNFBehavior == fltNonfiniteBehavior::NanOnly &&
9656 DstNanEnc == fltNanEncoding::AllOnes) {
9657 // E4M3FN-style: NaN is exp=all-ones, mant=all-ones.
9658 NaNResult = DAG.getConstant(Val: ((uint64_t)DstExpMax << DstMant) | DstMantMask,
9659 DL: dl, VT: IntVT);
9660 } else {
9661 // NaN -> poison for finite only values.
9662 NaNResult = DAG.getPOISON(VT: IntVT);
9663 }
9664
9665 // Inf handling.
9666 SDValue InfResult;
9667 if (DstNFBehavior == fltNonfiniteBehavior::IEEE754) {
9668 // Produce signed infinity.
9669 uint64_t InfBits = (uint64_t)DstExpMax << DstMant;
9670 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9671 N2: DAG.getConstant(Val: InfBits, DL: dl, VT: IntVT));
9672 } else if (Saturate) {
9673 // Inf saturates to max finite.
9674 uint64_t MaxFinite =
9675 ((uint64_t)DstExpMaxNormal << DstMant) | DstMaxMantAtMaxExp;
9676 InfResult = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9677 N2: DAG.getConstant(Val: MaxFinite, DL: dl, VT: IntVT));
9678 } else {
9679 // No Inf and not saturating -> poison.
9680 InfResult = DAG.getPOISON(VT: IntVT);
9681 }
9682
9683 SDValue ZeroResult = SignShifted;
9684
9685 // Final selection in an order: NaN takes priority, then Inf, then Zero.
9686 SDValue FiniteResult =
9687 DAG.getSelect(DL: dl, VT: IntVT, Cond: ExpIsNeg, LHS: DenormResult, RHS: NormResult);
9688 FiniteResult =
9689 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsOverflow, LHS: OverflowResult, RHS: FiniteResult);
9690
9691 SDValue Result = FiniteResult;
9692 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
9693 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
9694 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
9695
9696 // Truncate to destination integer type.
9697 return DAG.getZExtOrTrunc(Op: Result, DL: dl, VT: ResVT);
9698}
9699
9700SDValue
9701TargetLowering::expandCONVERT_FROM_ARBITRARY_FP(SDNode *Node,
9702 SelectionDAG &DAG) const {
9703 SDLoc dl(Node);
9704 EVT DstVT = Node->getValueType(ResNo: 0);
9705 EVT DstScalarVT = DstVT.getScalarType();
9706
9707 SDValue IntVal = Node->getOperand(Num: 0);
9708 const uint64_t SemEnum = Node->getConstantOperandVal(Num: 1);
9709 const auto Sem = static_cast<APFloatBase::Semantics>(SemEnum);
9710
9711 // Supported source formats.
9712 switch (Sem) {
9713 case APFloatBase::S_Float8E5M2:
9714 case APFloatBase::S_Float8E4M3FN:
9715 case APFloatBase::S_Float6E3M2FN:
9716 case APFloatBase::S_Float6E2M3FN:
9717 case APFloatBase::S_Float4E2M1FN:
9718 break;
9719 default:
9720 DAG.getContext()->emitError(ErrorStr: "CONVERT_FROM_ARBITRARY_FP: not implemented "
9721 "source format (semantics enum " +
9722 Twine(SemEnum) + ")");
9723 return SDValue();
9724 }
9725
9726 const fltSemantics &SrcSem = APFloatBase::EnumToSemantics(S: Sem);
9727 const unsigned SrcBits = APFloat::getSizeInBits(Sem: SrcSem);
9728 const unsigned SrcPrecision = APFloat::semanticsPrecision(SrcSem);
9729 const unsigned SrcMant = SrcPrecision - 1;
9730 const unsigned SrcExp = SrcBits - SrcMant - 1;
9731 const int SrcBias = 1 - APFloat::semanticsMinExponent(SrcSem);
9732 const fltNonfiniteBehavior NFBehavior = SrcSem.nonFiniteBehavior;
9733
9734 // Destination format parameters.
9735 const fltSemantics &DstSem = DstScalarVT.getFltSemantics();
9736 const unsigned DstBits = APFloat::getSizeInBits(Sem: DstSem);
9737 const unsigned DstMant = APFloat::semanticsPrecision(DstSem) - 1;
9738 const unsigned DstExpBits = DstBits - DstMant - 1;
9739 const int DstMinExp = APFloat::semanticsMinExponent(DstSem);
9740 const int DstBias = 1 - DstMinExp;
9741 const uint64_t DstExpAllOnes = (1ULL << DstExpBits) - 1;
9742
9743 // Work in an integer type matching the destination float width.
9744 EVT IntScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: DstBits);
9745 EVT IntVT = DstVT.isVector()
9746 ? EVT::getVectorVT(Context&: *DAG.getContext(), VT: IntScalarVT,
9747 EC: DstVT.getVectorElementCount())
9748 : IntScalarVT;
9749
9750 SDValue Src = DAG.getZExtOrTrunc(Op: IntVal, DL: dl, VT: IntVT);
9751
9752 EVT SetCCVT =
9753 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: IntVT);
9754
9755 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: IntVT);
9756 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: IntVT);
9757
9758 // Extract bit fields.
9759 const uint64_t MantMask = (SrcMant > 0) ? ((1ULL << SrcMant) - 1) : 0;
9760 const uint64_t ExpMask = (1ULL << SrcExp) - 1;
9761
9762 SDValue MantField = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Src,
9763 N2: DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT));
9764
9765 SDValue ExpField =
9766 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT,
9767 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9768 N2: DAG.getShiftAmountConstant(Val: SrcMant, VT: IntVT, DL: dl)),
9769 N2: DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT));
9770
9771 SDValue SignBit =
9772 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: Src,
9773 N2: DAG.getShiftAmountConstant(Val: SrcBits - 1, VT: IntVT, DL: dl));
9774
9775 SDValue SignShifted =
9776 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: SignBit,
9777 N2: DAG.getShiftAmountConstant(Val: DstBits - 1, VT: IntVT, DL: dl));
9778
9779 // Classify the input.
9780 SDValue ExpAllOnes = DAG.getConstant(Val: ExpMask, DL: dl, VT: IntVT);
9781 SDValue IsExpAllOnes =
9782 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: ExpAllOnes, Cond: ISD::SETEQ);
9783 SDValue IsExpZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: ExpField, RHS: Zero, Cond: ISD::SETEQ);
9784 SDValue IsMantZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETEQ);
9785 SDValue IsMantNonZero =
9786 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: Zero, Cond: ISD::SETNE);
9787
9788 SDValue IsNaN;
9789 if (NFBehavior == fltNonfiniteBehavior::FiniteOnly) {
9790 IsNaN = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
9791 } else if (NFBehavior == fltNonfiniteBehavior::IEEE754) {
9792 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantNonZero);
9793 } else {
9794 assert(SrcSem.nanEncoding == fltNanEncoding::AllOnes);
9795 SDValue MantAllOnes = DAG.getConstant(Val: MantMask, DL: dl, VT: IntVT);
9796 SDValue IsMantAllOnes =
9797 DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: MantField, RHS: MantAllOnes, Cond: ISD::SETEQ);
9798 IsNaN = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantAllOnes);
9799 }
9800
9801 SDValue IsInf;
9802 if (NFBehavior == fltNonfiniteBehavior::IEEE754)
9803 IsInf = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpAllOnes, N2: IsMantZero);
9804 else
9805 IsInf = DAG.getBoolConstant(V: false, DL: dl, VT: SetCCVT, OpVT: IntVT);
9806
9807 SDValue IsZero = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantZero);
9808 SDValue IsDenorm =
9809 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SetCCVT, N1: IsExpZero, N2: IsMantNonZero);
9810
9811 // Normal value conversion.
9812 const int BiasAdjust = DstBias - SrcBias;
9813 SDValue NormDstExp =
9814 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: IntVT, N1: ExpField,
9815 N2: DAG.getConstant(Val: APInt(DstBits, BiasAdjust, true), DL: dl, VT: IntVT));
9816
9817 SDValue NormDstMant;
9818 if (DstMant > SrcMant) {
9819 SDValue NormDstMantShift =
9820 DAG.getShiftAmountConstant(Val: DstMant - SrcMant, VT: IntVT, DL: dl);
9821 NormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: MantField, N2: NormDstMantShift);
9822 } else {
9823 NormDstMant = MantField;
9824 }
9825
9826 SDValue DstMantShift = DAG.getShiftAmountConstant(Val: DstMant, VT: IntVT, DL: dl);
9827 SDValue NormExpShifted =
9828 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: NormDstExp, N2: DstMantShift);
9829 SDValue NormResult =
9830 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
9831 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: NormExpShifted),
9832 N2: NormDstMant);
9833
9834 // Denormal value conversion.
9835 SDValue DenormResult;
9836 {
9837 const unsigned IntVTBits = DstBits;
9838 SDValue LeadingZeros =
9839 DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT: IntVT, Operand: MantField);
9840
9841 const int DenormExpConst =
9842 (int)IntVTBits + DstBias - SrcBias - (int)SrcMant;
9843 SDValue DenormDstExp = DAG.getNode(
9844 Opcode: ISD::SUB, DL: dl, VT: IntVT,
9845 N1: DAG.getConstant(Val: APInt(DstBits, DenormExpConst, true), DL: dl, VT: IntVT),
9846 N2: LeadingZeros);
9847
9848 SDValue MantMSB =
9849 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT,
9850 N1: DAG.getConstant(Val: IntVTBits - 1, DL: dl, VT: IntVT), N2: LeadingZeros);
9851
9852 SDValue LeadingOne = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: One, N2: MantMSB);
9853 SDValue Frac = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: IntVT, N1: MantField, N2: LeadingOne);
9854
9855 const unsigned ShiftSub = IntVTBits - 1 - DstMant;
9856 SDValue ShiftAmount = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: LeadingZeros,
9857 N2: DAG.getConstant(Val: ShiftSub, DL: dl, VT: IntVT));
9858
9859 SDValue DenormDstMant = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: Frac, N2: ShiftAmount);
9860
9861 SDValue DenormExpShifted =
9862 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: IntVT, N1: DenormDstExp, N2: DstMantShift);
9863 DenormResult = DAG.getNode(
9864 Opcode: ISD::OR, DL: dl, VT: IntVT,
9865 N1: DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted, N2: DenormExpShifted),
9866 N2: DenormDstMant);
9867 }
9868
9869 SDValue FiniteResult =
9870 DAG.getSelect(DL: dl, VT: IntVT, Cond: IsDenorm, LHS: DenormResult, RHS: NormResult);
9871
9872 const uint64_t QNaNBit = (DstMant > 0) ? (1ULL << (DstMant - 1)) : 0;
9873 SDValue NaNResult =
9874 DAG.getConstant(Val: (DstExpAllOnes << DstMant) | QNaNBit, DL: dl, VT: IntVT);
9875
9876 SDValue InfResult =
9877 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT, N1: SignShifted,
9878 N2: DAG.getConstant(Val: DstExpAllOnes << DstMant, DL: dl, VT: IntVT));
9879
9880 SDValue ZeroResult = SignShifted;
9881
9882 SDValue Result = FiniteResult;
9883 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsZero, LHS: ZeroResult, RHS: Result);
9884 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsInf, LHS: InfResult, RHS: Result);
9885 Result = DAG.getSelect(DL: dl, VT: IntVT, Cond: IsNaN, LHS: NaNResult, RHS: Result);
9886
9887 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DstVT, Operand: Result);
9888}
9889
9890bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
9891 SelectionDAG &DAG) const {
9892 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
9893 SDValue Src = Node->getOperand(Num: OpNo);
9894 EVT SrcVT = Src.getValueType();
9895 EVT DstVT = Node->getValueType(ResNo: 0);
9896 SDLoc dl(SDValue(Node, 0));
9897
9898 // FIXME: Only f32 to i64 conversions are supported.
9899 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
9900 return false;
9901
9902 if (Node->isStrictFPOpcode())
9903 // When a NaN is converted to an integer a trap is allowed. We can't
9904 // use this expansion here because it would eliminate that trap. Other
9905 // traps are also allowed and cannot be eliminated. See
9906 // IEEE 754-2008 sec 5.8.
9907 return false;
9908
9909 // Expand f32 -> i64 conversion
9910 // This algorithm comes from compiler-rt's implementation of fixsfdi:
9911 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
9912 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9913 EVT IntVT = SrcVT.changeTypeToInteger();
9914 EVT IntShVT = getShiftAmountTy(LHSTy: IntVT, DL: DAG.getDataLayout());
9915
9916 SDValue ExponentMask = DAG.getConstant(Val: 0x7F800000, DL: dl, VT: IntVT);
9917 SDValue ExponentLoBit = DAG.getConstant(Val: 23, DL: dl, VT: IntVT);
9918 SDValue Bias = DAG.getConstant(Val: 127, DL: dl, VT: IntVT);
9919 SDValue SignMask = DAG.getConstant(Val: APInt::getSignMask(BitWidth: SrcEltBits), DL: dl, VT: IntVT);
9920 SDValue SignLowBit = DAG.getConstant(Val: SrcEltBits - 1, DL: dl, VT: IntVT);
9921 SDValue MantissaMask = DAG.getConstant(Val: 0x007FFFFF, DL: dl, VT: IntVT);
9922
9923 SDValue Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntVT, Operand: Src);
9924
9925 SDValue ExponentBits = DAG.getNode(
9926 Opcode: ISD::SRL, DL: dl, VT: IntVT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: ExponentMask),
9927 N2: DAG.getZExtOrTrunc(Op: ExponentLoBit, DL: dl, VT: IntShVT));
9928 SDValue Exponent = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentBits, N2: Bias);
9929
9930 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: IntVT,
9931 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: SignMask),
9932 N2: DAG.getZExtOrTrunc(Op: SignLowBit, DL: dl, VT: IntShVT));
9933 Sign = DAG.getSExtOrTrunc(Op: Sign, DL: dl, VT: DstVT);
9934
9935 SDValue R = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: IntVT,
9936 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IntVT, N1: Bits, N2: MantissaMask),
9937 N2: DAG.getConstant(Val: 0x00800000, DL: dl, VT: IntVT));
9938
9939 R = DAG.getZExtOrTrunc(Op: R, DL: dl, VT: DstVT);
9940
9941 R = DAG.getSelectCC(
9942 DL: dl, LHS: Exponent, RHS: ExponentLoBit,
9943 True: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: DstVT, N1: R,
9944 N2: DAG.getZExtOrTrunc(
9945 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: Exponent, N2: ExponentLoBit),
9946 DL: dl, VT: IntShVT)),
9947 False: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: DstVT, N1: R,
9948 N2: DAG.getZExtOrTrunc(
9949 Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: IntVT, N1: ExponentLoBit, N2: Exponent),
9950 DL: dl, VT: IntShVT)),
9951 Cond: ISD::SETGT);
9952
9953 SDValue Ret = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: DstVT,
9954 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: R, N2: Sign), N2: Sign);
9955
9956 Result = DAG.getSelectCC(DL: dl, LHS: Exponent, RHS: DAG.getConstant(Val: 0, DL: dl, VT: IntVT),
9957 True: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), False: Ret, Cond: ISD::SETLT);
9958 return true;
9959}
9960
9961bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
9962 SDValue &Chain,
9963 SelectionDAG &DAG) const {
9964 SDLoc dl(SDValue(Node, 0));
9965 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
9966 SDValue Src = Node->getOperand(Num: OpNo);
9967
9968 EVT SrcVT = Src.getValueType();
9969 EVT DstVT = Node->getValueType(ResNo: 0);
9970 EVT SetCCVT =
9971 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
9972 EVT DstSetCCVT =
9973 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: DstVT);
9974
9975 // Only expand vector types if we have the appropriate vector bit operations.
9976 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
9977 ISD::FP_TO_SINT;
9978 if (DstVT.isVector() && (!isOperationLegalOrCustom(Op: SIntOpcode, VT: DstVT) ||
9979 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT: SrcVT)))
9980 return false;
9981
9982 // If the maximum float value is smaller then the signed integer range,
9983 // the destination signmask can't be represented by the float, so we can
9984 // just use FP_TO_SINT directly.
9985 const fltSemantics &APFSem = SrcVT.getFltSemantics();
9986 APFloat APF(APFSem, APInt::getZero(numBits: SrcVT.getScalarSizeInBits()));
9987 APInt SignMask = APInt::getSignMask(BitWidth: DstVT.getScalarSizeInBits());
9988 if (APFloat::opOverflow &
9989 APF.convertFromAPInt(Input: SignMask, IsSigned: false, RM: APFloat::rmNearestTiesToEven)) {
9990 if (Node->isStrictFPOpcode()) {
9991 Result = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
9992 Ops: { Node->getOperand(Num: 0), Src });
9993 Chain = Result.getValue(R: 1);
9994 } else
9995 Result = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
9996 return true;
9997 }
9998
9999 // Don't expand it if there isn't cheap fsub instruction.
10000 if (!isOperationLegalOrCustom(
10001 Op: Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, VT: SrcVT))
10002 return false;
10003
10004 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
10005 SDValue Sel;
10006
10007 if (Node->isStrictFPOpcode()) {
10008 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
10009 Chain: Node->getOperand(Num: 0), /*IsSignaling*/ true);
10010 Chain = Sel.getValue(R: 1);
10011 } else {
10012 Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT);
10013 }
10014
10015 bool Strict = Node->isStrictFPOpcode() ||
10016 shouldUseStrictFP_TO_INT(FpVT: SrcVT, IntVT: DstVT, /*IsSigned*/ false);
10017
10018 if (Strict) {
10019 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
10020 // signmask then offset (the result of which should be fully representable).
10021 // Sel = Src < 0x8000000000000000
10022 // FltOfs = select Sel, 0, 0x8000000000000000
10023 // IntOfs = select Sel, 0, 0x8000000000000000
10024 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
10025
10026 // TODO: Should any fast-math-flags be set for the FSUB?
10027 SDValue FltOfs = DAG.getSelect(DL: dl, VT: SrcVT, Cond: Sel,
10028 LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
10029 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10030 SDValue IntOfs = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel,
10031 LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT),
10032 RHS: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10033 SDValue SInt;
10034 if (Node->isStrictFPOpcode()) {
10035 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl, ResultTys: { SrcVT, MVT::Other },
10036 Ops: { Chain, Src, FltOfs });
10037 SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl, ResultTys: { DstVT, MVT::Other },
10038 Ops: { Val.getValue(R: 1), Val });
10039 Chain = SInt.getValue(R: 1);
10040 } else {
10041 SDValue Val = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: FltOfs);
10042 SInt = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Val);
10043 }
10044 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
10045 } else {
10046 // Expand based on maximum range of FP_TO_SINT:
10047 // True = fp_to_sint(Src)
10048 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
10049 // Result = select (Src < 0x8000000000000000), True, False
10050
10051 SDValue True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT, Operand: Src);
10052 // TODO: Should any fast-math-flags be set for the FSUB?
10053 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: DstVT,
10054 Operand: DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: SrcVT, N1: Src, N2: Cst));
10055 False = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: False,
10056 N2: DAG.getConstant(Val: SignMask, DL: dl, VT: DstVT));
10057 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
10058 Result = DAG.getSelect(DL: dl, VT: DstVT, Cond: Sel, LHS: True, RHS: False);
10059 }
10060 return true;
10061}
10062
10063bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
10064 SDValue &Chain, SelectionDAG &DAG) const {
10065 // This transform is not correct for converting 0 when rounding mode is set
10066 // to round toward negative infinity which will produce -0.0. So disable
10067 // under strictfp.
10068 if (Node->isStrictFPOpcode())
10069 return false;
10070
10071 SDValue Src = Node->getOperand(Num: 0);
10072 EVT SrcVT = Src.getValueType();
10073 EVT DstVT = Node->getValueType(ResNo: 0);
10074
10075 // If the input is known to be non-negative and SINT_TO_FP is legal then use
10076 // it.
10077 if (Node->getFlags().hasNonNeg() &&
10078 isOperationLegalOrCustom(Op: ISD::SINT_TO_FP, VT: SrcVT)) {
10079 Result =
10080 DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(Node), VT: DstVT, Operand: Node->getOperand(Num: 0));
10081 return true;
10082 }
10083
10084 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
10085 return false;
10086
10087 // Only expand vector types if we have the appropriate vector bit
10088 // operations.
10089 if (SrcVT.isVector() && (!isOperationLegalOrCustom(Op: ISD::SRL, VT: SrcVT) ||
10090 !isOperationLegalOrCustom(Op: ISD::FADD, VT: DstVT) ||
10091 !isOperationLegalOrCustom(Op: ISD::FSUB, VT: DstVT) ||
10092 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT: SrcVT) ||
10093 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT: SrcVT)))
10094 return false;
10095
10096 SDLoc dl(SDValue(Node, 0));
10097
10098 // Implementation of unsigned i64 to f64 following the algorithm in
10099 // __floatundidf in compiler_rt. This implementation performs rounding
10100 // correctly in all rounding modes with the exception of converting 0
10101 // when rounding toward negative infinity. In that case the fsub will
10102 // produce -0.0. This will be added to +0.0 and produce -0.0 which is
10103 // incorrect.
10104 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), DL: dl, VT: SrcVT);
10105 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
10106 Val: llvm::bit_cast<double>(UINT64_C(0x4530000000100000)), DL: dl, VT: DstVT);
10107 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), DL: dl, VT: SrcVT);
10108 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), DL: dl, VT: SrcVT);
10109 SDValue HiShift = DAG.getShiftAmountConstant(Val: 32, VT: SrcVT, DL: dl);
10110
10111 SDValue Lo = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: SrcVT, N1: Src, N2: LoMask);
10112 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: SrcVT, N1: Src, N2: HiShift);
10113 SDValue LoOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Lo, N2: TwoP52);
10114 SDValue HiOr = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: SrcVT, N1: Hi, N2: TwoP84);
10115 SDValue LoFlt = DAG.getBitcast(VT: DstVT, V: LoOr);
10116 SDValue HiFlt = DAG.getBitcast(VT: DstVT, V: HiOr);
10117 SDValue HiSub = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: DstVT, N1: HiFlt, N2: TwoP84PlusTwoP52);
10118 Result = DAG.getNode(Opcode: ISD::FADD, DL: dl, VT: DstVT, N1: LoFlt, N2: HiSub);
10119 return true;
10120}
10121
10122SDValue
10123TargetLowering::createSelectForFMINNUM_FMAXNUM(SDNode *Node,
10124 SelectionDAG &DAG) const {
10125 unsigned Opcode = Node->getOpcode();
10126 assert((Opcode == ISD::FMINNUM || Opcode == ISD::FMAXNUM ||
10127 Opcode == ISD::STRICT_FMINNUM || Opcode == ISD::STRICT_FMAXNUM) &&
10128 "Wrong opcode");
10129
10130 if (Node->getFlags().hasNoNaNs()) {
10131 ISD::CondCode Pred = Opcode == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
10132 EVT VT = Node->getValueType(ResNo: 0);
10133 if ((!isCondCodeLegal(CC: Pred, VT: VT.getSimpleVT()) ||
10134 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)) &&
10135 VT.isVector())
10136 return SDValue();
10137 SDValue Op1 = Node->getOperand(Num: 0);
10138 SDValue Op2 = Node->getOperand(Num: 1);
10139 return DAG.getSelectCC(DL: SDLoc(Node), LHS: Op1, RHS: Op2, True: Op1, False: Op2, Cond: Pred,
10140 Flags: Node->getFlags());
10141 }
10142
10143 return SDValue();
10144}
10145
10146SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
10147 SelectionDAG &DAG) const {
10148 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node, DAG))
10149 return Expanded;
10150
10151 EVT VT = Node->getValueType(ResNo: 0);
10152 if (VT.isScalableVector())
10153 report_fatal_error(
10154 reason: "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
10155
10156 SDLoc dl(Node);
10157 unsigned NewOp =
10158 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10159
10160 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10161 SDValue Quiet0 = Node->getOperand(Num: 0);
10162 SDValue Quiet1 = Node->getOperand(Num: 1);
10163
10164 if (!Node->getFlags().hasNoNaNs()) {
10165 // Insert canonicalizes if it's possible we need to quiet to get correct
10166 // sNaN behavior.
10167 if (!DAG.isKnownNeverSNaN(Op: Quiet0)) {
10168 Quiet0 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet0,
10169 Flags: Node->getFlags());
10170 }
10171 if (!DAG.isKnownNeverSNaN(Op: Quiet1)) {
10172 Quiet1 = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL: dl, VT, Operand: Quiet1,
10173 Flags: Node->getFlags());
10174 }
10175 }
10176
10177 return DAG.getNode(Opcode: NewOp, DL: dl, VT, N1: Quiet0, N2: Quiet1, Flags: Node->getFlags());
10178 }
10179
10180 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
10181 // instead if there are no NaNs.
10182 if (Node->getFlags().hasNoNaNs() ||
10183 (DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 0)) &&
10184 DAG.isKnownNeverNaN(Op: Node->getOperand(Num: 1)))) {
10185 unsigned IEEE2018Op =
10186 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10187 if (isOperationLegalOrCustom(Op: IEEE2018Op, VT))
10188 return DAG.getNode(Opcode: IEEE2018Op, DL: dl, VT, N1: Node->getOperand(Num: 0),
10189 N2: Node->getOperand(Num: 1), Flags: Node->getFlags());
10190 }
10191
10192 if (SDValue SelCC = createSelectForFMINNUM_FMAXNUM(Node, DAG))
10193 return SelCC;
10194
10195 return SDValue();
10196}
10197
10198static SDValue isSpecificZeroAfterMaybeRounding(SelectionDAG &DAG,
10199 const TargetLowering &TLI,
10200 const SDLoc &DL, SDValue Val,
10201 FPClassTest FPClass) {
10202 EVT VT = Val.getValueType();
10203 EVT CCVT = TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10204 EVT IntVT = VT.changeTypeToInteger();
10205 EVT FloatVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
10206 SDValue TestZero = DAG.getTargetConstant(Val: FPClass, DL, VT: MVT::i32);
10207 if (!TLI.isTypeLegal(VT: IntVT) &&
10208 !TLI.isOperationLegalOrCustom(Op: ISD::IS_FPCLASS, VT))
10209 Val = DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: FloatVT, N1: Val,
10210 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
10211 return DAG.getNode(Opcode: ISD::IS_FPCLASS, DL, VT: CCVT, N1: Val, N2: TestZero);
10212}
10213
10214SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
10215 SelectionDAG &DAG) const {
10216 if (SDValue Expanded = expandVectorNaryOpBySplitting(Node: N, DAG))
10217 return Expanded;
10218
10219 SDLoc DL(N);
10220 SDValue LHS = N->getOperand(Num: 0);
10221 SDValue RHS = N->getOperand(Num: 1);
10222 unsigned Opc = N->getOpcode();
10223 EVT VT = N->getValueType(ResNo: 0);
10224 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10225 bool IsMax = Opc == ISD::FMAXIMUM;
10226 SDNodeFlags Flags = N->getFlags();
10227
10228 // First, implement comparison not propagating NaN. If no native fmin or fmax
10229 // available, use plain select with setcc instead.
10230 SDValue MinMax;
10231 unsigned CompOpcIeee = IsMax ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
10232 unsigned CompOpc = IsMax ? ISD::FMAXNUM : ISD::FMINNUM;
10233
10234 // FIXME: We should probably define fminnum/fmaxnum variants with correct
10235 // signed zero behavior.
10236 bool MinMaxMustRespectOrderedZero = false;
10237
10238 if (isOperationLegalOrCustom(Op: CompOpcIeee, VT)) {
10239 MinMax = DAG.getNode(Opcode: CompOpcIeee, DL, VT, N1: LHS, N2: RHS, Flags);
10240 MinMaxMustRespectOrderedZero = true;
10241 } else if (isOperationLegalOrCustom(Op: CompOpc, VT)) {
10242 MinMax = DAG.getNode(Opcode: CompOpc, DL, VT, N1: LHS, N2: RHS, Flags);
10243 } else {
10244 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
10245 return DAG.UnrollVectorOp(N);
10246
10247 // NaN (if exists) will be propagated later, so orderness doesn't matter.
10248 SDValue Compare =
10249 DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: IsMax ? ISD::SETOGT : ISD::SETOLT);
10250 MinMax = DAG.getSelect(DL, VT, Cond: Compare, LHS, RHS, Flags);
10251 }
10252
10253 // Propagate any NaN of both operands
10254 if (!N->getFlags().hasNoNaNs() &&
10255 (!DAG.isKnownNeverNaN(Op: RHS) || !DAG.isKnownNeverNaN(Op: LHS))) {
10256 ConstantFP *FPNaN = ConstantFP::get(Context&: *DAG.getContext(),
10257 V: APFloat::getNaN(Sem: VT.getFltSemantics()));
10258 MinMax = DAG.getSelect(DL, VT, Cond: DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETUO),
10259 LHS: DAG.getConstantFP(V: *FPNaN, DL, VT), RHS: MinMax, Flags);
10260 }
10261
10262 // fminimum/fmaximum requires -0.0 less than +0.0
10263 if (!MinMaxMustRespectOrderedZero && !N->getFlags().hasNoSignedZeros() &&
10264 !DAG.isKnownNeverLogicalZero(Op: RHS) && !DAG.isKnownNeverLogicalZero(Op: LHS)) {
10265 SDValue IsEqual = DAG.getSetCC(DL, VT: CCVT, LHS, RHS, Cond: ISD::SETOEQ);
10266 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10267 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10268 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS, Flags);
10269 MinMax = DAG.getSelect(DL, VT, Cond: IsEqual, LHS: RetZero, RHS: MinMax, Flags);
10270 }
10271
10272 return MinMax;
10273}
10274
10275SDValue TargetLowering::expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *Node,
10276 SelectionDAG &DAG) const {
10277 SDLoc DL(Node);
10278 SDValue LHS = Node->getOperand(Num: 0);
10279 SDValue RHS = Node->getOperand(Num: 1);
10280 unsigned Opc = Node->getOpcode();
10281 EVT VT = Node->getValueType(ResNo: 0);
10282 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10283 bool IsMax = Opc == ISD::FMAXIMUMNUM;
10284 SDNodeFlags Flags = Node->getFlags();
10285
10286 unsigned NewOp =
10287 Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
10288
10289 if (isOperationLegalOrCustom(Op: NewOp, VT)) {
10290 if (!Flags.hasNoNaNs()) {
10291 // Insert canonicalizes if it's possible we need to quiet to get correct
10292 // sNaN behavior.
10293 if (!DAG.isKnownNeverSNaN(Op: LHS)) {
10294 LHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: LHS, Flags);
10295 }
10296 if (!DAG.isKnownNeverSNaN(Op: RHS)) {
10297 RHS = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT, Operand: RHS, Flags);
10298 }
10299 }
10300
10301 return DAG.getNode(Opcode: NewOp, DL, VT, N1: LHS, N2: RHS, Flags);
10302 }
10303
10304 // We can use FMINIMUM/FMAXIMUM if there is no NaN, since it has
10305 // same behaviors for all of other cases: +0.0 vs -0.0 included.
10306 if (Flags.hasNoNaNs() ||
10307 (DAG.isKnownNeverNaN(Op: LHS) && DAG.isKnownNeverNaN(Op: RHS))) {
10308 unsigned IEEE2019Op =
10309 Opc == ISD::FMINIMUMNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
10310 if (isOperationLegalOrCustom(Op: IEEE2019Op, VT))
10311 return DAG.getNode(Opcode: IEEE2019Op, DL, VT, N1: LHS, N2: RHS, Flags);
10312 }
10313
10314 // FMINNUM/FMAXMUM returns qNaN if either operand is sNaN, and it may return
10315 // either one for +0.0 vs -0.0.
10316 if ((Flags.hasNoNaNs() ||
10317 (DAG.isKnownNeverSNaN(Op: LHS) && DAG.isKnownNeverSNaN(Op: RHS))) &&
10318 (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10319 DAG.isKnownNeverLogicalZero(Op: RHS))) {
10320 unsigned IEEE2008Op = Opc == ISD::FMINIMUMNUM ? ISD::FMINNUM : ISD::FMAXNUM;
10321 if (isOperationLegalOrCustom(Op: IEEE2008Op, VT))
10322 return DAG.getNode(Opcode: IEEE2008Op, DL, VT, N1: LHS, N2: RHS, Flags);
10323 }
10324
10325 if (VT.isVector() &&
10326 (isOperationLegalOrCustomOrPromote(Op: Opc, VT: VT.getVectorElementType()) ||
10327 !isOperationLegalOrCustom(Op: ISD::VSELECT, VT)))
10328 return DAG.UnrollVectorOp(N: Node);
10329
10330 // If only one operand is NaN, override it with another operand.
10331 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: LHS)) {
10332 LHS = DAG.getSelectCC(DL, LHS, RHS: LHS, True: RHS, False: LHS, Cond: ISD::SETUO);
10333 }
10334 if (!Flags.hasNoNaNs() && !DAG.isKnownNeverNaN(Op: RHS)) {
10335 RHS = DAG.getSelectCC(DL, LHS: RHS, RHS, True: LHS, False: RHS, Cond: ISD::SETUO);
10336 }
10337
10338 // Always prefer RHS if equal.
10339 SDValue MinMax =
10340 DAG.getSelectCC(DL, LHS, RHS, True: LHS, False: RHS, Cond: IsMax ? ISD::SETGT : ISD::SETLT);
10341
10342 // TODO: We need quiet sNaN if strictfp.
10343
10344 // Fixup signed zero behavior.
10345 if (Flags.hasNoSignedZeros() || DAG.isKnownNeverLogicalZero(Op: LHS) ||
10346 DAG.isKnownNeverLogicalZero(Op: RHS)) {
10347 return MinMax;
10348 }
10349 SDValue IsZero = DAG.getSetCC(DL, VT: CCVT, LHS: MinMax,
10350 RHS: DAG.getConstantFP(Val: 0.0, DL, VT), Cond: ISD::SETEQ);
10351 SDValue IsSpecificZero = isSpecificZeroAfterMaybeRounding(
10352 DAG, TLI: *this, DL, Val: LHS, FPClass: IsMax ? fcPosZero : fcNegZero);
10353 // It's OK to select from LHS and MinMax, with only one ISD::IS_FPCLASS, as
10354 // we preferred RHS when generate MinMax, if the operands are equal.
10355 SDValue RetZero = DAG.getSelect(DL, VT, Cond: IsSpecificZero, LHS, RHS: MinMax, Flags);
10356 return DAG.getSelect(DL, VT, Cond: IsZero, LHS: RetZero, RHS: MinMax, Flags);
10357}
10358
10359/// Returns a true value if if this FPClassTest can be performed with an ordered
10360/// fcmp to 0, and a false value if it's an unordered fcmp to 0. Returns
10361/// std::nullopt if it cannot be performed as a compare with 0.
10362static std::optional<bool> isFCmpEqualZero(FPClassTest Test,
10363 const fltSemantics &Semantics,
10364 const MachineFunction &MF) {
10365 FPClassTest OrderedMask = Test & ~fcNan;
10366 FPClassTest NanTest = Test & fcNan;
10367 bool IsOrdered = NanTest == fcNone;
10368 bool IsUnordered = NanTest == fcNan;
10369
10370 // Skip cases that are testing for only a qnan or snan.
10371 if (!IsOrdered && !IsUnordered)
10372 return std::nullopt;
10373
10374 if (OrderedMask == fcZero &&
10375 MF.getDenormalMode(FPType: Semantics).Input == DenormalMode::IEEE)
10376 return IsOrdered;
10377 if (OrderedMask == (fcZero | fcSubnormal) &&
10378 MF.getDenormalMode(FPType: Semantics).inputsAreZero())
10379 return IsOrdered;
10380 return std::nullopt;
10381}
10382
10383SDValue TargetLowering::expandIS_FPCLASS(EVT ResultVT, SDValue Op,
10384 const FPClassTest OrigTestMask,
10385 SDNodeFlags Flags, const SDLoc &DL,
10386 SelectionDAG &DAG) const {
10387 EVT OperandVT = Op.getValueType();
10388 assert(OperandVT.isFloatingPoint());
10389 FPClassTest Test = OrigTestMask;
10390
10391 // Degenerated cases.
10392 if (Test == fcNone)
10393 return DAG.getBoolConstant(V: false, DL, VT: ResultVT, OpVT: OperandVT);
10394 if (Test == fcAllFlags)
10395 return DAG.getBoolConstant(V: true, DL, VT: ResultVT, OpVT: OperandVT);
10396
10397 // PPC double double is a pair of doubles, of which the higher part determines
10398 // the value class.
10399 if (OperandVT == MVT::ppcf128) {
10400 Op = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::f64, N1: Op,
10401 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
10402 OperandVT = MVT::f64;
10403 }
10404
10405 // Floating-point type properties.
10406 EVT ScalarFloatVT = OperandVT.getScalarType();
10407 const Type *FloatTy = ScalarFloatVT.getTypeForEVT(Context&: *DAG.getContext());
10408 const llvm::fltSemantics &Semantics = FloatTy->getFltSemantics();
10409 bool IsF80 = (ScalarFloatVT == MVT::f80);
10410
10411 // Some checks can be implemented using float comparisons, if floating point
10412 // exceptions are ignored.
10413 if (Flags.hasNoFPExcept() &&
10414 isOperationLegalOrCustom(Op: ISD::SETCC, VT: OperandVT.getScalarType())) {
10415 FPClassTest FPTestMask = Test;
10416 bool IsInvertedFP = false;
10417
10418 if (FPClassTest InvertedFPCheck =
10419 invertFPClassTestIfSimpler(Test: FPTestMask, UseFCmp: true)) {
10420 FPTestMask = InvertedFPCheck;
10421 IsInvertedFP = true;
10422 }
10423
10424 ISD::CondCode OrderedCmpOpcode = IsInvertedFP ? ISD::SETUNE : ISD::SETOEQ;
10425 ISD::CondCode UnorderedCmpOpcode = IsInvertedFP ? ISD::SETONE : ISD::SETUEQ;
10426
10427 // See if we can fold an | fcNan into an unordered compare.
10428 FPClassTest OrderedFPTestMask = FPTestMask & ~fcNan;
10429
10430 // Can't fold the ordered check if we're only testing for snan or qnan
10431 // individually.
10432 if ((FPTestMask & fcNan) != fcNan)
10433 OrderedFPTestMask = FPTestMask;
10434
10435 const bool IsOrdered = FPTestMask == OrderedFPTestMask;
10436
10437 if (std::optional<bool> IsCmp0 =
10438 isFCmpEqualZero(Test: FPTestMask, Semantics, MF: DAG.getMachineFunction());
10439 IsCmp0 && (isCondCodeLegalOrCustom(
10440 CC: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode,
10441 VT: OperandVT.getScalarType().getSimpleVT()))) {
10442
10443 // If denormals could be implicitly treated as 0, this is not equivalent
10444 // to a compare with 0 since it will also be true for denormals.
10445 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op,
10446 RHS: DAG.getConstantFP(Val: 0.0, DL, VT: OperandVT),
10447 Cond: *IsCmp0 ? OrderedCmpOpcode : UnorderedCmpOpcode);
10448 }
10449
10450 if (FPTestMask == fcNan &&
10451 isCondCodeLegalOrCustom(CC: IsInvertedFP ? ISD::SETO : ISD::SETUO,
10452 VT: OperandVT.getScalarType().getSimpleVT()))
10453 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Op,
10454 Cond: IsInvertedFP ? ISD::SETO : ISD::SETUO);
10455
10456 bool IsOrderedInf = FPTestMask == fcInf;
10457 if ((FPTestMask == fcInf || FPTestMask == (fcInf | fcNan)) &&
10458 isCondCodeLegalOrCustom(CC: IsOrderedInf ? OrderedCmpOpcode
10459 : UnorderedCmpOpcode,
10460 VT: OperandVT.getScalarType().getSimpleVT()) &&
10461 isOperationLegalOrCustom(Op: ISD::FABS, VT: OperandVT.getScalarType()) &&
10462 (isOperationLegal(Op: ISD::ConstantFP, VT: OperandVT.getScalarType()) ||
10463 (OperandVT.isVector() &&
10464 isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: OperandVT)))) {
10465 // isinf(x) --> fabs(x) == inf
10466 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10467 SDValue Inf =
10468 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10469 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf,
10470 Cond: IsOrderedInf ? OrderedCmpOpcode : UnorderedCmpOpcode);
10471 }
10472
10473 if ((OrderedFPTestMask == fcPosInf || OrderedFPTestMask == fcNegInf) &&
10474 isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedCmpOpcode
10475 : UnorderedCmpOpcode,
10476 VT: OperandVT.getSimpleVT())) {
10477 // isposinf(x) --> x == inf
10478 // isneginf(x) --> x == -inf
10479 // isposinf(x) || nan --> x u== inf
10480 // isneginf(x) || nan --> x u== -inf
10481
10482 SDValue Inf = DAG.getConstantFP(
10483 Val: APFloat::getInf(Sem: Semantics, Negative: OrderedFPTestMask == fcNegInf), DL,
10484 VT: OperandVT);
10485 return DAG.getSetCC(DL, VT: ResultVT, LHS: Op, RHS: Inf,
10486 Cond: IsOrdered ? OrderedCmpOpcode : UnorderedCmpOpcode);
10487 }
10488
10489 if (OrderedFPTestMask == (fcSubnormal | fcZero) && !IsOrdered) {
10490 // TODO: Could handle ordered case, but it produces worse code for
10491 // x86. Maybe handle ordered if fabs is free?
10492
10493 ISD::CondCode OrderedOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10494 ISD::CondCode UnorderedOp = IsInvertedFP ? ISD::SETOGE : ISD::SETULT;
10495
10496 if (isCondCodeLegalOrCustom(CC: IsOrdered ? OrderedOp : UnorderedOp,
10497 VT: OperandVT.getScalarType().getSimpleVT())) {
10498 // (issubnormal(x) || iszero(x)) --> fabs(x) < smallest_normal
10499
10500 // TODO: Maybe only makes sense if fabs is free. Integer test of
10501 // exponent bits seems better for x86.
10502 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10503 SDValue SmallestNormal = DAG.getConstantFP(
10504 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10505 return DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal,
10506 Cond: IsOrdered ? OrderedOp : UnorderedOp);
10507 }
10508 }
10509
10510 if (FPTestMask == fcNormal) {
10511 // TODO: Handle unordered
10512 ISD::CondCode IsFiniteOp = IsInvertedFP ? ISD::SETUGE : ISD::SETOLT;
10513 ISD::CondCode IsNormalOp = IsInvertedFP ? ISD::SETOLT : ISD::SETUGE;
10514
10515 if (isCondCodeLegalOrCustom(CC: IsFiniteOp,
10516 VT: OperandVT.getScalarType().getSimpleVT()) &&
10517 isCondCodeLegalOrCustom(CC: IsNormalOp,
10518 VT: OperandVT.getScalarType().getSimpleVT()) &&
10519 isFAbsFree(VT: OperandVT)) {
10520 // isnormal(x) --> fabs(x) < infinity && !(fabs(x) < smallest_normal)
10521 SDValue Inf =
10522 DAG.getConstantFP(Val: APFloat::getInf(Sem: Semantics), DL, VT: OperandVT);
10523 SDValue SmallestNormal = DAG.getConstantFP(
10524 Val: APFloat::getSmallestNormalized(Sem: Semantics), DL, VT: OperandVT);
10525
10526 SDValue Abs = DAG.getNode(Opcode: ISD::FABS, DL, VT: OperandVT, Operand: Op);
10527 SDValue IsFinite = DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: Inf, Cond: IsFiniteOp);
10528 SDValue IsNormal =
10529 DAG.getSetCC(DL, VT: ResultVT, LHS: Abs, RHS: SmallestNormal, Cond: IsNormalOp);
10530 unsigned LogicOp = IsInvertedFP ? ISD::OR : ISD::AND;
10531 return DAG.getNode(Opcode: LogicOp, DL, VT: ResultVT, N1: IsFinite, N2: IsNormal);
10532 }
10533 }
10534 }
10535
10536 // Some checks may be represented as inversion of simpler check, for example
10537 // "inf|normal|subnormal|zero" => !"nan".
10538 bool IsInverted = false;
10539
10540 if (FPClassTest InvertedCheck = invertFPClassTestIfSimpler(Test, UseFCmp: false)) {
10541 Test = InvertedCheck;
10542 IsInverted = true;
10543 }
10544
10545 // In the general case use integer operations.
10546 unsigned BitSize = OperandVT.getScalarSizeInBits();
10547 EVT IntVT = OperandVT.changeElementType(
10548 Context&: *DAG.getContext(), EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BitSize));
10549 SDValue OpAsInt = DAG.getBitcast(VT: IntVT, V: Op);
10550
10551 // Various masks.
10552 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
10553 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
10554 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
10555 const unsigned ExplicitIntBitInF80 = 63;
10556 APInt ExpMask = Inf;
10557 if (IsF80)
10558 ExpMask.clearBit(BitPosition: ExplicitIntBitInF80);
10559 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
10560 APInt QNaNBitMask =
10561 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
10562 APInt InversionMask = APInt::getAllOnes(numBits: ResultVT.getScalarSizeInBits());
10563
10564 SDValue ValueMaskV = DAG.getConstant(Val: ValueMask, DL, VT: IntVT);
10565 SDValue SignBitV = DAG.getConstant(Val: SignBit, DL, VT: IntVT);
10566 SDValue ExpMaskV = DAG.getConstant(Val: ExpMask, DL, VT: IntVT);
10567 SDValue ZeroV = DAG.getConstant(Val: 0, DL, VT: IntVT);
10568 SDValue InfV = DAG.getConstant(Val: Inf, DL, VT: IntVT);
10569 SDValue ResultInversionMask = DAG.getConstant(Val: InversionMask, DL, VT: ResultVT);
10570
10571 SDValue Res;
10572 const auto appendResult = [&](SDValue PartialRes) {
10573 if (PartialRes) {
10574 if (Res)
10575 Res = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: Res, N2: PartialRes);
10576 else
10577 Res = PartialRes;
10578 }
10579 };
10580
10581 SDValue IntBitIsSetV; // Explicit integer bit in f80 mantissa is set.
10582 const auto getIntBitIsSet = [&]() -> SDValue {
10583 if (!IntBitIsSetV) {
10584 APInt IntBitMask(BitSize, 0);
10585 IntBitMask.setBit(ExplicitIntBitInF80);
10586 SDValue IntBitMaskV = DAG.getConstant(Val: IntBitMask, DL, VT: IntVT);
10587 SDValue IntBitV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: IntBitMaskV);
10588 IntBitIsSetV = DAG.getSetCC(DL, VT: ResultVT, LHS: IntBitV, RHS: ZeroV, Cond: ISD::SETNE);
10589 }
10590 return IntBitIsSetV;
10591 };
10592
10593 // Split the value into sign bit and absolute value.
10594 SDValue AbsV = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ValueMaskV);
10595 SDValue SignV = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt,
10596 RHS: DAG.getConstant(Val: 0, DL, VT: IntVT), Cond: ISD::SETLT);
10597
10598 // Tests that involve more than one class should be processed first.
10599 SDValue PartialRes;
10600
10601 if (IsF80)
10602 ; // Detect finite numbers of f80 by checking individual classes because
10603 // they have different settings of the explicit integer bit.
10604 else if ((Test & fcFinite) == fcFinite) {
10605 // finite(V) ==> (a << 1) < (inf << 1)
10606 //
10607 // See https://github.com/llvm/llvm-project/issues/169270, this is slightly
10608 // shorter than the `finite(V) ==> abs(V) < exp_mask` formula used before.
10609
10610 assert(APFloat::isIEEELikeFP(OperandVT.getFltSemantics()) &&
10611 "finite check requires IEEE-like FP");
10612
10613 SDValue One = DAG.getShiftAmountConstant(Val: 1, VT: IntVT, DL);
10614 SDValue TwiceOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: OpAsInt, N2: One);
10615 SDValue TwiceInf = DAG.getNode(Opcode: ISD::SHL, DL, VT: IntVT, N1: ExpMaskV, N2: One);
10616
10617 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: TwiceOp, RHS: TwiceInf, Cond: ISD::SETULT);
10618 Test &= ~fcFinite;
10619 } else if ((Test & fcFinite) == fcPosFinite) {
10620 // finite(V) && V > 0 ==> V < exp_mask
10621 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ExpMaskV, Cond: ISD::SETULT);
10622 Test &= ~fcPosFinite;
10623 } else if ((Test & fcFinite) == fcNegFinite) {
10624 // finite(V) && V < 0 ==> abs(V) < exp_mask && signbit == 1
10625 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ExpMaskV, Cond: ISD::SETLT);
10626 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10627 Test &= ~fcNegFinite;
10628 }
10629 appendResult(PartialRes);
10630
10631 if (FPClassTest PartialCheck = Test & (fcZero | fcSubnormal)) {
10632 // fcZero | fcSubnormal => test all exponent bits are 0
10633 // TODO: Handle sign bit specific cases
10634 if (PartialCheck == (fcZero | fcSubnormal)) {
10635 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: OpAsInt, N2: ExpMaskV);
10636 SDValue ExpIsZero =
10637 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
10638 appendResult(ExpIsZero);
10639 Test &= ~PartialCheck & fcAllFlags;
10640 }
10641 }
10642
10643 // Check for individual classes.
10644
10645 if (unsigned PartialCheck = Test & fcZero) {
10646 if (PartialCheck == fcPosZero)
10647 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: ZeroV, Cond: ISD::SETEQ);
10648 else if (PartialCheck == fcZero)
10649 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: ZeroV, Cond: ISD::SETEQ);
10650 else // ISD::fcNegZero
10651 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: SignBitV, Cond: ISD::SETEQ);
10652 appendResult(PartialRes);
10653 }
10654
10655 if (unsigned PartialCheck = Test & fcSubnormal) {
10656 // issubnormal(V) ==> unsigned(abs(V) - 1) < (all mantissa bits set)
10657 // issubnormal(V) && V>0 ==> unsigned(V - 1) < (all mantissa bits set)
10658 SDValue V = (PartialCheck == fcPosSubnormal) ? OpAsInt : AbsV;
10659 SDValue MantissaV = DAG.getConstant(Val: AllOneMantissa, DL, VT: IntVT);
10660 SDValue VMinusOneV =
10661 DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: V, N2: DAG.getConstant(Val: 1, DL, VT: IntVT));
10662 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: VMinusOneV, RHS: MantissaV, Cond: ISD::SETULT);
10663 if (PartialCheck == fcNegSubnormal)
10664 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10665 appendResult(PartialRes);
10666 }
10667
10668 if (unsigned PartialCheck = Test & fcInf) {
10669 if (PartialCheck == fcPosInf)
10670 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: InfV, Cond: ISD::SETEQ);
10671 else if (PartialCheck == fcInf)
10672 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETEQ);
10673 else { // ISD::fcNegInf
10674 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
10675 SDValue NegInfV = DAG.getConstant(Val: NegInf, DL, VT: IntVT);
10676 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: OpAsInt, RHS: NegInfV, Cond: ISD::SETEQ);
10677 }
10678 appendResult(PartialRes);
10679 }
10680
10681 if (unsigned PartialCheck = Test & fcNan) {
10682 APInt InfWithQnanBit = Inf | QNaNBitMask;
10683 SDValue InfWithQnanBitV = DAG.getConstant(Val: InfWithQnanBit, DL, VT: IntVT);
10684 if (PartialCheck == fcNan) {
10685 // isnan(V) ==> abs(V) > int(inf)
10686 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
10687 if (IsF80) {
10688 // Recognize unsupported values as NaNs for compatibility with glibc.
10689 // In them (exp(V)==0) == int_bit.
10690 SDValue ExpBits = DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: AbsV, N2: ExpMaskV);
10691 SDValue ExpIsZero =
10692 DAG.getSetCC(DL, VT: ResultVT, LHS: ExpBits, RHS: ZeroV, Cond: ISD::SETEQ);
10693 SDValue IsPseudo =
10694 DAG.getSetCC(DL, VT: ResultVT, LHS: getIntBitIsSet(), RHS: ExpIsZero, Cond: ISD::SETEQ);
10695 PartialRes = DAG.getNode(Opcode: ISD::OR, DL, VT: ResultVT, N1: PartialRes, N2: IsPseudo);
10696 }
10697 } else if (PartialCheck == fcQNan) {
10698 // isquiet(V) ==> abs(V) >= (unsigned(Inf) | quiet_bit)
10699 PartialRes =
10700 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETGE);
10701 } else { // ISD::fcSNan
10702 // issignaling(V) ==> abs(V) > unsigned(Inf) &&
10703 // abs(V) < (unsigned(Inf) | quiet_bit)
10704 SDValue IsNan = DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfV, Cond: ISD::SETGT);
10705 SDValue IsNotQnan =
10706 DAG.getSetCC(DL, VT: ResultVT, LHS: AbsV, RHS: InfWithQnanBitV, Cond: ISD::SETLT);
10707 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: IsNan, N2: IsNotQnan);
10708 }
10709 appendResult(PartialRes);
10710 }
10711
10712 if (unsigned PartialCheck = Test & fcNormal) {
10713 // isnormal(V) ==> (0 < exp < max_exp) ==> (unsigned(exp-1) < (max_exp-1))
10714 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
10715 SDValue ExpLSBV = DAG.getConstant(Val: ExpLSB, DL, VT: IntVT);
10716 SDValue ExpMinus1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: IntVT, N1: AbsV, N2: ExpLSBV);
10717 APInt ExpLimit = ExpMask - ExpLSB;
10718 SDValue ExpLimitV = DAG.getConstant(Val: ExpLimit, DL, VT: IntVT);
10719 PartialRes = DAG.getSetCC(DL, VT: ResultVT, LHS: ExpMinus1, RHS: ExpLimitV, Cond: ISD::SETULT);
10720 if (PartialCheck == fcNegNormal)
10721 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: SignV);
10722 else if (PartialCheck == fcPosNormal) {
10723 SDValue PosSignV =
10724 DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: SignV, N2: ResultInversionMask);
10725 PartialRes = DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: PosSignV);
10726 }
10727 if (IsF80)
10728 PartialRes =
10729 DAG.getNode(Opcode: ISD::AND, DL, VT: ResultVT, N1: PartialRes, N2: getIntBitIsSet());
10730 appendResult(PartialRes);
10731 }
10732
10733 if (!Res)
10734 return DAG.getConstant(Val: IsInverted, DL, VT: ResultVT);
10735 if (IsInverted)
10736 Res = DAG.getNode(Opcode: ISD::XOR, DL, VT: ResultVT, N1: Res, N2: ResultInversionMask);
10737 return Res;
10738}
10739
10740// Only expand vector types if we have the appropriate vector bit operations.
10741static bool canExpandVectorCTPOP(const TargetLowering &TLI, EVT VT) {
10742 assert(VT.isVector() && "Expected vector type");
10743 unsigned Len = VT.getScalarSizeInBits();
10744 return TLI.isOperationLegalOrCustom(Op: ISD::ADD, VT) &&
10745 TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT) &&
10746 TLI.isOperationLegalOrCustom(Op: ISD::SRL, VT) &&
10747 (Len == 8 || TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT)) &&
10748 TLI.isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT);
10749}
10750
10751SDValue TargetLowering::expandCTPOP(SDNode *Node, SelectionDAG &DAG) const {
10752 SDLoc dl(Node);
10753 EVT VT = Node->getValueType(ResNo: 0);
10754 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
10755 SDValue Op = Node->getOperand(Num: 0);
10756 unsigned Len = VT.getScalarSizeInBits();
10757 assert(VT.isInteger() && "CTPOP not implemented for this type.");
10758
10759 // TODO: Add support for irregular type lengths.
10760 if (!(Len <= 128 && Len % 8 == 0))
10761 return SDValue();
10762
10763 // Only expand vector types if we have the appropriate vector bit operations.
10764 if (VT.isVector() && !canExpandVectorCTPOP(TLI: *this, VT))
10765 return SDValue();
10766
10767 // This is the "best" algorithm from
10768 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
10769 SDValue Mask55 =
10770 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x55)), DL: dl, VT);
10771 SDValue Mask33 =
10772 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x33)), DL: dl, VT);
10773 SDValue Mask0F =
10774 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x0F)), DL: dl, VT);
10775
10776 // v = v - ((v >> 1) & 0x55555555...)
10777 Op = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op,
10778 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10779 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10780 N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT)),
10781 N2: Mask55));
10782 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
10783 Op = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op, N2: Mask33),
10784 N2: DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10785 N1: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10786 N2: DAG.getConstant(Val: 2, DL: dl, VT: ShVT)),
10787 N2: Mask33));
10788 // v = (v + (v >> 4)) & 0x0F0F0F0F...
10789 Op = DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10790 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
10791 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10792 N2: DAG.getConstant(Val: 4, DL: dl, VT: ShVT))),
10793 N2: Mask0F);
10794
10795 if (Len <= 8)
10796 return Op;
10797
10798 // Avoid the multiply if we only have 2 bytes to add.
10799 // TODO: Only doing this for scalars because vectors weren't as obviously
10800 // improved.
10801 if (Len == 16 && !VT.isVector()) {
10802 // v = (v + (v >> 8)) & 0x00FF;
10803 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT,
10804 N1: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Op,
10805 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op,
10806 N2: DAG.getConstant(Val: 8, DL: dl, VT: ShVT))),
10807 N2: DAG.getConstant(Val: 0xFF, DL: dl, VT));
10808 }
10809
10810 // v = (v * 0x01010101...) >> (Len - 8)
10811 SDValue V;
10812 if (isOperationLegalOrCustomOrPromote(
10813 Op: ISD::MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
10814 SDValue Mask01 =
10815 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x01)), DL: dl, VT);
10816 V = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Op, N2: Mask01);
10817 } else {
10818 V = Op;
10819 for (unsigned Shift = 8; Shift < Len; Shift *= 2) {
10820 SDValue ShiftC = DAG.getShiftAmountConstant(Val: Shift, VT, DL: dl);
10821 V = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: V,
10822 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: ShiftC));
10823 }
10824 }
10825 return DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: V, N2: DAG.getConstant(Val: Len - 8, DL: dl, VT: ShVT));
10826}
10827
10828SDValue TargetLowering::expandVPCTPOP(SDNode *Node, SelectionDAG &DAG) const {
10829 SDLoc dl(Node);
10830 EVT VT = Node->getValueType(ResNo: 0);
10831 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
10832 SDValue Op = Node->getOperand(Num: 0);
10833 SDValue Mask = Node->getOperand(Num: 1);
10834 SDValue VL = Node->getOperand(Num: 2);
10835 unsigned Len = VT.getScalarSizeInBits();
10836 assert(VT.isInteger() && "VP_CTPOP not implemented for this type.");
10837
10838 // TODO: Add support for irregular type lengths.
10839 if (!(Len <= 128 && Len % 8 == 0))
10840 return SDValue();
10841
10842 // This is same algorithm of expandCTPOP from
10843 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
10844 SDValue Mask55 =
10845 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x55)), DL: dl, VT);
10846 SDValue Mask33 =
10847 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x33)), DL: dl, VT);
10848 SDValue Mask0F =
10849 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x0F)), DL: dl, VT);
10850
10851 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5;
10852
10853 // v = v - ((v >> 1) & 0x55555555...)
10854 Tmp1 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT,
10855 N1: DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op,
10856 N2: DAG.getConstant(Val: 1, DL: dl, VT: ShVT), N3: Mask, N4: VL),
10857 N2: Mask55, N3: Mask, N4: VL);
10858 Op = DAG.getNode(Opcode: ISD::VP_SUB, DL: dl, VT, N1: Op, N2: Tmp1, N3: Mask, N4: VL);
10859
10860 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
10861 Tmp2 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Op, N2: Mask33, N3: Mask, N4: VL);
10862 Tmp3 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT,
10863 N1: DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op,
10864 N2: DAG.getConstant(Val: 2, DL: dl, VT: ShVT), N3: Mask, N4: VL),
10865 N2: Mask33, N3: Mask, N4: VL);
10866 Op = DAG.getNode(Opcode: ISD::VP_ADD, DL: dl, VT, N1: Tmp2, N2: Tmp3, N3: Mask, N4: VL);
10867
10868 // v = (v + (v >> 4)) & 0x0F0F0F0F...
10869 Tmp4 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 4, DL: dl, VT: ShVT),
10870 N3: Mask, N4: VL),
10871 Tmp5 = DAG.getNode(Opcode: ISD::VP_ADD, DL: dl, VT, N1: Op, N2: Tmp4, N3: Mask, N4: VL);
10872 Op = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp5, N2: Mask0F, N3: Mask, N4: VL);
10873
10874 if (Len <= 8)
10875 return Op;
10876
10877 // v = (v * 0x01010101...) >> (Len - 8)
10878 SDValue V;
10879 if (isOperationLegalOrCustomOrPromote(
10880 Op: ISD::VP_MUL, VT: getTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
10881 SDValue Mask01 =
10882 DAG.getConstant(Val: APInt::getSplat(NewLen: Len, V: APInt(8, 0x01)), DL: dl, VT);
10883 V = DAG.getNode(Opcode: ISD::VP_MUL, DL: dl, VT, N1: Op, N2: Mask01, N3: Mask, N4: VL);
10884 } else {
10885 V = Op;
10886 for (unsigned Shift = 8; Shift < Len; Shift *= 2) {
10887 SDValue ShiftC = DAG.getShiftAmountConstant(Val: Shift, VT, DL: dl);
10888 V = DAG.getNode(Opcode: ISD::VP_ADD, DL: dl, VT, N1: V,
10889 N2: DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: V, N2: ShiftC, N3: Mask, N4: VL),
10890 N3: Mask, N4: VL);
10891 }
10892 }
10893 return DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: V, N2: DAG.getConstant(Val: Len - 8, DL: dl, VT: ShVT),
10894 N3: Mask, N4: VL);
10895}
10896
10897SDValue TargetLowering::expandCTLZ(SDNode *Node, SelectionDAG &DAG) const {
10898 SDLoc dl(Node);
10899 EVT VT = Node->getValueType(ResNo: 0);
10900 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
10901 SDValue Op = Node->getOperand(Num: 0);
10902 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
10903
10904 // If the non-ZERO_POISON version is supported we can use that instead.
10905 if (Node->getOpcode() == ISD::CTLZ_ZERO_POISON &&
10906 isOperationLegalOrCustom(Op: ISD::CTLZ, VT))
10907 return DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Op);
10908
10909 // If the ZERO_POISON version is supported use that and handle the zero case.
10910 if (isOperationLegalOrCustom(Op: ISD::CTLZ_ZERO_POISON, VT)) {
10911 EVT SetCCVT =
10912 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
10913 SDValue CTLZ = DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Op);
10914 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
10915 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
10916 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
10917 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTLZ);
10918 }
10919
10920 // Only expand vector types if we have the appropriate vector bit operations.
10921 // This includes the operations needed to expand CTPOP if it isn't supported.
10922 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
10923 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
10924 !canExpandVectorCTPOP(TLI: *this, VT)) ||
10925 !isOperationLegalOrCustom(Op: ISD::SRL, VT) ||
10926 !isOperationLegalOrCustomOrPromote(Op: ISD::OR, VT)))
10927 return SDValue();
10928
10929 // for now, we do this:
10930 // x = x | (x >> 1);
10931 // x = x | (x >> 2);
10932 // ...
10933 // x = x | (x >>16);
10934 // x = x | (x >>32); // for 64-bit input
10935 // return popcount(~x);
10936 //
10937 // Ref: "Hacker's Delight" by Henry Warren
10938 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
10939 SDValue Tmp = DAG.getConstant(Val: 1ULL << i, DL: dl, VT: ShVT);
10940 Op = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Op,
10941 N2: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: Tmp));
10942 }
10943 Op = DAG.getNOT(DL: dl, Val: Op, VT);
10944 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Op);
10945}
10946
10947SDValue TargetLowering::expandVPCTLZ(SDNode *Node, SelectionDAG &DAG) const {
10948 SDLoc dl(Node);
10949 EVT VT = Node->getValueType(ResNo: 0);
10950 EVT ShVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
10951 SDValue Op = Node->getOperand(Num: 0);
10952 SDValue Mask = Node->getOperand(Num: 1);
10953 SDValue VL = Node->getOperand(Num: 2);
10954 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
10955
10956 // do this:
10957 // x = x | (x >> 1);
10958 // x = x | (x >> 2);
10959 // ...
10960 // x = x | (x >>16);
10961 // x = x | (x >>32); // for 64-bit input
10962 // return popcount(~x);
10963 for (unsigned i = 0; (1U << i) < NumBitsPerElt; ++i) {
10964 SDValue Tmp = DAG.getConstant(Val: 1ULL << i, DL: dl, VT: ShVT);
10965 Op = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Op,
10966 N2: DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: Tmp, N3: Mask, N4: VL), N3: Mask,
10967 N4: VL);
10968 }
10969 Op = DAG.getNode(Opcode: ISD::VP_XOR, DL: dl, VT, N1: Op, N2: DAG.getAllOnesConstant(DL: dl, VT),
10970 N3: Mask, N4: VL);
10971 return DAG.getNode(Opcode: ISD::VP_CTPOP, DL: dl, VT, N1: Op, N2: Mask, N3: VL);
10972}
10973
10974SDValue TargetLowering::expandCTLS(SDNode *Node, SelectionDAG &DAG) const {
10975 SDLoc dl(Node);
10976 EVT VT = Node->getValueType(ResNo: 0);
10977 SDValue Op = DAG.getFreeze(V: Node->getOperand(Num: 0));
10978 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
10979
10980 // CTLS(x) = CTLZ(OR(SHL(XOR(x, SRA(x, BW-1)), 1), 1))
10981 // This transforms the sign bits into leading zeros that can be counted.
10982 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: NumBitsPerElt - 1, VT, DL: dl);
10983 SDValue SignBit = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Op, N2: ShiftAmt);
10984 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: SignBit);
10985 SDValue Shl =
10986 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
10987 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Shl, N2: DAG.getConstant(Val: 1, DL: dl, VT));
10988 return DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL: dl, VT, Operand: Or);
10989}
10990
10991SDValue TargetLowering::CTTZTableLookup(SDNode *Node, SelectionDAG &DAG,
10992 const SDLoc &DL, EVT VT, SDValue Op,
10993 unsigned BitWidth) const {
10994 if (BitWidth != 32 && BitWidth != 64)
10995 return SDValue();
10996
10997 const DataLayout &TD = DAG.getDataLayout();
10998 if (!isOperationCustom(Op: ISD::ConstantPool, VT: getPointerTy(DL: TD)))
10999 return SDValue();
11000
11001 APInt DeBruijn = BitWidth == 32 ? APInt(32, 0x077CB531U)
11002 : APInt(64, 0x0218A392CD3D5DBFULL);
11003 MachinePointerInfo PtrInfo =
11004 MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction());
11005 unsigned ShiftAmt = BitWidth - Log2_32(Value: BitWidth);
11006 SDValue Neg = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
11007 SDValue Lookup = DAG.getNode(
11008 Opcode: ISD::SRL, DL, VT,
11009 N1: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Op, N2: Neg),
11010 N2: DAG.getConstant(Val: DeBruijn, DL, VT)),
11011 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
11012 Lookup = DAG.getSExtOrTrunc(Op: Lookup, DL, VT: getPointerTy(DL: TD));
11013
11014 SmallVector<uint8_t> Table(BitWidth, 0);
11015 for (unsigned i = 0; i < BitWidth; i++) {
11016 APInt Shl = DeBruijn.shl(shiftAmt: i);
11017 APInt Lshr = Shl.lshr(shiftAmt: ShiftAmt);
11018 Table[Lshr.getZExtValue()] = i;
11019 }
11020
11021 // Create a ConstantArray in Constant Pool
11022 auto *CA = ConstantDataArray::get(Context&: *DAG.getContext(), Elts&: Table);
11023 SDValue CPIdx = DAG.getConstantPool(C: CA, VT: getPointerTy(DL: TD),
11024 Align: TD.getPrefTypeAlign(Ty: CA->getType()));
11025 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: DL, VT, Chain: DAG.getEntryNode(),
11026 Ptr: DAG.getMemBasePlusOffset(Base: CPIdx, Offset: Lookup, DL),
11027 PtrInfo, MemVT: MVT::i8);
11028 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON)
11029 return ExtLoad;
11030
11031 EVT SetCCVT =
11032 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11033 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
11034 SDValue SrcIsZero = DAG.getSetCC(DL, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11035 return DAG.getSelect(DL, VT, Cond: SrcIsZero,
11036 LHS: DAG.getConstant(Val: BitWidth, DL, VT), RHS: ExtLoad);
11037}
11038
11039SDValue TargetLowering::expandCTTZ(SDNode *Node, SelectionDAG &DAG) const {
11040 SDLoc dl(Node);
11041 EVT VT = Node->getValueType(ResNo: 0);
11042 SDValue Op = Node->getOperand(Num: 0);
11043 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
11044
11045 // If the non-ZERO_POISON version is supported we can use that instead.
11046 if (Node->getOpcode() == ISD::CTTZ_ZERO_POISON &&
11047 isOperationLegalOrCustom(Op: ISD::CTTZ, VT))
11048 return DAG.getNode(Opcode: ISD::CTTZ, DL: dl, VT, Operand: Op);
11049
11050 // If the ZERO_POISON version is supported use that and handle the zero case.
11051 if (isOperationLegalOrCustom(Op: ISD::CTTZ_ZERO_POISON, VT)) {
11052 EVT SetCCVT =
11053 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11054 SDValue CTTZ = DAG.getNode(Opcode: ISD::CTTZ_ZERO_POISON, DL: dl, VT, Operand: Op);
11055 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11056 SDValue SrcIsZero = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Op, RHS: Zero, Cond: ISD::SETEQ);
11057 return DAG.getSelect(DL: dl, VT, Cond: SrcIsZero,
11058 LHS: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT), RHS: CTTZ);
11059 }
11060
11061 // Only expand vector types if we have the appropriate vector bit operations.
11062 // This includes the operations needed to expand CTPOP if it isn't supported.
11063 if (VT.isVector() && (!isPowerOf2_32(Value: NumBitsPerElt) ||
11064 (!isOperationLegalOrCustom(Op: ISD::CTPOP, VT) &&
11065 !isOperationLegalOrCustom(Op: ISD::CTLZ, VT) &&
11066 !canExpandVectorCTPOP(TLI: *this, VT)) ||
11067 !isOperationLegalOrCustom(Op: ISD::SUB, VT) ||
11068 !isOperationLegalOrCustomOrPromote(Op: ISD::AND, VT) ||
11069 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11070 return SDValue();
11071
11072 // Emit Table Lookup if ISD::CTPOP used in the fallback path below is going
11073 // to be expanded or converted to a libcall.
11074 if (!VT.isVector() && !isOperationLegalOrCustomOrPromote(Op: ISD::CTPOP, VT) &&
11075 !isOperationLegal(Op: ISD::CTLZ, VT))
11076 if (SDValue V = CTTZTableLookup(Node, DAG, DL: dl, VT, Op, BitWidth: NumBitsPerElt))
11077 return V;
11078
11079 // for now, we use: { return popcount(~x & (x - 1)); }
11080 // unless the target has ctlz but not ctpop, in which case we use:
11081 // { return 32 - nlz(~x & (x-1)); }
11082 // Ref: "Hacker's Delight" by Henry Warren
11083 SDValue Tmp = DAG.getNode(
11084 Opcode: ISD::AND, DL: dl, VT, N1: DAG.getNOT(DL: dl, Val: Op, VT),
11085 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 1, DL: dl, VT)));
11086
11087 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
11088 if (isOperationLegal(Op: ISD::CTLZ, VT) && !isOperationLegal(Op: ISD::CTPOP, VT)) {
11089 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: DAG.getConstant(Val: NumBitsPerElt, DL: dl, VT),
11090 N2: DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Tmp));
11091 }
11092
11093 return DAG.getNode(Opcode: ISD::CTPOP, DL: dl, VT, Operand: Tmp);
11094}
11095
11096SDValue TargetLowering::expandVPCTTZ(SDNode *Node, SelectionDAG &DAG) const {
11097 SDValue Op = Node->getOperand(Num: 0);
11098 SDValue Mask = Node->getOperand(Num: 1);
11099 SDValue VL = Node->getOperand(Num: 2);
11100 SDLoc dl(Node);
11101 EVT VT = Node->getValueType(ResNo: 0);
11102
11103 // Same as the vector part of expandCTTZ, use: popcount(~x & (x - 1))
11104 SDValue Not = DAG.getNode(Opcode: ISD::VP_XOR, DL: dl, VT, N1: Op,
11105 N2: DAG.getAllOnesConstant(DL: dl, VT), N3: Mask, N4: VL);
11106 SDValue MinusOne = DAG.getNode(Opcode: ISD::VP_SUB, DL: dl, VT, N1: Op,
11107 N2: DAG.getConstant(Val: 1, DL: dl, VT), N3: Mask, N4: VL);
11108 SDValue Tmp = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Not, N2: MinusOne, N3: Mask, N4: VL);
11109 return DAG.getNode(Opcode: ISD::VP_CTPOP, DL: dl, VT, N1: Tmp, N2: Mask, N3: VL);
11110}
11111
11112SDValue TargetLowering::expandVPCTTZElements(SDNode *N,
11113 SelectionDAG &DAG) const {
11114 // %cond = to_bool_vec %source
11115 // %splat = splat /*val=*/VL
11116 // %tz = step_vector
11117 // %v = vp.select %cond, /*true=*/tz, /*false=*/%splat
11118 // %r = vp.reduce.umin %v
11119 SDLoc DL(N);
11120 SDValue Source = N->getOperand(Num: 0);
11121 SDValue Mask = N->getOperand(Num: 1);
11122 SDValue EVL = N->getOperand(Num: 2);
11123 EVT SrcVT = Source.getValueType();
11124 EVT ResVT = N->getValueType(ResNo: 0);
11125 EVT ResVecVT =
11126 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ResVT, EC: SrcVT.getVectorElementCount());
11127
11128 // Convert to boolean vector.
11129 if (SrcVT.getScalarType() != MVT::i1) {
11130 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
11131 SrcVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
11132 EC: SrcVT.getVectorElementCount());
11133 Source = DAG.getNode(Opcode: ISD::VP_SETCC, DL, VT: SrcVT, N1: Source, N2: AllZero,
11134 N3: DAG.getCondCode(Cond: ISD::SETNE), N4: Mask, N5: EVL);
11135 }
11136
11137 SDValue ExtEVL = DAG.getZExtOrTrunc(Op: EVL, DL, VT: ResVT);
11138 SDValue Splat = DAG.getSplat(VT: ResVecVT, DL, Op: ExtEVL);
11139 SDValue StepVec = DAG.getStepVector(DL, ResVT: ResVecVT);
11140 SDValue Select =
11141 DAG.getNode(Opcode: ISD::VP_SELECT, DL, VT: ResVecVT, N1: Source, N2: StepVec, N3: Splat, N4: EVL);
11142 return DAG.getNode(Opcode: ISD::VP_REDUCE_UMIN, DL, VT: ResVT, N1: ExtEVL, N2: Select, N3: Mask, N4: EVL);
11143}
11144
11145/// Returns a type-legalized version of \p Mask as the first item in the
11146/// pair. The second item contains a type-legalized step vector that's
11147/// guaranteed to fit the number of elements in \p Mask.
11148/// If the stepvector would require splitting, returns an empty SDValue
11149/// as the second item to signal that the operation should be split instead.
11150static std::pair<SDValue, SDValue>
11151getLegalMaskAndStepVector(SDValue Mask, bool ZeroIsPoison, SDLoc DL,
11152 SelectionDAG &DAG) {
11153 EVT MaskVT = Mask.getValueType();
11154 EVT BoolVT = MaskVT.getScalarType();
11155
11156 // Find a suitable type for a stepvector.
11157 // If zero is poison, we can assume the upper limit of the result is VF-1.
11158 ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
11159 if (MaskVT.isScalableVector())
11160 VScaleRange = getVScaleRange(F: &DAG.getMachineFunction().getFunction(), BitWidth: 64);
11161 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11162 uint64_t EltWidth = TLI.getBitWidthForCttzElements(
11163 RetVT: EVT(TLI.getVectorIdxTy(DL: DAG.getDataLayout())),
11164 EC: MaskVT.getVectorElementCount(), ZeroIsPoison, VScaleRange: &VScaleRange);
11165 // If the step vector element type is smaller than the mask element type,
11166 // use the mask type directly to avoid widening issues.
11167 EltWidth = std::max(a: EltWidth, b: BoolVT.getFixedSizeInBits());
11168 EVT StepVT = MVT::getIntegerVT(BitWidth: EltWidth);
11169 EVT StepVecVT = MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: StepVT);
11170
11171 // If promotion or widening is required to make the type legal, do it here.
11172 // Promotion of integers within LegalizeVectorOps is looking for types of
11173 // the same size but with a smaller number of larger elements, not the usual
11174 // larger size with the same number of larger elements.
11175 TargetLowering::LegalizeTypeAction TypeAction =
11176 TLI.getTypeAction(Context&: *DAG.getContext(), VT: StepVecVT);
11177 SDValue StepVec;
11178 if (TypeAction == TargetLowering::TypePromoteInteger) {
11179 StepVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11180 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11181 } else if (TypeAction == TargetLowering::TypeWidenVector) {
11182 // For widening, the element count changes. Create a step vector with only
11183 // the original elements valid and zeros for padding. Also widen the mask.
11184 EVT WideVecVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVecVT);
11185 unsigned WideNumElts = WideVecVT.getVectorNumElements();
11186
11187 // Build widened step vector: <0, 1, ..., OrigNumElts-1, poison, poison, ..>
11188 SDValue OrigStepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11189 SDValue UndefStep = DAG.getPOISON(VT: WideVecVT);
11190 StepVec = DAG.getInsertSubvector(DL, Vec: UndefStep, SubVec: OrigStepVec, Idx: 0);
11191
11192 // Widen mask: pad with zeros.
11193 EVT WideMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: BoolVT, NumElements: WideNumElts);
11194 SDValue ZeroMask = DAG.getConstant(Val: 0, DL, VT: WideMaskVT);
11195 Mask = DAG.getInsertSubvector(DL, Vec: ZeroMask, SubVec: Mask, Idx: 0);
11196 } else if (TypeAction == TargetLowering::TypeSplitVector) {
11197 // The stepvector type would require splitting. Signal to the caller
11198 // that the operation should be split instead of expanded.
11199 return {Mask, SDValue()};
11200 } else {
11201 StepVec = DAG.getStepVector(DL, ResVT: StepVecVT);
11202 }
11203
11204 return {Mask, StepVec};
11205}
11206
11207SDValue TargetLowering::expandVectorFindLastActive(SDNode *N,
11208 SelectionDAG &DAG) const {
11209 SDLoc DL(N);
11210 auto [Mask, StepVec] = getLegalMaskAndStepVector(
11211 Mask: N->getOperand(Num: 0), /*ZeroIsPoison=*/true, DL, DAG);
11212
11213 // If StepVec is empty, the stepvector would require splitting.
11214 // Split the operation instead and let it be recursively legalized.
11215 if (!StepVec) {
11216 EVT MaskVT = N->getOperand(Num: 0).getValueType();
11217 EVT ResVT = N->getValueType(ResNo: 0);
11218
11219 // Split the mask
11220 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: MaskVT);
11221 auto [MaskLo, MaskHi] = DAG.SplitVector(N: N->getOperand(Num: 0), DL);
11222
11223 // Create split VECTOR_FIND_LAST_ACTIVE operations
11224 SDValue LoResult =
11225 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskLo);
11226 SDValue HiResult =
11227 DAG.getNode(Opcode: ISD::VECTOR_FIND_LAST_ACTIVE, DL, VT: ResVT, Operand: MaskHi);
11228
11229 // Check if any lane is active in the high mask.
11230 SDValue AnyHiActive = DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: MVT::i1, Operand: MaskHi);
11231 SDValue Cond = DAG.getBoolExtOrTrunc(
11232 Op: AnyHiActive, SL: DL,
11233 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: MVT::i1),
11234 OpVT: MVT::i1);
11235
11236 // Adjust HiResult by adding the number of elements in Lo
11237 SDValue LoNumElts =
11238 DAG.getElementCount(DL, VT: ResVT, EC: LoVT.getVectorElementCount());
11239 SDValue AdjustedHiResult =
11240 DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: HiResult, N2: LoNumElts);
11241
11242 // Return: AnyHiActive ? AdjustedHiResult : LoResult;
11243 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: ResVT, N1: Cond, N2: AdjustedHiResult,
11244 N3: LoResult);
11245 }
11246
11247 EVT StepVecVT = StepVec.getValueType();
11248 EVT StepVT = StepVec.getValueType().getVectorElementType();
11249
11250 // Zero out lanes with inactive elements, then find the highest remaining
11251 // value from the stepvector.
11252 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
11253 SDValue ActiveElts = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
11254 SDValue HighestIdx = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL, VT: StepVT, Operand: ActiveElts);
11255 return DAG.getZExtOrTrunc(Op: HighestIdx, DL, VT: N->getValueType(ResNo: 0));
11256}
11257
11258SDValue TargetLowering::expandLoopDependenceMask(SDNode *N,
11259 SelectionDAG &DAG) const {
11260 SDLoc DL(N);
11261 EVT VT = N->getValueType(ResNo: 0);
11262 SDValue SourceValue = N->getOperand(Num: 0);
11263 SDValue SinkValue = N->getOperand(Num: 1);
11264 SDValue EltSizeInBytes = N->getOperand(Num: 2);
11265
11266 // Note: The lane offset is scalable if the mask is scalable.
11267 ElementCount LaneOffsetEC =
11268 ElementCount::get(MinVal: N->getConstantOperandVal(Num: 3), Scalable: VT.isScalableVT());
11269
11270 EVT AddrVT = SourceValue->getValueType(ResNo: 0);
11271 bool IsReadAfterWrite = N->getOpcode() == ISD::LOOP_DEPENDENCE_RAW_MASK;
11272
11273 EVT CmpVT =
11274 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: AddrVT);
11275
11276 // Unsigned compare: Source >= Sink.
11277 SDValue SourceAheadOfOrEqualToSink =
11278 DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETUGE);
11279
11280 // Take the difference between the pointers and divided by the element size,
11281 // to see how many lanes separate them.
11282 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: AddrVT, N1: SinkValue, N2: SourceValue);
11283
11284 // RAW_MASK: Diff = Source >= Sink ? (Source - Sink) : (Sink - Source)
11285 if (IsReadAfterWrite)
11286 Diff = DAG.getSelect(DL, VT: AddrVT, Cond: SourceAheadOfOrEqualToSink,
11287 LHS: DAG.getNegative(Val: Diff, DL, VT: AddrVT), RHS: Diff);
11288
11289 Diff = DAG.getNode(Opcode: ISD::SDIV, DL, VT: AddrVT, N1: Diff, N2: EltSizeInBytes);
11290
11291 // The pointers do not alias if:
11292 // - Source >= Sink (WAR_MASK)
11293 // - Source == Sink (RAW_MASK)
11294 SDValue NoAlias = SourceAheadOfOrEqualToSink;
11295 if (IsReadAfterWrite)
11296 NoAlias = DAG.getSetCC(DL, VT: CmpVT, LHS: SourceValue, RHS: SinkValue, Cond: ISD::SETEQ);
11297
11298 // The pointers do not alias if:
11299 // Lane + LaneOffset < Diff (WAR/RAW_MASK)
11300 SDValue LaneOffset = DAG.getElementCount(DL, VT: AddrVT, EC: LaneOffsetEC);
11301 SDValue MaskN = DAG.getSelect(
11302 DL, VT: AddrVT, Cond: NoAlias,
11303 LHS: DAG.getConstant(Val: APInt::getMaxValue(numBits: AddrVT.getScalarSizeInBits()), DL,
11304 VT: AddrVT),
11305 RHS: Diff);
11306
11307 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT, N1: LaneOffset, N2: MaskN);
11308}
11309
11310SDValue TargetLowering::expandABS(SDNode *N, SelectionDAG &DAG,
11311 bool IsNegative) const {
11312 SDLoc dl(N);
11313 EVT VT = N->getValueType(ResNo: 0);
11314 SDValue Op = N->getOperand(Num: 0);
11315
11316 // If expanding ABS_MIN_POISON, fall back to ABS if the target supports it.
11317 if (N->getOpcode() == ISD::ABS_MIN_POISON &&
11318 isOperationLegalOrCustom(Op: ISD::ABS, VT)) {
11319 SDValue AbsVal = DAG.getNode(Opcode: ISD::ABS, DL: dl, VT, Operand: Op);
11320 if (IsNegative)
11321 return DAG.getNegative(Val: AbsVal, DL: dl, VT);
11322 return AbsVal;
11323 }
11324
11325 // abs(x) -> smax(x,sub(0,x))
11326 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11327 isOperationLegal(Op: ISD::SMAX, VT)) {
11328 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11329 Op = DAG.getFreeze(V: Op);
11330 return DAG.getNode(Opcode: ISD::SMAX, DL: dl, VT, N1: Op,
11331 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11332 }
11333
11334 // abs(x) -> umin(x,sub(0,x))
11335 if (!IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11336 isOperationLegal(Op: ISD::UMIN, VT)) {
11337 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11338 Op = DAG.getFreeze(V: Op);
11339 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: Op,
11340 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11341 }
11342
11343 // 0 - abs(x) -> smin(x, sub(0,x))
11344 if (IsNegative && isOperationLegal(Op: ISD::SUB, VT) &&
11345 isOperationLegal(Op: ISD::SMIN, VT)) {
11346 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
11347 Op = DAG.getFreeze(V: Op);
11348 return DAG.getNode(Opcode: ISD::SMIN, DL: dl, VT, N1: Op,
11349 N2: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Zero, N2: Op));
11350 }
11351
11352 // Only expand vector types if we have the appropriate vector operations.
11353 if (VT.isVector() &&
11354 (!isOperationLegalOrCustom(Op: ISD::SRA, VT) ||
11355 (!IsNegative && !isOperationLegalOrCustom(Op: ISD::ADD, VT)) ||
11356 (IsNegative && !isOperationLegalOrCustom(Op: ISD::SUB, VT)) ||
11357 !isOperationLegalOrCustomOrPromote(Op: ISD::XOR, VT)))
11358 return SDValue();
11359
11360 Op = DAG.getFreeze(V: Op);
11361 SDValue Shift = DAG.getNode(
11362 Opcode: ISD::SRA, DL: dl, VT, N1: Op,
11363 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11364 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Shift);
11365
11366 // abs(x) -> Y = sra (X, size(X)-1); sub (xor (X, Y), Y)
11367 if (!IsNegative)
11368 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Shift);
11369
11370 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
11371 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Shift, N2: Xor);
11372}
11373
11374SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
11375 SDLoc dl(N);
11376 EVT VT = N->getValueType(ResNo: 0);
11377 SDValue LHS = N->getOperand(Num: 0);
11378 SDValue RHS = N->getOperand(Num: 1);
11379 bool IsSigned = N->getOpcode() == ISD::ABDS;
11380
11381 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
11382 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
11383 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
11384 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
11385 if (isOperationLegal(Op: MaxOpc, VT) && isOperationLegal(Op: MinOpc, VT)) {
11386 LHS = DAG.getFreeze(V: LHS);
11387 RHS = DAG.getFreeze(V: RHS);
11388 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
11389 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
11390 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
11391 }
11392
11393 // abdu(lhs, rhs) -> or(usubsat(lhs,rhs), usubsat(rhs,lhs))
11394 if (!IsSigned && isOperationLegal(Op: ISD::USUBSAT, VT)) {
11395 LHS = DAG.getFreeze(V: LHS);
11396 RHS = DAG.getFreeze(V: RHS);
11397 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT,
11398 N1: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: LHS, N2: RHS),
11399 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL: dl, VT, N1: RHS, N2: LHS));
11400 }
11401
11402 // If the subtract doesn't overflow then just use abs(sub())
11403 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
11404
11405 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: LHS, N1: RHS))
11406 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11407 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS));
11408
11409 if (DAG.willNotOverflowSub(IsSigned: IsSigned || IsNonNegative, N0: RHS, N1: LHS))
11410 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT,
11411 Operand: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11412
11413 EVT CCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
11414 ISD::CondCode CC = IsSigned ? ISD::CondCode::SETGT : ISD::CondCode::SETUGT;
11415 LHS = DAG.getFreeze(V: LHS);
11416 RHS = DAG.getFreeze(V: RHS);
11417 SDValue Cmp = DAG.getSetCC(DL: dl, VT: CCVT, LHS, RHS, Cond: CC);
11418
11419 // Branchless expansion iff cmp result is allbits:
11420 // abds(lhs, rhs) -> sub(sgt(lhs, rhs), xor(sgt(lhs, rhs), sub(lhs, rhs)))
11421 // abdu(lhs, rhs) -> sub(ugt(lhs, rhs), xor(ugt(lhs, rhs), sub(lhs, rhs)))
11422 if (CCVT == VT && getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
11423 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
11424 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Diff, N2: Cmp);
11425 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Cmp, N2: Xor);
11426 }
11427
11428 // Similar to the branchless expansion, if we don't prefer selects, use the
11429 // (sign-extended) usubo overflow flag if the (scalar) type is illegal as this
11430 // is more likely to legalize cleanly: abdu(lhs, rhs) -> sub(xor(sub(lhs,
11431 // rhs), uof(lhs, rhs)), uof(lhs, rhs))
11432 if (!IsSigned && VT.isScalarInteger() && !isTypeLegal(VT) &&
11433 !preferSelectsOverBooleanArithmetic(VT)) {
11434 SDValue USubO =
11435 DAG.getNode(Opcode: ISD::USUBO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {LHS, RHS});
11436 SDValue Cmp = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT, Operand: USubO.getValue(R: 1));
11437 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: USubO.getValue(R: 0), N2: Cmp);
11438 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Xor, N2: Cmp);
11439 }
11440
11441 // FIXME: Should really try to split the vector in case it's legal on a
11442 // subvector.
11443 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
11444 return DAG.UnrollVectorOp(N);
11445
11446 // abds(lhs, rhs) -> select(sgt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11447 // abdu(lhs, rhs) -> select(ugt(lhs,rhs), sub(lhs,rhs), sub(rhs,lhs))
11448 return DAG.getSelect(DL: dl, VT, Cond: Cmp, LHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS),
11449 RHS: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: RHS, N2: LHS));
11450}
11451
11452SDValue TargetLowering::expandAVG(SDNode *N, SelectionDAG &DAG) const {
11453 SDLoc dl(N);
11454 EVT VT = N->getValueType(ResNo: 0);
11455 SDValue LHS = N->getOperand(Num: 0);
11456 SDValue RHS = N->getOperand(Num: 1);
11457
11458 unsigned Opc = N->getOpcode();
11459 bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
11460 bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
11461 unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
11462 unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
11463 unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
11464 unsigned ExtOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
11465 assert((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS ||
11466 Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
11467 "Unknown AVG node");
11468
11469 // If the operands are already extended, we can add+shift.
11470 bool IsExt =
11471 (IsSigned && DAG.ComputeNumSignBits(Op: LHS) >= 2 &&
11472 DAG.ComputeNumSignBits(Op: RHS) >= 2) ||
11473 (!IsSigned && DAG.computeKnownBits(Op: LHS).countMinLeadingZeros() >= 1 &&
11474 DAG.computeKnownBits(Op: RHS).countMinLeadingZeros() >= 1);
11475 if (IsExt) {
11476 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
11477 if (!IsFloor)
11478 Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Sum, N2: DAG.getConstant(Val: 1, DL: dl, VT));
11479 return DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Sum,
11480 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11481 }
11482
11483 // For scalars, see if we can efficiently extend/truncate to use add+shift.
11484 if (VT.isScalarInteger()) {
11485 EVT ExtVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
11486 if (isTypeLegal(VT: ExtVT) && isTruncateFree(FromVT: ExtVT, ToVT: VT)) {
11487 LHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: LHS);
11488 RHS = DAG.getNode(Opcode: ExtOpc, DL: dl, VT: ExtVT, Operand: RHS);
11489 SDValue Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: LHS, N2: RHS);
11490 if (!IsFloor)
11491 Avg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ExtVT, N1: Avg,
11492 N2: DAG.getConstant(Val: 1, DL: dl, VT: ExtVT));
11493 // Just use SRL as we will be truncating away the extended sign bits.
11494 Avg = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: ExtVT, N1: Avg,
11495 N2: DAG.getShiftAmountConstant(Val: 1, VT: ExtVT, DL: dl));
11496 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Avg);
11497 }
11498 }
11499
11500 // avgflooru(lhs, rhs) -> or(lshr(add(lhs, rhs),1),shl(overflow, typesize-1))
11501 if (Opc == ISD::AVGFLOORU && VT.isScalarInteger() && !isTypeLegal(VT) &&
11502 isOperationLegalOrCustom(
11503 Op: ISD::UADDO, VT: getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT))) {
11504 SDValue UAddWithOverflow =
11505 DAG.getNode(Opcode: ISD::UADDO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i1), Ops: {RHS, LHS});
11506
11507 SDValue Sum = UAddWithOverflow.getValue(R: 0);
11508 SDValue Overflow = UAddWithOverflow.getValue(R: 1);
11509
11510 // Right shift the sum by 1
11511 SDValue LShrVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Sum,
11512 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11513
11514 SDValue ZeroExtOverflow = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT, Operand: Overflow);
11515 SDValue OverflowShl = DAG.getNode(
11516 Opcode: ISD::SHL, DL: dl, VT, N1: ZeroExtOverflow,
11517 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL: dl));
11518
11519 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: LShrVal, N2: OverflowShl);
11520 }
11521
11522 // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
11523 // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
11524 // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
11525 // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
11526 LHS = DAG.getFreeze(V: LHS);
11527 RHS = DAG.getFreeze(V: RHS);
11528 SDValue Sign = DAG.getNode(Opcode: SignOpc, DL: dl, VT, N1: LHS, N2: RHS);
11529 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
11530 SDValue Shift =
11531 DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: Xor, N2: DAG.getShiftAmountConstant(Val: 1, VT, DL: dl));
11532 return DAG.getNode(Opcode: SumOpc, DL: dl, VT, N1: Sign, N2: Shift);
11533}
11534
11535SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
11536 SDLoc dl(N);
11537 EVT VT = N->getValueType(ResNo: 0);
11538 SDValue Op = N->getOperand(Num: 0);
11539
11540 if (!VT.isSimple())
11541 return SDValue();
11542
11543 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11544 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
11545 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
11546 default:
11547 return SDValue();
11548 case MVT::i16:
11549 // Use a rotate by 8. This can be further expanded if necessary.
11550 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11551 case MVT::i32:
11552 // This is meant for ARM specifically, which has ROTR but no ROTL.
11553 // t = x ^ rotr(x, 16)
11554 // t = bic(t, 0x00ff0000)
11555 // t = lshr(t, 8)
11556 // x = t ^ rotr(x, 8)
11557 if (isOperationLegalOrCustom(Op: ISD::ROTR, VT)) {
11558 SDValue Rotr16 =
11559 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 16, DL: dl, VT: SHVT));
11560 SDValue Tmp = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Op, N2: Rotr16);
11561 Tmp = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp,
11562 N2: DAG.getConstant(Val: 0xFF00FFFF, DL: dl, VT));
11563 Tmp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11564 SDValue Rotr8 =
11565 DAG.getNode(Opcode: ISD::ROTR, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11566 return DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Tmp, N2: Rotr8);
11567 }
11568 Tmp4 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11569 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11570 N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11571 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11572 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11573 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT));
11574 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11575 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11576 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11577 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11578 case MVT::i64:
11579 Tmp8 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11580 Tmp7 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11581 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11582 Tmp7 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp7, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11583 Tmp6 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11584 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11585 Tmp6 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp6, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11586 Tmp5 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Op,
11587 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11588 Tmp5 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp5, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11589 Tmp4 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT));
11590 Tmp4 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp4,
11591 N2: DAG.getConstant(Val: 255ULL<<24, DL: dl, VT));
11592 Tmp3 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT));
11593 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp3,
11594 N2: DAG.getConstant(Val: 255ULL<<16, DL: dl, VT));
11595 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT));
11596 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2,
11597 N2: DAG.getConstant(Val: 255ULL<<8, DL: dl, VT));
11598 Tmp1 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT));
11599 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp7);
11600 Tmp6 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp6, N2: Tmp5);
11601 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp3);
11602 Tmp2 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp1);
11603 Tmp8 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp6);
11604 Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp2);
11605 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp8, N2: Tmp4);
11606 }
11607}
11608
11609SDValue TargetLowering::expandVPBSWAP(SDNode *N, SelectionDAG &DAG) const {
11610 SDLoc dl(N);
11611 EVT VT = N->getValueType(ResNo: 0);
11612 SDValue Op = N->getOperand(Num: 0);
11613 SDValue Mask = N->getOperand(Num: 1);
11614 SDValue EVL = N->getOperand(Num: 2);
11615
11616 if (!VT.isSimple())
11617 return SDValue();
11618
11619 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11620 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
11621 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
11622 default:
11623 return SDValue();
11624 case MVT::i16:
11625 Tmp1 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT),
11626 N3: Mask, N4: EVL);
11627 Tmp2 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT),
11628 N3: Mask, N4: EVL);
11629 return DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp1, N2: Tmp2, N3: Mask, N4: EVL);
11630 case MVT::i32:
11631 Tmp4 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT),
11632 N3: Mask, N4: EVL);
11633 Tmp3 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT),
11634 N3: Mask, N4: EVL);
11635 Tmp3 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT),
11636 N3: Mask, N4: EVL);
11637 Tmp2 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT),
11638 N3: Mask, N4: EVL);
11639 Tmp2 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp2,
11640 N2: DAG.getConstant(Val: 0xFF00, DL: dl, VT), N3: Mask, N4: EVL);
11641 Tmp1 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT),
11642 N3: Mask, N4: EVL);
11643 Tmp4 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp4, N2: Tmp3, N3: Mask, N4: EVL);
11644 Tmp2 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp2, N2: Tmp1, N3: Mask, N4: EVL);
11645 return DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp4, N2: Tmp2, N3: Mask, N4: EVL);
11646 case MVT::i64:
11647 Tmp8 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT),
11648 N3: Mask, N4: EVL);
11649 Tmp7 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Op,
11650 N2: DAG.getConstant(Val: 255ULL << 8, DL: dl, VT), N3: Mask, N4: EVL);
11651 Tmp7 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Tmp7, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT),
11652 N3: Mask, N4: EVL);
11653 Tmp6 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Op,
11654 N2: DAG.getConstant(Val: 255ULL << 16, DL: dl, VT), N3: Mask, N4: EVL);
11655 Tmp6 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Tmp6, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT),
11656 N3: Mask, N4: EVL);
11657 Tmp5 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Op,
11658 N2: DAG.getConstant(Val: 255ULL << 24, DL: dl, VT), N3: Mask, N4: EVL);
11659 Tmp5 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Tmp5, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT),
11660 N3: Mask, N4: EVL);
11661 Tmp4 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 8, DL: dl, VT: SHVT),
11662 N3: Mask, N4: EVL);
11663 Tmp4 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp4,
11664 N2: DAG.getConstant(Val: 255ULL << 24, DL: dl, VT), N3: Mask, N4: EVL);
11665 Tmp3 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 24, DL: dl, VT: SHVT),
11666 N3: Mask, N4: EVL);
11667 Tmp3 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp3,
11668 N2: DAG.getConstant(Val: 255ULL << 16, DL: dl, VT), N3: Mask, N4: EVL);
11669 Tmp2 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 40, DL: dl, VT: SHVT),
11670 N3: Mask, N4: EVL);
11671 Tmp2 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp2,
11672 N2: DAG.getConstant(Val: 255ULL << 8, DL: dl, VT), N3: Mask, N4: EVL);
11673 Tmp1 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: 56, DL: dl, VT: SHVT),
11674 N3: Mask, N4: EVL);
11675 Tmp8 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp8, N2: Tmp7, N3: Mask, N4: EVL);
11676 Tmp6 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp6, N2: Tmp5, N3: Mask, N4: EVL);
11677 Tmp4 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp4, N2: Tmp3, N3: Mask, N4: EVL);
11678 Tmp2 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp2, N2: Tmp1, N3: Mask, N4: EVL);
11679 Tmp8 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp8, N2: Tmp6, N3: Mask, N4: EVL);
11680 Tmp4 = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp4, N2: Tmp2, N3: Mask, N4: EVL);
11681 return DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp8, N2: Tmp4, N3: Mask, N4: EVL);
11682 }
11683}
11684
11685SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
11686 SDLoc dl(N);
11687 EVT VT = N->getValueType(ResNo: 0);
11688 SDValue Op = N->getOperand(Num: 0);
11689 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11690 unsigned Sz = VT.getScalarSizeInBits();
11691
11692 SDValue Tmp, Tmp2, Tmp3;
11693
11694 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
11695 // and finally the i1 pairs.
11696 // TODO: We can easily support i4/i2 legal types if any target ever does.
11697 if (Sz >= 8 && isPowerOf2_32(Value: Sz)) {
11698 // Create the masks - repeating the pattern every byte.
11699 APInt Mask4 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x0F));
11700 APInt Mask2 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x33));
11701 APInt Mask1 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x55));
11702
11703 // BSWAP if the type is wider than a single byte.
11704 Tmp = (Sz > 8 ? DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT, Operand: Op) : Op);
11705
11706 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
11707 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11708 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11709 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask4, DL: dl, VT));
11710 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT));
11711 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11712
11713 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
11714 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11715 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11716 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask2, DL: dl, VT));
11717 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT));
11718 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11719
11720 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
11721 Tmp2 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11722 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11723 Tmp3 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask1, DL: dl, VT));
11724 Tmp3 = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT));
11725 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
11726 return Tmp;
11727 }
11728
11729 Tmp = DAG.getConstant(Val: 0, DL: dl, VT);
11730 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
11731 if (I < J)
11732 Tmp2 =
11733 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: J - I, DL: dl, VT: SHVT));
11734 else
11735 Tmp2 =
11736 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Op, N2: DAG.getConstant(Val: I - J, DL: dl, VT: SHVT));
11737
11738 APInt Shift = APInt::getOneBitSet(numBits: Sz, BitNo: J);
11739 Tmp2 = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Tmp2, N2: DAG.getConstant(Val: Shift, DL: dl, VT));
11740 Tmp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp, N2: Tmp2);
11741 }
11742
11743 return Tmp;
11744}
11745
11746SDValue TargetLowering::expandVPBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
11747 assert(N->getOpcode() == ISD::VP_BITREVERSE);
11748
11749 SDLoc dl(N);
11750 EVT VT = N->getValueType(ResNo: 0);
11751 SDValue Op = N->getOperand(Num: 0);
11752 SDValue Mask = N->getOperand(Num: 1);
11753 SDValue EVL = N->getOperand(Num: 2);
11754 EVT SHVT = getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
11755 unsigned Sz = VT.getScalarSizeInBits();
11756
11757 SDValue Tmp, Tmp2, Tmp3;
11758
11759 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
11760 // and finally the i1 pairs.
11761 // TODO: We can easily support i4/i2 legal types if any target ever does.
11762 if (Sz >= 8 && isPowerOf2_32(Value: Sz)) {
11763 // Create the masks - repeating the pattern every byte.
11764 APInt Mask4 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x0F));
11765 APInt Mask2 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x33));
11766 APInt Mask1 = APInt::getSplat(NewLen: Sz, V: APInt(8, 0x55));
11767
11768 // BSWAP if the type is wider than a single byte.
11769 Tmp = (Sz > 8 ? DAG.getNode(Opcode: ISD::VP_BSWAP, DL: dl, VT, N1: Op, N2: Mask, N3: EVL) : Op);
11770
11771 // swap i4: ((V >> 4) & 0x0F) | ((V & 0x0F) << 4)
11772 Tmp2 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT),
11773 N3: Mask, N4: EVL);
11774 Tmp2 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp2,
11775 N2: DAG.getConstant(Val: Mask4, DL: dl, VT), N3: Mask, N4: EVL);
11776 Tmp3 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask4, DL: dl, VT),
11777 N3: Mask, N4: EVL);
11778 Tmp3 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 4, DL: dl, VT: SHVT),
11779 N3: Mask, N4: EVL);
11780 Tmp = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp2, N2: Tmp3, N3: Mask, N4: EVL);
11781
11782 // swap i2: ((V >> 2) & 0x33) | ((V & 0x33) << 2)
11783 Tmp2 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT),
11784 N3: Mask, N4: EVL);
11785 Tmp2 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp2,
11786 N2: DAG.getConstant(Val: Mask2, DL: dl, VT), N3: Mask, N4: EVL);
11787 Tmp3 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask2, DL: dl, VT),
11788 N3: Mask, N4: EVL);
11789 Tmp3 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 2, DL: dl, VT: SHVT),
11790 N3: Mask, N4: EVL);
11791 Tmp = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp2, N2: Tmp3, N3: Mask, N4: EVL);
11792
11793 // swap i1: ((V >> 1) & 0x55) | ((V & 0x55) << 1)
11794 Tmp2 = DAG.getNode(Opcode: ISD::VP_SRL, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT),
11795 N3: Mask, N4: EVL);
11796 Tmp2 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp2,
11797 N2: DAG.getConstant(Val: Mask1, DL: dl, VT), N3: Mask, N4: EVL);
11798 Tmp3 = DAG.getNode(Opcode: ISD::VP_AND, DL: dl, VT, N1: Tmp, N2: DAG.getConstant(Val: Mask1, DL: dl, VT),
11799 N3: Mask, N4: EVL);
11800 Tmp3 = DAG.getNode(Opcode: ISD::VP_SHL, DL: dl, VT, N1: Tmp3, N2: DAG.getConstant(Val: 1, DL: dl, VT: SHVT),
11801 N3: Mask, N4: EVL);
11802 Tmp = DAG.getNode(Opcode: ISD::VP_OR, DL: dl, VT, N1: Tmp2, N2: Tmp3, N3: Mask, N4: EVL);
11803 return Tmp;
11804 }
11805 return SDValue();
11806}
11807
11808std::pair<SDValue, SDValue>
11809TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
11810 SelectionDAG &DAG) const {
11811 SDLoc SL(LD);
11812 SDValue Chain = LD->getChain();
11813 SDValue BasePTR = LD->getBasePtr();
11814 EVT SrcVT = LD->getMemoryVT();
11815 EVT DstVT = LD->getValueType(ResNo: 0);
11816 ISD::LoadExtType ExtType = LD->getExtensionType();
11817
11818 if (SrcVT.isScalableVector())
11819 report_fatal_error(reason: "Cannot scalarize scalable vector loads");
11820
11821 unsigned NumElem = SrcVT.getVectorNumElements();
11822
11823 EVT SrcEltVT = SrcVT.getScalarType();
11824 EVT DstEltVT = DstVT.getScalarType();
11825
11826 // A vector must always be stored in memory as-is, i.e. without any padding
11827 // between the elements, since various code depend on it, e.g. in the
11828 // handling of a bitcast of a vector type to int, which may be done with a
11829 // vector store followed by an integer load. A vector that does not have
11830 // elements that are byte-sized must therefore be stored as an integer
11831 // built out of the extracted vector elements.
11832 if (!SrcEltVT.isByteSized()) {
11833 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
11834 EVT LoadVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumLoadBits);
11835
11836 unsigned NumSrcBits = SrcVT.getSizeInBits();
11837 EVT SrcIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumSrcBits);
11838
11839 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
11840 SDValue SrcEltBitMask = DAG.getConstant(
11841 Val: APInt::getLowBitsSet(numBits: NumLoadBits, loBitsSet: SrcEltBits), DL: SL, VT: LoadVT);
11842
11843 // Load the whole vector and avoid masking off the top bits as it makes
11844 // the codegen worse.
11845 SDValue Load =
11846 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: SL, VT: LoadVT, Chain, Ptr: BasePTR,
11847 PtrInfo: LD->getPointerInfo(), MemVT: SrcIntVT, Alignment: LD->getBaseAlign(),
11848 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
11849
11850 SmallVector<SDValue, 8> Vals;
11851 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11852 unsigned ShiftIntoIdx =
11853 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
11854 SDValue ShiftAmount = DAG.getShiftAmountConstant(
11855 Val: ShiftIntoIdx * SrcEltVT.getSizeInBits(), VT: LoadVT, DL: SL);
11856 SDValue ShiftedElt = DAG.getNode(Opcode: ISD::SRL, DL: SL, VT: LoadVT, N1: Load, N2: ShiftAmount);
11857 SDValue Elt =
11858 DAG.getNode(Opcode: ISD::AND, DL: SL, VT: LoadVT, N1: ShiftedElt, N2: SrcEltBitMask);
11859 SDValue Scalar = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: SrcEltVT, Operand: Elt);
11860
11861 if (ExtType != ISD::NON_EXTLOAD) {
11862 unsigned ExtendOp = ISD::getExtForLoadExtType(IsFP: false, ExtType);
11863 Scalar = DAG.getNode(Opcode: ExtendOp, DL: SL, VT: DstEltVT, Operand: Scalar);
11864 }
11865
11866 Vals.push_back(Elt: Scalar);
11867 }
11868
11869 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
11870 return std::make_pair(x&: Value, y: Load.getValue(R: 1));
11871 }
11872
11873 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
11874 assert(SrcEltVT.isByteSized());
11875
11876 SmallVector<SDValue, 8> Vals;
11877 SmallVector<SDValue, 8> LoadChains;
11878
11879 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11880 SDValue ScalarLoad = DAG.getExtLoad(
11881 ExtType, dl: SL, VT: DstEltVT, Chain, Ptr: BasePTR,
11882 PtrInfo: LD->getPointerInfo().getWithOffset(O: Idx * Stride), MemVT: SrcEltVT,
11883 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
11884
11885 BasePTR = DAG.getObjectPtrOffset(SL, Ptr: BasePTR, Offset: TypeSize::getFixed(ExactSize: Stride));
11886
11887 Vals.push_back(Elt: ScalarLoad.getValue(R: 0));
11888 LoadChains.push_back(Elt: ScalarLoad.getValue(R: 1));
11889 }
11890
11891 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: LoadChains);
11892 SDValue Value = DAG.getBuildVector(VT: DstVT, DL: SL, Ops: Vals);
11893
11894 return std::make_pair(x&: Value, y&: NewChain);
11895}
11896
11897SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
11898 SelectionDAG &DAG) const {
11899 SDLoc SL(ST);
11900
11901 SDValue Chain = ST->getChain();
11902 SDValue BasePtr = ST->getBasePtr();
11903 SDValue Value = ST->getValue();
11904 EVT StVT = ST->getMemoryVT();
11905
11906 if (StVT.isScalableVector())
11907 report_fatal_error(reason: "Cannot scalarize scalable vector stores");
11908
11909 // The type of the data we want to save
11910 EVT RegVT = Value.getValueType();
11911 EVT RegSclVT = RegVT.getScalarType();
11912
11913 // The type of data as saved in memory.
11914 EVT MemSclVT = StVT.getScalarType();
11915
11916 unsigned NumElem = StVT.getVectorNumElements();
11917
11918 // A vector must always be stored in memory as-is, i.e. without any padding
11919 // between the elements, since various code depend on it, e.g. in the
11920 // handling of a bitcast of a vector type to int, which may be done with a
11921 // vector store followed by an integer load. A vector that does not have
11922 // elements that are byte-sized must therefore be stored as an integer
11923 // built out of the extracted vector elements.
11924 if (!MemSclVT.isByteSized()) {
11925 unsigned NumBits = StVT.getSizeInBits();
11926 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits);
11927
11928 SDValue CurrVal = DAG.getConstant(Val: 0, DL: SL, VT: IntVT);
11929
11930 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11931 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
11932 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT: MemSclVT, Operand: Elt);
11933 SDValue ExtElt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SL, VT: IntVT, Operand: Trunc);
11934 unsigned ShiftIntoIdx =
11935 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
11936 SDValue ShiftAmount =
11937 DAG.getConstant(Val: ShiftIntoIdx * MemSclVT.getSizeInBits(), DL: SL, VT: IntVT);
11938 SDValue ShiftedElt =
11939 DAG.getNode(Opcode: ISD::SHL, DL: SL, VT: IntVT, N1: ExtElt, N2: ShiftAmount);
11940 CurrVal = DAG.getNode(Opcode: ISD::OR, DL: SL, VT: IntVT, N1: CurrVal, N2: ShiftedElt);
11941 }
11942
11943 return DAG.getStore(Chain, dl: SL, Val: CurrVal, Ptr: BasePtr, PtrInfo: ST->getPointerInfo(),
11944 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
11945 AAInfo: ST->getAAInfo());
11946 }
11947
11948 // Store Stride in bytes
11949 unsigned Stride = MemSclVT.getSizeInBits() / 8;
11950 assert(Stride && "Zero stride!");
11951 // Extract each of the elements from the original vector and save them into
11952 // memory individually.
11953 SmallVector<SDValue, 8> Stores;
11954 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
11955 SDValue Elt = DAG.getExtractVectorElt(DL: SL, VT: RegSclVT, Vec: Value, Idx);
11956
11957 SDValue Ptr =
11958 DAG.getObjectPtrOffset(SL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: Idx * Stride));
11959
11960 // This scalar TruncStore may be illegal, but we legalize it later.
11961 SDValue Store = DAG.getTruncStore(
11962 Chain, dl: SL, Val: Elt, Ptr, PtrInfo: ST->getPointerInfo().getWithOffset(O: Idx * Stride),
11963 SVT: MemSclVT, Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(),
11964 AAInfo: ST->getAAInfo());
11965
11966 Stores.push_back(Elt: Store);
11967 }
11968
11969 return DAG.getNode(Opcode: ISD::TokenFactor, DL: SL, VT: MVT::Other, Ops: Stores);
11970}
11971
11972std::pair<SDValue, SDValue>
11973TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
11974 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
11975 "unaligned indexed loads not implemented!");
11976 SDValue Chain = LD->getChain();
11977 SDValue Ptr = LD->getBasePtr();
11978 EVT VT = LD->getValueType(ResNo: 0);
11979 EVT LoadedVT = LD->getMemoryVT();
11980 SDLoc dl(LD);
11981 auto &MF = DAG.getMachineFunction();
11982
11983 if (VT.isFloatingPoint() || VT.isVector()) {
11984 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadedVT.getSizeInBits());
11985 if (isTypeLegal(VT: intVT) && isTypeLegal(VT: LoadedVT)) {
11986 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: intVT) &&
11987 LoadedVT.isVector()) {
11988 // Scalarize the load and let the individual components be handled.
11989 return scalarizeVectorLoad(LD, DAG);
11990 }
11991
11992 // Expand to a (misaligned) integer load of the same size,
11993 // then bitconvert to floating point or vector.
11994 SDValue newLoad = DAG.getLoad(VT: intVT, dl, Chain, Ptr,
11995 MMO: LD->getMemOperand());
11996 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: LoadedVT, Operand: newLoad);
11997 if (LoadedVT != VT)
11998 Result = DAG.getNode(Opcode: VT.isFloatingPoint() ? ISD::FP_EXTEND :
11999 ISD::ANY_EXTEND, DL: dl, VT, Operand: Result);
12000
12001 return std::make_pair(x&: Result, y: newLoad.getValue(R: 1));
12002 }
12003
12004 // Copy the value to a (aligned) stack slot using (unaligned) integer
12005 // loads and stores, then do a (aligned) load from the stack slot.
12006 MVT RegVT = getRegisterType(Context&: *DAG.getContext(), VT: intVT);
12007 unsigned LoadedBytes = LoadedVT.getStoreSize();
12008 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12009 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
12010
12011 // Make sure the stack slot is also aligned for the register type.
12012 SDValue StackBase = DAG.CreateStackTemporary(VT1: LoadedVT, VT2: RegVT);
12013 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackBase.getNode())->getIndex();
12014 SmallVector<SDValue, 8> Stores;
12015 SDValue StackPtr = StackBase;
12016 unsigned Offset = 0;
12017
12018 EVT PtrVT = Ptr.getValueType();
12019 EVT StackPtrVT = StackPtr.getValueType();
12020
12021 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12022 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12023
12024 // Do all but one copies using the full register width.
12025 for (unsigned i = 1; i < NumRegs; i++) {
12026 // Load one integer register's worth from the original location.
12027 SDValue Load = DAG.getLoad(
12028 VT: RegVT, dl, Chain, Ptr, PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset),
12029 Alignment: LD->getBaseAlign(), MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
12030 // Follow the load with a store to the stack slot. Remember the store.
12031 Stores.push_back(Elt: DAG.getStore(
12032 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12033 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset)));
12034 // Increment the pointers.
12035 Offset += RegBytes;
12036
12037 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12038 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12039 }
12040
12041 // The last copy may be partial. Do an extending load.
12042 EVT MemVT = EVT::getIntegerVT(Context&: *DAG.getContext(),
12043 BitWidth: 8 * (LoadedBytes - Offset));
12044 SDValue Load = DAG.getExtLoad(
12045 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain, Ptr,
12046 PtrInfo: LD->getPointerInfo().getWithOffset(O: Offset), MemVT, Alignment: LD->getBaseAlign(),
12047 MMOFlags: LD->getMemOperand()->getFlags(), AAInfo: LD->getAAInfo());
12048 // Follow the load with a store to the stack slot. Remember the store.
12049 // On big-endian machines this requires a truncating store to ensure
12050 // that the bits end up in the right place.
12051 Stores.push_back(Elt: DAG.getTruncStore(
12052 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr: StackPtr,
12053 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), SVT: MemVT));
12054
12055 // The order of the stores doesn't matter - say it with a TokenFactor.
12056 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12057
12058 // Finally, perform the original load only redirected to the stack slot.
12059 Load = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl, VT, Chain: TF, Ptr: StackBase,
12060 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0),
12061 MemVT: LoadedVT);
12062
12063 // Callers expect a MERGE_VALUES node.
12064 return std::make_pair(x&: Load, y&: TF);
12065 }
12066
12067 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
12068 "Unaligned load of unsupported type.");
12069
12070 // Compute the new VT that is half the size of the old one. This is an
12071 // integer MVT.
12072 unsigned NumBits = LoadedVT.getSizeInBits();
12073 EVT NewLoadedVT;
12074 NewLoadedVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits/2);
12075 NumBits >>= 1;
12076
12077 Align Alignment = LD->getBaseAlign();
12078 unsigned IncrementSize = NumBits / 8;
12079 ISD::LoadExtType HiExtType = LD->getExtensionType();
12080
12081 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
12082 if (HiExtType == ISD::NON_EXTLOAD)
12083 HiExtType = ISD::ZEXTLOAD;
12084
12085 // Load the value in two parts
12086 SDValue Lo, Hi;
12087 if (DAG.getDataLayout().isLittleEndian()) {
12088 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12089 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12090 AAInfo: LD->getAAInfo());
12091
12092 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12093 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr,
12094 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12095 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12096 AAInfo: LD->getAAInfo());
12097 } else {
12098 Hi = DAG.getExtLoad(ExtType: HiExtType, dl, VT, Chain, Ptr, PtrInfo: LD->getPointerInfo(),
12099 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12100 AAInfo: LD->getAAInfo());
12101
12102 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12103 Lo = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
12104 PtrInfo: LD->getPointerInfo().getWithOffset(O: IncrementSize),
12105 MemVT: NewLoadedVT, Alignment, MMOFlags: LD->getMemOperand()->getFlags(),
12106 AAInfo: LD->getAAInfo());
12107 }
12108
12109 // aggregate the two parts
12110 SDValue ShiftAmount = DAG.getShiftAmountConstant(Val: NumBits, VT, DL: dl);
12111 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Hi, N2: ShiftAmount);
12112 Result = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Result, N2: Lo);
12113
12114 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Lo.getValue(R: 1),
12115 N2: Hi.getValue(R: 1));
12116
12117 return std::make_pair(x&: Result, y&: TF);
12118}
12119
12120SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
12121 SelectionDAG &DAG) const {
12122 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
12123 "unaligned indexed stores not implemented!");
12124 SDValue Chain = ST->getChain();
12125 SDValue Ptr = ST->getBasePtr();
12126 SDValue Val = ST->getValue();
12127 EVT VT = Val.getValueType();
12128 Align Alignment = ST->getBaseAlign();
12129 auto &MF = DAG.getMachineFunction();
12130 EVT StoreMemVT = ST->getMemoryVT();
12131
12132 SDLoc dl(ST);
12133 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
12134 EVT intVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getSizeInBits());
12135 if (isTypeLegal(VT: intVT)) {
12136 if (!isOperationLegalOrCustom(Op: ISD::STORE, VT: intVT) &&
12137 StoreMemVT.isVector()) {
12138 // Scalarize the store and let the individual components be handled.
12139 SDValue Result = scalarizeVectorStore(ST, DAG);
12140 return Result;
12141 }
12142 // Expand to a bitconvert of the value to the integer type of the
12143 // same size, then a (misaligned) int store.
12144 // FIXME: Does not handle truncating floating point stores!
12145 SDValue Result = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: intVT, Operand: Val);
12146 Result = DAG.getStore(Chain, dl, Val: Result, Ptr, PtrInfo: ST->getPointerInfo(),
12147 Alignment, MMOFlags: ST->getMemOperand()->getFlags());
12148 return Result;
12149 }
12150 // Do a (aligned) store to a stack slot, then copy from the stack slot
12151 // to the final destination using (unaligned) integer loads and stores.
12152 MVT RegVT = getRegisterType(
12153 Context&: *DAG.getContext(),
12154 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreMemVT.getSizeInBits()));
12155 EVT PtrVT = Ptr.getValueType();
12156 unsigned StoredBytes = StoreMemVT.getStoreSize();
12157 unsigned RegBytes = RegVT.getSizeInBits() / 8;
12158 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
12159
12160 // Make sure the stack slot is also aligned for the register type.
12161 SDValue StackPtr = DAG.CreateStackTemporary(VT1: StoreMemVT, VT2: RegVT);
12162 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
12163
12164 // Perform the original store, only redirected to the stack slot.
12165 SDValue Store = DAG.getTruncStore(
12166 Chain, dl, Val, Ptr: StackPtr,
12167 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset: 0), SVT: StoreMemVT);
12168
12169 EVT StackPtrVT = StackPtr.getValueType();
12170
12171 SDValue PtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: PtrVT);
12172 SDValue StackPtrIncrement = DAG.getConstant(Val: RegBytes, DL: dl, VT: StackPtrVT);
12173 SmallVector<SDValue, 8> Stores;
12174 unsigned Offset = 0;
12175
12176 // Do all but one copies using the full register width.
12177 for (unsigned i = 1; i < NumRegs; i++) {
12178 // Load one integer register's worth from the stack slot.
12179 SDValue Load = DAG.getLoad(
12180 VT: RegVT, dl, Chain: Store, Ptr: StackPtr,
12181 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset));
12182 // Store it to the final location. Remember the store.
12183 Stores.push_back(Elt: DAG.getStore(Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12184 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset),
12185 Alignment: ST->getBaseAlign(),
12186 MMOFlags: ST->getMemOperand()->getFlags()));
12187 // Increment the pointers.
12188 Offset += RegBytes;
12189 StackPtr = DAG.getObjectPtrOffset(SL: dl, Ptr: StackPtr, Offset: StackPtrIncrement);
12190 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: PtrIncrement);
12191 }
12192
12193 // The last store may be partial. Do a truncating store. On big-endian
12194 // machines this requires an extending load from the stack slot to ensure
12195 // that the bits are in the right place.
12196 EVT LoadMemVT =
12197 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: 8 * (StoredBytes - Offset));
12198
12199 // Load from the stack slot.
12200 SDValue Load = DAG.getExtLoad(
12201 ExtType: ISD::EXTLOAD, dl, VT: RegVT, Chain: Store, Ptr: StackPtr,
12202 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: FrameIndex, Offset), MemVT: LoadMemVT);
12203
12204 Stores.push_back(Elt: DAG.getTruncStore(
12205 Chain: Load.getValue(R: 1), dl, Val: Load, Ptr,
12206 PtrInfo: ST->getPointerInfo().getWithOffset(O: Offset), SVT: LoadMemVT,
12207 Alignment: ST->getBaseAlign(), MMOFlags: ST->getMemOperand()->getFlags(), AAInfo: ST->getAAInfo()));
12208 // The order of the stores doesn't matter - say it with a TokenFactor.
12209 SDValue Result = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Stores);
12210 return Result;
12211 }
12212
12213 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
12214 "Unaligned store of unknown type.");
12215 // Get the half-size VT
12216 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(Context&: *DAG.getContext());
12217 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
12218 unsigned IncrementSize = NumBits / 8;
12219
12220 // Divide the stored value in two parts.
12221 SDValue ShiftAmount =
12222 DAG.getShiftAmountConstant(Val: NumBits, VT: Val.getValueType(), DL: dl);
12223 SDValue Lo = Val;
12224 // If Val is a constant, replace the upper bits with 0. The SRL will constant
12225 // fold and not use the upper bits. A smaller constant may be easier to
12226 // materialize.
12227 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Lo); C && !C->isOpaque())
12228 Lo = DAG.getNode(
12229 Opcode: ISD::AND, DL: dl, VT, N1: Lo,
12230 N2: DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: NumBits), DL: dl,
12231 VT));
12232 SDValue Hi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Val, N2: ShiftAmount);
12233
12234 // Store the two parts
12235 SDValue Store1, Store2;
12236 Store1 = DAG.getTruncStore(Chain, dl,
12237 Val: DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
12238 Ptr, PtrInfo: ST->getPointerInfo(), SVT: NewStoredVT, Alignment,
12239 MMOFlags: ST->getMemOperand()->getFlags());
12240
12241 Ptr = DAG.getObjectPtrOffset(SL: dl, Ptr, Offset: TypeSize::getFixed(ExactSize: IncrementSize));
12242 Store2 = DAG.getTruncStore(
12243 Chain, dl, Val: DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
12244 PtrInfo: ST->getPointerInfo().getWithOffset(O: IncrementSize), SVT: NewStoredVT, Alignment,
12245 MMOFlags: ST->getMemOperand()->getFlags(), AAInfo: ST->getAAInfo());
12246
12247 SDValue Result =
12248 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: Store1, N2: Store2);
12249 return Result;
12250}
12251
12252SDValue
12253TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
12254 const SDLoc &DL, EVT DataVT,
12255 SelectionDAG &DAG,
12256 bool IsCompressedMemory) const {
12257 SDValue Increment;
12258 EVT AddrVT = Addr.getValueType();
12259 EVT MaskVT = Mask.getValueType();
12260 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
12261 "Incompatible types of Data and Mask");
12262 if (IsCompressedMemory) {
12263 // Incrementing the pointer according to number of '1's in the mask.
12264 if (DataVT.isScalableVector()) {
12265 EVT MaskExtVT = MaskVT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i32);
12266 SDValue MaskExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MaskExtVT, Operand: Mask);
12267 Increment = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: MVT::i32, Operand: MaskExt);
12268 } else {
12269 EVT MaskIntVT =
12270 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MaskVT.getSizeInBits());
12271 SDValue MaskInIntReg = DAG.getBitcast(VT: MaskIntVT, V: Mask);
12272 if (MaskIntVT.getSizeInBits() < 32) {
12273 MaskInIntReg =
12274 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: MaskInIntReg);
12275 MaskIntVT = MVT::i32;
12276 }
12277 Increment = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: MaskIntVT, Operand: MaskInIntReg);
12278 }
12279 // Scale is an element size in bytes.
12280 SDValue Scale = DAG.getConstant(Val: DataVT.getScalarSizeInBits() / 8, DL,
12281 VT: AddrVT);
12282 Increment = DAG.getZExtOrTrunc(Op: Increment, DL, VT: AddrVT);
12283 Increment = DAG.getNode(Opcode: ISD::MUL, DL, VT: AddrVT, N1: Increment, N2: Scale);
12284 } else
12285 Increment = DAG.getTypeSize(DL, VT: AddrVT, TS: DataVT.getStoreSize());
12286
12287 return DAG.getNode(Opcode: ISD::ADD, DL, VT: AddrVT, N1: Addr, N2: Increment);
12288}
12289
12290static SDValue clampDynamicVectorIndex(SelectionDAG &DAG, SDValue Idx,
12291 EVT VecVT, const SDLoc &dl,
12292 ElementCount SubEC) {
12293 assert(!(SubEC.isScalable() && VecVT.isFixedLengthVector()) &&
12294 "Cannot index a scalable vector within a fixed-width vector");
12295
12296 unsigned NElts = VecVT.getVectorMinNumElements();
12297 unsigned NumSubElts = SubEC.getKnownMinValue();
12298 EVT IdxVT = Idx.getValueType();
12299
12300 if (VecVT.isScalableVector() && !SubEC.isScalable()) {
12301 // If this is a constant index and we know the value plus the number of the
12302 // elements in the subvector minus one is less than the minimum number of
12303 // elements then it's safe to return Idx.
12304 if (auto *IdxCst = dyn_cast<ConstantSDNode>(Val&: Idx))
12305 if (IdxCst->getZExtValue() + (NumSubElts - 1) < NElts)
12306 return Idx;
12307 SDValue VS =
12308 DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getFixedSizeInBits(), NElts));
12309 unsigned SubOpcode = NumSubElts <= NElts ? ISD::SUB : ISD::USUBSAT;
12310 SDValue Sub = DAG.getNode(Opcode: SubOpcode, DL: dl, VT: IdxVT, N1: VS,
12311 N2: DAG.getConstant(Val: NumSubElts, DL: dl, VT: IdxVT));
12312 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx, N2: Sub);
12313 }
12314 if (isPowerOf2_32(Value: NElts) && NumSubElts == 1) {
12315 APInt Imm = APInt::getLowBitsSet(numBits: IdxVT.getSizeInBits(), loBitsSet: Log2_32(Value: NElts));
12316 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: IdxVT, N1: Idx,
12317 N2: DAG.getConstant(Val: Imm, DL: dl, VT: IdxVT));
12318 }
12319 unsigned MaxIndex = NumSubElts < NElts ? NElts - NumSubElts : 0;
12320 return DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT: IdxVT, N1: Idx,
12321 N2: DAG.getConstant(Val: MaxIndex, DL: dl, VT: IdxVT));
12322}
12323
12324SDValue
12325TargetLowering::getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr,
12326 EVT VecVT, SDValue Index,
12327 const SDNodeFlags PtrArithFlags) const {
12328 return getVectorSubVecPointer(
12329 DAG, VecPtr, VecVT,
12330 SubVecVT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(), NumElements: 1),
12331 Index, PtrArithFlags);
12332}
12333
12334SDValue
12335TargetLowering::getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr,
12336 EVT VecVT, EVT SubVecVT, SDValue Index,
12337 const SDNodeFlags PtrArithFlags) const {
12338 SDLoc dl(Index);
12339 // Make sure the index type is big enough to compute in.
12340 Index = DAG.getZExtOrTrunc(Op: Index, DL: dl, VT: VecPtr.getValueType());
12341
12342 EVT EltVT = VecVT.getVectorElementType();
12343
12344 // Calculate the element offset and add it to the pointer.
12345 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
12346 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
12347 "Converting bits to bytes lost precision");
12348 assert(SubVecVT.getVectorElementType() == EltVT &&
12349 "Sub-vector must be a vector with matching element type");
12350 Index = clampDynamicVectorIndex(DAG, Idx: Index, VecVT, dl,
12351 SubEC: SubVecVT.getVectorElementCount());
12352
12353 EVT IdxVT = Index.getValueType();
12354 if (SubVecVT.isScalableVector())
12355 Index =
12356 DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12357 N2: DAG.getVScale(DL: dl, VT: IdxVT, MulImm: APInt(IdxVT.getSizeInBits(), 1)));
12358
12359 Index = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: IdxVT, N1: Index,
12360 N2: DAG.getConstant(Val: EltSize, DL: dl, VT: IdxVT));
12361 return DAG.getMemBasePlusOffset(Base: VecPtr, Offset: Index, DL: dl, Flags: PtrArithFlags);
12362}
12363
12364//===----------------------------------------------------------------------===//
12365// Implementation of Emulated TLS Model
12366//===----------------------------------------------------------------------===//
12367
12368SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
12369 SelectionDAG &DAG) const {
12370 // Access to address of TLS varialbe xyz is lowered to a function call:
12371 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
12372 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12373 PointerType *VoidPtrType = PointerType::get(C&: *DAG.getContext(), AddressSpace: 0);
12374 SDLoc dl(GA);
12375
12376 ArgListTy Args;
12377 const GlobalValue *GV =
12378 cast<GlobalValue>(Val: GA->getGlobal()->stripPointerCastsAndAliases());
12379 SmallString<32> NameString("__emutls_v.");
12380 NameString += GV->getName();
12381 StringRef EmuTlsVarName(NameString);
12382 const GlobalVariable *EmuTlsVar =
12383 GV->getParent()->getNamedGlobal(Name: EmuTlsVarName);
12384 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
12385 Args.emplace_back(args: DAG.getGlobalAddress(GV: EmuTlsVar, DL: dl, VT: PtrVT), args&: VoidPtrType);
12386
12387 SDValue EmuTlsGetAddr = DAG.getExternalSymbol(Sym: "__emutls_get_address", VT: PtrVT);
12388
12389 TargetLowering::CallLoweringInfo CLI(DAG);
12390 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
12391 CLI.setLibCallee(CC: CallingConv::C, ResultType: VoidPtrType, Target: EmuTlsGetAddr, ArgsList: std::move(Args));
12392 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
12393
12394 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
12395 // At last for X86 targets, maybe good for other targets too?
12396 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
12397 MFI.setAdjustsStack(true); // Is this only for X86 target?
12398 MFI.setHasCalls(true);
12399
12400 assert((GA->getOffset() == 0) &&
12401 "Emulated TLS must have zero offset in GlobalAddressSDNode");
12402 return CallResult.first;
12403}
12404
12405SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
12406 SelectionDAG &DAG) const {
12407 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
12408 if (!isCtlzFast())
12409 return SDValue();
12410 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
12411 SDLoc dl(Op);
12412 if (isNullConstant(V: Op.getOperand(i: 1)) && CC == ISD::SETEQ) {
12413 EVT VT = Op.getOperand(i: 0).getValueType();
12414 SDValue Zext = Op.getOperand(i: 0);
12415 if (VT.bitsLT(VT: MVT::i32)) {
12416 VT = MVT::i32;
12417 Zext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT, Operand: Op.getOperand(i: 0));
12418 }
12419 unsigned Log2b = Log2_32(Value: VT.getSizeInBits());
12420 SDValue Clz = DAG.getNode(Opcode: ISD::CTLZ, DL: dl, VT, Operand: Zext);
12421 SDValue Scc = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: Clz,
12422 N2: DAG.getConstant(Val: Log2b, DL: dl, VT: MVT::i32));
12423 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Scc);
12424 }
12425 return SDValue();
12426}
12427
12428SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
12429 SDValue Op0 = Node->getOperand(Num: 0);
12430 SDValue Op1 = Node->getOperand(Num: 1);
12431 EVT VT = Op0.getValueType();
12432 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12433 unsigned Opcode = Node->getOpcode();
12434 SDLoc DL(Node);
12435
12436 // If both sign bits are zero, flip UMIN/UMAX <-> SMIN/SMAX if legal.
12437 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(MinMaxOpc: Opcode);
12438 if (isOperationLegal(Op: AltOpcode, VT) && DAG.SignBitIsZero(Op: Op0) &&
12439 DAG.SignBitIsZero(Op: Op1))
12440 return DAG.getNode(Opcode: AltOpcode, DL, VT, N1: Op0, N2: Op1);
12441
12442 // umax(x,1) --> sub(x,cmpeq(x,0)) iff cmp result is allbits
12443 if (Opcode == ISD::UMAX && llvm::isOneOrOneSplat(V: Op1, AllowUndefs: true) && BoolVT == VT &&
12444 getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12445 Op0 = DAG.getFreeze(V: Op0);
12446 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
12447 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12448 N2: DAG.getSetCC(DL, VT, LHS: Op0, RHS: Zero, Cond: ISD::SETEQ));
12449 }
12450
12451 // umin(x,y) -> sub(x,usubsat(x,y))
12452 // TODO: Missing freeze(Op0)?
12453 if (Opcode == ISD::UMIN && isOperationLegal(Op: ISD::SUB, VT) &&
12454 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12455 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0,
12456 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op0, N2: Op1));
12457 }
12458
12459 // umax(x,y) -> add(x,usubsat(y,x))
12460 // TODO: Missing freeze(Op0)?
12461 if (Opcode == ISD::UMAX && isOperationLegal(Op: ISD::ADD, VT) &&
12462 isOperationLegal(Op: ISD::USUBSAT, VT)) {
12463 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0,
12464 N2: DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: Op1, N2: Op0));
12465 }
12466
12467 // FIXME: Should really try to split the vector in case it's legal on a
12468 // subvector.
12469 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12470 return DAG.UnrollVectorOp(N: Node);
12471
12472 // Attempt to find an existing SETCC node that we can reuse.
12473 // TODO: Do we need a generic doesSETCCNodeExist?
12474 // TODO: Missing freeze(Op0)/freeze(Op1)?
12475 auto buildMinMax = [&](ISD::CondCode PrefCC, ISD::CondCode AltCC,
12476 ISD::CondCode PrefCommuteCC,
12477 ISD::CondCode AltCommuteCC) {
12478 SDVTList BoolVTList = DAG.getVTList(VT: BoolVT);
12479 for (ISD::CondCode CC : {PrefCC, AltCC}) {
12480 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12481 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12482 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12483 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12484 }
12485 }
12486 for (ISD::CondCode CC : {PrefCommuteCC, AltCommuteCC}) {
12487 if (DAG.doesNodeExist(Opcode: ISD::SETCC, VTList: BoolVTList,
12488 Ops: {Op0, Op1, DAG.getCondCode(Cond: CC)})) {
12489 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: CC);
12490 return DAG.getSelect(DL, VT, Cond, LHS: Op1, RHS: Op0);
12491 }
12492 }
12493 SDValue Cond = DAG.getSetCC(DL, VT: BoolVT, LHS: Op0, RHS: Op1, Cond: PrefCC);
12494 return DAG.getSelect(DL, VT, Cond, LHS: Op0, RHS: Op1);
12495 };
12496
12497 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
12498 // -> Y = (A < B) ? B : A
12499 // -> Y = (A >= B) ? A : B
12500 // -> Y = (A <= B) ? B : A
12501 switch (Opcode) {
12502 case ISD::SMAX:
12503 return buildMinMax(ISD::SETGT, ISD::SETGE, ISD::SETLT, ISD::SETLE);
12504 case ISD::SMIN:
12505 return buildMinMax(ISD::SETLT, ISD::SETLE, ISD::SETGT, ISD::SETGE);
12506 case ISD::UMAX:
12507 return buildMinMax(ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE);
12508 case ISD::UMIN:
12509 return buildMinMax(ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE);
12510 }
12511
12512 llvm_unreachable("How did we get here?");
12513}
12514
12515SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
12516 unsigned Opcode = Node->getOpcode();
12517 SDValue LHS = Node->getOperand(Num: 0);
12518 SDValue RHS = Node->getOperand(Num: 1);
12519 EVT VT = LHS.getValueType();
12520 SDLoc dl(Node);
12521
12522 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
12523 assert(VT.isInteger() && "Expected operands to be integers");
12524
12525 // usub.sat(a, b) -> umax(a, b) - b
12526 if (Opcode == ISD::USUBSAT && isOperationLegal(Op: ISD::UMAX, VT)) {
12527 SDValue Max = DAG.getNode(Opcode: ISD::UMAX, DL: dl, VT, N1: LHS, N2: RHS);
12528 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: RHS);
12529 }
12530
12531 // usub.sat(a, 1) -> sub(a, zext(a != 0))
12532 // Prefer this on targets without legal/cost-effective overflow-carry nodes.
12533 if (Opcode == ISD::USUBSAT && isOneOrOneSplat(V: RHS) &&
12534 !isOperationLegalOrCustom(Op: ISD::USUBO_CARRY, VT)) {
12535 LHS = DAG.getFreeze(V: LHS);
12536 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12537 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12538 SDValue IsNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETNE);
12539 SDValue Subtrahend = DAG.getBoolExtOrTrunc(Op: IsNonZero, SL: dl, VT, OpVT: BoolVT);
12540 Subtrahend =
12541 DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Subtrahend, N2: DAG.getConstant(Val: 1, DL: dl, VT));
12542 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: Subtrahend);
12543 }
12544
12545 // uadd.sat(a, b) -> umin(a, ~b) + b
12546 if (Opcode == ISD::UADDSAT && isOperationLegal(Op: ISD::UMIN, VT)) {
12547 SDValue InvRHS = DAG.getNOT(DL: dl, Val: RHS, VT);
12548 SDValue Min = DAG.getNode(Opcode: ISD::UMIN, DL: dl, VT, N1: LHS, N2: InvRHS);
12549 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Min, N2: RHS);
12550 }
12551
12552 unsigned OverflowOp;
12553 switch (Opcode) {
12554 case ISD::SADDSAT:
12555 OverflowOp = ISD::SADDO;
12556 break;
12557 case ISD::UADDSAT:
12558 OverflowOp = ISD::UADDO;
12559 break;
12560 case ISD::SSUBSAT:
12561 OverflowOp = ISD::SSUBO;
12562 break;
12563 case ISD::USUBSAT:
12564 OverflowOp = ISD::USUBO;
12565 break;
12566 default:
12567 llvm_unreachable("Expected method to receive signed or unsigned saturation "
12568 "addition or subtraction node.");
12569 }
12570
12571 // FIXME: Should really try to split the vector in case it's legal on a
12572 // subvector.
12573 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12574 return DAG.UnrollVectorOp(N: Node);
12575
12576 unsigned BitWidth = LHS.getScalarValueSizeInBits();
12577 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12578 SDValue Result = DAG.getNode(Opcode: OverflowOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12579 SDValue SumDiff = Result.getValue(R: 0);
12580 SDValue Overflow = Result.getValue(R: 1);
12581 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12582 SDValue AllOnes = DAG.getAllOnesConstant(DL: dl, VT);
12583
12584 if (Opcode == ISD::UADDSAT) {
12585 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12586 // (LHS + RHS) | OverflowMask
12587 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12588 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: SumDiff, N2: OverflowMask);
12589 }
12590 // Overflow ? 0xffff.... : (LHS + RHS)
12591 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: AllOnes, RHS: SumDiff);
12592 }
12593
12594 if (Opcode == ISD::USUBSAT) {
12595 if (getBooleanContents(Type: VT) == ZeroOrNegativeOneBooleanContent) {
12596 // (LHS - RHS) & ~OverflowMask
12597 SDValue OverflowMask = DAG.getSExtOrTrunc(Op: Overflow, DL: dl, VT);
12598 SDValue Not = DAG.getNOT(DL: dl, Val: OverflowMask, VT);
12599 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SumDiff, N2: Not);
12600 }
12601 // Overflow ? 0 : (LHS - RHS)
12602 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Zero, RHS: SumDiff);
12603 }
12604
12605 assert((Opcode == ISD::SADDSAT || Opcode == ISD::SSUBSAT) &&
12606 "Expected signed saturating add/sub opcode");
12607
12608 const APInt MinVal = APInt::getSignedMinValue(numBits: BitWidth);
12609 const APInt MaxVal = APInt::getSignedMaxValue(numBits: BitWidth);
12610
12611 KnownBits KnownLHS = DAG.computeKnownBits(Op: LHS);
12612 KnownBits KnownRHS = DAG.computeKnownBits(Op: RHS);
12613
12614 // If either of the operand signs are known, then they are guaranteed to
12615 // only saturate in one direction. If non-negative they will saturate
12616 // towards SIGNED_MAX, if negative they will saturate towards SIGNED_MIN.
12617 //
12618 // In the case of ISD::SSUBSAT, 'x - y' is equivalent to 'x + (-y)', so the
12619 // sign of 'y' has to be flipped.
12620
12621 bool LHSIsNonNegative = KnownLHS.isNonNegative();
12622 bool RHSIsNonNegative =
12623 Opcode == ISD::SADDSAT ? KnownRHS.isNonNegative() : KnownRHS.isNegative();
12624 if (LHSIsNonNegative || RHSIsNonNegative) {
12625 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12626 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: SumDiff);
12627 }
12628
12629 bool LHSIsNegative = KnownLHS.isNegative();
12630 bool RHSIsNegative =
12631 Opcode == ISD::SADDSAT ? KnownRHS.isNegative() : KnownRHS.isNonNegative();
12632 if (LHSIsNegative || RHSIsNegative) {
12633 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12634 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMin, RHS: SumDiff);
12635 }
12636
12637 // Overflow ? (SumDiff >> BW) ^ MinVal : SumDiff
12638 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12639 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: SumDiff,
12640 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT));
12641 Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Shift, N2: SatMin);
12642 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: SumDiff);
12643}
12644
12645SDValue TargetLowering::expandCMP(SDNode *Node, SelectionDAG &DAG) const {
12646 unsigned Opcode = Node->getOpcode();
12647 SDValue LHS = Node->getOperand(Num: 0);
12648 SDValue RHS = Node->getOperand(Num: 1);
12649 EVT VT = LHS.getValueType();
12650 EVT ResVT = Node->getValueType(ResNo: 0);
12651 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12652 SDLoc dl(Node);
12653
12654 auto LTPredicate = (Opcode == ISD::UCMP ? ISD::SETULT : ISD::SETLT);
12655 auto GTPredicate = (Opcode == ISD::UCMP ? ISD::SETUGT : ISD::SETGT);
12656 SDValue IsLT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: LTPredicate);
12657 SDValue IsGT = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS, Cond: GTPredicate);
12658
12659 // We can't perform arithmetic on i1 values. Extending them would
12660 // probably result in worse codegen, so let's just use two selects instead.
12661 // Some targets are also just better off using selects rather than subtraction
12662 // because one of the conditions can be merged with one of the selects.
12663 // And finally, if we don't know the contents of high bits of a boolean value
12664 // we can't perform any arithmetic either.
12665 if (preferSelectsOverBooleanArithmetic(VT) ||
12666 BoolVT.getScalarSizeInBits() == 1 ||
12667 getBooleanContents(Type: BoolVT) == UndefinedBooleanContent) {
12668 SDValue SelectZeroOrOne =
12669 DAG.getSelect(DL: dl, VT: ResVT, Cond: IsGT, LHS: DAG.getConstant(Val: 1, DL: dl, VT: ResVT),
12670 RHS: DAG.getConstant(Val: 0, DL: dl, VT: ResVT));
12671 return DAG.getSelect(DL: dl, VT: ResVT, Cond: IsLT, LHS: DAG.getAllOnesConstant(DL: dl, VT: ResVT),
12672 RHS: SelectZeroOrOne);
12673 }
12674
12675 if (getBooleanContents(Type: BoolVT) == ZeroOrNegativeOneBooleanContent)
12676 std::swap(a&: IsGT, b&: IsLT);
12677 return DAG.getSExtOrTrunc(Op: DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: BoolVT, N1: IsGT, N2: IsLT), DL: dl,
12678 VT: ResVT);
12679}
12680
12681SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
12682 unsigned Opcode = Node->getOpcode();
12683 bool IsSigned = Opcode == ISD::SSHLSAT;
12684 SDValue LHS = Node->getOperand(Num: 0);
12685 SDValue RHS = Node->getOperand(Num: 1);
12686 EVT VT = LHS.getValueType();
12687 SDLoc dl(Node);
12688
12689 assert((Node->getOpcode() == ISD::SSHLSAT ||
12690 Node->getOpcode() == ISD::USHLSAT) &&
12691 "Expected a SHLSAT opcode");
12692 assert(VT.isInteger() && "Expected operands to be integers");
12693
12694 if (VT.isVector() && !isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
12695 return DAG.UnrollVectorOp(N: Node);
12696
12697 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
12698
12699 unsigned BW = VT.getScalarSizeInBits();
12700 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12701 SDValue Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: RHS);
12702 SDValue Orig =
12703 DAG.getNode(Opcode: IsSigned ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: Result, N2: RHS);
12704
12705 SDValue SatVal;
12706 if (IsSigned) {
12707 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: BW), DL: dl, VT);
12708 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: BW), DL: dl, VT);
12709 SDValue Cond =
12710 DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETLT);
12711 SatVal = DAG.getSelect(DL: dl, VT, Cond, LHS: SatMin, RHS: SatMax);
12712 } else {
12713 SatVal = DAG.getConstant(Val: APInt::getMaxValue(numBits: BW), DL: dl, VT);
12714 }
12715 SDValue Cond = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Orig, Cond: ISD::SETNE);
12716 return DAG.getSelect(DL: dl, VT, Cond, LHS: SatVal, RHS: Result);
12717}
12718
12719void TargetLowering::forceExpandMultiply(SelectionDAG &DAG, const SDLoc &dl,
12720 bool Signed, SDValue &Lo, SDValue &Hi,
12721 SDValue LHS, SDValue RHS,
12722 SDValue HiLHS, SDValue HiRHS) const {
12723 EVT VT = LHS.getValueType();
12724 assert(RHS.getValueType() == VT && "Mismatching operand types");
12725
12726 assert((HiLHS && HiRHS) || (!HiLHS && !HiRHS));
12727 assert((!Signed || !HiLHS) &&
12728 "Signed flag should only be set when HiLHS and RiRHS are null");
12729
12730 // We'll expand the multiplication by brute force because we have no other
12731 // options. This is a trivially-generalized version of the code from
12732 // Hacker's Delight (itself derived from Knuth's Algorithm M from section
12733 // 4.3.1). If Signed is set, we can use arithmetic right shifts to propagate
12734 // sign bits while calculating the Hi half.
12735 unsigned Bits = VT.getScalarSizeInBits();
12736 unsigned HalfBits = Bits / 2;
12737 SDValue Mask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: Bits, loBitsSet: HalfBits), DL: dl, VT);
12738 SDValue LL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: LHS, N2: Mask);
12739 SDValue RL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: RHS, N2: Mask);
12740
12741 SDValue T = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RL);
12742 SDValue TL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: T, N2: Mask);
12743
12744 SDValue Shift = DAG.getShiftAmountConstant(Val: HalfBits, VT, DL: dl);
12745 // This is always an unsigned shift.
12746 SDValue TH = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: T, N2: Shift);
12747
12748 unsigned ShiftOpc = Signed ? ISD::SRA : ISD::SRL;
12749 SDValue LH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: LHS, N2: Shift);
12750 SDValue RH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: RHS, N2: Shift);
12751
12752 SDValue U =
12753 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RL), N2: TH);
12754 SDValue UL = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: U, N2: Mask);
12755 SDValue UH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: U, N2: Shift);
12756
12757 SDValue V =
12758 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LL, N2: RH), N2: UL);
12759 SDValue VH = DAG.getNode(Opcode: ShiftOpc, DL: dl, VT, N1: V, N2: Shift);
12760
12761 Lo = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: TL,
12762 N2: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: V, N2: Shift));
12763
12764 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LH, N2: RH),
12765 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: UH, N2: VH));
12766
12767 // If HiLHS and HiRHS are set, multiply them by the opposite low part and add
12768 // the products to Hi.
12769 if (HiLHS) {
12770 SDValue RHLL = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: HiRHS, N2: LHS);
12771 SDValue RLLH = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: RHS, N2: HiLHS);
12772 Hi = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: Hi,
12773 N2: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: RHLL, N2: RLLH));
12774 }
12775}
12776
12777void TargetLowering::forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl,
12778 bool Signed, const SDValue LHS,
12779 const SDValue RHS, SDValue &Lo,
12780 SDValue &Hi) const {
12781 EVT VT = LHS.getValueType();
12782 assert(RHS.getValueType() == VT && "Mismatching operand types");
12783 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
12784 // We can fall back to a libcall with an illegal type for the MUL if we
12785 // have a libcall big enough.
12786 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
12787 if (WideVT == MVT::i16)
12788 LC = RTLIB::MUL_I16;
12789 else if (WideVT == MVT::i32)
12790 LC = RTLIB::MUL_I32;
12791 else if (WideVT == MVT::i64)
12792 LC = RTLIB::MUL_I64;
12793 else if (WideVT == MVT::i128)
12794 LC = RTLIB::MUL_I128;
12795
12796 RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(Call: LC);
12797 if (LibcallImpl == RTLIB::Unsupported) {
12798 forceExpandMultiply(DAG, dl, Signed, Lo, Hi, LHS, RHS);
12799 return;
12800 }
12801
12802 SDValue HiLHS, HiRHS;
12803 if (Signed) {
12804 // The high part is obtained by SRA'ing all but one of the bits of low
12805 // part.
12806 unsigned LoSize = VT.getFixedSizeInBits();
12807 SDValue Shift = DAG.getShiftAmountConstant(Val: LoSize - 1, VT, DL: dl);
12808 HiLHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: LHS, N2: Shift);
12809 HiRHS = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: RHS, N2: Shift);
12810 } else {
12811 HiLHS = DAG.getConstant(Val: 0, DL: dl, VT);
12812 HiRHS = DAG.getConstant(Val: 0, DL: dl, VT);
12813 }
12814
12815 // Attempt a libcall.
12816 SDValue Ret;
12817 TargetLowering::MakeLibCallOptions CallOptions;
12818 CallOptions.setIsSigned(Signed);
12819 CallOptions.setIsPostTypeLegalization(true);
12820 if (shouldSplitFunctionArgumentsAsLittleEndian(DL: DAG.getDataLayout())) {
12821 // Halves of WideVT are packed into registers in different order
12822 // depending on platform endianness. This is usually handled by
12823 // the C calling convention, but we can't defer to it in
12824 // the legalizer.
12825 SDValue Args[] = {LHS, HiLHS, RHS, HiRHS};
12826 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12827 } else {
12828 SDValue Args[] = {HiLHS, LHS, HiRHS, RHS};
12829 Ret = makeLibCall(DAG, LC, RetVT: WideVT, Ops: Args, CallOptions, dl).first;
12830 }
12831 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
12832 "Ret value is a collection of constituent nodes holding result.");
12833 if (DAG.getDataLayout().isLittleEndian()) {
12834 // Same as above.
12835 Lo = Ret.getOperand(i: 0);
12836 Hi = Ret.getOperand(i: 1);
12837 } else {
12838 Lo = Ret.getOperand(i: 1);
12839 Hi = Ret.getOperand(i: 0);
12840 }
12841}
12842
12843SDValue
12844TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
12845 assert((Node->getOpcode() == ISD::SMULFIX ||
12846 Node->getOpcode() == ISD::UMULFIX ||
12847 Node->getOpcode() == ISD::SMULFIXSAT ||
12848 Node->getOpcode() == ISD::UMULFIXSAT) &&
12849 "Expected a fixed point multiplication opcode");
12850
12851 SDLoc dl(Node);
12852 SDValue LHS = Node->getOperand(Num: 0);
12853 SDValue RHS = Node->getOperand(Num: 1);
12854 EVT VT = LHS.getValueType();
12855 unsigned Scale = Node->getConstantOperandVal(Num: 2);
12856 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
12857 Node->getOpcode() == ISD::UMULFIXSAT);
12858 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
12859 Node->getOpcode() == ISD::SMULFIXSAT);
12860 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
12861 unsigned VTSize = VT.getScalarSizeInBits();
12862
12863 if (!Scale) {
12864 // [us]mul.fix(a, b, 0) -> mul(a, b)
12865 if (!Saturating) {
12866 if (isOperationLegalOrCustom(Op: ISD::MUL, VT))
12867 return DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
12868 } else if (Signed && isOperationLegalOrCustom(Op: ISD::SMULO, VT)) {
12869 SDValue Result =
12870 DAG.getNode(Opcode: ISD::SMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12871 SDValue Product = Result.getValue(R: 0);
12872 SDValue Overflow = Result.getValue(R: 1);
12873 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12874
12875 APInt MinVal = APInt::getSignedMinValue(numBits: VTSize);
12876 APInt MaxVal = APInt::getSignedMaxValue(numBits: VTSize);
12877 SDValue SatMin = DAG.getConstant(Val: MinVal, DL: dl, VT);
12878 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12879 // Xor the inputs, if resulting sign bit is 0 the product will be
12880 // positive, else negative.
12881 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
12882 SDValue ProdNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Xor, RHS: Zero, Cond: ISD::SETLT);
12883 Result = DAG.getSelect(DL: dl, VT, Cond: ProdNeg, LHS: SatMin, RHS: SatMax);
12884 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: Result, RHS: Product);
12885 } else if (!Signed && isOperationLegalOrCustom(Op: ISD::UMULO, VT)) {
12886 SDValue Result =
12887 DAG.getNode(Opcode: ISD::UMULO, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: BoolVT), N1: LHS, N2: RHS);
12888 SDValue Product = Result.getValue(R: 0);
12889 SDValue Overflow = Result.getValue(R: 1);
12890
12891 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
12892 SDValue SatMax = DAG.getConstant(Val: MaxVal, DL: dl, VT);
12893 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: SatMax, RHS: Product);
12894 }
12895 }
12896
12897 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
12898 "Expected scale to be less than the number of bits if signed or at "
12899 "most the number of bits if unsigned.");
12900 assert(LHS.getValueType() == RHS.getValueType() &&
12901 "Expected both operands to be the same type");
12902
12903 // Get the upper and lower bits of the result.
12904 SDValue Lo, Hi;
12905 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
12906 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
12907 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
12908 if (isOperationLegalOrCustom(Op: LoHiOp, VT)) {
12909 SDValue Result = DAG.getNode(Opcode: LoHiOp, DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS, N2: RHS);
12910 Lo = Result.getValue(R: 0);
12911 Hi = Result.getValue(R: 1);
12912 } else if (isOperationLegalOrCustom(Op: HiOp, VT)) {
12913 Lo = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
12914 Hi = DAG.getNode(Opcode: HiOp, DL: dl, VT, N1: LHS, N2: RHS);
12915 } else if (isOperationLegalOrCustom(Op: ISD::MUL, VT: WideVT)) {
12916 // Try for a multiplication using a wider type.
12917 unsigned Ext = Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
12918 SDValue LHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: LHS);
12919 SDValue RHSExt = DAG.getNode(Opcode: Ext, DL: dl, VT: WideVT, Operand: RHS);
12920 SDValue Res = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHSExt, N2: RHSExt);
12921 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Res);
12922 SDValue Shifted =
12923 DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: WideVT, N1: Res,
12924 N2: DAG.getShiftAmountConstant(Val: VTSize, VT: WideVT, DL: dl));
12925 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Shifted);
12926 } else if (VT.isVector()) {
12927 return SDValue();
12928 } else {
12929 forceExpandWideMUL(DAG, dl, Signed, LHS, RHS, Lo, Hi);
12930 }
12931
12932 if (Scale == VTSize)
12933 // Result is just the top half since we'd be shifting by the width of the
12934 // operand. Overflow impossible so this works for both UMULFIX and
12935 // UMULFIXSAT.
12936 return Hi;
12937
12938 // The result will need to be shifted right by the scale since both operands
12939 // are scaled. The result is given to us in 2 halves, so we only want part of
12940 // both in the result.
12941 SDValue Result = DAG.getNode(Opcode: ISD::FSHR, DL: dl, VT, N1: Hi, N2: Lo,
12942 N3: DAG.getShiftAmountConstant(Val: Scale, VT, DL: dl));
12943 if (!Saturating)
12944 return Result;
12945
12946 if (!Signed) {
12947 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
12948 // widened multiplication) aren't all zeroes.
12949
12950 // Saturate to max if ((Hi >> Scale) != 0),
12951 // which is the same as if (Hi > ((1 << Scale) - 1))
12952 APInt MaxVal = APInt::getMaxValue(numBits: VTSize);
12953 SDValue LowMask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale),
12954 DL: dl, VT);
12955 Result = DAG.getSelectCC(DL: dl, LHS: Hi, RHS: LowMask,
12956 True: DAG.getConstant(Val: MaxVal, DL: dl, VT), False: Result,
12957 Cond: ISD::SETUGT);
12958
12959 return Result;
12960 }
12961
12962 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
12963 // widened multiplication) aren't all ones or all zeroes.
12964
12965 SDValue SatMin = DAG.getConstant(Val: APInt::getSignedMinValue(numBits: VTSize), DL: dl, VT);
12966 SDValue SatMax = DAG.getConstant(Val: APInt::getSignedMaxValue(numBits: VTSize), DL: dl, VT);
12967
12968 if (Scale == 0) {
12969 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: Lo,
12970 N2: DAG.getShiftAmountConstant(Val: VTSize - 1, VT, DL: dl));
12971 SDValue Overflow = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Hi, RHS: Sign, Cond: ISD::SETNE);
12972 // Saturated to SatMin if wide product is negative, and SatMax if wide
12973 // product is positive ...
12974 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
12975 SDValue ResultIfOverflow = DAG.getSelectCC(DL: dl, LHS: Hi, RHS: Zero, True: SatMin, False: SatMax,
12976 Cond: ISD::SETLT);
12977 // ... but only if we overflowed.
12978 return DAG.getSelect(DL: dl, VT, Cond: Overflow, LHS: ResultIfOverflow, RHS: Result);
12979 }
12980
12981 // We handled Scale==0 above so all the bits to examine is in Hi.
12982
12983 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
12984 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
12985 SDValue LowMask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: VTSize, loBitsSet: Scale - 1),
12986 DL: dl, VT);
12987 Result = DAG.getSelectCC(DL: dl, LHS: Hi, RHS: LowMask, True: SatMax, False: Result, Cond: ISD::SETGT);
12988 // Saturate to min if (Hi >> (Scale - 1)) < -1),
12989 // which is the same as if (HI < (-1 << (Scale - 1))
12990 SDValue HighMask =
12991 DAG.getConstant(Val: APInt::getHighBitsSet(numBits: VTSize, hiBitsSet: VTSize - Scale + 1),
12992 DL: dl, VT);
12993 Result = DAG.getSelectCC(DL: dl, LHS: Hi, RHS: HighMask, True: SatMin, False: Result, Cond: ISD::SETLT);
12994 return Result;
12995}
12996
12997SDValue
12998TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
12999 SDValue LHS, SDValue RHS,
13000 unsigned Scale, SelectionDAG &DAG) const {
13001 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
13002 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
13003 "Expected a fixed point division opcode");
13004
13005 EVT VT = LHS.getValueType();
13006 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
13007 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
13008 EVT BoolVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13009
13010 // If there is enough room in the type to upscale the LHS or downscale the
13011 // RHS before the division, we can perform it in this type without having to
13012 // resize. For signed operations, the LHS headroom is the number of
13013 // redundant sign bits, and for unsigned ones it is the number of zeroes.
13014 // The headroom for the RHS is the number of trailing zeroes.
13015 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(Op: LHS) - 1
13016 : DAG.computeKnownBits(Op: LHS).countMinLeadingZeros();
13017 unsigned RHSTrail = DAG.computeKnownBits(Op: RHS).countMinTrailingZeros();
13018
13019 // For signed saturating operations, we need to be able to detect true integer
13020 // division overflow; that is, when you have MIN / -EPS. However, this
13021 // is undefined behavior and if we emit divisions that could take such
13022 // values it may cause undesired behavior (arithmetic exceptions on x86, for
13023 // example).
13024 // Avoid this by requiring an extra bit so that we never get this case.
13025 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
13026 // signed saturating division, we need to emit a whopping 32-bit division.
13027 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
13028 return SDValue();
13029
13030 unsigned LHSShift = std::min(a: LHSLead, b: Scale);
13031 unsigned RHSShift = Scale - LHSShift;
13032
13033 // At this point, we know that if we shift the LHS up by LHSShift and the
13034 // RHS down by RHSShift, we can emit a regular division with a final scaling
13035 // factor of Scale.
13036
13037 if (LHSShift)
13038 LHS = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS,
13039 N2: DAG.getShiftAmountConstant(Val: LHSShift, VT, DL: dl));
13040 if (RHSShift)
13041 RHS = DAG.getNode(Opcode: Signed ? ISD::SRA : ISD::SRL, DL: dl, VT, N1: RHS,
13042 N2: DAG.getShiftAmountConstant(Val: RHSShift, VT, DL: dl));
13043
13044 SDValue Quot;
13045 if (Signed) {
13046 // For signed operations, if the resulting quotient is negative and the
13047 // remainder is nonzero, subtract 1 from the quotient to round towards
13048 // negative infinity.
13049 SDValue Rem;
13050 // FIXME: Ideally we would always produce an SDIVREM here, but if the
13051 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
13052 // we couldn't just form a libcall, but the type legalizer doesn't do it.
13053 if (isTypeLegal(VT) &&
13054 isOperationLegalOrCustom(Op: ISD::SDIVREM, VT)) {
13055 Quot = DAG.getNode(Opcode: ISD::SDIVREM, DL: dl,
13056 VTList: DAG.getVTList(VT1: VT, VT2: VT),
13057 N1: LHS, N2: RHS);
13058 Rem = Quot.getValue(R: 1);
13059 Quot = Quot.getValue(R: 0);
13060 } else {
13061 Quot = DAG.getNode(Opcode: ISD::SDIV, DL: dl, VT,
13062 N1: LHS, N2: RHS);
13063 Rem = DAG.getNode(Opcode: ISD::SREM, DL: dl, VT,
13064 N1: LHS, N2: RHS);
13065 }
13066 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT);
13067 SDValue RemNonZero = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: Rem, RHS: Zero, Cond: ISD::SETNE);
13068 SDValue LHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS, RHS: Zero, Cond: ISD::SETLT);
13069 SDValue RHSNeg = DAG.getSetCC(DL: dl, VT: BoolVT, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13070 SDValue QuotNeg = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: BoolVT, N1: LHSNeg, N2: RHSNeg);
13071 SDValue Sub1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Quot,
13072 N2: DAG.getConstant(Val: 1, DL: dl, VT));
13073 Quot = DAG.getSelect(DL: dl, VT,
13074 Cond: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: BoolVT, N1: RemNonZero, N2: QuotNeg),
13075 LHS: Sub1, RHS: Quot);
13076 } else
13077 Quot = DAG.getNode(Opcode: ISD::UDIV, DL: dl, VT,
13078 N1: LHS, N2: RHS);
13079
13080 return Quot;
13081}
13082
13083void TargetLowering::expandUADDSUBO(
13084 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13085 SDLoc dl(Node);
13086 SDValue LHS = Node->getOperand(Num: 0);
13087 SDValue RHS = Node->getOperand(Num: 1);
13088 bool IsAdd = Node->getOpcode() == ISD::UADDO;
13089
13090 // If UADDO_CARRY/SUBO_CARRY is legal, use that instead.
13091 unsigned OpcCarry = IsAdd ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
13092 if (isOperationLegalOrCustom(Op: OpcCarry, VT: Node->getValueType(ResNo: 0))) {
13093 SDValue CarryIn = DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 1));
13094 SDValue NodeCarry = DAG.getNode(Opcode: OpcCarry, DL: dl, VTList: Node->getVTList(),
13095 Ops: { LHS, RHS, CarryIn });
13096 Result = SDValue(NodeCarry.getNode(), 0);
13097 Overflow = SDValue(NodeCarry.getNode(), 1);
13098 return;
13099 }
13100
13101 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13102 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13103
13104 EVT ResultType = Node->getValueType(ResNo: 1);
13105 EVT SetCCType = getSetCCResultType(
13106 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13107 SDValue SetCC;
13108 if (IsAdd && isOneConstant(V: RHS)) {
13109 // Special case: uaddo X, 1 overflowed if X+1 is 0. This potential reduces
13110 // the live range of X. We assume comparing with 0 is cheap.
13111 // The general case (X + C) < C is not necessarily beneficial. Although we
13112 // reduce the live range of X, we may introduce the materialization of
13113 // constant C.
13114 SetCC =
13115 DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result,
13116 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETEQ);
13117 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
13118 // Special case: uaddo X, -1 overflows if X != 0.
13119 SetCC =
13120 DAG.getSetCC(DL: dl, VT: SetCCType, LHS,
13121 RHS: DAG.getConstant(Val: 0, DL: dl, VT: Node->getValueType(ResNo: 0)), Cond: ISD::SETNE);
13122 } else {
13123 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
13124 SetCC = DAG.getSetCC(DL: dl, VT: SetCCType, LHS: Result, RHS: LHS, Cond: CC);
13125 }
13126 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13127}
13128
13129void TargetLowering::expandSADDSUBO(
13130 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
13131 SDLoc dl(Node);
13132 SDValue LHS = Node->getOperand(Num: 0);
13133 SDValue RHS = Node->getOperand(Num: 1);
13134 bool IsAdd = Node->getOpcode() == ISD::SADDO;
13135
13136 Result = DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL: dl,
13137 VT: LHS.getValueType(), N1: LHS, N2: RHS);
13138
13139 EVT ResultType = Node->getValueType(ResNo: 1);
13140 EVT OType = getSetCCResultType(
13141 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: Node->getValueType(ResNo: 0));
13142
13143 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
13144 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
13145 if (isOperationLegal(Op: OpcSat, VT: LHS.getValueType())) {
13146 SDValue Sat = DAG.getNode(Opcode: OpcSat, DL: dl, VT: LHS.getValueType(), N1: LHS, N2: RHS);
13147 SDValue SetCC = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Sat, Cond: ISD::SETNE);
13148 Overflow = DAG.getBoolExtOrTrunc(Op: SetCC, SL: dl, VT: ResultType, OpVT: ResultType);
13149 return;
13150 }
13151
13152 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
13153
13154 if (IsAdd) {
13155 // For an addition, the result should be less than one of the operands (LHS)
13156 // if and only if the other operand (RHS) is negative, otherwise there will
13157 // be overflow.
13158 SDValue ResultLowerThanLHS =
13159 DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: LHS, Cond: ISD::SETLT);
13160 SDValue RHSNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
13161 Overflow = DAG.getBoolExtOrTrunc(
13162 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: RHSNegative, N2: ResultLowerThanLHS), SL: dl,
13163 VT: ResultType, OpVT: ResultType);
13164 } else {
13165 // For subtraction, overflow occurs when the signed comparison of operands
13166 // doesn't match the sign of the result.
13167 SDValue LHSLessThanRHS = DAG.getSetCC(DL: dl, VT: OType, LHS, RHS, Cond: ISD::SETLT);
13168 SDValue ResultNegative = DAG.getSetCC(DL: dl, VT: OType, LHS: Result, RHS: Zero, Cond: ISD::SETLT);
13169 Overflow = DAG.getBoolExtOrTrunc(
13170 Op: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: OType, N1: LHSLessThanRHS, N2: ResultNegative), SL: dl,
13171 VT: ResultType, OpVT: ResultType);
13172 }
13173}
13174
13175bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
13176 SDValue &Overflow, SelectionDAG &DAG) const {
13177 SDLoc dl(Node);
13178 EVT VT = Node->getValueType(ResNo: 0);
13179 EVT SetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
13180 SDValue LHS = Node->getOperand(Num: 0);
13181 SDValue RHS = Node->getOperand(Num: 1);
13182 bool isSigned = Node->getOpcode() == ISD::SMULO;
13183
13184 // For power-of-two multiplications we can use a simpler shift expansion.
13185 if (ConstantSDNode *RHSC = isConstOrConstSplat(N: RHS)) {
13186 const APInt &C = RHSC->getAPIntValue();
13187 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
13188 if (C.isPowerOf2()) {
13189 // smulo(x, signed_min) is same as umulo(x, signed_min).
13190 bool UseArithShift = isSigned && !C.isMinSignedValue();
13191 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: C.logBase2(), VT, DL: dl);
13192 Result = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: LHS, N2: ShiftAmt);
13193 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT,
13194 LHS: DAG.getNode(Opcode: UseArithShift ? ISD::SRA : ISD::SRL,
13195 DL: dl, VT, N1: Result, N2: ShiftAmt),
13196 RHS: LHS, Cond: ISD::SETNE);
13197 return true;
13198 }
13199 }
13200
13201 SDValue BottomHalf;
13202 SDValue TopHalf;
13203 EVT WideVT = VT.widenIntegerElementType(Context&: *DAG.getContext());
13204
13205 static const unsigned Ops[2][3] =
13206 { { ISD::UMUL_LOHI, ISD::MULHU, ISD::ZERO_EXTEND },
13207 { ISD::SMUL_LOHI, ISD::MULHS, ISD::SIGN_EXTEND }};
13208 if (isOperationLegalOrCustom(Op: Ops[isSigned][0], VT)) {
13209 BottomHalf = DAG.getNode(Opcode: Ops[isSigned][0], DL: dl, VTList: DAG.getVTList(VT1: VT, VT2: VT), N1: LHS,
13210 N2: RHS);
13211 TopHalf = BottomHalf.getValue(R: 1);
13212 } else if (isOperationLegalOrCustom(Op: Ops[isSigned][1], VT)) {
13213 BottomHalf = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: LHS, N2: RHS);
13214 TopHalf = DAG.getNode(Opcode: Ops[isSigned][1], DL: dl, VT, N1: LHS, N2: RHS);
13215 } else if (isTypeLegal(VT: WideVT)) {
13216 LHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: LHS);
13217 RHS = DAG.getNode(Opcode: Ops[isSigned][2], DL: dl, VT: WideVT, Operand: RHS);
13218 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: WideVT, N1: LHS, N2: RHS);
13219 BottomHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Mul);
13220 SDValue ShiftAmt =
13221 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits(), VT: WideVT, DL: dl);
13222 TopHalf = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT,
13223 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: WideVT, N1: Mul, N2: ShiftAmt));
13224 } else {
13225 if (VT.isVector())
13226 return false;
13227
13228 forceExpandWideMUL(DAG, dl, Signed: isSigned, LHS, RHS, Lo&: BottomHalf, Hi&: TopHalf);
13229 }
13230
13231 Result = BottomHalf;
13232 if (isSigned) {
13233 SDValue ShiftAmt = DAG.getShiftAmountConstant(
13234 Val: VT.getScalarSizeInBits() - 1, VT: BottomHalf.getValueType(), DL: dl);
13235 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT, N1: BottomHalf, N2: ShiftAmt);
13236 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf, RHS: Sign, Cond: ISD::SETNE);
13237 } else {
13238 Overflow = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: TopHalf,
13239 RHS: DAG.getConstant(Val: 0, DL: dl, VT), Cond: ISD::SETNE);
13240 }
13241
13242 // Truncate the result if SetCC returns a larger type than needed.
13243 EVT RType = Node->getValueType(ResNo: 1);
13244 if (RType.bitsLT(VT: Overflow.getValueType()))
13245 Overflow = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: RType, Operand: Overflow);
13246
13247 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
13248 "Unexpected result type for S/UMULO legalization");
13249 return true;
13250}
13251
13252SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
13253 SDLoc dl(Node);
13254 ISD::NodeType BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13255 SDValue Op = Node->getOperand(Num: 0);
13256 SDNodeFlags Flags = Node->getFlags();
13257 EVT VT = Op.getValueType();
13258
13259 // Try to use a shuffle reduction for power of two vectors.
13260 if (VT.isPow2VectorType()) {
13261 // See if the reduction opcode is safe to use with widened types.
13262 bool WidenSrc = false;
13263 switch (Node->getOpcode()) {
13264 case ISD::VECREDUCE_FADD:
13265 case ISD::VECREDUCE_FMUL:
13266 case ISD::VECREDUCE_ADD:
13267 case ISD::VECREDUCE_MUL:
13268 case ISD::VECREDUCE_AND:
13269 case ISD::VECREDUCE_OR:
13270 case ISD::VECREDUCE_XOR:
13271 case ISD::VECREDUCE_SMAX:
13272 case ISD::VECREDUCE_SMIN:
13273 case ISD::VECREDUCE_UMAX:
13274 case ISD::VECREDUCE_UMIN:
13275 WidenSrc = VT.isFixedLengthVector();
13276 break;
13277 }
13278
13279 while (VT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
13280 EVT HalfVT = VT.getHalfNumVectorElementsVT(Context&: *DAG.getContext());
13281 if (!isOperationLegalOrCustom(Op: BaseOpcode, VT: HalfVT)) {
13282 if (WidenSrc && Op.getOpcode() != ISD::BUILD_VECTOR) {
13283 // Attempt to widen the source vectors to a legal op.
13284 EVT WideVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: HalfVT);
13285 if (WideVT.isVector() &&
13286 WideVT.getScalarType() == HalfVT.getScalarType() &&
13287 WideVT.getVectorNumElements() >= HalfVT.getVectorNumElements() &&
13288 isOperationLegalOrCustom(Op: BaseOpcode, VT: WideVT)) {
13289 SDValue Lo, Hi;
13290 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13291 Lo = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Lo, Idx: 0);
13292 Hi = DAG.getInsertSubvector(DL: dl, Vec: DAG.getPOISON(VT: WideVT), SubVec: Hi, Idx: 0);
13293 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: WideVT, N1: Lo, N2: Hi, Flags);
13294 Op = DAG.getExtractSubvector(DL: dl, VT: HalfVT, Vec: Op, Idx: 0);
13295 VT = HalfVT;
13296 continue;
13297 }
13298 }
13299 break;
13300 }
13301
13302 SDValue Lo, Hi;
13303 std::tie(args&: Lo, args&: Hi) = DAG.SplitVector(N: Op, DL: dl);
13304 Op = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: HalfVT, N1: Lo, N2: Hi, Flags);
13305 VT = HalfVT;
13306
13307 // Stop if splitting is enough to make the reduction legal.
13308 if (isOperationLegalOrCustom(Op: Node->getOpcode(), VT: HalfVT))
13309 return DAG.getNode(Opcode: Node->getOpcode(), DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Op,
13310 Flags);
13311 }
13312 }
13313
13314 if (VT.isScalableVector())
13315 reportFatalInternalError(
13316 reason: "Expanding reductions for scalable vectors is undefined.");
13317
13318 EVT EltVT = VT.getVectorElementType();
13319 unsigned NumElts = VT.getVectorNumElements();
13320
13321 SmallVector<SDValue, 8> Ops;
13322 DAG.ExtractVectorElements(Op, Args&: Ops, Start: 0, Count: NumElts);
13323
13324 SDValue Res = Ops[0];
13325 for (unsigned i = 1; i < NumElts; i++)
13326 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13327
13328 // Result type may be wider than element type.
13329 if (EltVT != Node->getValueType(ResNo: 0))
13330 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: Node->getValueType(ResNo: 0), Operand: Res);
13331 return Res;
13332}
13333
13334SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
13335 SDLoc dl(Node);
13336 SDValue AccOp = Node->getOperand(Num: 0);
13337 SDValue VecOp = Node->getOperand(Num: 1);
13338 SDNodeFlags Flags = Node->getFlags();
13339
13340 EVT VT = VecOp.getValueType();
13341 EVT EltVT = VT.getVectorElementType();
13342
13343 if (VT.isScalableVector())
13344 report_fatal_error(
13345 reason: "Expanding reductions for scalable vectors is undefined.");
13346
13347 unsigned NumElts = VT.getVectorNumElements();
13348
13349 SmallVector<SDValue, 8> Ops;
13350 DAG.ExtractVectorElements(Op: VecOp, Args&: Ops, Start: 0, Count: NumElts);
13351
13352 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Node->getOpcode());
13353
13354 SDValue Res = AccOp;
13355 for (unsigned i = 0; i < NumElts; i++)
13356 Res = DAG.getNode(Opcode: BaseOpcode, DL: dl, VT: EltVT, N1: Res, N2: Ops[i], Flags);
13357
13358 return Res;
13359}
13360
13361bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
13362 SelectionDAG &DAG) const {
13363 EVT VT = Node->getValueType(ResNo: 0);
13364 SDLoc dl(Node);
13365 bool isSigned = Node->getOpcode() == ISD::SREM;
13366 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
13367 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
13368 SDValue Dividend = Node->getOperand(Num: 0);
13369 SDValue Divisor = Node->getOperand(Num: 1);
13370 if (isOperationLegalOrCustom(Op: DivRemOpc, VT)) {
13371 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
13372 Result = DAG.getNode(Opcode: DivRemOpc, DL: dl, VTList: VTs, N1: Dividend, N2: Divisor).getValue(R: 1);
13373 return true;
13374 }
13375 if (isOperationLegalOrCustom(Op: DivOpc, VT)) {
13376 // X % Y -> X-X/Y*Y
13377 SDValue Divide = DAG.getNode(Opcode: DivOpc, DL: dl, VT, N1: Dividend, N2: Divisor);
13378 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT, N1: Divide, N2: Divisor);
13379 Result = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Dividend, N2: Mul);
13380 return true;
13381 }
13382 return false;
13383}
13384
13385SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
13386 SelectionDAG &DAG) const {
13387 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
13388 SDLoc dl(SDValue(Node, 0));
13389 SDValue Src = Node->getOperand(Num: 0);
13390
13391 // DstVT is the result type, while SatVT is the size to which we saturate
13392 EVT SrcVT = Src.getValueType();
13393 EVT DstVT = Node->getValueType(ResNo: 0);
13394
13395 EVT SatVT = cast<VTSDNode>(Val: Node->getOperand(Num: 1))->getVT();
13396 unsigned SatWidth = SatVT.getScalarSizeInBits();
13397 unsigned DstWidth = DstVT.getScalarSizeInBits();
13398 assert(SatWidth <= DstWidth &&
13399 "Expected saturation width smaller than result width");
13400
13401 // Determine minimum and maximum integer values and their corresponding
13402 // floating-point values.
13403 APInt MinInt, MaxInt;
13404 if (IsSigned) {
13405 MinInt = APInt::getSignedMinValue(numBits: SatWidth).sext(width: DstWidth);
13406 MaxInt = APInt::getSignedMaxValue(numBits: SatWidth).sext(width: DstWidth);
13407 } else {
13408 MinInt = APInt::getMinValue(numBits: SatWidth).zext(width: DstWidth);
13409 MaxInt = APInt::getMaxValue(numBits: SatWidth).zext(width: DstWidth);
13410 }
13411
13412 // We cannot risk emitting FP_TO_XINT nodes with a source VT of [b]f16, as
13413 // libcall emission cannot handle this. Large result types will fail.
13414 if (SrcVT == MVT::f16 || SrcVT == MVT::bf16) {
13415 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f32, Operand: Src);
13416 SrcVT = Src.getValueType();
13417 }
13418
13419 const fltSemantics &Sem = SrcVT.getFltSemantics();
13420 APFloat MinFloat(Sem);
13421 APFloat MaxFloat(Sem);
13422
13423 APFloat::opStatus MinStatus =
13424 MinFloat.convertFromAPInt(Input: MinInt, IsSigned, RM: APFloat::rmTowardZero);
13425 APFloat::opStatus MaxStatus =
13426 MaxFloat.convertFromAPInt(Input: MaxInt, IsSigned, RM: APFloat::rmTowardZero);
13427 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
13428 !(MaxStatus & APFloat::opStatus::opInexact);
13429
13430 SDValue MinFloatNode = DAG.getConstantFP(Val: MinFloat, DL: dl, VT: SrcVT);
13431 SDValue MaxFloatNode = DAG.getConstantFP(Val: MaxFloat, DL: dl, VT: SrcVT);
13432
13433 // If the integer bounds are exactly representable as floats and min/max are
13434 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
13435 // of comparisons and selects.
13436 auto EmitMinMax = [&](unsigned MinOpcode, unsigned MaxOpcode,
13437 bool MayPropagateNaN) {
13438 bool MinMaxLegal = isOperationLegalOrCustom(Op: MinOpcode, VT: SrcVT) &&
13439 isOperationLegalOrCustom(Op: MaxOpcode, VT: SrcVT);
13440 if (!MinMaxLegal)
13441 return SDValue();
13442
13443 SDValue Clamped = Src;
13444
13445 // Clamp Src by MinFloat from below. If !MayPropagateNaN and Src is NaN
13446 // then the result is MinFloat.
13447 Clamped = DAG.getNode(Opcode: MaxOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MinFloatNode);
13448 // Clamp by MaxFloat from above. If !MayPropagateNaN then NaN cannot occur.
13449 Clamped = DAG.getNode(Opcode: MinOpcode, DL: dl, VT: SrcVT, N1: Clamped, N2: MaxFloatNode);
13450 // Convert clamped value to integer.
13451 SDValue FpToInt = DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
13452 DL: dl, VT: DstVT, Operand: Clamped);
13453
13454 // If !MayPropagateNan and the conversion is unsigned case we're done,
13455 // because we mapped NaN to MinFloat, which will cast to zero.
13456 if (!MayPropagateNaN && !IsSigned)
13457 return FpToInt;
13458
13459 // Otherwise, select 0 if Src is NaN.
13460 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13461 EVT SetCCVT =
13462 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13463 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13464 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: FpToInt);
13465 };
13466 if (AreExactFloatBounds) {
13467 if (SDValue Res = EmitMinMax(ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM,
13468 /*MayPropagateNaN=*/false))
13469 return Res;
13470 // These may propagate NaN for sNaN operands.
13471 if (SDValue Res =
13472 EmitMinMax(ISD::FMINNUM, ISD::FMAXNUM, /*MayPropagateNaN=*/true))
13473 return Res;
13474 // These always propagate NaN.
13475 if (SDValue Res =
13476 EmitMinMax(ISD::FMINIMUM, ISD::FMAXIMUM, /*MayPropagateNaN=*/true))
13477 return Res;
13478 }
13479
13480 SDValue MinIntNode = DAG.getConstant(Val: MinInt, DL: dl, VT: DstVT);
13481 SDValue MaxIntNode = DAG.getConstant(Val: MaxInt, DL: dl, VT: DstVT);
13482
13483 // Result of direct conversion. The assumption here is that the operation is
13484 // non-trapping and it's fine to apply it to an out-of-range value if we
13485 // select it away later.
13486 SDValue FpToInt =
13487 DAG.getNode(Opcode: IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, DL: dl, VT: DstVT, Operand: Src);
13488
13489 SDValue Select = FpToInt;
13490
13491 EVT SetCCVT =
13492 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
13493
13494 // If Src ULT MinFloat, select MinInt. In particular, this also selects
13495 // MinInt if Src is NaN.
13496 SDValue ULT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MinFloatNode, Cond: ISD::SETULT);
13497 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: ULT, LHS: MinIntNode, RHS: Select);
13498 // If Src OGT MaxFloat, select MaxInt.
13499 SDValue OGT = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: MaxFloatNode, Cond: ISD::SETOGT);
13500 Select = DAG.getSelect(DL: dl, VT: DstVT, Cond: OGT, LHS: MaxIntNode, RHS: Select);
13501
13502 // In the unsigned case we are done, because we mapped NaN to MinInt, which
13503 // is already zero.
13504 if (!IsSigned)
13505 return Select;
13506
13507 // Otherwise, select 0 if Src is NaN.
13508 SDValue ZeroInt = DAG.getConstant(Val: 0, DL: dl, VT: DstVT);
13509 SDValue IsNan = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Src, Cond: ISD::CondCode::SETUO);
13510 return DAG.getSelect(DL: dl, VT: DstVT, Cond: IsNan, LHS: ZeroInt, RHS: Select);
13511}
13512
13513SDValue TargetLowering::expandRoundInexactToOdd(EVT ResultVT, SDValue Op,
13514 const SDLoc &dl,
13515 SelectionDAG &DAG) const {
13516 EVT OperandVT = Op.getValueType();
13517 if (OperandVT.getScalarType() == ResultVT.getScalarType())
13518 return Op;
13519 EVT ResultIntVT = ResultVT.changeTypeToInteger();
13520 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13521 // can induce double-rounding which may alter the results. We can
13522 // correct for this using a trick explained in: Boldo, Sylvie, and
13523 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13524 // World Congress. 2005.
13525 SDValue Narrow = DAG.getFPExtendOrRound(Op, DL: dl, VT: ResultVT);
13526 SDValue NarrowAsWide = DAG.getFPExtendOrRound(Op: Narrow, DL: dl, VT: OperandVT);
13527
13528 // We can keep the narrow value as-is if narrowing was exact (no
13529 // rounding error), the wide value was NaN (the narrow value is also
13530 // NaN and should be preserved) or if we rounded to the odd value.
13531 SDValue NarrowBits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultIntVT, Operand: Narrow);
13532 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: ResultIntVT);
13533 SDValue NegativeOne = DAG.getAllOnesConstant(DL: dl, VT: ResultIntVT);
13534 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: One);
13535 EVT ResultIntVTCCVT = getSetCCResultType(
13536 DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: And.getValueType());
13537 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: ResultIntVT);
13538 // The result is already odd so we don't need to do anything.
13539 SDValue AlreadyOdd = DAG.getSetCC(DL: dl, VT: ResultIntVTCCVT, LHS: And, RHS: Zero, Cond: ISD::SETNE);
13540
13541 EVT WideSetCCVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(),
13542 VT: Op.getValueType());
13543 // We keep results which are exact, odd or NaN.
13544 SDValue KeepNarrow =
13545 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: Op, RHS: NarrowAsWide, Cond: ISD::SETUEQ);
13546 KeepNarrow = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: WideSetCCVT, N1: KeepNarrow, N2: AlreadyOdd);
13547 // We morally performed a round-down if AbsNarrow is smaller than
13548 // AbsWide.
13549 SDValue AbsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: Op);
13550 SDValue AbsNarrowAsWide = DAG.getNode(Opcode: ISD::FABS, DL: dl, VT: OperandVT, Operand: NarrowAsWide);
13551 SDValue NarrowIsRd =
13552 DAG.getSetCC(DL: dl, VT: WideSetCCVT, LHS: AbsWide, RHS: AbsNarrowAsWide, Cond: ISD::SETOGT);
13553 // If the narrow value is odd or exact, pick it.
13554 // Otherwise, narrow is even and corresponds to either the rounded-up
13555 // or rounded-down value. If narrow is the rounded-down value, we want
13556 // the rounded-up value as it will be odd.
13557 SDValue Adjust = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: NarrowIsRd, LHS: One, RHS: NegativeOne);
13558 SDValue Adjusted = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ResultIntVT, N1: NarrowBits, N2: Adjust);
13559 Op = DAG.getSelect(DL: dl, VT: ResultIntVT, Cond: KeepNarrow, LHS: NarrowBits, RHS: Adjusted);
13560 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ResultVT, Operand: Op);
13561}
13562
13563SDValue TargetLowering::expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const {
13564 assert(Node->getOpcode() == ISD::FP_ROUND && "Unexpected opcode!");
13565 SDValue Op = Node->getOperand(Num: 0);
13566 EVT VT = Node->getValueType(ResNo: 0);
13567 SDLoc dl(Node);
13568 if (VT.getScalarType() == MVT::bf16) {
13569 if (Node->getConstantOperandVal(Num: 1) == 1) {
13570 return DAG.getNode(Opcode: ISD::FP_TO_BF16, DL: dl, VT, Operand: Node->getOperand(Num: 0));
13571 }
13572 EVT OperandVT = Op.getValueType();
13573 SDValue IsNaN = DAG.getSetCC(
13574 DL: dl,
13575 VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: OperandVT),
13576 LHS: Op, RHS: Op, Cond: ISD::SETUO);
13577
13578 // We are rounding binary64/binary128 -> binary32 -> bfloat16. This
13579 // can induce double-rounding which may alter the results. We can
13580 // correct for this using a trick explained in: Boldo, Sylvie, and
13581 // Guillaume Melquiond. "When double rounding is odd." 17th IMACS
13582 // World Congress. 2005.
13583 EVT F32 = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::f32);
13584 EVT I32 = F32.changeTypeToInteger();
13585 Op = expandRoundInexactToOdd(ResultVT: F32, Op, dl, DAG);
13586 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: I32, Operand: Op);
13587
13588 // Conversions should set NaN's quiet bit. This also prevents NaNs from
13589 // turning into infinities.
13590 SDValue NaN =
13591 DAG.getNode(Opcode: ISD::OR, DL: dl, VT: I32, N1: Op, N2: DAG.getConstant(Val: 0x400000, DL: dl, VT: I32));
13592
13593 // Factor in the contribution of the low 16 bits.
13594 SDValue One = DAG.getConstant(Val: 1, DL: dl, VT: I32);
13595 SDValue Lsb = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13596 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13597 Lsb = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: I32, N1: Lsb, N2: One);
13598 SDValue RoundingBias =
13599 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Lsb, N2: DAG.getConstant(Val: 0x7fff, DL: dl, VT: I32));
13600 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: I32, N1: Op, N2: RoundingBias);
13601
13602 // Don't round if we had a NaN, we don't want to turn 0x7fffffff into
13603 // 0x80000000.
13604 Op = DAG.getSelect(DL: dl, VT: I32, Cond: IsNaN, LHS: NaN, RHS: Add);
13605
13606 // Now that we have rounded, shift the bits into position.
13607 Op = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: I32, N1: Op,
13608 N2: DAG.getShiftAmountConstant(Val: 16, VT: I32, DL: dl));
13609 EVT I16 = I32.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i16);
13610 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: I16, Operand: Op);
13611 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Op);
13612 }
13613 return SDValue();
13614}
13615
13616SDValue TargetLowering::expandVectorSplice(SDNode *Node,
13617 SelectionDAG &DAG) const {
13618 assert((Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT ||
13619 Node->getOpcode() == ISD::VECTOR_SPLICE_RIGHT) &&
13620 "Unexpected opcode!");
13621 assert((Node->getValueType(0).isScalableVector() ||
13622 !isa<ConstantSDNode>(Node->getOperand(2))) &&
13623 "Fixed length vector types with constant offsets expected to use "
13624 "SHUFFLE_VECTOR!");
13625
13626 EVT VT = Node->getValueType(ResNo: 0);
13627 SDValue V1 = Node->getOperand(Num: 0);
13628 SDValue V2 = Node->getOperand(Num: 1);
13629 SDValue Offset = Node->getOperand(Num: 2);
13630 SDLoc DL(Node);
13631
13632 // Expand through memory thusly:
13633 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
13634 // Store V1, Ptr
13635 // Store V2, Ptr + sizeof(V1)
13636 // if (VECTOR_SPLICE_LEFT)
13637 // Ptr = Ptr + (Offset * sizeof(VT.Elt))
13638 // else
13639 // Ptr = Ptr + sizeof(V1) - (Offset * size(VT.Elt))
13640 // Res = Load Ptr
13641
13642 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
13643
13644 EVT MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getVectorElementType(),
13645 EC: VT.getVectorElementCount() * 2);
13646 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
13647 EVT PtrVT = StackPtr.getValueType();
13648 auto &MF = DAG.getMachineFunction();
13649 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13650 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13651
13652 // Store the lo part of CONCAT_VECTORS(V1, V2)
13653 SDValue StoreV1 =
13654 DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: V1, Ptr: StackPtr, PtrInfo, Alignment);
13655 // Store the hi part of CONCAT_VECTORS(V1, V2)
13656 SDValue VTBytes = DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getStoreSize());
13657 SDValue StackPtr2 = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: VTBytes);
13658 SDValue StoreV2 =
13659 DAG.getStore(Chain: StoreV1, dl: DL, Val: V2, Ptr: StackPtr2, PtrInfo, Alignment);
13660
13661 // NOTE: TrailingBytes must be clamped so as not to read outside of V1:V2.
13662 SDValue EltByteSize =
13663 DAG.getTypeSize(DL, VT: PtrVT, TS: VT.getVectorElementType().getStoreSize());
13664 Offset = DAG.getZExtOrTrunc(Op: Offset, DL, VT: PtrVT);
13665 SDValue TrailingBytes = DAG.getNode(Opcode: ISD::MUL, DL, VT: PtrVT, N1: Offset, N2: EltByteSize);
13666
13667 TrailingBytes = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PtrVT, N1: TrailingBytes, N2: VTBytes);
13668
13669 if (Node->getOpcode() == ISD::VECTOR_SPLICE_LEFT)
13670 StackPtr = DAG.getMemBasePlusOffset(Base: StackPtr, Offset: TrailingBytes, DL);
13671 else
13672 StackPtr = DAG.getNode(Opcode: ISD::SUB, DL, VT: PtrVT, N1: StackPtr2, N2: TrailingBytes);
13673
13674 // Load the spliced result
13675 return DAG.getLoad(VT, dl: DL, Chain: StoreV2, Ptr: StackPtr,
13676 PtrInfo: MachinePointerInfo::getUnknownStack(MF), Alignment);
13677}
13678
13679SDValue TargetLowering::expandVECTOR_COMPRESS(SDNode *Node,
13680 SelectionDAG &DAG) const {
13681 SDLoc DL(Node);
13682 SDValue Vec = Node->getOperand(Num: 0);
13683 SDValue Mask = Node->getOperand(Num: 1);
13684 SDValue Passthru = Node->getOperand(Num: 2);
13685
13686 EVT VecVT = Vec.getValueType();
13687 EVT ScalarVT = VecVT.getScalarType();
13688 EVT MaskVT = Mask.getValueType();
13689 EVT MaskScalarVT = MaskVT.getScalarType();
13690
13691 // Needs to be handled by targets that have scalable vector types.
13692 if (VecVT.isScalableVector())
13693 report_fatal_error(reason: "Cannot expand masked_compress for scalable vectors.");
13694
13695 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
13696 SDValue StackPtr = DAG.CreateStackTemporary(Bytes: VecVT.getStoreSize(), Alignment);
13697 int FI = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13698 MachinePointerInfo PtrInfo =
13699 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
13700
13701 MVT PositionVT = getVectorIdxTy(DL: DAG.getDataLayout());
13702 SDValue Chain = DAG.getEntryNode();
13703 SDValue OutPos = DAG.getConstant(Val: 0, DL, VT: PositionVT);
13704
13705 bool HasPassthru = !Passthru.isUndef();
13706
13707 // If we have a passthru vector, store it on the stack, overwrite the matching
13708 // positions and then re-write the last element that was potentially
13709 // overwritten even though mask[i] = false.
13710 if (HasPassthru)
13711 Chain = DAG.getStore(Chain, dl: DL, Val: Passthru, Ptr: StackPtr, PtrInfo, Alignment);
13712
13713 SDValue LastWriteVal;
13714 APInt PassthruSplatVal;
13715 bool IsSplatPassthru =
13716 ISD::isConstantSplatVector(N: Passthru.getNode(), SplatValue&: PassthruSplatVal);
13717
13718 if (IsSplatPassthru) {
13719 // As we do not know which position we wrote to last, we cannot simply
13720 // access that index from the passthru vector. So we first check if passthru
13721 // is a splat vector, to use any element ...
13722 LastWriteVal = DAG.getConstant(Val: PassthruSplatVal, DL, VT: ScalarVT);
13723 } else if (HasPassthru) {
13724 // ... if it is not a splat vector, we need to get the passthru value at
13725 // position = popcount(mask) and re-load it from the stack before it is
13726 // overwritten in the loop below.
13727 EVT PopcountVT = ScalarVT.changeTypeToInteger();
13728 SDValue Popcount = DAG.getNode(
13729 Opcode: ISD::TRUNCATE, DL,
13730 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1), Operand: Mask);
13731 Popcount = DAG.getNode(
13732 Opcode: ISD::ZERO_EXTEND, DL,
13733 VT: MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: PopcountVT),
13734 Operand: Popcount);
13735 Popcount = DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT: PopcountVT, Operand: Popcount);
13736 SDValue LastElmtPtr =
13737 getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: Popcount);
13738 LastWriteVal = DAG.getLoad(
13739 VT: ScalarVT, dl: DL, Chain, Ptr: LastElmtPtr,
13740 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13741 Chain = LastWriteVal.getValue(R: 1);
13742 }
13743
13744 unsigned NumElms = VecVT.getVectorNumElements();
13745 for (unsigned I = 0; I < NumElms; I++) {
13746 SDValue ValI = DAG.getExtractVectorElt(DL, VT: ScalarVT, Vec, Idx: I);
13747 SDValue OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
13748 Chain = DAG.getStore(
13749 Chain, dl: DL, Val: ValI, Ptr: OutPtr,
13750 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13751
13752 // Get the mask value and add it to the current output position. This
13753 // either increments by 1 if MaskI is true or adds 0 otherwise.
13754 // Freeze in case we have poison/undef mask entries.
13755 SDValue MaskI = DAG.getExtractVectorElt(DL, VT: MaskScalarVT, Vec: Mask, Idx: I);
13756 MaskI = DAG.getFreeze(V: MaskI);
13757 MaskI = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: MaskI);
13758 MaskI = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: PositionVT, Operand: MaskI);
13759 OutPos = DAG.getNode(Opcode: ISD::ADD, DL, VT: PositionVT, N1: OutPos, N2: MaskI);
13760
13761 if (HasPassthru && I == NumElms - 1) {
13762 SDValue EndOfVector =
13763 DAG.getConstant(Val: VecVT.getVectorNumElements() - 1, DL, VT: PositionVT);
13764 SDValue AllLanesSelected =
13765 DAG.getSetCC(DL, VT: MVT::i1, LHS: OutPos, RHS: EndOfVector, Cond: ISD::CondCode::SETUGT);
13766 OutPos = DAG.getNode(Opcode: ISD::UMIN, DL, VT: PositionVT, N1: OutPos, N2: EndOfVector);
13767 OutPtr = getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT, Index: OutPos);
13768
13769 // Re-write the last ValI if all lanes were selected. Otherwise,
13770 // overwrite the last write it with the passthru value.
13771 LastWriteVal = DAG.getSelect(DL, VT: ScalarVT, Cond: AllLanesSelected, LHS: ValI,
13772 RHS: LastWriteVal, Flags: SDNodeFlags::Unpredictable);
13773 Chain = DAG.getStore(
13774 Chain, dl: DL, Val: LastWriteVal, Ptr: OutPtr,
13775 PtrInfo: MachinePointerInfo::getUnknownStack(MF&: DAG.getMachineFunction()));
13776 }
13777 }
13778
13779 return DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo, Alignment);
13780}
13781
13782SDValue TargetLowering::expandCttzElts(SDNode *Node, SelectionDAG &DAG) const {
13783 SDLoc DL(Node);
13784 EVT VT = Node->getValueType(ResNo: 0);
13785
13786 bool ZeroIsPoison = Node->getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON;
13787 auto [Mask, StepVec] =
13788 getLegalMaskAndStepVector(Mask: Node->getOperand(Num: 0), ZeroIsPoison, DL, DAG);
13789
13790 // No legal step vector: split mask in half and recombine results.
13791 // LoNumElts uses the non-poison CTTZ_ELTS so its result is well-defined
13792 // (== LoNumElts when no active lane), allowing the SETNE comparison.
13793 // Result: (ResLo != LoNumElts) ? ResLo : (LoNumElts + ResHi)
13794 if (!StepVec) {
13795 EVT ResVT = Node->getValueType(ResNo: 0);
13796 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Node->getOperand(Num: 0), DL);
13797 SDValue LoNumElts = DAG.getElementCount(
13798 DL, VT: ResVT, EC: MaskLo.getValueType().getVectorElementCount());
13799 SDValue ResLo = DAG.getNode(Opcode: ISD::CTTZ_ELTS, DL, VT: ResVT, Operand: MaskLo);
13800 SDValue ResHi = DAG.getNode(Opcode: Node->getOpcode(), DL, VT: ResVT, Operand: MaskHi);
13801 SDValue ResLoNotNumElts = DAG.getSetCC(
13802 DL, VT: getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: ResVT),
13803 LHS: ResLo, RHS: LoNumElts, Cond: ISD::SETNE);
13804 // Per LangRef, ResVT must be wide enough to hold the total element count,
13805 // so the sum cannot wrap as an unsigned add. NSW is not guaranteed since
13806 // the count is only required to fit unsigned.
13807 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ResVT, N1: LoNumElts, N2: ResHi,
13808 Flags: SDNodeFlags::NoUnsignedWrap);
13809 return DAG.getSelect(DL, VT: ResVT, Cond: ResLoNotNumElts, LHS: ResLo, RHS: Sum);
13810 }
13811
13812 EVT StepVecVT = StepVec.getValueType();
13813 EVT StepVT = StepVecVT.getVectorElementType();
13814
13815 // Promote the scalar result type early to avoid redundant zexts.
13816 if (getTypeAction(VT: StepVT.getSimpleVT()) == TypePromoteInteger)
13817 StepVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT: StepVT);
13818
13819 SDValue VL =
13820 DAG.getElementCount(DL, VT: StepVT, EC: StepVecVT.getVectorElementCount());
13821 SDValue SplatVL = DAG.getSplat(VT: StepVecVT, DL, Op: VL);
13822 StepVec = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVecVT, N1: SplatVL, N2: StepVec);
13823 SDValue Zeroes = DAG.getConstant(Val: 0, DL, VT: StepVecVT);
13824 SDValue Select = DAG.getSelect(DL, VT: StepVecVT, Cond: Mask, LHS: StepVec, RHS: Zeroes);
13825 SDValue Max = DAG.getNode(Opcode: ISD::VECREDUCE_UMAX, DL,
13826 VT: StepVecVT.getVectorElementType(), Operand: Select);
13827 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT: StepVT, N1: VL,
13828 N2: DAG.getZExtOrTrunc(Op: Max, DL, VT: StepVT));
13829
13830 return DAG.getZExtOrTrunc(Op: Sub, DL, VT);
13831}
13832
13833SDValue TargetLowering::expandPartialReduceMLA(SDNode *N,
13834 SelectionDAG &DAG) const {
13835 SDLoc DL(N);
13836 SDValue Acc = N->getOperand(Num: 0);
13837 SDValue MulLHS = N->getOperand(Num: 1);
13838 SDValue MulRHS = N->getOperand(Num: 2);
13839 EVT AccVT = Acc.getValueType();
13840 EVT MulOpVT = MulLHS.getValueType();
13841
13842 EVT ExtMulOpVT =
13843 EVT::getVectorVT(Context&: *DAG.getContext(), VT: AccVT.getVectorElementType(),
13844 EC: MulOpVT.getVectorElementCount());
13845
13846 unsigned ExtOpcLHS, ExtOpcRHS;
13847 switch (N->getOpcode()) {
13848 default:
13849 llvm_unreachable("Unexpected opcode");
13850 case ISD::PARTIAL_REDUCE_UMLA:
13851 ExtOpcLHS = ExtOpcRHS = ISD::ZERO_EXTEND;
13852 break;
13853 case ISD::PARTIAL_REDUCE_SMLA:
13854 ExtOpcLHS = ExtOpcRHS = ISD::SIGN_EXTEND;
13855 break;
13856 case ISD::PARTIAL_REDUCE_FMLA:
13857 ExtOpcLHS = ExtOpcRHS = ISD::FP_EXTEND;
13858 break;
13859 }
13860
13861 if (ExtMulOpVT != MulOpVT) {
13862 MulLHS = DAG.getNode(Opcode: ExtOpcLHS, DL, VT: ExtMulOpVT, Operand: MulLHS);
13863 MulRHS = DAG.getNode(Opcode: ExtOpcRHS, DL, VT: ExtMulOpVT, Operand: MulRHS);
13864 }
13865 SDValue Input = MulLHS;
13866 if (N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA) {
13867 if (!llvm::isOneOrOneSplatFP(V: MulRHS))
13868 Input = DAG.getNode(Opcode: ISD::FMUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
13869 } else if (!llvm::isOneOrOneSplat(V: MulRHS)) {
13870 Input = DAG.getNode(Opcode: ISD::MUL, DL, VT: ExtMulOpVT, N1: MulLHS, N2: MulRHS);
13871 }
13872
13873 unsigned Stride = AccVT.getVectorMinNumElements();
13874 unsigned ScaleFactor = MulOpVT.getVectorMinNumElements() / Stride;
13875
13876 // Collect all of the subvectors
13877 std::deque<SDValue> Subvectors = {Acc};
13878 for (unsigned I = 0; I < ScaleFactor; I++)
13879 Subvectors.push_back(x: DAG.getExtractSubvector(DL, VT: AccVT, Vec: Input, Idx: I * Stride));
13880
13881 unsigned FlatNode =
13882 N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA ? ISD::FADD : ISD::ADD;
13883
13884 // Flatten the subvector tree
13885 while (Subvectors.size() > 1) {
13886 Subvectors.push_back(
13887 x: DAG.getNode(Opcode: FlatNode, DL, VT: AccVT, Ops: {Subvectors[0], Subvectors[1]}));
13888 Subvectors.pop_front();
13889 Subvectors.pop_front();
13890 }
13891
13892 assert(Subvectors.size() == 1 &&
13893 "There should only be one subvector after tree flattening");
13894
13895 return Subvectors[0];
13896}
13897
13898/// Given a store node \p StoreNode, return true if it is safe to fold that node
13899/// into \p FPNode, which expands to a library call with output pointers.
13900static bool canFoldStoreIntoLibCallOutputPointers(StoreSDNode *StoreNode,
13901 SDNode *FPNode) {
13902 SmallVector<const SDNode *, 8> Worklist;
13903 SmallVector<const SDNode *, 8> DeferredNodes;
13904 SmallPtrSet<const SDNode *, 16> Visited;
13905
13906 // Skip FPNode use by StoreNode (that's the use we want to fold into FPNode).
13907 for (SDValue Op : StoreNode->ops())
13908 if (Op.getNode() != FPNode)
13909 Worklist.push_back(Elt: Op.getNode());
13910
13911 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
13912 while (!Worklist.empty()) {
13913 const SDNode *Node = Worklist.pop_back_val();
13914 auto [_, Inserted] = Visited.insert(Ptr: Node);
13915 if (!Inserted)
13916 continue;
13917
13918 if (MaxSteps > 0 && Visited.size() >= MaxSteps)
13919 return false;
13920
13921 // Reached the FPNode (would result in a cycle).
13922 // OR Reached CALLSEQ_START (would result in nested call sequences).
13923 if (Node == FPNode || Node->getOpcode() == ISD::CALLSEQ_START)
13924 return false;
13925
13926 if (Node->getOpcode() == ISD::CALLSEQ_END) {
13927 // Defer looking into call sequences (so we can check we're outside one).
13928 // We still need to look through these for the predecessor check.
13929 DeferredNodes.push_back(Elt: Node);
13930 continue;
13931 }
13932
13933 for (SDValue Op : Node->ops())
13934 Worklist.push_back(Elt: Op.getNode());
13935 }
13936
13937 // True if we're outside a call sequence and don't have the FPNode as a
13938 // predecessor. No cycles or nested call sequences possible.
13939 return !SDNode::hasPredecessorHelper(N: FPNode, Visited, Worklist&: DeferredNodes,
13940 MaxSteps);
13941}
13942
13943bool TargetLowering::expandMultipleResultFPLibCall(
13944 SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node,
13945 SmallVectorImpl<SDValue> &Results,
13946 std::optional<unsigned> CallRetResNo) const {
13947 if (LC == RTLIB::UNKNOWN_LIBCALL)
13948 return false;
13949
13950 RTLIB::LibcallImpl LibcallImpl = getLibcallImpl(Call: LC);
13951 if (LibcallImpl == RTLIB::Unsupported)
13952 return false;
13953
13954 LLVMContext &Ctx = *DAG.getContext();
13955 EVT VT = Node->getValueType(ResNo: 0);
13956 unsigned NumResults = Node->getNumValues();
13957
13958 // Find users of the node that store the results (and share input chains). The
13959 // destination pointers can be used instead of creating stack allocations.
13960 SDValue StoresInChain;
13961 SmallVector<StoreSDNode *, 2> ResultStores(NumResults);
13962 for (SDNode *User : Node->users()) {
13963 if (!ISD::isNormalStore(N: User))
13964 continue;
13965 auto *ST = cast<StoreSDNode>(Val: User);
13966 SDValue StoreValue = ST->getValue();
13967 unsigned ResNo = StoreValue.getResNo();
13968 // Ensure the store corresponds to an output pointer.
13969 if (CallRetResNo == ResNo)
13970 continue;
13971 // Ensure the store to the default address space and not atomic or volatile.
13972 if (!ST->isSimple() || ST->getAddressSpace() != 0)
13973 continue;
13974 // Ensure all store chains are the same (so they don't alias).
13975 if (StoresInChain && ST->getChain() != StoresInChain)
13976 continue;
13977 // Ensure the store is properly aligned.
13978 Type *StoreType = StoreValue.getValueType().getTypeForEVT(Context&: Ctx);
13979 if (ST->getAlign() <
13980 DAG.getDataLayout().getABITypeAlign(Ty: StoreType->getScalarType()))
13981 continue;
13982 // Avoid:
13983 // 1. Creating cyclic dependencies.
13984 // 2. Expanding the node to a call within a call sequence.
13985 if (!canFoldStoreIntoLibCallOutputPointers(StoreNode: ST, FPNode: Node))
13986 continue;
13987 ResultStores[ResNo] = ST;
13988 StoresInChain = ST->getChain();
13989 }
13990
13991 ArgListTy Args;
13992
13993 // Pass the arguments.
13994 for (const SDValue &Op : Node->op_values()) {
13995 EVT ArgVT = Op.getValueType();
13996 Type *ArgTy = ArgVT.getTypeForEVT(Context&: Ctx);
13997 Args.emplace_back(args: Op, args&: ArgTy);
13998 }
13999
14000 // Pass the output pointers.
14001 SmallVector<SDValue, 2> ResultPtrs(NumResults);
14002 Type *PointerTy = PointerType::getUnqual(C&: Ctx);
14003 for (auto [ResNo, ST] : llvm::enumerate(First&: ResultStores)) {
14004 if (ResNo == CallRetResNo)
14005 continue;
14006 EVT ResVT = Node->getValueType(ResNo);
14007 SDValue ResultPtr = ST ? ST->getBasePtr() : DAG.CreateStackTemporary(VT: ResVT);
14008 ResultPtrs[ResNo] = ResultPtr;
14009 Args.emplace_back(args&: ResultPtr, args&: PointerTy);
14010 }
14011
14012 SDLoc DL(Node);
14013
14014 if (RTLIB::RuntimeLibcallsInfo::hasVectorMaskArgument(Impl: LibcallImpl)) {
14015 // Pass the vector mask (if required).
14016 EVT MaskVT = getSetCCResultType(DL: DAG.getDataLayout(), Context&: Ctx, VT);
14017 SDValue Mask = DAG.getBoolConstant(V: true, DL, VT: MaskVT, OpVT: VT);
14018 Args.emplace_back(args&: Mask, args: MaskVT.getTypeForEVT(Context&: Ctx));
14019 }
14020
14021 Type *RetType = CallRetResNo.has_value()
14022 ? Node->getValueType(ResNo: *CallRetResNo).getTypeForEVT(Context&: Ctx)
14023 : Type::getVoidTy(C&: Ctx);
14024 SDValue InChain = StoresInChain ? StoresInChain : DAG.getEntryNode();
14025 SDValue Callee =
14026 DAG.getExternalSymbol(LCImpl: LibcallImpl, VT: getPointerTy(DL: DAG.getDataLayout()));
14027 TargetLowering::CallLoweringInfo CLI(DAG);
14028 CLI.setDebugLoc(DL).setChain(InChain).setLibCallee(
14029 CC: getLibcallImplCallingConv(Call: LibcallImpl), ResultType: RetType, Target: Callee, ArgsList: std::move(Args));
14030
14031 auto [Call, CallChain] = LowerCallTo(CLI);
14032
14033 for (auto [ResNo, ResultPtr] : llvm::enumerate(First&: ResultPtrs)) {
14034 if (ResNo == CallRetResNo) {
14035 Results.push_back(Elt: Call);
14036 continue;
14037 }
14038 MachinePointerInfo PtrInfo;
14039 SDValue LoadResult = DAG.getLoad(VT: Node->getValueType(ResNo), dl: DL, Chain: CallChain,
14040 Ptr: ResultPtr, PtrInfo);
14041 SDValue OutChain = LoadResult.getValue(R: 1);
14042
14043 if (StoreSDNode *ST = ResultStores[ResNo]) {
14044 // Replace store with the library call.
14045 DAG.ReplaceAllUsesOfValueWith(From: SDValue(ST, 0), To: OutChain);
14046 PtrInfo = ST->getPointerInfo();
14047 } else {
14048 PtrInfo = MachinePointerInfo::getFixedStack(
14049 MF&: DAG.getMachineFunction(),
14050 FI: cast<FrameIndexSDNode>(Val&: ResultPtr)->getIndex());
14051 }
14052
14053 Results.push_back(Elt: LoadResult);
14054 }
14055
14056 return true;
14057}
14058
14059bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
14060 SDValue &LHS, SDValue &RHS,
14061 SDValue &CC, SDValue Mask,
14062 SDValue EVL, bool &NeedInvert,
14063 const SDLoc &dl, SDValue &Chain,
14064 bool IsSignaling) const {
14065 MVT OpVT = LHS.getSimpleValueType();
14066 ISD::CondCode CCCode = cast<CondCodeSDNode>(Val&: CC)->get();
14067 NeedInvert = false;
14068 assert(!EVL == !Mask && "VP Mask and EVL must either both be set or unset");
14069 bool IsNonVP = !EVL;
14070 switch (getCondCodeAction(CC: CCCode, VT: OpVT)) {
14071 default:
14072 llvm_unreachable("Unknown condition code action!");
14073 case TargetLowering::Legal:
14074 // Nothing to do.
14075 break;
14076 case TargetLowering::Expand: {
14077 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(Operation: CCCode);
14078 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14079 std::swap(a&: LHS, b&: RHS);
14080 CC = DAG.getCondCode(Cond: InvCC);
14081 return true;
14082 }
14083 // Swapping operands didn't work. Try inverting the condition.
14084 bool NeedSwap = false;
14085 InvCC = getSetCCInverse(Operation: CCCode, Type: OpVT);
14086 if (!isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14087 // If inverting the condition is not enough, try swapping operands
14088 // on top of it.
14089 InvCC = ISD::getSetCCSwappedOperands(Operation: InvCC);
14090 NeedSwap = true;
14091 }
14092 if (isCondCodeLegalOrCustom(CC: InvCC, VT: OpVT)) {
14093 CC = DAG.getCondCode(Cond: InvCC);
14094 NeedInvert = true;
14095 if (NeedSwap)
14096 std::swap(a&: LHS, b&: RHS);
14097 return true;
14098 }
14099
14100 // Special case: expand i1 comparisons using logical operations.
14101 if (OpVT == MVT::i1) {
14102 SDValue Ret;
14103 switch (CCCode) {
14104 default:
14105 llvm_unreachable("Unknown integer setcc!");
14106 case ISD::SETEQ: // X == Y --> ~(X ^ Y)
14107 Ret = DAG.getNOT(DL: dl, Val: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS),
14108 VT: MVT::i1);
14109 break;
14110 case ISD::SETNE: // X != Y --> (X ^ Y)
14111 Ret = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i1, N1: LHS, N2: RHS);
14112 break;
14113 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
14114 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
14115 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: RHS,
14116 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14117 break;
14118 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
14119 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
14120 Ret = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i1, N1: LHS,
14121 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14122 break;
14123 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
14124 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
14125 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: RHS,
14126 N2: DAG.getNOT(DL: dl, Val: LHS, VT: MVT::i1));
14127 break;
14128 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
14129 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
14130 Ret = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i1, N1: LHS,
14131 N2: DAG.getNOT(DL: dl, Val: RHS, VT: MVT::i1));
14132 break;
14133 }
14134
14135 LHS = DAG.getZExtOrTrunc(Op: Ret, DL: dl, VT);
14136 RHS = SDValue();
14137 CC = SDValue();
14138 return true;
14139 }
14140
14141 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
14142 unsigned Opc = 0;
14143 switch (CCCode) {
14144 default:
14145 llvm_unreachable("Don't know how to expand this condition!");
14146 case ISD::SETUO:
14147 if (isCondCodeLegal(CC: ISD::SETUNE, VT: OpVT)) {
14148 CC1 = ISD::SETUNE;
14149 CC2 = ISD::SETUNE;
14150 Opc = ISD::OR;
14151 break;
14152 }
14153 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14154 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
14155 NeedInvert = true;
14156 [[fallthrough]];
14157 case ISD::SETO:
14158 assert(isCondCodeLegal(ISD::SETOEQ, OpVT) &&
14159 "If SETO is expanded, SETOEQ must be legal!");
14160 CC1 = ISD::SETOEQ;
14161 CC2 = ISD::SETOEQ;
14162 Opc = ISD::AND;
14163 break;
14164 case ISD::SETONE:
14165 case ISD::SETUEQ:
14166 // If the SETUO or SETO CC isn't legal, we might be able to use
14167 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
14168 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
14169 // the operands.
14170 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14171 if (!isCondCodeLegal(CC: CC2, VT: OpVT) && (isCondCodeLegal(CC: ISD::SETOGT, VT: OpVT) ||
14172 isCondCodeLegal(CC: ISD::SETOLT, VT: OpVT))) {
14173 CC1 = ISD::SETOGT;
14174 CC2 = ISD::SETOLT;
14175 Opc = ISD::OR;
14176 NeedInvert = ((unsigned)CCCode & 0x8U);
14177 break;
14178 }
14179 [[fallthrough]];
14180 case ISD::SETOEQ:
14181 case ISD::SETOGT:
14182 case ISD::SETOGE:
14183 case ISD::SETOLT:
14184 case ISD::SETOLE:
14185 case ISD::SETUNE:
14186 case ISD::SETUGT:
14187 case ISD::SETUGE:
14188 case ISD::SETULT:
14189 case ISD::SETULE:
14190 // If we are floating point, assign and break, otherwise fall through.
14191 if (!OpVT.isInteger()) {
14192 // We can use the 4th bit to tell if we are the unordered
14193 // or ordered version of the opcode.
14194 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
14195 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
14196 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
14197 break;
14198 }
14199 // Fallthrough if we are unsigned integer.
14200 [[fallthrough]];
14201 case ISD::SETLE:
14202 case ISD::SETGT:
14203 case ISD::SETGE:
14204 case ISD::SETLT:
14205 case ISD::SETNE:
14206 case ISD::SETEQ:
14207 // If all combinations of inverting the condition and swapping operands
14208 // didn't work then we have no means to expand the condition.
14209 llvm_unreachable("Don't know how to expand this condition!");
14210 }
14211
14212 SDValue SetCC1, SetCC2;
14213 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
14214 // If we aren't the ordered or unorder operation,
14215 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
14216 if (IsNonVP) {
14217 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC1, Chain, IsSignaling);
14218 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS, RHS, Cond: CC2, Chain, IsSignaling);
14219 } else {
14220 SetCC1 = DAG.getSetCCVP(DL: dl, VT, LHS, RHS, Cond: CC1, Mask, EVL);
14221 SetCC2 = DAG.getSetCCVP(DL: dl, VT, LHS, RHS, Cond: CC2, Mask, EVL);
14222 }
14223 } else {
14224 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
14225 if (IsNonVP) {
14226 SetCC1 = DAG.getSetCC(DL: dl, VT, LHS, RHS: LHS, Cond: CC1, Chain, IsSignaling);
14227 SetCC2 = DAG.getSetCC(DL: dl, VT, LHS: RHS, RHS, Cond: CC2, Chain, IsSignaling);
14228 } else {
14229 SetCC1 = DAG.getSetCCVP(DL: dl, VT, LHS, RHS: LHS, Cond: CC1, Mask, EVL);
14230 SetCC2 = DAG.getSetCCVP(DL: dl, VT, LHS: RHS, RHS, Cond: CC2, Mask, EVL);
14231 }
14232 }
14233 if (Chain)
14234 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, N1: SetCC1.getValue(R: 1),
14235 N2: SetCC2.getValue(R: 1));
14236 if (IsNonVP)
14237 LHS = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SetCC1, N2: SetCC2);
14238 else {
14239 // Transform the binary opcode to the VP equivalent.
14240 assert((Opc == ISD::OR || Opc == ISD::AND) && "Unexpected opcode");
14241 Opc = Opc == ISD::OR ? ISD::VP_OR : ISD::VP_AND;
14242 LHS = DAG.getNode(Opcode: Opc, DL: dl, VT, N1: SetCC1, N2: SetCC2, N3: Mask, N4: EVL);
14243 }
14244 RHS = SDValue();
14245 CC = SDValue();
14246 return true;
14247 }
14248 }
14249 return false;
14250}
14251
14252SDValue TargetLowering::expandVectorNaryOpBySplitting(SDNode *Node,
14253 SelectionDAG &DAG) const {
14254 EVT VT = Node->getValueType(ResNo: 0);
14255 // Despite its documentation, GetSplitDestVTs will assert if VT cannot be
14256 // split into two equal parts.
14257 if (!VT.isVector() || !VT.getVectorElementCount().isKnownMultipleOf(RHS: 2))
14258 return SDValue();
14259
14260 // Restrict expansion to cases where both parts can be concatenated.
14261 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT);
14262 if (LoVT != HiVT || !isTypeLegal(VT: LoVT))
14263 return SDValue();
14264
14265 SDLoc DL(Node);
14266 unsigned Opcode = Node->getOpcode();
14267
14268 // Don't expand if the result is likely to be unrolled anyway.
14269 if (!isOperationLegalOrCustomOrPromote(Op: Opcode, VT: LoVT))
14270 return SDValue();
14271
14272 SmallVector<SDValue, 4> LoOps, HiOps;
14273 for (const SDValue &V : Node->op_values()) {
14274 auto [Lo, Hi] = DAG.SplitVector(N: V, DL, LoVT, HiVT);
14275 LoOps.push_back(Elt: Lo);
14276 HiOps.push_back(Elt: Hi);
14277 }
14278
14279 SDValue SplitOpLo = DAG.getNode(Opcode, DL, VT: LoVT, Ops: LoOps, Flags: Node->getFlags());
14280 SDValue SplitOpHi = DAG.getNode(Opcode, DL, VT: HiVT, Ops: HiOps, Flags: Node->getFlags());
14281 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: SplitOpLo, N2: SplitOpHi);
14282}
14283
14284SDValue TargetLowering::scalarizeExtractedVectorLoad(EVT ResultVT,
14285 const SDLoc &DL,
14286 EVT InVecVT, SDValue EltNo,
14287 LoadSDNode *OriginalLoad,
14288 SelectionDAG &DAG) const {
14289 assert(OriginalLoad->isSimple());
14290
14291 EVT VecEltVT = InVecVT.getVectorElementType();
14292
14293 // If the vector element type is not a multiple of a byte then we are unable
14294 // to correctly compute an address to load only the extracted element as a
14295 // scalar.
14296 if (!VecEltVT.isByteSized())
14297 return SDValue();
14298
14299 ISD::LoadExtType ExtTy =
14300 ResultVT.bitsGT(VT: VecEltVT) ? ISD::EXTLOAD : ISD::NON_EXTLOAD;
14301 if (!isOperationLegalOrCustom(Op: ISD::LOAD, VT: VecEltVT))
14302 return SDValue();
14303
14304 std::optional<unsigned> ByteOffset;
14305 Align Alignment = OriginalLoad->getAlign();
14306 MachinePointerInfo MPI;
14307 if (auto *ConstEltNo = dyn_cast<ConstantSDNode>(Val&: EltNo)) {
14308 int Elt = ConstEltNo->getZExtValue();
14309 ByteOffset = VecEltVT.getSizeInBits() * Elt / 8;
14310 MPI = OriginalLoad->getPointerInfo().getWithOffset(O: *ByteOffset);
14311 Alignment = commonAlignment(A: Alignment, Offset: *ByteOffset);
14312 } else {
14313 // Discard the pointer info except the address space because the memory
14314 // operand can't represent this new access since the offset is variable.
14315 MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
14316 Alignment = commonAlignment(A: Alignment, Offset: VecEltVT.getSizeInBits() / 8);
14317 }
14318
14319 if (!shouldReduceLoadWidth(Load: OriginalLoad, ExtTy, NewVT: VecEltVT, ByteOffset))
14320 return SDValue();
14321
14322 unsigned IsFast = 0;
14323 if (!allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: VecEltVT,
14324 AddrSpace: OriginalLoad->getAddressSpace(), Alignment,
14325 Flags: OriginalLoad->getMemOperand()->getFlags(), Fast: &IsFast) ||
14326 !IsFast)
14327 return SDValue();
14328
14329 // The original DAG loaded the entire vector from memory, so arithmetic
14330 // within it must be inbounds.
14331 SDValue NewPtr = getInboundsVectorElementPointer(
14332 DAG, VecPtr: OriginalLoad->getBasePtr(), VecVT: InVecVT, Index: EltNo);
14333
14334 // We are replacing a vector load with a scalar load. The new load must have
14335 // identical memory op ordering to the original.
14336 SDValue Load;
14337 if (ResultVT.bitsGT(VT: VecEltVT)) {
14338 // If the result type of vextract is wider than the load, then issue an
14339 // extending load instead.
14340 ISD::LoadExtType ExtType =
14341 isLoadLegal(ValVT: ResultVT, MemVT: VecEltVT, Alignment,
14342 AddrSpace: OriginalLoad->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false)
14343 ? ISD::ZEXTLOAD
14344 : ISD::EXTLOAD;
14345 Load = DAG.getExtLoad(ExtType, dl: DL, VT: ResultVT, Chain: OriginalLoad->getChain(),
14346 Ptr: NewPtr, PtrInfo: MPI, MemVT: VecEltVT, Alignment,
14347 MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14348 AAInfo: OriginalLoad->getAAInfo());
14349 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14350 } else {
14351 // The result type is narrower or the same width as the vector element
14352 Load = DAG.getLoad(VT: VecEltVT, dl: DL, Chain: OriginalLoad->getChain(), Ptr: NewPtr, PtrInfo: MPI,
14353 Alignment, MMOFlags: OriginalLoad->getMemOperand()->getFlags(),
14354 AAInfo: OriginalLoad->getAAInfo());
14355 DAG.makeEquivalentMemoryOrdering(OldLoad: OriginalLoad, NewMemOp: Load);
14356 if (ResultVT.bitsLT(VT: VecEltVT))
14357 Load = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResultVT, Operand: Load);
14358 else
14359 Load = DAG.getBitcast(VT: ResultVT, V: Load);
14360 }
14361
14362 return Load;
14363}
14364
14365// Set type id for call site info and metadata 'call_target'.
14366// We are filtering for:
14367// a) The call-graph-section use case that wants to know about indirect
14368// calls, or
14369// b) We want to annotate indirect calls.
14370void TargetLowering::setTypeIdForCallsiteInfo(
14371 const CallBase *CB, MachineFunction &MF,
14372 MachineFunction::CallSiteInfo &CSInfo) const {
14373 if (CB && CB->isIndirectCall() &&
14374 (MF.getTarget().Options.EmitCallGraphSection ||
14375 MF.getTarget().Options.EmitCallSiteInfo))
14376 CSInfo = MachineFunction::CallSiteInfo(*CB);
14377}
14378