1//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass combines dag nodes to form fewer, simpler DAG nodes. It can be run
10// both before and after the DAG is legalized.
11//
12// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
13// primarily intended to handle simplification opportunities that are implicit
14// in the LLVM IR and exposed by the various codegen lowering phases.
15//
16//===----------------------------------------------------------------------===//
17
18#include "llvm/ADT/APFloat.h"
19#include "llvm/ADT/APInt.h"
20#include "llvm/ADT/APSInt.h"
21#include "llvm/ADT/ArrayRef.h"
22#include "llvm/ADT/DenseMap.h"
23#include "llvm/ADT/IntervalMap.h"
24#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/SetVector.h"
26#include "llvm/ADT/SmallBitVector.h"
27#include "llvm/ADT/SmallPtrSet.h"
28#include "llvm/ADT/SmallSet.h"
29#include "llvm/ADT/SmallVector.h"
30#include "llvm/ADT/Statistic.h"
31#include "llvm/Analysis/AliasAnalysis.h"
32#include "llvm/Analysis/MemoryLocation.h"
33#include "llvm/Analysis/TargetLibraryInfo.h"
34#include "llvm/Analysis/ValueTracking.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/CodeGen/ByteProvider.h"
37#include "llvm/CodeGen/DAGCombine.h"
38#include "llvm/CodeGen/ISDOpcodes.h"
39#include "llvm/CodeGen/MachineFrameInfo.h"
40#include "llvm/CodeGen/MachineFunction.h"
41#include "llvm/CodeGen/MachineMemOperand.h"
42#include "llvm/CodeGen/SDPatternMatch.h"
43#include "llvm/CodeGen/SelectionDAG.h"
44#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
45#include "llvm/CodeGen/SelectionDAGNodes.h"
46#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
47#include "llvm/CodeGen/TargetLowering.h"
48#include "llvm/CodeGen/TargetRegisterInfo.h"
49#include "llvm/CodeGen/TargetSubtargetInfo.h"
50#include "llvm/CodeGen/ValueTypes.h"
51#include "llvm/CodeGenTypes/MachineValueType.h"
52#include "llvm/IR/Attributes.h"
53#include "llvm/IR/Constant.h"
54#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugInfoMetadata.h"
56#include "llvm/IR/DerivedTypes.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/Metadata.h"
59#include "llvm/Support/Casting.h"
60#include "llvm/Support/CodeGen.h"
61#include "llvm/Support/CommandLine.h"
62#include "llvm/Support/Compiler.h"
63#include "llvm/Support/Debug.h"
64#include "llvm/Support/DebugCounter.h"
65#include "llvm/Support/ErrorHandling.h"
66#include "llvm/Support/KnownBits.h"
67#include "llvm/Support/MathExtras.h"
68#include "llvm/Support/raw_ostream.h"
69#include "llvm/Target/TargetMachine.h"
70#include "llvm/Target/TargetOptions.h"
71#include <algorithm>
72#include <cassert>
73#include <cstdint>
74#include <functional>
75#include <iterator>
76#include <optional>
77#include <string>
78#include <tuple>
79#include <utility>
80#include <variant>
81
82#include "SDNodeDbgValue.h"
83
84using namespace llvm;
85using namespace llvm::SDPatternMatch;
86
87#define DEBUG_TYPE "dagcombine"
88
89STATISTIC(NodesCombined , "Number of dag nodes combined");
90STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
91STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
92STATISTIC(OpsNarrowed , "Number of load/op/store narrowed");
93STATISTIC(LdStFP2Int , "Number of fp load/store pairs transformed to int");
94STATISTIC(SlicedLoads, "Number of load sliced");
95STATISTIC(NumFPLogicOpsConv, "Number of logic ops converted to fp ops");
96
97DEBUG_COUNTER(DAGCombineCounter, "dagcombine",
98 "Controls whether a DAG combine is performed for a node");
99
100static cl::opt<bool>
101CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
102 cl::desc("Enable DAG combiner's use of IR alias analysis"));
103
104static cl::opt<bool>
105UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(Val: true),
106 cl::desc("Enable DAG combiner's use of TBAA"));
107
108#ifndef NDEBUG
109static cl::opt<std::string>
110CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
111 cl::desc("Only use DAG-combiner alias analysis in this"
112 " function"));
113#endif
114
115/// Hidden option to stress test load slicing, i.e., when this option
116/// is enabled, load slicing bypasses most of its profitability guards.
117static cl::opt<bool>
118StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
119 cl::desc("Bypass the profitability model of load slicing"),
120 cl::init(Val: false));
121
122static cl::opt<bool>
123 MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(Val: true),
124 cl::desc("DAG combiner may split indexing from loads"));
125
126static cl::opt<bool>
127 EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(Val: true),
128 cl::desc("DAG combiner enable merging multiple stores "
129 "into a wider store"));
130
131static cl::opt<unsigned> TokenFactorInlineLimit(
132 "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(Val: 2048),
133 cl::desc("Limit the number of operands to inline for Token Factors"));
134
135static cl::opt<unsigned> StoreMergeDependenceLimit(
136 "combiner-store-merge-dependence-limit", cl::Hidden, cl::init(Val: 10),
137 cl::desc("Limit the number of times for the same StoreNode and RootNode "
138 "to bail out in store merging dependence check"));
139
140static cl::opt<bool> EnableReduceLoadOpStoreWidth(
141 "combiner-reduce-load-op-store-width", cl::Hidden, cl::init(Val: true),
142 cl::desc("DAG combiner enable reducing the width of load/op/store "
143 "sequence"));
144static cl::opt<bool> ReduceLoadOpStoreWidthForceNarrowingProfitable(
145 "combiner-reduce-load-op-store-width-force-narrowing-profitable",
146 cl::Hidden, cl::init(Val: false),
147 cl::desc("DAG combiner force override the narrowing profitable check when "
148 "reducing the width of load/op/store sequences"));
149
150static cl::opt<bool> EnableShrinkLoadReplaceStoreWithStore(
151 "combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(Val: true),
152 cl::desc("DAG combiner enable load/<replace bytes>/store with "
153 "a narrower store"));
154
155static cl::opt<bool> EnableTopologicalSorting(
156 "combiner-topological-sorting", cl::Hidden, cl::init(Val: false),
157 cl::desc("DAG combiner nodes consistently processed in topological order"));
158
159static cl::opt<bool> DisableCombines("combiner-disabled", cl::Hidden,
160 cl::init(Val: false),
161 cl::desc("Disable the DAG combiner"));
162
163namespace {
164
165 class DAGCombiner {
166 SelectionDAG &DAG;
167 const TargetLowering &TLI;
168 const SelectionDAGTargetInfo *STI;
169 CombineLevel Level = BeforeLegalizeTypes;
170 CodeGenOptLevel OptLevel;
171 bool LegalDAG = false;
172 bool LegalOperations = false;
173 bool LegalTypes = false;
174 bool ForCodeSize;
175 bool DisableGenericCombines;
176
177 /// Worklist of all of the nodes that need to be simplified.
178 ///
179 /// This must behave as a stack -- new nodes to process are pushed onto the
180 /// back and when processing we pop off of the back.
181 ///
182 /// The worklist will not contain duplicates but may contain null entries
183 /// due to nodes being deleted from the underlying DAG. For fast lookup and
184 /// deduplication, the index of the node in this vector is stored in the
185 /// node in SDNode::CombinerWorklistIndex.
186 SmallVector<SDNode *, 64> Worklist;
187
188 /// This records all nodes attempted to be added to the worklist since we
189 /// considered a new worklist entry. As we keep do not add duplicate nodes
190 /// in the worklist, this is different from the tail of the worklist.
191 SmallSetVector<SDNode *, 32> PruningList;
192
193 /// Map from candidate StoreNode to the pair of RootNode and count.
194 /// The count is used to track how many times we have seen the StoreNode
195 /// with the same RootNode bail out in dependence check. If we have seen
196 /// the bail out for the same pair many times over a limit, we won't
197 /// consider the StoreNode with the same RootNode as store merging
198 /// candidate again.
199 DenseMap<SDNode *, std::pair<SDNode *, unsigned>> StoreRootCountMap;
200
201 // BatchAA - Used for DAG load/store alias analysis.
202 BatchAAResults *BatchAA;
203
204 /// This caches all chains that have already been processed in
205 /// DAGCombiner::getStoreMergeCandidates() and found to have no mergeable
206 /// stores candidates.
207 SmallPtrSet<SDNode *, 4> ChainsWithoutMergeableStores;
208
209 /// When an instruction is simplified, add all users of the instruction to
210 /// the work lists because they might get more simplified now.
211 void AddUsersToWorklist(SDNode *N) {
212 for (SDNode *Node : N->users())
213 AddToWorklist(N: Node);
214 }
215
216 /// Convenient shorthand to add a node and all of its user to the worklist.
217 void AddToWorklistWithUsers(SDNode *N) {
218 AddUsersToWorklist(N);
219 AddToWorklist(N);
220 }
221
222 // Prune potentially dangling nodes. This is called after
223 // any visit to a node, but should also be called during a visit after any
224 // failed combine which may have created a DAG node.
225 void clearAddedDanglingWorklistEntries() {
226 // Check any nodes added to the worklist to see if they are prunable.
227 while (!PruningList.empty()) {
228 auto *N = PruningList.pop_back_val();
229 if (N->use_empty())
230 recursivelyDeleteUnusedNodes(N);
231 }
232 }
233
234 SDNode *getNextWorklistEntry() {
235 // Before we do any work, remove nodes that are not in use.
236 clearAddedDanglingWorklistEntries();
237 SDNode *N = nullptr;
238 // The Worklist holds the SDNodes in order, but it may contain null
239 // entries.
240 while (!N && !Worklist.empty()) {
241 N = Worklist.pop_back_val();
242 }
243
244 if (N) {
245 assert(N->getCombinerWorklistIndex() >= 0 &&
246 "Found a worklist entry without a corresponding map entry!");
247 // Set to -2 to indicate that we combined the node.
248 N->setCombinerWorklistIndex(-2);
249 }
250 return N;
251 }
252
253 /// Call the node-specific routine that folds each particular type of node.
254 SDValue visit(SDNode *N);
255
256 public:
257 DAGCombiner(SelectionDAG &D, BatchAAResults *BatchAA, CodeGenOptLevel OL)
258 : DAG(D), TLI(D.getTargetLoweringInfo()),
259 STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL),
260 BatchAA(BatchAA) {
261 ForCodeSize = DAG.shouldOptForSize();
262 DisableGenericCombines =
263 DisableCombines || (STI && STI->disableGenericCombines(OptLevel));
264 }
265
266 void ConsiderForPruning(SDNode *N) {
267 // Mark this for potential pruning.
268 PruningList.insert(X: N);
269 }
270
271 /// Add to the worklist making sure its instance is at the back (next to be
272 /// processed.)
273 void AddToWorklist(SDNode *N, bool IsCandidateForPruning = true,
274 bool SkipIfCombinedBefore = false) {
275 assert(N->getOpcode() != ISD::DELETED_NODE &&
276 "Deleted Node added to Worklist");
277
278 // Skip handle nodes as they can't usefully be combined and confuse the
279 // zero-use deletion strategy.
280 if (N->getOpcode() == ISD::HANDLENODE)
281 return;
282
283 if (SkipIfCombinedBefore && N->getCombinerWorklistIndex() == -2)
284 return;
285
286 if (IsCandidateForPruning)
287 ConsiderForPruning(N);
288
289 if (N->getCombinerWorklistIndex() < 0) {
290 N->setCombinerWorklistIndex(Worklist.size());
291 Worklist.push_back(Elt: N);
292 }
293 }
294
295 /// Remove all instances of N from the worklist.
296 void removeFromWorklist(SDNode *N) {
297 PruningList.remove(X: N);
298 StoreRootCountMap.erase(Val: N);
299
300 int WorklistIndex = N->getCombinerWorklistIndex();
301 // If not in the worklist, the index might be -1 or -2 (was combined
302 // before). As the node gets deleted anyway, there's no need to update
303 // the index.
304 if (WorklistIndex < 0)
305 return; // Not in the worklist.
306
307 // Null out the entry rather than erasing it to avoid a linear operation.
308 Worklist[WorklistIndex] = nullptr;
309 N->setCombinerWorklistIndex(-1);
310 }
311
312 void deleteAndRecombine(SDNode *N);
313 bool recursivelyDeleteUnusedNodes(SDNode *N);
314
315 /// Replaces all uses of the results of one DAG node with new values.
316 SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
317 bool AddTo = true);
318
319 /// Replaces all uses of the results of one DAG node with new values.
320 SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
321 return CombineTo(N, To: &Res, NumTo: 1, AddTo);
322 }
323
324 /// Replaces all uses of the results of one DAG node with new values.
325 SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
326 bool AddTo = true) {
327 SDValue To[] = { Res0, Res1 };
328 return CombineTo(N, To, NumTo: 2, AddTo);
329 }
330
331 SDValue CombineTo(SDNode *N, SmallVectorImpl<SDValue> *To,
332 bool AddTo = true) {
333 return CombineTo(N, To: To->data(), NumTo: To->size(), AddTo);
334 }
335
336 void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
337
338 private:
339 /// Check the specified integer node value to see if it can be simplified or
340 /// if things it uses can be simplified by bit propagation.
341 /// If so, return true.
342 bool SimplifyDemandedBits(SDValue Op) {
343 unsigned BitWidth = Op.getScalarValueSizeInBits();
344 APInt DemandedBits = APInt::getAllOnes(numBits: BitWidth);
345 return SimplifyDemandedBits(Op, DemandedBits);
346 }
347
348 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
349 EVT VT = Op.getValueType();
350 APInt DemandedElts = VT.isFixedLengthVector()
351 ? APInt::getAllOnes(numBits: VT.getVectorNumElements())
352 : APInt(1, 1);
353 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, AssumeSingleUse: false);
354 }
355
356 /// Check the specified vector node value to see if it can be simplified or
357 /// if things it uses can be simplified as it only uses some of the
358 /// elements. If so, return true.
359 bool SimplifyDemandedVectorElts(SDValue Op) {
360 // TODO: For now just pretend it cannot be simplified.
361 if (Op.getValueType().isScalableVector())
362 return false;
363
364 unsigned NumElts = Op.getValueType().getVectorNumElements();
365 APInt DemandedElts = APInt::getAllOnes(numBits: NumElts);
366 return SimplifyDemandedVectorElts(Op, DemandedElts);
367 }
368
369 bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
370 const APInt &DemandedElts,
371 bool AssumeSingleUse = false);
372 bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
373 bool AssumeSingleUse = false);
374
375 bool CombineToPreIndexedLoadStore(SDNode *N);
376 bool CombineToPostIndexedLoadStore(SDNode *N);
377 SDValue SplitIndexingFromLoad(LoadSDNode *LD);
378 bool SliceUpLoad(SDNode *N);
379
380 // Looks up the chain to find a unique (unaliased) store feeding the passed
381 // load. If no such store is found, returns a nullptr.
382 // Note: This will look past a CALLSEQ_START if the load is chained to it so
383 // so that it can find stack stores for byval params.
384 StoreSDNode *getUniqueStoreFeeding(LoadSDNode *LD, int64_t &Offset);
385 // Scalars have size 0 to distinguish from singleton vectors.
386 SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
387 bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
388 bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
389
390 void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
391 SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
392 SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
393 SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
394 SDValue PromoteIntBinOp(SDValue Op);
395 SDValue PromoteIntShiftOp(SDValue Op);
396 SDValue PromoteExtend(SDValue Op);
397 bool PromoteLoad(SDValue Op);
398
399 SDValue foldShiftToAvg(SDNode *N, const SDLoc &DL);
400 // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
401 SDValue foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT);
402
403 SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
404 SDValue RHS, SDValue True, SDValue False,
405 ISD::CondCode CC);
406
407 /// Call the node-specific routine that knows how to fold each
408 /// particular type of node. If that doesn't do anything, try the
409 /// target-specific DAG combines.
410 SDValue combine(SDNode *N);
411
412 // Visitation implementation - Implement dag node combining for different
413 // node types. The semantics are as follows:
414 // Return Value:
415 // SDValue.getNode() == 0 - No change was made
416 // SDValue.getNode() == N - N was replaced, is dead and has been handled.
417 // otherwise - N should be replaced by the returned Operand.
418 //
419 SDValue visitTokenFactor(SDNode *N);
420 SDValue visitMERGE_VALUES(SDNode *N);
421 SDValue visitADD(SDNode *N);
422 SDValue visitADDLike(SDNode *N);
423 SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, const SDLoc &DL);
424 SDValue visitPTRADD(SDNode *N);
425 SDValue visitSUB(SDNode *N);
426 SDValue visitADDSAT(SDNode *N);
427 SDValue visitSUBSAT(SDNode *N);
428 SDValue visitADDC(SDNode *N);
429 SDValue visitADDO(SDNode *N);
430 SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
431 SDValue visitSUBC(SDNode *N);
432 SDValue visitSUBO(SDNode *N);
433 SDValue visitADDE(SDNode *N);
434 SDValue visitUADDO_CARRY(SDNode *N);
435 SDValue visitSADDO_CARRY(SDNode *N);
436 SDValue visitUADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
437 SDNode *N);
438 SDValue visitSADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
439 SDNode *N);
440 SDValue visitSUBE(SDNode *N);
441 SDValue visitUSUBO_CARRY(SDNode *N);
442 SDValue visitSSUBO_CARRY(SDNode *N);
443 SDValue visitMUL(SDNode *N);
444 SDValue visitMULFIX(SDNode *N);
445 SDValue useDivRem(SDNode *N);
446 SDValue visitSDIV(SDNode *N);
447 SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
448 SDValue visitUDIV(SDNode *N);
449 SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
450 SDValue visitREM(SDNode *N);
451 SDValue visitMULHU(SDNode *N);
452 SDValue visitMULHS(SDNode *N);
453 SDValue visitAVG(SDNode *N);
454 SDValue visitABD(SDNode *N);
455 SDValue visitSMUL_LOHI(SDNode *N);
456 SDValue visitUMUL_LOHI(SDNode *N);
457 SDValue visitMULO(SDNode *N);
458 SDValue visitIMINMAX(SDNode *N);
459 SDValue visitAND(SDNode *N);
460 SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
461 SDValue visitOR(SDNode *N);
462 SDValue visitORLike(SDValue N0, SDValue N1, const SDLoc &DL);
463 SDValue visitXOR(SDNode *N);
464 SDValue SimplifyVCastOp(SDNode *N, const SDLoc &DL);
465 SDValue SimplifyVBinOp(SDNode *N, const SDLoc &DL);
466 SDValue visitSHL(SDNode *N);
467 SDValue visitSRA(SDNode *N);
468 SDValue visitSRL(SDNode *N);
469 SDValue visitFunnelShift(SDNode *N);
470 SDValue visitSHLSAT(SDNode *N);
471 SDValue visitRotate(SDNode *N);
472 SDValue visitABS(SDNode *N);
473 SDValue visitABS_MIN_POISON(SDNode *N);
474 SDValue visitCLMUL(SDNode *N);
475 SDValue visitPEXT(SDNode *N);
476 SDValue visitPDEP(SDNode *N);
477 SDValue visitBSWAP(SDNode *N);
478 SDValue visitBITREVERSE(SDNode *N);
479 SDValue visitCTLZ(SDNode *N);
480 SDValue visitCTLZ_ZERO_POISON(SDNode *N);
481 SDValue visitCTTZ(SDNode *N);
482 SDValue visitCTTZ_ZERO_POISON(SDNode *N);
483 SDValue visitCTPOP(SDNode *N);
484 SDValue visitPARITY(SDNode *N);
485 SDValue visitSELECT(SDNode *N);
486 SDValue visitVSELECT(SDNode *N);
487 SDValue visitSELECT_CC(SDNode *N);
488 SDValue visitSETCC(SDNode *N);
489 SDValue visitSETCCCARRY(SDNode *N);
490 SDValue visitSIGN_EXTEND(SDNode *N);
491 SDValue visitZERO_EXTEND(SDNode *N);
492 SDValue visitANY_EXTEND(SDNode *N);
493 SDValue visitAssertExt(SDNode *N);
494 SDValue visitAssertAlign(SDNode *N);
495 SDValue visitIS_FPCLASS(SDNode *N);
496 SDValue visitSIGN_EXTEND_INREG(SDNode *N);
497 SDValue visitEXTEND_VECTOR_INREG(SDNode *N);
498 SDValue visitTRUNCATE(SDNode *N);
499 SDValue visitTRUNCATE_USAT_U(SDNode *N);
500 SDValue visitBITCAST(SDNode *N);
501 SDValue visitFREEZE(SDNode *N);
502 SDValue visitBUILD_PAIR(SDNode *N);
503 SDValue visitFADD(SDNode *N);
504 SDValue visitSTRICT_FADD(SDNode *N);
505 SDValue visitFSUB(SDNode *N);
506 SDValue visitFMUL(SDNode *N);
507 SDValue visitFMA(SDNode *N);
508 SDValue visitFMAD(SDNode *N);
509 SDValue visitFMULADD(SDNode *N);
510 SDValue visitFDIV(SDNode *N);
511 SDValue visitFREM(SDNode *N);
512 SDValue visitFSQRT(SDNode *N);
513 SDValue visitFCOPYSIGN(SDNode *N);
514 SDValue visitFPOW(SDNode *N);
515 SDValue visitFCANONICALIZE(SDNode *N);
516 SDValue visitSINT_TO_FP(SDNode *N);
517 SDValue visitUINT_TO_FP(SDNode *N);
518 SDValue visitFP_TO_SINT(SDNode *N);
519 SDValue visitFP_TO_UINT(SDNode *N);
520 SDValue visitXROUND(SDNode *N);
521 SDValue visitFP_ROUND(SDNode *N);
522 SDValue visitFP_EXTEND(SDNode *N);
523 SDValue visitFNEG(SDNode *N);
524 SDValue visitFABS(SDNode *N);
525 SDValue visitFCEIL(SDNode *N);
526 SDValue visitFTRUNC(SDNode *N);
527 SDValue visitFFREXP(SDNode *N);
528 SDValue visitFFLOOR(SDNode *N);
529 SDValue visitFMinMax(SDNode *N);
530 SDValue visitBRCOND(SDNode *N);
531 SDValue visitBR_CC(SDNode *N);
532 SDValue visitLOAD(SDNode *N);
533
534 SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
535 SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
536 SDValue replaceStoreOfInsertLoad(StoreSDNode *ST);
537
538 bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(SDNode *N);
539 SDValue combineStoreConcatTruncVector(StoreSDNode *N);
540 SDValue visitSTORE(SDNode *N);
541 SDValue visitATOMIC_STORE(SDNode *N);
542 SDValue visitLIFETIME_END(SDNode *N);
543 SDValue visitINSERT_VECTOR_ELT(SDNode *N);
544 SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
545 SDValue visitBUILD_VECTOR(SDNode *N);
546 SDValue visitCONCAT_VECTORS(SDNode *N);
547 SDValue visitVECTOR_INTERLEAVE(SDNode *N);
548 SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
549 SDValue visitVECTOR_SHUFFLE(SDNode *N);
550 SDValue visitSCALAR_TO_VECTOR(SDNode *N);
551 SDValue visitINSERT_SUBVECTOR(SDNode *N);
552 SDValue visitVECTOR_COMPRESS(SDNode *N);
553 SDValue visitMLOAD(SDNode *N);
554 SDValue visitMSTORE(SDNode *N);
555 SDValue visitMGATHER(SDNode *N);
556 SDValue visitMSCATTER(SDNode *N);
557 SDValue visitMHISTOGRAM(SDNode *N);
558 SDValue visitPARTIAL_REDUCE_MLA(SDNode *N);
559 SDValue visitLOOP_DEPENDENCE_MASK(SDNode *N);
560 SDValue visitVPGATHER(SDNode *N);
561 SDValue visitVPSCATTER(SDNode *N);
562 SDValue visitVP_STRIDED_LOAD(SDNode *N);
563 SDValue visitVP_STRIDED_STORE(SDNode *N);
564 SDValue visitFP_TO_FP16(SDNode *N);
565 SDValue visitFP16_TO_FP(SDNode *N);
566 SDValue visitFP_TO_BF16(SDNode *N);
567 SDValue visitBF16_TO_FP(SDNode *N);
568 SDValue visitVECREDUCE(SDNode *N);
569 SDValue visitVPOp(SDNode *N);
570 SDValue visitGET_FPENV_MEM(SDNode *N);
571 SDValue visitSET_FPENV_MEM(SDNode *N);
572
573 SDValue visitFADDForFMACombine(SDNode *N);
574 SDValue visitFSUBForFMACombine(SDNode *N);
575 SDValue visitFMULForFMADistributiveCombine(SDNode *N);
576
577 SDValue XformToShuffleWithZero(SDNode *N);
578 bool reassociationCanBreakAddressingModePattern(unsigned Opc,
579 const SDLoc &DL,
580 SDNode *N,
581 SDValue N0,
582 SDValue N1);
583 SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
584 SDValue N1, SDNodeFlags Flags);
585 SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
586 SDValue N1, SDNodeFlags Flags);
587 SDValue reassociateReduction(unsigned RedOpc, unsigned Opc, const SDLoc &DL,
588 EVT VT, SDValue N0, SDValue N1,
589 SDNodeFlags Flags = SDNodeFlags());
590
591 SDValue visitShiftByConstant(SDNode *N);
592
593 SDValue foldSelectOfConstants(SDNode *N);
594 SDValue foldVSelectOfConstants(SDNode *N);
595 SDValue foldBinOpIntoSelect(SDNode *BO);
596 bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
597 SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
598 SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
599 SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
600 SDValue N2, SDValue N3, ISD::CondCode CC,
601 bool NotExtCompare = false);
602 SDValue convertSelectOfFPConstantsToLoadOffset(
603 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
604 ISD::CondCode CC);
605 SDValue foldSignChangeInBitcast(SDNode *N);
606 SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
607 SDValue N2, SDValue N3, ISD::CondCode CC);
608 SDValue foldSelectOfBinops(SDNode *N);
609 SDValue foldSextSetcc(SDNode *N);
610 SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
611 const SDLoc &DL);
612 SDValue foldSubToUSubSat(EVT DstVT, SDNode *N, const SDLoc &DL);
613 SDValue foldABSToABD(SDNode *N, const SDLoc &DL);
614 SDValue foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
615 SDValue False, ISD::CondCode CC, const SDLoc &DL);
616 SDValue foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
617 SDValue False, ISD::CondCode CC, const SDLoc &DL);
618 SDValue unfoldMaskedMerge(SDNode *N);
619 SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
620 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
621 const SDLoc &DL, bool foldBooleans);
622 SDValue rebuildSetCC(SDValue N);
623
624 bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
625 SDValue &CC, bool MatchStrict = false) const;
626 bool isOneUseSetCC(SDValue N) const;
627
628 SDValue foldAddToAvg(SDNode *N, const SDLoc &DL);
629 SDValue foldSubToAvg(SDNode *N, const SDLoc &DL);
630
631 SDValue foldCTLZToCTLS(SDValue Src, const SDLoc &DL);
632
633 SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
634 unsigned HiOp);
635 SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
636 SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
637 const TargetLowering &TLI);
638 SDValue foldPartialReduceMLAMulOp(SDNode *N);
639 SDValue foldPartialReduceAdd(SDNode *N);
640
641 SDValue CombineExtLoad(SDNode *N);
642 SDValue CombineZExtLogicopShiftLoad(SDNode *N);
643 SDValue combineRepeatedFPDivisors(SDNode *N);
644 SDValue combineFMulOrFDivWithIntPow2(SDNode *N);
645 SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf);
646 SDValue mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex);
647 SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
648 SDValue combineInsertEltToLoad(SDNode *N, unsigned InsIndex);
649 SDValue foldExtractSubvectorFromConcatVectors(EVT VT, SDValue V,
650 uint64_t ExtIdx,
651 const SDLoc &DL);
652 SDValue BuildSDIV(SDNode *N);
653 SDValue BuildSDIVPow2(SDNode *N);
654 SDValue BuildUDIV(SDNode *N);
655 SDValue BuildSREMPow2(SDNode *N);
656 SDValue buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N);
657 SDValue BuildLogBase2(SDValue V, const SDLoc &DL,
658 bool KnownNeverZero = false,
659 bool InexpensiveOnly = false,
660 std::optional<EVT> OutVT = std::nullopt);
661 SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
662 SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
663 SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
664 SDValue buildSqrtEstimateImpl(SDValue Op, bool Recip, SDNodeFlags Flags);
665 SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
666 bool Reciprocal);
667 SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
668 bool Reciprocal);
669 SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
670 bool DemandHighBits = true);
671 SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
672 SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
673 SDValue InnerPos, SDValue InnerNeg, bool FromAdd,
674 bool HasPos, unsigned PosOpcode,
675 unsigned NegOpcode, const SDLoc &DL);
676 SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
677 SDValue InnerPos, SDValue InnerNeg, bool FromAdd,
678 bool HasPos, unsigned PosOpcode,
679 unsigned NegOpcode, const SDLoc &DL);
680 SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
681 bool FromAdd);
682 SDValue MatchLoadCombine(SDNode *N);
683 SDValue mergeTruncStores(StoreSDNode *N);
684 SDValue reduceLoadWidth(SDNode *N);
685 SDValue ReduceLoadOpStoreWidth(SDNode *N);
686 SDValue splitMergedValStore(StoreSDNode *ST);
687 SDValue TransformFPLoadStorePair(SDNode *N);
688 SDValue convertBuildVecExtToExt(SDNode *N);
689 SDValue convertBuildVecZextToBuildVecWithZeros(SDNode *N);
690 SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
691 SDValue reduceBuildVecTruncToBitCast(SDNode *N);
692 SDValue reduceBuildVecToShuffle(SDNode *N);
693 SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
694 ArrayRef<int> VectorMask, SDValue VecIn1,
695 SDValue VecIn2, unsigned LeftIdx,
696 bool DidSplitVec);
697 SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
698
699 /// Walk up chain skipping non-aliasing memory nodes,
700 /// looking for aliasing nodes and adding them to the Aliases vector.
701 void GatherAllAliases(SDNode *N, SDValue OriginalChain,
702 SmallVectorImpl<SDValue> &Aliases);
703
704 /// Return true if there is any possibility that the two addresses overlap.
705 bool mayAlias(SDNode *Op0, SDNode *Op1) const;
706
707 /// Walk up chain skipping non-aliasing memory nodes, looking for a better
708 /// chain (aliasing node.)
709 SDValue FindBetterChain(SDNode *N, SDValue Chain);
710
711 /// Try to replace a store and any possibly adjacent stores on
712 /// consecutive chains with better chains. Return true only if St is
713 /// replaced.
714 ///
715 /// Notice that other chains may still be replaced even if the function
716 /// returns false.
717 bool findBetterNeighborChains(StoreSDNode *St);
718
719 // Helper for findBetterNeighborChains. Walk up store chain add additional
720 // chained stores that do not overlap and can be parallelized.
721 bool parallelizeChainedStores(StoreSDNode *St);
722
723 /// Holds a pointer to an LSBaseSDNode as well as information on where it
724 /// is located in a sequence of memory operations connected by a chain.
725 struct MemOpLink {
726 // Ptr to the mem node.
727 LSBaseSDNode *MemNode;
728
729 // Offset from the base ptr.
730 int64_t OffsetFromBase;
731
732 MemOpLink(LSBaseSDNode *N, int64_t Offset)
733 : MemNode(N), OffsetFromBase(Offset) {}
734 };
735
736 // Classify the origin of a stored value.
737 enum class StoreSource { Unknown, Constant, Extract, Load };
738 StoreSource getStoreSource(SDValue StoreVal) {
739 switch (StoreVal.getOpcode()) {
740 case ISD::Constant:
741 case ISD::ConstantFP:
742 return StoreSource::Constant;
743 case ISD::BUILD_VECTOR:
744 if (ISD::isBuildVectorOfConstantSDNodes(N: StoreVal.getNode()) ||
745 ISD::isBuildVectorOfConstantFPSDNodes(N: StoreVal.getNode()))
746 return StoreSource::Constant;
747 return StoreSource::Unknown;
748 case ISD::EXTRACT_VECTOR_ELT:
749 case ISD::EXTRACT_SUBVECTOR:
750 return StoreSource::Extract;
751 case ISD::LOAD:
752 return StoreSource::Load;
753 default:
754 return StoreSource::Unknown;
755 }
756 }
757
758 /// This is a helper function for visitMUL to check the profitability
759 /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
760 /// MulNode is the original multiply, AddNode is (add x, c1),
761 /// and ConstNode is c2.
762 bool isMulAddWithConstProfitable(SDNode *MulNode, SDValue AddNode,
763 SDValue ConstNode);
764
765 /// This is a helper function for visitAND and visitZERO_EXTEND. Returns
766 /// true if the (and (load x) c) pattern matches an extload. ExtVT returns
767 /// the type of the loaded value to be extended.
768 bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
769 EVT LoadResultTy, EVT &ExtVT);
770
771 /// Helper function to calculate whether the given Load/Store can have its
772 /// width reduced to ExtVT.
773 bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
774 EVT &MemVT, unsigned ShAmt = 0);
775
776 /// Used by BackwardsPropagateMask to find suitable loads.
777 bool SearchForAndLoads(SDNode *N, SmallVectorImpl<LoadSDNode*> &Loads,
778 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
779 ConstantSDNode *Mask, SDNode *&NodeToMask);
780 /// Attempt to propagate a given AND node back to load leaves so that they
781 /// can be combined into narrow loads.
782 bool BackwardsPropagateMask(SDNode *N);
783
784 /// Helper function for mergeConsecutiveStores which merges the component
785 /// store chains.
786 SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
787 unsigned NumStores);
788
789 /// Helper function for mergeConsecutiveStores which checks if all the store
790 /// nodes have the same underlying object. We can still reuse the first
791 /// store's pointer info if all the stores are from the same object.
792 bool hasSameUnderlyingObj(ArrayRef<MemOpLink> StoreNodes);
793
794 /// This is a helper function for mergeConsecutiveStores. When the source
795 /// elements of the consecutive stores are all constants or all extracted
796 /// vector elements, try to merge them into one larger store introducing
797 /// bitcasts if necessary. \return True if a merged store was created.
798 bool mergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
799 EVT MemVT, unsigned NumStores,
800 bool IsConstantSrc, bool UseVector,
801 bool UseTrunc);
802
803 /// This is a helper function for mergeConsecutiveStores. Stores that
804 /// potentially may be merged with St are placed in StoreNodes. On success,
805 /// returns a chain predecessor to all store candidates.
806 SDNode *getStoreMergeCandidates(StoreSDNode *St,
807 SmallVectorImpl<MemOpLink> &StoreNodes);
808
809 /// Helper function for mergeConsecutiveStores. Checks if candidate stores
810 /// have indirect dependency through their operands. RootNode is the
811 /// predecessor to all stores calculated by getStoreMergeCandidates and is
812 /// used to prune the dependency check. \return True if safe to merge.
813 bool checkMergeStoreCandidatesForDependencies(
814 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
815 SDNode *RootNode);
816
817 /// Helper function for tryStoreMergeOfLoads. Checks if the load/store
818 /// chain has a call in it. \return True if a call is found.
819 bool hasCallInLdStChain(StoreSDNode *St, LoadSDNode *Ld);
820
821 /// This is a helper function for mergeConsecutiveStores. Given a list of
822 /// store candidates, find the first N that are consecutive in memory.
823 /// Returns 0 if there are not at least 2 consecutive stores to try merging.
824 unsigned getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
825 int64_t ElementSizeBytes) const;
826
827 /// This is a helper function for mergeConsecutiveStores. It is used for
828 /// store chains that are composed entirely of constant values.
829 bool tryStoreMergeOfConstants(SmallVectorImpl<MemOpLink> &StoreNodes,
830 unsigned NumConsecutiveStores,
831 EVT MemVT, SDNode *Root, bool AllowVectors);
832
833 /// This is a helper function for mergeConsecutiveStores. It is used for
834 /// store chains that are composed entirely of extracted vector elements.
835 /// When extracting multiple vector elements, try to store them in one
836 /// vector store rather than a sequence of scalar stores.
837 bool tryStoreMergeOfExtracts(SmallVectorImpl<MemOpLink> &StoreNodes,
838 unsigned NumConsecutiveStores, EVT MemVT,
839 SDNode *Root);
840
841 /// This is a helper function for mergeConsecutiveStores. It is used for
842 /// store chains that are composed entirely of loaded values.
843 bool tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
844 unsigned NumConsecutiveStores, EVT MemVT,
845 SDNode *Root, bool AllowVectors,
846 bool IsNonTemporalStore, bool IsNonTemporalLoad);
847
848 /// Merge consecutive store operations into a wide store.
849 /// This optimization uses wide integers or vectors when possible.
850 /// \return true if stores were merged.
851 bool mergeConsecutiveStores(StoreSDNode *St);
852
853 /// Try to transform a truncation where C is a constant:
854 /// (trunc (and X, C)) -> (and (trunc X), (trunc C))
855 ///
856 /// \p N needs to be a truncation and its first operand an AND. Other
857 /// requirements are checked by the function (e.g. that trunc is
858 /// single-use) and if missed an empty SDValue is returned.
859 SDValue distributeTruncateThroughAnd(SDNode *N);
860
861 /// Helper function to determine whether the target supports operation
862 /// given by \p Opcode for type \p VT, that is, whether the operation
863 /// is legal or custom before legalizing operations, and whether is
864 /// legal (but not custom) after legalization.
865 bool hasOperation(unsigned Opcode, EVT VT) {
866 return TLI.isOperationLegalOrCustom(Op: Opcode, VT, LegalOnly: LegalOperations);
867 }
868
869 bool hasUMin(EVT VT) const {
870 auto LK = TLI.getTypeConversion(Context&: *DAG.getContext(), VT);
871 return (LK.first == TargetLoweringBase::TypeLegal ||
872 LK.first == TargetLoweringBase::TypePromoteInteger) &&
873 TLI.isOperationLegalOrCustom(Op: ISD::UMIN, VT: LK.second);
874 }
875
876 public:
877 /// Runs the dag combiner on all nodes in the work list
878 void Run(CombineLevel AtLevel);
879
880 SelectionDAG &getDAG() const { return DAG; }
881
882 /// Convenience wrapper around TargetLowering::getShiftAmountTy.
883 EVT getShiftAmountTy(EVT LHSTy) {
884 return TLI.getShiftAmountTy(LHSTy, DL: DAG.getDataLayout());
885 }
886
887 /// This method returns true if we are running before type legalization or
888 /// if the specified VT is legal.
889 bool isTypeLegal(const EVT &VT) {
890 if (!LegalTypes) return true;
891 return TLI.isTypeLegal(VT);
892 }
893
894 /// Convenience wrapper around TargetLowering::getSetCCResultType
895 EVT getSetCCResultType(EVT VT) const {
896 return TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT);
897 }
898
899 void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
900 SDValue OrigLoad, SDValue ExtLoad,
901 ISD::NodeType ExtType);
902 };
903
904/// This class is a DAGUpdateListener that removes any deleted
905/// nodes from the worklist.
906class WorklistRemover : public SelectionDAG::DAGUpdateListener {
907 DAGCombiner &DC;
908
909public:
910 explicit WorklistRemover(DAGCombiner &dc)
911 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
912
913 void NodeDeleted(SDNode *N, SDNode *E) override {
914 DC.removeFromWorklist(N);
915 }
916};
917
918class WorklistInserter : public SelectionDAG::DAGUpdateListener {
919 DAGCombiner &DC;
920
921public:
922 explicit WorklistInserter(DAGCombiner &dc)
923 : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
924
925 // FIXME: Ideally we could add N to the worklist, but this causes exponential
926 // compile time costs in large DAGs, e.g. Halide.
927 void NodeInserted(SDNode *N) override { DC.ConsiderForPruning(N); }
928};
929
930} // end anonymous namespace
931
932//===----------------------------------------------------------------------===//
933// TargetLowering::DAGCombinerInfo implementation
934//===----------------------------------------------------------------------===//
935
936void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
937 ((DAGCombiner*)DC)->AddToWorklist(N);
938}
939
940SDValue TargetLowering::DAGCombinerInfo::
941CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
942 return ((DAGCombiner*)DC)->CombineTo(N, To: &To[0], NumTo: To.size(), AddTo);
943}
944
945SDValue TargetLowering::DAGCombinerInfo::
946CombineTo(SDNode *N, SDValue Res, bool AddTo) {
947 return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
948}
949
950SDValue TargetLowering::DAGCombinerInfo::
951CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
952 return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
953}
954
955bool TargetLowering::DAGCombinerInfo::
956recursivelyDeleteUnusedNodes(SDNode *N) {
957 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N);
958}
959
960void TargetLowering::DAGCombinerInfo::
961CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
962 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
963}
964
965//===----------------------------------------------------------------------===//
966// Helper Functions
967//===----------------------------------------------------------------------===//
968
969void DAGCombiner::deleteAndRecombine(SDNode *N) {
970 removeFromWorklist(N);
971
972 // If the operands of this node are only used by the node, they will now be
973 // dead. Make sure to re-visit them and recursively delete dead nodes.
974 for (const SDValue &Op : N->ops())
975 // For an operand generating multiple values, one of the values may
976 // become dead allowing further simplification (e.g. split index
977 // arithmetic from an indexed load).
978 if (Op->hasOneUse() || Op->getNumValues() > 1)
979 AddToWorklist(N: Op.getNode());
980
981 DAG.DeleteNode(N);
982}
983
984// APInts must be the same size for most operations, this helper
985// function zero extends the shorter of the pair so that they match.
986// We provide an Offset so that we can create bitwidths that won't overflow.
987static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
988 unsigned Bits = Offset + std::max(a: LHS.getBitWidth(), b: RHS.getBitWidth());
989 LHS = LHS.zext(width: Bits);
990 RHS = RHS.zext(width: Bits);
991}
992
993// Return true if this node is a setcc, or is a select_cc
994// that selects between the target values used for true and false, making it
995// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
996// the appropriate nodes based on the type of node we are checking. This
997// simplifies life a bit for the callers.
998bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
999 SDValue &CC, bool MatchStrict) const {
1000 if (N.getOpcode() == ISD::SETCC) {
1001 LHS = N.getOperand(i: 0);
1002 RHS = N.getOperand(i: 1);
1003 CC = N.getOperand(i: 2);
1004 return true;
1005 }
1006
1007 if (MatchStrict &&
1008 (N.getOpcode() == ISD::STRICT_FSETCC ||
1009 N.getOpcode() == ISD::STRICT_FSETCCS)) {
1010 LHS = N.getOperand(i: 1);
1011 RHS = N.getOperand(i: 2);
1012 CC = N.getOperand(i: 3);
1013 return true;
1014 }
1015
1016 if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N: N.getOperand(i: 2)) ||
1017 !TLI.isConstFalseVal(N: N.getOperand(i: 3)))
1018 return false;
1019
1020 if (TLI.getBooleanContents(Type: N.getValueType()) ==
1021 TargetLowering::UndefinedBooleanContent)
1022 return false;
1023
1024 LHS = N.getOperand(i: 0);
1025 RHS = N.getOperand(i: 1);
1026 CC = N.getOperand(i: 4);
1027 return true;
1028}
1029
1030/// Return true if this is a SetCC-equivalent operation with only one use.
1031/// If this is true, it allows the users to invert the operation for free when
1032/// it is profitable to do so.
1033bool DAGCombiner::isOneUseSetCC(SDValue N) const {
1034 SDValue N0, N1, N2;
1035 if (isSetCCEquivalent(N, LHS&: N0, RHS&: N1, CC&: N2) && N->hasOneUse())
1036 return true;
1037 return false;
1038}
1039
1040static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy) {
1041 if (!ScalarTy.isSimple())
1042 return false;
1043
1044 uint64_t MaskForTy = 0ULL;
1045 switch (ScalarTy.getSimpleVT().SimpleTy) {
1046 case MVT::i8:
1047 MaskForTy = 0xFFULL;
1048 break;
1049 case MVT::i16:
1050 MaskForTy = 0xFFFFULL;
1051 break;
1052 case MVT::i32:
1053 MaskForTy = 0xFFFFFFFFULL;
1054 break;
1055 default:
1056 return false;
1057 break;
1058 }
1059
1060 APInt Val;
1061 if (ISD::isConstantSplatVector(N, SplatValue&: Val))
1062 return Val.getLimitedValue() == MaskForTy;
1063
1064 return false;
1065}
1066
1067// Determines if it is a constant integer or a splat/build vector of constant
1068// integers (and undefs).
1069// Do not permit build vector implicit truncation unless AllowTruncation is set.
1070static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false,
1071 bool AllowTruncation = false) {
1072 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: N))
1073 return !(Const->isOpaque() && NoOpaques);
1074 if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR)
1075 return false;
1076 unsigned BitWidth = N.getScalarValueSizeInBits();
1077 for (const SDValue &Op : N->op_values()) {
1078 if (Op.isUndef())
1079 continue;
1080 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val: Op);
1081 if (!Const || (Const->isOpaque() && NoOpaques))
1082 return false;
1083 // When AllowTruncation is true, allow constants that have been promoted
1084 // during type legalization as long as the value fits in the target type.
1085 if ((AllowTruncation &&
1086 Const->getAPIntValue().getActiveBits() > BitWidth) ||
1087 (!AllowTruncation && Const->getAPIntValue().getBitWidth() != BitWidth))
1088 return false;
1089 }
1090 return true;
1091}
1092
1093// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
1094// undef's.
1095static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
1096 if (V.getOpcode() != ISD::BUILD_VECTOR)
1097 return false;
1098 return isConstantOrConstantVector(N: V, NoOpaques) ||
1099 ISD::isBuildVectorOfConstantFPSDNodes(N: V.getNode());
1100}
1101
1102// Determine if this an indexed load with an opaque target constant index.
1103static bool canSplitIdx(LoadSDNode *LD) {
1104 return MaySplitLoadIndex &&
1105 (LD->getOperand(Num: 2).getOpcode() != ISD::TargetConstant ||
1106 !cast<ConstantSDNode>(Val: LD->getOperand(Num: 2))->isOpaque());
1107}
1108
1109bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
1110 const SDLoc &DL,
1111 SDNode *N,
1112 SDValue N0,
1113 SDValue N1) {
1114 // Currently this only tries to ensure we don't undo the GEP splits done by
1115 // CodeGenPrepare when shouldConsiderGEPOffsetSplit is true. To ensure this,
1116 // we check if the following transformation would be problematic:
1117 // (load/store (add, (add, x, offset1), offset2)) ->
1118 // (load/store (add, x, offset1+offset2)).
1119
1120 // (load/store (add, (add, x, y), offset2)) ->
1121 // (load/store (add, (add, x, offset2), y)).
1122
1123 if (!N0.isAnyAdd())
1124 return false;
1125
1126 // Check for vscale addressing modes.
1127 // (load/store (add/sub (add x, y), vscale))
1128 // (load/store (add/sub (add x, y), (lsl vscale, C)))
1129 // (load/store (add/sub (add x, y), (mul vscale, C)))
1130 if ((N1.getOpcode() == ISD::VSCALE ||
1131 ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::MUL) &&
1132 N1.getOperand(i: 0).getOpcode() == ISD::VSCALE &&
1133 isa<ConstantSDNode>(Val: N1.getOperand(i: 1)))) &&
1134 N1.getValueType().getFixedSizeInBits() <= 64) {
1135 int64_t ScalableOffset = N1.getOpcode() == ISD::VSCALE
1136 ? N1.getConstantOperandVal(i: 0)
1137 : (N1.getOperand(i: 0).getConstantOperandVal(i: 0) *
1138 (N1.getOpcode() == ISD::SHL
1139 ? (1LL << N1.getConstantOperandVal(i: 1))
1140 : N1.getConstantOperandVal(i: 1)));
1141 if (Opc == ISD::SUB)
1142 ScalableOffset = -ScalableOffset;
1143 if (all_of(Range: N->users(), P: [&](SDNode *Node) {
1144 if (auto *LoadStore = dyn_cast<MemSDNode>(Val: Node);
1145 LoadStore && LoadStore->hasUniqueMemOperand() &&
1146 LoadStore->getBasePtr().getNode() == N) {
1147 TargetLoweringBase::AddrMode AM;
1148 AM.HasBaseReg = true;
1149 AM.ScalableOffset = ScalableOffset;
1150 EVT VT = LoadStore->getMemoryVT();
1151 unsigned AS = LoadStore->getAddressSpace();
1152 Type *AccessTy = VT.getTypeForEVT(Context&: *DAG.getContext());
1153 return TLI.isLegalAddressingMode(DL: DAG.getDataLayout(), AM, Ty: AccessTy,
1154 AddrSpace: AS);
1155 }
1156 return false;
1157 }))
1158 return true;
1159 }
1160
1161 if (Opc != ISD::ADD && Opc != ISD::PTRADD)
1162 return false;
1163
1164 auto *C2 = dyn_cast<ConstantSDNode>(Val&: N1);
1165 if (!C2)
1166 return false;
1167
1168 const APInt &C2APIntVal = C2->getAPIntValue();
1169 if (C2APIntVal.getSignificantBits() > 64)
1170 return false;
1171
1172 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
1173 if (N0.hasOneUse())
1174 return false;
1175
1176 const APInt &C1APIntVal = C1->getAPIntValue();
1177 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1178 if (CombinedValueIntVal.getSignificantBits() > 64)
1179 return false;
1180 const int64_t CombinedValue = CombinedValueIntVal.getSExtValue();
1181
1182 for (SDNode *Node : N->users()) {
1183 if (auto *LoadStore = dyn_cast<MemSDNode>(Val: Node)) {
1184 if (!LoadStore->hasUniqueMemOperand())
1185 continue;
1186 // Is x[offset2] already not a legal addressing mode? If so then
1187 // reassociating the constants breaks nothing (we test offset2 because
1188 // that's the one we hope to fold into the load or store).
1189 TargetLoweringBase::AddrMode AM;
1190 AM.HasBaseReg = true;
1191 AM.BaseOffs = C2APIntVal.getSExtValue();
1192 EVT VT = LoadStore->getMemoryVT();
1193 unsigned AS = LoadStore->getAddressSpace();
1194 Type *AccessTy = VT.getTypeForEVT(Context&: *DAG.getContext());
1195 if (!TLI.isLegalAddressingMode(DL: DAG.getDataLayout(), AM, Ty: AccessTy, AddrSpace: AS))
1196 continue;
1197
1198 // Would x[offset1+offset2] still be a legal addressing mode?
1199 AM.BaseOffs = CombinedValue;
1200 if (!TLI.isLegalAddressingMode(DL: DAG.getDataLayout(), AM, Ty: AccessTy, AddrSpace: AS))
1201 return true;
1202 }
1203 }
1204 } else {
1205 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Val: N0.getOperand(i: 1)))
1206 if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA))
1207 return false;
1208
1209 for (SDNode *Node : N->users()) {
1210 auto *LoadStore = dyn_cast<MemSDNode>(Val: Node);
1211 if (!LoadStore || !LoadStore->hasUniqueMemOperand())
1212 return false;
1213
1214 // Is x[offset2] a legal addressing mode? If so then
1215 // reassociating the constants breaks address pattern
1216 TargetLoweringBase::AddrMode AM;
1217 AM.HasBaseReg = true;
1218 AM.BaseOffs = C2APIntVal.getSExtValue();
1219 EVT VT = LoadStore->getMemoryVT();
1220 unsigned AS = LoadStore->getAddressSpace();
1221 Type *AccessTy = VT.getTypeForEVT(Context&: *DAG.getContext());
1222 if (!TLI.isLegalAddressingMode(DL: DAG.getDataLayout(), AM, Ty: AccessTy, AddrSpace: AS))
1223 return false;
1224 }
1225 return true;
1226 }
1227
1228 return false;
1229}
1230
1231/// Helper for DAGCombiner::reassociateOps. Try to reassociate (Opc N0, N1) if
1232/// \p N0 is the same kind of operation as \p Opc.
1233SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
1234 SDValue N0, SDValue N1,
1235 SDNodeFlags Flags) {
1236 EVT VT = N0.getValueType();
1237
1238 if (N0.getOpcode() != Opc)
1239 return SDValue();
1240
1241 SDValue N00 = N0.getOperand(i: 0);
1242 SDValue N01 = N0.getOperand(i: 1);
1243
1244 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N01)) {
1245 SDNodeFlags NewFlags;
1246 if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() &&
1247 Flags.hasNoUnsignedWrap())
1248 NewFlags |= SDNodeFlags::NoUnsignedWrap;
1249
1250 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N1)) {
1251 // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
1252 if (SDValue OpNode = DAG.FoldConstantArithmetic(Opcode: Opc, DL, VT, Ops: {N01, N1})) {
1253 NewFlags.setDisjoint(Flags.hasDisjoint() &&
1254 N0->getFlags().hasDisjoint());
1255 return DAG.getNode(Opcode: Opc, DL, VT, N1: N00, N2: OpNode, Flags: NewFlags);
1256 }
1257 return SDValue();
1258 }
1259 if (TLI.isReassocProfitable(DAG, N0, N1)) {
1260 // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
1261 // iff (op x, c1) has one use
1262 SDValue OpNode = DAG.getNode(Opcode: Opc, DL: SDLoc(N0), VT, N1: N00, N2: N1, Flags: NewFlags);
1263 return DAG.getNode(Opcode: Opc, DL, VT, N1: OpNode, N2: N01, Flags: NewFlags);
1264 }
1265 }
1266
1267 // Check for repeated operand logic simplifications.
1268 if (Opc == ISD::AND || Opc == ISD::OR) {
1269 // (N00 & N01) & N00 --> N00 & N01
1270 // (N00 & N01) & N01 --> N00 & N01
1271 // (N00 | N01) | N00 --> N00 | N01
1272 // (N00 | N01) | N01 --> N00 | N01
1273 if (N1 == N00 || N1 == N01)
1274 return N0;
1275 }
1276 if (Opc == ISD::XOR) {
1277 // (N00 ^ N01) ^ N00 --> N01
1278 if (N1 == N00)
1279 return N01;
1280 // (N00 ^ N01) ^ N01 --> N00
1281 if (N1 == N01)
1282 return N00;
1283 }
1284
1285 if (TLI.isReassocProfitable(DAG, N0, N1)) {
1286 if (N1 != N01) {
1287 // Reassociate if (op N00, N1) already exist
1288 if (SDNode *NE = DAG.getNodeIfExists(Opcode: Opc, VTList: DAG.getVTList(VT), Ops: {N00, N1})) {
1289 // if Op (Op N00, N1), N01 already exist
1290 // we need to stop reassciate to avoid dead loop
1291 if (!DAG.doesNodeExist(Opcode: Opc, VTList: DAG.getVTList(VT), Ops: {SDValue(NE, 0), N01}))
1292 return DAG.getNode(Opcode: Opc, DL, VT, N1: SDValue(NE, 0), N2: N01);
1293 }
1294 }
1295
1296 if (N1 != N00) {
1297 // Reassociate if (op N01, N1) already exist
1298 if (SDNode *NE = DAG.getNodeIfExists(Opcode: Opc, VTList: DAG.getVTList(VT), Ops: {N01, N1})) {
1299 // if Op (Op N01, N1), N00 already exist
1300 // we need to stop reassciate to avoid dead loop
1301 if (!DAG.doesNodeExist(Opcode: Opc, VTList: DAG.getVTList(VT), Ops: {SDValue(NE, 0), N00}))
1302 return DAG.getNode(Opcode: Opc, DL, VT, N1: SDValue(NE, 0), N2: N00);
1303 }
1304 }
1305
1306 // Reassociate the operands from (OR/AND (OR/AND(N00, N001)), N1) to (OR/AND
1307 // (OR/AND(N00, N1)), N01) when N00 and N1 are comparisons with the same
1308 // predicate or to (OR/AND (OR/AND(N1, N01)), N00) when N01 and N1 are
1309 // comparisons with the same predicate. This enables optimizations as the
1310 // following one:
1311 // CMP(A,C)||CMP(B,C) => CMP(MIN/MAX(A,B), C)
1312 // CMP(A,C)&&CMP(B,C) => CMP(MIN/MAX(A,B), C)
1313 if (Opc == ISD::AND || Opc == ISD::OR) {
1314 if (N1->getOpcode() == ISD::SETCC && N00->getOpcode() == ISD::SETCC &&
1315 N01->getOpcode() == ISD::SETCC) {
1316 ISD::CondCode CC1 = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
1317 ISD::CondCode CC00 = cast<CondCodeSDNode>(Val: N00.getOperand(i: 2))->get();
1318 ISD::CondCode CC01 = cast<CondCodeSDNode>(Val: N01.getOperand(i: 2))->get();
1319 if (CC1 == CC00 && CC1 != CC01) {
1320 SDValue OpNode = DAG.getNode(Opcode: Opc, DL: SDLoc(N0), VT, N1: N00, N2: N1, Flags);
1321 return DAG.getNode(Opcode: Opc, DL, VT, N1: OpNode, N2: N01, Flags);
1322 }
1323 if (CC1 == CC01 && CC1 != CC00) {
1324 SDValue OpNode = DAG.getNode(Opcode: Opc, DL: SDLoc(N0), VT, N1: N01, N2: N1, Flags);
1325 return DAG.getNode(Opcode: Opc, DL, VT, N1: OpNode, N2: N00, Flags);
1326 }
1327 }
1328 }
1329 }
1330
1331 return SDValue();
1332}
1333
1334/// Try to reassociate commutative (Opc N0, N1) if either \p N0 or \p N1 is the
1335/// same kind of operation as \p Opc.
1336SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
1337 SDValue N1, SDNodeFlags Flags) {
1338 assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative.");
1339
1340 // Floating-point reassociation is not allowed without loose FP math.
1341 if (N0.getValueType().isFloatingPoint() ||
1342 N1.getValueType().isFloatingPoint())
1343 if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros())
1344 return SDValue();
1345
1346 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1, Flags))
1347 return Combined;
1348 if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0: N1, N1: N0, Flags))
1349 return Combined;
1350 return SDValue();
1351}
1352
1353// Try to fold Opc(vecreduce(x), vecreduce(y)) -> vecreduce(Opc(x, y))
1354// Note that we only expect Flags to be passed from FP operations. For integer
1355// operations they need to be dropped.
1356SDValue DAGCombiner::reassociateReduction(unsigned RedOpc, unsigned Opc,
1357 const SDLoc &DL, EVT VT, SDValue N0,
1358 SDValue N1, SDNodeFlags Flags) {
1359 if (N0.getOpcode() == RedOpc && N1.getOpcode() == RedOpc &&
1360 N0.getOperand(i: 0).getValueType() == N1.getOperand(i: 0).getValueType() &&
1361 N0->hasOneUse() && N1->hasOneUse() &&
1362 TLI.isOperationLegalOrCustom(Op: Opc, VT: N0.getOperand(i: 0).getValueType()) &&
1363 TLI.shouldReassociateReduction(RedOpc, VT: N0.getOperand(i: 0).getValueType())) {
1364 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
1365 return DAG.getNode(Opcode: RedOpc, DL, VT,
1366 Operand: DAG.getNode(Opcode: Opc, DL, VT: N0.getOperand(i: 0).getValueType(),
1367 N1: N0.getOperand(i: 0), N2: N1.getOperand(i: 0)));
1368 }
1369
1370 // Reassociate op(op(vecreduce(a), b), op(vecreduce(c), d)) into
1371 // op(vecreduce(op(a, c)), op(b, d)), to combine the reductions into a
1372 // single node.
1373 SDValue A, B, C, D, RedA, RedB;
1374 if (sd_match(N: N0,
1375 P: m_OneUse(P: m_c_BinOp(
1376 Opc, L: m_Value(N&: RedA, P: m_OneUse(P: m_UnaryOp(Opc: RedOpc, Op: m_Value(N&: A)))),
1377 R: m_Value(N&: B, P: m_Unless(P: m_UnaryOp(Opc: RedOpc, Op: m_Value())))))) &&
1378 sd_match(N: N1,
1379 P: m_OneUse(P: m_c_BinOp(
1380 Opc, L: m_Value(N&: RedB, P: m_OneUse(P: m_UnaryOp(Opc: RedOpc, Op: m_Value(N&: C)))),
1381 R: m_Value(N&: D, P: m_Unless(P: m_UnaryOp(Opc: RedOpc, Op: m_Value())))))) &&
1382 A.getValueType() == C.getValueType() &&
1383 hasOperation(Opcode: Opc, VT: A.getValueType()) &&
1384 TLI.shouldReassociateReduction(RedOpc, VT)) {
1385 if ((Opc == ISD::FADD || Opc == ISD::FMUL) &&
1386 (!N0->getFlags().hasAllowReassociation() ||
1387 !N1->getFlags().hasAllowReassociation() ||
1388 !RedA->getFlags().hasAllowReassociation() ||
1389 !RedB->getFlags().hasAllowReassociation()))
1390 return SDValue();
1391 SelectionDAG::FlagInserter FlagsInserter(
1392 DAG, Flags & N0->getFlags() & N1->getFlags() & RedA->getFlags() &
1393 RedB->getFlags());
1394 SDValue Op = DAG.getNode(Opcode: Opc, DL, VT: A.getValueType(), N1: A, N2: C);
1395 SDValue Red = DAG.getNode(Opcode: RedOpc, DL, VT, Operand: Op);
1396 SDValue Op2 = DAG.getNode(Opcode: Opc, DL, VT, N1: B, N2: D);
1397 return DAG.getNode(Opcode: Opc, DL, VT, N1: Red, N2: Op2);
1398 }
1399
1400 // Reassociate a reduction chain so two reductions become adjacent and the
1401 // folds above can merge them:
1402 // op(vecreduce(X), op(vecreduce(Y), Z))
1403 // -> op(vecreduce(op(X, Y)), Z)
1404 // Applied to fixpoint by the combiner worklist, this collapses an
1405 // arbitrarily long chain of reductions (such as the left-leaning chain SLP
1406 // emits) into a single reduction.
1407 auto FoldReductionChain = [&](SDValue Red0, SDValue Chain) -> SDValue {
1408 SDValue X, Y, Z, RedY;
1409 if (!sd_match(N: Red0, P: m_OneUse(P: m_UnaryOp(Opc: RedOpc, Op: m_Value(N&: X)))) ||
1410 !sd_match(
1411 N: Chain,
1412 P: m_OneUse(P: m_c_BinOp(
1413 Opc, L: m_Value(N&: RedY, P: m_OneUse(P: m_UnaryOp(Opc: RedOpc, Op: m_Value(N&: Y)))),
1414 R: m_Value(N&: Z, P: m_Unless(P: m_UnaryOp(Opc: RedOpc, Op: m_Value())))))) ||
1415 X.getValueType() != Y.getValueType() ||
1416 !hasOperation(Opcode: Opc, VT: X.getValueType()) ||
1417 !TLI.shouldReassociateReduction(RedOpc, VT))
1418 return SDValue();
1419 if ((Opc == ISD::FADD || Opc == ISD::FMUL) &&
1420 (!Chain->getFlags().hasAllowReassociation() ||
1421 !Red0->getFlags().hasAllowReassociation() ||
1422 !RedY->getFlags().hasAllowReassociation()))
1423 return SDValue();
1424 SelectionDAG::FlagInserter FlagsInserter(
1425 DAG, Flags & Chain->getFlags() & Red0->getFlags() & RedY->getFlags());
1426 SDValue Op = DAG.getNode(Opcode: Opc, DL, VT: X.getValueType(), N1: X, N2: Y);
1427 SDValue Red = DAG.getNode(Opcode: RedOpc, DL, VT, Operand: Op);
1428 return DAG.getNode(Opcode: Opc, DL, VT, N1: Red, N2: Z);
1429 };
1430 if (SDValue V = FoldReductionChain(N0, N1))
1431 return V;
1432 if (SDValue V = FoldReductionChain(N1, N0))
1433 return V;
1434
1435 return SDValue();
1436}
1437
1438SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
1439 bool AddTo) {
1440 assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
1441 ++NodesCombined;
1442 LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
1443 To[0].dump(&DAG);
1444 dbgs() << " and " << NumTo - 1 << " other values\n");
1445 for (unsigned i = 0, e = NumTo; i != e; ++i)
1446 assert((!To[i].getNode() ||
1447 N->getValueType(i) == To[i].getValueType()) &&
1448 "Cannot combine value to value of different type!");
1449
1450 WorklistRemover DeadNodes(*this);
1451 DAG.ReplaceAllUsesWith(From: N, To);
1452 if (AddTo) {
1453 // Push the new nodes and any users onto the worklist
1454 for (unsigned i = 0, e = NumTo; i != e; ++i) {
1455 if (To[i].getNode())
1456 AddToWorklistWithUsers(N: To[i].getNode());
1457 }
1458 }
1459
1460 // Finally, if the node is now dead, remove it from the graph. The node
1461 // may not be dead if the replacement process recursively simplified to
1462 // something else needing this node.
1463 if (N->use_empty())
1464 deleteAndRecombine(N);
1465 return SDValue(N, 0);
1466}
1467
1468void DAGCombiner::
1469CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
1470 // Replace the old value with the new one.
1471 ++NodesCombined;
1472 LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.dump(&DAG);
1473 dbgs() << "\nWith: "; TLO.New.dump(&DAG); dbgs() << '\n');
1474
1475 // Replace all uses.
1476 DAG.ReplaceAllUsesOfValueWith(From: TLO.Old, To: TLO.New);
1477
1478 // Push the new node and any (possibly new) users onto the worklist.
1479 AddToWorklistWithUsers(N: TLO.New.getNode());
1480
1481 // Finally, if the node is now dead, remove it from the graph.
1482 recursivelyDeleteUnusedNodes(N: TLO.Old.getNode());
1483}
1484
1485/// Check the specified integer node value to see if it can be simplified or if
1486/// things it uses can be simplified by bit propagation. If so, return true.
1487bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
1488 const APInt &DemandedElts,
1489 bool AssumeSingleUse) {
1490 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1491 KnownBits Known;
1492 if (!TLI.SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth: 0,
1493 AssumeSingleUse))
1494 return false;
1495
1496 // Revisit the node.
1497 AddToWorklist(N: Op.getNode());
1498
1499 CommitTargetLoweringOpt(TLO);
1500 return true;
1501}
1502
1503/// Check the specified vector node value to see if it can be simplified or
1504/// if things it uses can be simplified as it only uses some of the elements.
1505/// If so, return true.
1506bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
1507 const APInt &DemandedElts,
1508 bool AssumeSingleUse) {
1509 TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
1510 APInt KnownUndef, KnownZero;
1511 if (!TLI.SimplifyDemandedVectorElts(Op, DemandedEltMask: DemandedElts, KnownUndef, KnownZero,
1512 TLO, Depth: 0, AssumeSingleUse))
1513 return false;
1514
1515 // Revisit the node.
1516 AddToWorklist(N: Op.getNode());
1517
1518 CommitTargetLoweringOpt(TLO);
1519 return true;
1520}
1521
1522void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
1523 SDLoc DL(Load);
1524 EVT VT = Load->getValueType(ResNo: 0);
1525 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: SDValue(ExtLoad, 0));
1526
1527 LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
1528 Trunc.dump(&DAG); dbgs() << '\n');
1529
1530 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 0), To: Trunc);
1531 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: SDValue(ExtLoad, 1));
1532
1533 AddToWorklist(N: Trunc.getNode());
1534 recursivelyDeleteUnusedNodes(N: Load);
1535}
1536
1537SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
1538 Replace = false;
1539 SDLoc DL(Op);
1540 if (ISD::isUNINDEXEDLoad(N: Op.getNode())) {
1541 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
1542 EVT MemVT = LD->getMemoryVT();
1543 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(N: LD) ? ISD::EXTLOAD
1544 : LD->getExtensionType();
1545 Replace = true;
1546 return DAG.getExtLoad(ExtType, dl: DL, VT: PVT,
1547 Chain: LD->getChain(), Ptr: LD->getBasePtr(),
1548 MemVT, MMO: LD->getMemOperand());
1549 }
1550
1551 unsigned Opc = Op.getOpcode();
1552 switch (Opc) {
1553 default: break;
1554 case ISD::AssertSext:
1555 if (SDValue Op0 = SExtPromoteOperand(Op: Op.getOperand(i: 0), PVT))
1556 return DAG.getNode(Opcode: ISD::AssertSext, DL, VT: PVT, N1: Op0, N2: Op.getOperand(i: 1));
1557 break;
1558 case ISD::AssertZext:
1559 if (SDValue Op0 = ZExtPromoteOperand(Op: Op.getOperand(i: 0), PVT))
1560 return DAG.getNode(Opcode: ISD::AssertZext, DL, VT: PVT, N1: Op0, N2: Op.getOperand(i: 1));
1561 break;
1562 case ISD::Constant: {
1563 unsigned ExtOpc =
1564 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1565 return DAG.getNode(Opcode: ExtOpc, DL, VT: PVT, Operand: Op);
1566 }
1567 }
1568
1569 if (!TLI.isOperationLegal(Op: ISD::ANY_EXTEND, VT: PVT))
1570 return SDValue();
1571 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: PVT, Operand: Op);
1572}
1573
1574SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
1575 if (!TLI.isOperationLegal(Op: ISD::SIGN_EXTEND_INREG, VT: PVT))
1576 return SDValue();
1577 EVT OldVT = Op.getValueType();
1578 SDLoc DL(Op);
1579 bool Replace = false;
1580 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1581 if (!NewOp.getNode())
1582 return SDValue();
1583 AddToWorklist(N: NewOp.getNode());
1584
1585 if (Replace)
1586 ReplaceLoadWithPromotedLoad(Load: Op.getNode(), ExtLoad: NewOp.getNode());
1587 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: NewOp.getValueType(), N1: NewOp,
1588 N2: DAG.getValueType(OldVT));
1589}
1590
1591SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
1592 EVT OldVT = Op.getValueType();
1593 SDLoc DL(Op);
1594 bool Replace = false;
1595 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1596 if (!NewOp.getNode())
1597 return SDValue();
1598 AddToWorklist(N: NewOp.getNode());
1599
1600 if (Replace)
1601 ReplaceLoadWithPromotedLoad(Load: Op.getNode(), ExtLoad: NewOp.getNode());
1602 return DAG.getZeroExtendInReg(Op: NewOp, DL, VT: OldVT);
1603}
1604
1605/// Promote the specified integer binary operation if the target indicates it is
1606/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1607/// i32 since i16 instructions are longer.
1608SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
1609 if (!LegalOperations)
1610 return SDValue();
1611
1612 EVT VT = Op.getValueType();
1613 if (VT.isVector() || !VT.isInteger())
1614 return SDValue();
1615
1616 // If operation type is 'undesirable', e.g. i16 on x86, consider
1617 // promoting it.
1618 unsigned Opc = Op.getOpcode();
1619 if (TLI.isTypeDesirableForOp(Opc, VT))
1620 return SDValue();
1621
1622 EVT PVT = VT;
1623 // Consult target whether it is a good idea to promote this operation and
1624 // what's the right type to promote it to.
1625 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1626 assert(PVT != VT && "Don't know what type to promote to!");
1627
1628 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
1629
1630 bool Replace0 = false;
1631 SDValue N0 = Op.getOperand(i: 0);
1632 SDValue NN0 = PromoteOperand(Op: N0, PVT, Replace&: Replace0);
1633
1634 bool Replace1 = false;
1635 SDValue N1 = Op.getOperand(i: 1);
1636 SDValue NN1 = PromoteOperand(Op: N1, PVT, Replace&: Replace1);
1637 SDLoc DL(Op);
1638
1639 SDValue RV =
1640 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: DAG.getNode(Opcode: Opc, DL, VT: PVT, N1: NN0, N2: NN1));
1641
1642 // We are always replacing N0/N1's use in N and only need additional
1643 // replacements if there are additional uses.
1644 // Note: We are checking uses of the *nodes* (SDNode) rather than values
1645 // (SDValue) here because the node may reference multiple values
1646 // (for example, the chain value of a load node).
1647 Replace0 &= !N0->hasOneUse();
1648 Replace1 &= (N0 != N1) && !N1->hasOneUse();
1649
1650 // Combine Op here so it is preserved past replacements.
1651 CombineTo(N: Op.getNode(), Res: RV);
1652
1653 // If operands have a use ordering, make sure we deal with
1654 // predecessor first.
1655 if (Replace0 && Replace1 && N0->isPredecessorOf(N: N1.getNode())) {
1656 std::swap(a&: N0, b&: N1);
1657 std::swap(a&: NN0, b&: NN1);
1658 }
1659
1660 if (Replace0) {
1661 AddToWorklist(N: NN0.getNode());
1662 ReplaceLoadWithPromotedLoad(Load: N0.getNode(), ExtLoad: NN0.getNode());
1663 }
1664 if (Replace1) {
1665 AddToWorklist(N: NN1.getNode());
1666 ReplaceLoadWithPromotedLoad(Load: N1.getNode(), ExtLoad: NN1.getNode());
1667 }
1668 return Op;
1669 }
1670 return SDValue();
1671}
1672
1673/// Promote the specified integer shift operation if the target indicates it is
1674/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
1675/// i32 since i16 instructions are longer.
1676SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
1677 if (!LegalOperations)
1678 return SDValue();
1679
1680 EVT VT = Op.getValueType();
1681 if (VT.isVector() || !VT.isInteger())
1682 return SDValue();
1683
1684 // If operation type is 'undesirable', e.g. i16 on x86, consider
1685 // promoting it.
1686 unsigned Opc = Op.getOpcode();
1687 if (TLI.isTypeDesirableForOp(Opc, VT))
1688 return SDValue();
1689
1690 EVT PVT = VT;
1691 // Consult target whether it is a good idea to promote this operation and
1692 // what's the right type to promote it to.
1693 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1694 assert(PVT != VT && "Don't know what type to promote to!");
1695
1696 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
1697
1698 SDNodeFlags TruncFlags;
1699 bool Replace = false;
1700 SDValue N0 = Op.getOperand(i: 0);
1701 if (Opc == ISD::SRA) {
1702 N0 = SExtPromoteOperand(Op: N0, PVT);
1703 } else if (Opc == ISD::SRL) {
1704 N0 = ZExtPromoteOperand(Op: N0, PVT);
1705 } else {
1706 if (Op->getFlags().hasNoUnsignedWrap()) {
1707 N0 = ZExtPromoteOperand(Op: N0, PVT);
1708 TruncFlags = SDNodeFlags::NoUnsignedWrap;
1709 } else if (Op->getFlags().hasNoSignedWrap()) {
1710 N0 = SExtPromoteOperand(Op: N0, PVT);
1711 TruncFlags = SDNodeFlags::NoSignedWrap;
1712 } else {
1713 N0 = PromoteOperand(Op: N0, PVT, Replace);
1714 }
1715 }
1716
1717 if (!N0.getNode())
1718 return SDValue();
1719
1720 SDLoc DL(Op);
1721 SDValue N1 = Op.getOperand(i: 1);
1722 SDValue RV = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT,
1723 Operand: DAG.getNode(Opcode: Opc, DL, VT: PVT, N1: N0, N2: N1), Flags: TruncFlags);
1724
1725 if (Replace)
1726 ReplaceLoadWithPromotedLoad(Load: Op.getOperand(i: 0).getNode(), ExtLoad: N0.getNode());
1727
1728 // Deal with Op being deleted.
1729 if (Op && Op.getOpcode() != ISD::DELETED_NODE)
1730 return RV;
1731 }
1732 return SDValue();
1733}
1734
1735SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1736 if (!LegalOperations)
1737 return SDValue();
1738
1739 EVT VT = Op.getValueType();
1740 if (VT.isVector() || !VT.isInteger())
1741 return SDValue();
1742
1743 // If operation type is 'undesirable', e.g. i16 on x86, consider
1744 // promoting it.
1745 unsigned Opc = Op.getOpcode();
1746 if (TLI.isTypeDesirableForOp(Opc, VT))
1747 return SDValue();
1748
1749 EVT PVT = VT;
1750 // Consult target whether it is a good idea to promote this operation and
1751 // what's the right type to promote it to.
1752 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1753 assert(PVT != VT && "Don't know what type to promote to!");
1754 // fold (aext (aext x)) -> (aext x)
1755 // fold (aext (zext x)) -> (zext x)
1756 // fold (aext (sext x)) -> (sext x)
1757 LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
1758 return DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT, Operand: Op.getOperand(i: 0));
1759 }
1760 return SDValue();
1761}
1762
1763bool DAGCombiner::PromoteLoad(SDValue Op) {
1764 if (!LegalOperations)
1765 return false;
1766
1767 if (!ISD::isUNINDEXEDLoad(N: Op.getNode()))
1768 return false;
1769
1770 EVT VT = Op.getValueType();
1771 if (VT.isVector() || !VT.isInteger())
1772 return false;
1773
1774 // If operation type is 'undesirable', e.g. i16 on x86, consider
1775 // promoting it.
1776 unsigned Opc = Op.getOpcode();
1777 if (TLI.isTypeDesirableForOp(Opc, VT))
1778 return false;
1779
1780 EVT PVT = VT;
1781 // Consult target whether it is a good idea to promote this operation and
1782 // what's the right type to promote it to.
1783 if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1784 assert(PVT != VT && "Don't know what type to promote to!");
1785
1786 SDLoc DL(Op);
1787 SDNode *N = Op.getNode();
1788 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1789 EVT MemVT = LD->getMemoryVT();
1790 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(N: LD) ? ISD::EXTLOAD
1791 : LD->getExtensionType();
1792 SDValue NewLD = DAG.getExtLoad(ExtType, dl: DL, VT: PVT,
1793 Chain: LD->getChain(), Ptr: LD->getBasePtr(),
1794 MemVT, MMO: LD->getMemOperand());
1795 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: NewLD);
1796
1797 LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
1798 Result.dump(&DAG); dbgs() << '\n');
1799
1800 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
1801 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: NewLD.getValue(R: 1));
1802
1803 AddToWorklist(N: Result.getNode());
1804 recursivelyDeleteUnusedNodes(N);
1805 return true;
1806 }
1807
1808 return false;
1809}
1810
1811/// Recursively delete a node which has no uses and any operands for
1812/// which it is the only use.
1813///
1814/// Note that this both deletes the nodes and removes them from the worklist.
1815/// It also adds any nodes who have had a user deleted to the worklist as they
1816/// may now have only one use and subject to other combines.
1817bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
1818 if (!N->use_empty())
1819 return false;
1820
1821 SmallSetVector<SDNode *, 16> Nodes;
1822 Nodes.insert(X: N);
1823 do {
1824 N = Nodes.pop_back_val();
1825 if (!N)
1826 continue;
1827
1828 if (N->use_empty()) {
1829 for (const SDValue &ChildN : N->op_values())
1830 Nodes.insert(X: ChildN.getNode());
1831
1832 removeFromWorklist(N);
1833 DAG.DeleteNode(N);
1834 } else {
1835 AddToWorklist(N);
1836 }
1837 } while (!Nodes.empty());
1838 return true;
1839}
1840
1841//===----------------------------------------------------------------------===//
1842// Main DAG Combiner implementation
1843//===----------------------------------------------------------------------===//
1844
1845void DAGCombiner::Run(CombineLevel AtLevel) {
1846 // set the instance variables, so that the various visit routines may use it.
1847 Level = AtLevel;
1848 LegalDAG = Level >= AfterLegalizeDAG;
1849 LegalOperations = Level >= AfterLegalizeVectorOps;
1850 LegalTypes = Level >= AfterLegalizeTypes;
1851
1852 bool UseTopologicalSorting = EnableTopologicalSorting.getNumOccurrences() > 0
1853 ? EnableTopologicalSorting
1854 : TLI.useTopologicalSorting();
1855
1856 WorklistInserter AddNodes(*this);
1857
1858 if (UseTopologicalSorting)
1859 DAG.AssignTopologicalOrder();
1860
1861 // Add all the dag nodes to the worklist.
1862 //
1863 // Note: All nodes are not added to PruningList here, this is because the only
1864 // nodes which can be deleted are those which have no uses and all other nodes
1865 // which would otherwise be added to the worklist by the first call to
1866 // getNextWorklistEntry are already present in it.
1867 if (UseTopologicalSorting) {
1868 for (SDNode &Node : reverse(C: DAG.allnodes()))
1869 AddToWorklist(N: &Node, /* IsCandidateForPruning */ Node.use_empty());
1870 } else {
1871 for (SDNode &Node : DAG.allnodes())
1872 AddToWorklist(N: &Node, /* IsCandidateForPruning */ Node.use_empty());
1873 }
1874
1875 // Create a dummy node (which is not added to allnodes), that adds a reference
1876 // to the root node, preventing it from being deleted, and tracking any
1877 // changes of the root.
1878 HandleSDNode Dummy(DAG.getRoot());
1879
1880 // While we have a valid worklist entry node, try to combine it.
1881 while (SDNode *N = getNextWorklistEntry()) {
1882 // If N has no uses, it is dead. Make sure to revisit all N's operands once
1883 // N is deleted from the DAG, since they too may now be dead or may have a
1884 // reduced number of uses, allowing other xforms.
1885 if (recursivelyDeleteUnusedNodes(N))
1886 continue;
1887
1888 WorklistRemover DeadNodes(*this);
1889
1890 // If this combine is running after legalizing the DAG, re-legalize any
1891 // nodes pulled off the worklist.
1892 if (LegalDAG) {
1893 SmallSetVector<SDNode *, 16> UpdatedNodes;
1894 bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
1895
1896 for (SDNode *LN : UpdatedNodes)
1897 AddToWorklistWithUsers(N: LN);
1898
1899 if (!NIsValid)
1900 continue;
1901 }
1902
1903 LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
1904
1905 // Add any operands of the new node which have not yet been combined to the
1906 // worklist as well. getNextWorklistEntry flags nodes that have been
1907 // combined before. Because the worklist uniques things already, this won't
1908 // repeatedly process the same operand.
1909 for (const SDValue &ChildN : N->op_values())
1910 AddToWorklist(N: ChildN.getNode(), /*IsCandidateForPruning=*/true,
1911 /*SkipIfCombinedBefore=*/true);
1912
1913 SDValue RV = combine(N);
1914
1915 if (!RV.getNode())
1916 continue;
1917
1918 ++NodesCombined;
1919
1920 // Invalidate cached info.
1921 ChainsWithoutMergeableStores.clear();
1922
1923 // If we get back the same node we passed in, rather than a new node or
1924 // zero, we know that the node must have defined multiple values and
1925 // CombineTo was used. Since CombineTo takes care of the worklist
1926 // mechanics for us, we have no work to do in this case.
1927 if (RV.getNode() == N)
1928 continue;
1929
1930 assert(N->getOpcode() != ISD::DELETED_NODE &&
1931 RV.getOpcode() != ISD::DELETED_NODE &&
1932 "Node was deleted but visit returned new node!");
1933
1934 LLVM_DEBUG(dbgs() << " ... into: "; RV.dump(&DAG));
1935
1936 if (N->getNumValues() == RV->getNumValues())
1937 DAG.ReplaceAllUsesWith(From: N, To: RV.getNode());
1938 else {
1939 assert(N->getValueType(0) == RV.getValueType() &&
1940 N->getNumValues() == 1 && "Type mismatch");
1941 DAG.ReplaceAllUsesWith(From: N, To: &RV);
1942 }
1943
1944 // Push the new node and any users onto the worklist. Omit this if the
1945 // new node is the EntryToken (e.g. if a store managed to get optimized
1946 // out), because re-visiting the EntryToken and its users will not uncover
1947 // any additional opportunities, but there may be a large number of such
1948 // users, potentially causing compile time explosion.
1949 if (RV.getOpcode() != ISD::EntryToken)
1950 AddToWorklistWithUsers(N: RV.getNode());
1951
1952 // Finally, if the node is now dead, remove it from the graph. The node
1953 // may not be dead if the replacement process recursively simplified to
1954 // something else needing this node. This will also take care of adding any
1955 // operands which have lost a user to the worklist.
1956 recursivelyDeleteUnusedNodes(N);
1957 }
1958
1959 // If the root changed (e.g. it was a dead load, update the root).
1960 DAG.setRoot(Dummy.getValue());
1961 DAG.RemoveDeadNodes();
1962}
1963
1964SDValue DAGCombiner::visit(SDNode *N) {
1965 // clang-format off
1966 switch (N->getOpcode()) {
1967 default: break;
1968 case ISD::TokenFactor: return visitTokenFactor(N);
1969 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N);
1970 case ISD::ADD: return visitADD(N);
1971 case ISD::PTRADD: return visitPTRADD(N);
1972 case ISD::SUB: return visitSUB(N);
1973 case ISD::SADDSAT:
1974 case ISD::UADDSAT: return visitADDSAT(N);
1975 case ISD::SSUBSAT:
1976 case ISD::USUBSAT: return visitSUBSAT(N);
1977 case ISD::ADDC: return visitADDC(N);
1978 case ISD::SADDO:
1979 case ISD::UADDO: return visitADDO(N);
1980 case ISD::SUBC: return visitSUBC(N);
1981 case ISD::SSUBO:
1982 case ISD::USUBO: return visitSUBO(N);
1983 case ISD::ADDE: return visitADDE(N);
1984 case ISD::UADDO_CARRY: return visitUADDO_CARRY(N);
1985 case ISD::SADDO_CARRY: return visitSADDO_CARRY(N);
1986 case ISD::SUBE: return visitSUBE(N);
1987 case ISD::USUBO_CARRY: return visitUSUBO_CARRY(N);
1988 case ISD::SSUBO_CARRY: return visitSSUBO_CARRY(N);
1989 case ISD::SMULFIX:
1990 case ISD::SMULFIXSAT:
1991 case ISD::UMULFIX:
1992 case ISD::UMULFIXSAT: return visitMULFIX(N);
1993 case ISD::MUL: return visitMUL(N);
1994 case ISD::SDIV: return visitSDIV(N);
1995 case ISD::UDIV: return visitUDIV(N);
1996 case ISD::SREM:
1997 case ISD::UREM: return visitREM(N);
1998 case ISD::MULHU: return visitMULHU(N);
1999 case ISD::MULHS: return visitMULHS(N);
2000 case ISD::AVGFLOORS:
2001 case ISD::AVGFLOORU:
2002 case ISD::AVGCEILS:
2003 case ISD::AVGCEILU: return visitAVG(N);
2004 case ISD::ABDS:
2005 case ISD::ABDU: return visitABD(N);
2006 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N);
2007 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N);
2008 case ISD::SMULO:
2009 case ISD::UMULO: return visitMULO(N);
2010 case ISD::SMIN:
2011 case ISD::SMAX:
2012 case ISD::UMIN:
2013 case ISD::UMAX: return visitIMINMAX(N);
2014 case ISD::AND: return visitAND(N);
2015 case ISD::OR: return visitOR(N);
2016 case ISD::XOR: return visitXOR(N);
2017 case ISD::SHL: return visitSHL(N);
2018 case ISD::SRA: return visitSRA(N);
2019 case ISD::SRL: return visitSRL(N);
2020 case ISD::ROTR:
2021 case ISD::ROTL: return visitRotate(N);
2022 case ISD::FSHL:
2023 case ISD::FSHR: return visitFunnelShift(N);
2024 case ISD::SSHLSAT:
2025 case ISD::USHLSAT: return visitSHLSAT(N);
2026 case ISD::ABS: return visitABS(N);
2027 case ISD::ABS_MIN_POISON: return visitABS_MIN_POISON(N);
2028 case ISD::CLMUL:
2029 case ISD::CLMULR:
2030 case ISD::CLMULH: return visitCLMUL(N);
2031 case ISD::PEXT: return visitPEXT(N);
2032 case ISD::PDEP: return visitPDEP(N);
2033 case ISD::BSWAP: return visitBSWAP(N);
2034 case ISD::BITREVERSE: return visitBITREVERSE(N);
2035 case ISD::CTLZ: return visitCTLZ(N);
2036 case ISD::CTLZ_ZERO_POISON: return visitCTLZ_ZERO_POISON(N);
2037 case ISD::CTTZ: return visitCTTZ(N);
2038 case ISD::CTTZ_ZERO_POISON: return visitCTTZ_ZERO_POISON(N);
2039 case ISD::CTPOP: return visitCTPOP(N);
2040 case ISD::PARITY: return visitPARITY(N);
2041 case ISD::SELECT: return visitSELECT(N);
2042 case ISD::VSELECT: return visitVSELECT(N);
2043 case ISD::SELECT_CC: return visitSELECT_CC(N);
2044 case ISD::SETCC: return visitSETCC(N);
2045 case ISD::SETCCCARRY: return visitSETCCCARRY(N);
2046 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N);
2047 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N);
2048 case ISD::ANY_EXTEND: return visitANY_EXTEND(N);
2049 case ISD::AssertSext:
2050 case ISD::AssertZext: return visitAssertExt(N);
2051 case ISD::AssertAlign: return visitAssertAlign(N);
2052 case ISD::IS_FPCLASS: return visitIS_FPCLASS(N);
2053 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N);
2054 case ISD::SIGN_EXTEND_VECTOR_INREG:
2055 case ISD::ZERO_EXTEND_VECTOR_INREG:
2056 case ISD::ANY_EXTEND_VECTOR_INREG: return visitEXTEND_VECTOR_INREG(N);
2057 case ISD::TRUNCATE: return visitTRUNCATE(N);
2058 case ISD::TRUNCATE_USAT_U: return visitTRUNCATE_USAT_U(N);
2059 case ISD::BITCAST: return visitBITCAST(N);
2060 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N);
2061 case ISD::FADD: return visitFADD(N);
2062 case ISD::STRICT_FADD: return visitSTRICT_FADD(N);
2063 case ISD::FSUB: return visitFSUB(N);
2064 case ISD::FMUL: return visitFMUL(N);
2065 case ISD::FMA: return visitFMA(N);
2066 case ISD::FMAD: return visitFMAD(N);
2067 case ISD::FMULADD: return visitFMULADD(N);
2068 case ISD::FDIV: return visitFDIV(N);
2069 case ISD::FREM: return visitFREM(N);
2070 case ISD::FSQRT: return visitFSQRT(N);
2071 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N);
2072 case ISD::FPOW: return visitFPOW(N);
2073 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N);
2074 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N);
2075 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N);
2076 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N);
2077 case ISD::LROUND:
2078 case ISD::LLROUND:
2079 case ISD::LRINT:
2080 case ISD::LLRINT: return visitXROUND(N);
2081 case ISD::FP_ROUND: return visitFP_ROUND(N);
2082 case ISD::FP_EXTEND: return visitFP_EXTEND(N);
2083 case ISD::FNEG: return visitFNEG(N);
2084 case ISD::FABS: return visitFABS(N);
2085 case ISD::FFLOOR: return visitFFLOOR(N);
2086 case ISD::FMINNUM:
2087 case ISD::FMAXNUM:
2088 case ISD::FMINIMUM:
2089 case ISD::FMAXIMUM:
2090 case ISD::FMINIMUMNUM:
2091 case ISD::FMAXIMUMNUM: return visitFMinMax(N);
2092 case ISD::FCEIL: return visitFCEIL(N);
2093 case ISD::FTRUNC: return visitFTRUNC(N);
2094 case ISD::FFREXP: return visitFFREXP(N);
2095 case ISD::BRCOND: return visitBRCOND(N);
2096 case ISD::BR_CC: return visitBR_CC(N);
2097 case ISD::LOAD: return visitLOAD(N);
2098 case ISD::STORE: return visitSTORE(N);
2099 case ISD::ATOMIC_STORE: return visitATOMIC_STORE(N);
2100 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
2101 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
2102 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
2103 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N);
2104 case ISD::VECTOR_INTERLEAVE: return visitVECTOR_INTERLEAVE(N);
2105 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N);
2106 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N);
2107 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N);
2108 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N);
2109 case ISD::MGATHER: return visitMGATHER(N);
2110 case ISD::MLOAD: return visitMLOAD(N);
2111 case ISD::MSCATTER: return visitMSCATTER(N);
2112 case ISD::MSTORE: return visitMSTORE(N);
2113 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: return visitMHISTOGRAM(N);
2114 case ISD::PARTIAL_REDUCE_SMLA:
2115 case ISD::PARTIAL_REDUCE_UMLA:
2116 case ISD::PARTIAL_REDUCE_SUMLA:
2117 case ISD::PARTIAL_REDUCE_FMLA:
2118 return visitPARTIAL_REDUCE_MLA(N);
2119 case ISD::LOOP_DEPENDENCE_RAW_MASK:
2120 case ISD::LOOP_DEPENDENCE_WAR_MASK:
2121 return visitLOOP_DEPENDENCE_MASK(N);
2122 case ISD::VECTOR_COMPRESS: return visitVECTOR_COMPRESS(N);
2123 case ISD::LIFETIME_END: return visitLIFETIME_END(N);
2124 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N);
2125 case ISD::FP16_TO_FP: return visitFP16_TO_FP(N);
2126 case ISD::FP_TO_BF16: return visitFP_TO_BF16(N);
2127 case ISD::BF16_TO_FP: return visitBF16_TO_FP(N);
2128 case ISD::FREEZE: return visitFREEZE(N);
2129 case ISD::GET_FPENV_MEM: return visitGET_FPENV_MEM(N);
2130 case ISD::SET_FPENV_MEM: return visitSET_FPENV_MEM(N);
2131 case ISD::FCANONICALIZE: return visitFCANONICALIZE(N);
2132 case ISD::VECREDUCE_FADD:
2133 case ISD::VECREDUCE_FMUL:
2134 case ISD::VECREDUCE_ADD:
2135 case ISD::VECREDUCE_MUL:
2136 case ISD::VECREDUCE_AND:
2137 case ISD::VECREDUCE_OR:
2138 case ISD::VECREDUCE_XOR:
2139 case ISD::VECREDUCE_SMAX:
2140 case ISD::VECREDUCE_SMIN:
2141 case ISD::VECREDUCE_UMAX:
2142 case ISD::VECREDUCE_UMIN:
2143 case ISD::VECREDUCE_FMAX:
2144 case ISD::VECREDUCE_FMIN:
2145 case ISD::VECREDUCE_FMAXIMUM:
2146 case ISD::VECREDUCE_FMINIMUM:
2147 case ISD::VECREDUCE_FMAXIMUMNUM:
2148 case ISD::VECREDUCE_FMINIMUMNUM: return visitVECREDUCE(N);
2149#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
2150#include "llvm/IR/VPIntrinsics.def"
2151 return visitVPOp(N);
2152 }
2153 // clang-format on
2154 return SDValue();
2155}
2156
2157SDValue DAGCombiner::combine(SDNode *N) {
2158 if (!DebugCounter::shouldExecute(Counter&: DAGCombineCounter))
2159 return SDValue();
2160
2161 SDValue RV;
2162 if (!DisableGenericCombines)
2163 RV = visit(N);
2164
2165 // If nothing happened, try a target-specific DAG combine.
2166 if (!RV.getNode()) {
2167 assert(N->getOpcode() != ISD::DELETED_NODE &&
2168 "Node was deleted but visit returned NULL!");
2169
2170 if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
2171 TLI.hasTargetDAGCombine(NT: (ISD::NodeType)N->getOpcode())) {
2172
2173 // Expose the DAG combiner to the target combiner impls.
2174 TargetLowering::DAGCombinerInfo
2175 DagCombineInfo(DAG, Level, false, this);
2176
2177 RV = TLI.PerformDAGCombine(N, DCI&: DagCombineInfo);
2178 }
2179 }
2180
2181 // If nothing happened still, try promoting the operation.
2182 if (!RV.getNode()) {
2183 switch (N->getOpcode()) {
2184 default: break;
2185 case ISD::ADD:
2186 case ISD::SUB:
2187 case ISD::MUL:
2188 case ISD::AND:
2189 case ISD::OR:
2190 case ISD::XOR:
2191 RV = PromoteIntBinOp(Op: SDValue(N, 0));
2192 break;
2193 case ISD::SHL:
2194 case ISD::SRA:
2195 case ISD::SRL:
2196 RV = PromoteIntShiftOp(Op: SDValue(N, 0));
2197 break;
2198 case ISD::SIGN_EXTEND:
2199 case ISD::ZERO_EXTEND:
2200 case ISD::ANY_EXTEND:
2201 RV = PromoteExtend(Op: SDValue(N, 0));
2202 break;
2203 case ISD::LOAD:
2204 if (PromoteLoad(Op: SDValue(N, 0)))
2205 RV = SDValue(N, 0);
2206 break;
2207 }
2208 }
2209
2210 // If N is a commutative binary node, try to eliminate it if the commuted
2211 // version is already present in the DAG.
2212 if (!RV.getNode() && TLI.isCommutativeBinOp(Opcode: N->getOpcode())) {
2213 SDValue N0 = N->getOperand(Num: 0);
2214 SDValue N1 = N->getOperand(Num: 1);
2215
2216 // Constant operands are canonicalized to RHS.
2217 if (N0 != N1 && (isa<ConstantSDNode>(Val: N0) || !isa<ConstantSDNode>(Val: N1))) {
2218 SDValue Ops[] = {N1, N0};
2219 SDNode *CSENode = DAG.getNodeIfExists(Opcode: N->getOpcode(), VTList: N->getVTList(), Ops,
2220 Flags: N->getFlags());
2221 if (CSENode)
2222 return SDValue(CSENode, 0);
2223 }
2224 }
2225
2226 return RV;
2227}
2228
2229/// Given a node, return its input chain if it has one, otherwise return a null
2230/// sd operand.
2231static SDValue getInputChainForNode(SDNode *N) {
2232 if (unsigned NumOps = N->getNumOperands()) {
2233 if (N->getOperand(Num: 0).getValueType() == MVT::Other)
2234 return N->getOperand(Num: 0);
2235 if (N->getOperand(Num: NumOps-1).getValueType() == MVT::Other)
2236 return N->getOperand(Num: NumOps-1);
2237 for (unsigned i = 1; i < NumOps-1; ++i)
2238 if (N->getOperand(Num: i).getValueType() == MVT::Other)
2239 return N->getOperand(Num: i);
2240 }
2241 return SDValue();
2242}
2243
2244SDValue DAGCombiner::visitFCANONICALIZE(SDNode *N) {
2245 SDValue Operand = N->getOperand(Num: 0);
2246 EVT VT = Operand.getValueType();
2247 SDLoc dl(N);
2248
2249 // Canonicalize undef to quiet NaN.
2250 if (Operand.isUndef()) {
2251 APFloat CanonicalQNaN = APFloat::getQNaN(Sem: VT.getFltSemantics());
2252 return DAG.getConstantFP(Val: CanonicalQNaN, DL: dl, VT);
2253 }
2254 return SDValue();
2255}
2256
2257SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
2258 // If N has two operands, where one has an input chain equal to the other,
2259 // the 'other' chain is redundant.
2260 if (N->getNumOperands() == 2) {
2261 if (getInputChainForNode(N: N->getOperand(Num: 0).getNode()) == N->getOperand(Num: 1))
2262 return N->getOperand(Num: 0);
2263 if (getInputChainForNode(N: N->getOperand(Num: 1).getNode()) == N->getOperand(Num: 0))
2264 return N->getOperand(Num: 1);
2265 }
2266
2267 // Don't simplify token factors if optnone.
2268 if (OptLevel == CodeGenOptLevel::None)
2269 return SDValue();
2270
2271 // Don't simplify the token factor if the node itself has too many operands.
2272 if (N->getNumOperands() > TokenFactorInlineLimit)
2273 return SDValue();
2274
2275 // If the sole user is a token factor, we should make sure we have a
2276 // chance to merge them together. This prevents TF chains from inhibiting
2277 // optimizations.
2278 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TokenFactor)
2279 AddToWorklist(N: *(N->user_begin()));
2280
2281 SmallVector<SDNode *, 8> TFs; // List of token factors to visit.
2282 SmallVector<SDValue, 8> Ops; // Ops for replacing token factor.
2283 SmallPtrSet<SDNode*, 16> SeenOps;
2284 bool Changed = false; // If we should replace this token factor.
2285
2286 // Start out with this token factor.
2287 TFs.push_back(Elt: N);
2288
2289 // Iterate through token factors. The TFs grows when new token factors are
2290 // encountered.
2291 for (unsigned i = 0; i < TFs.size(); ++i) {
2292 // Limit number of nodes to inline, to avoid quadratic compile times.
2293 // We have to add the outstanding Token Factors to Ops, otherwise we might
2294 // drop Ops from the resulting Token Factors.
2295 if (Ops.size() > TokenFactorInlineLimit) {
2296 for (unsigned j = i; j < TFs.size(); j++)
2297 Ops.emplace_back(Args&: TFs[j], Args: 0);
2298 // Drop unprocessed Token Factors from TFs, so we do not add them to the
2299 // combiner worklist later.
2300 TFs.resize(N: i);
2301 break;
2302 }
2303
2304 SDNode *TF = TFs[i];
2305 // Check each of the operands.
2306 for (const SDValue &Op : TF->op_values()) {
2307 switch (Op.getOpcode()) {
2308 case ISD::EntryToken:
2309 // Entry tokens don't need to be added to the list. They are
2310 // redundant.
2311 Changed = true;
2312 break;
2313
2314 case ISD::TokenFactor:
2315 if (Op.hasOneUse() && !is_contained(Range&: TFs, Element: Op.getNode())) {
2316 // Queue up for processing.
2317 TFs.push_back(Elt: Op.getNode());
2318 Changed = true;
2319 break;
2320 }
2321 [[fallthrough]];
2322
2323 default:
2324 // Only add if it isn't already in the list.
2325 if (SeenOps.insert(Ptr: Op.getNode()).second)
2326 Ops.push_back(Elt: Op);
2327 else
2328 Changed = true;
2329 break;
2330 }
2331 }
2332 }
2333
2334 // Re-visit inlined Token Factors, to clean them up in case they have been
2335 // removed. Skip the first Token Factor, as this is the current node.
2336 for (unsigned i = 1, e = TFs.size(); i < e; i++)
2337 AddToWorklist(N: TFs[i]);
2338
2339 // Remove Nodes that are chained to another node in the list. Do so
2340 // by walking up chains breath-first stopping when we've seen
2341 // another operand. In general we must climb to the EntryNode, but we can exit
2342 // early if we find all remaining work is associated with just one operand as
2343 // no further pruning is possible.
2344
2345 // List of nodes to search through and original Ops from which they originate.
2346 SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
2347 SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
2348 SmallPtrSet<SDNode *, 16> SeenChains;
2349 bool DidPruneOps = false;
2350
2351 unsigned NumLeftToConsider = 0;
2352 for (const SDValue &Op : Ops) {
2353 Worklist.push_back(Elt: std::make_pair(x: Op.getNode(), y: NumLeftToConsider++));
2354 OpWorkCount.push_back(Elt: 1);
2355 }
2356
2357 auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
2358 // If this is an Op, we can remove the op from the list. Remark any
2359 // search associated with it as from the current OpNumber.
2360 if (SeenOps.contains(Ptr: Op)) {
2361 Changed = true;
2362 DidPruneOps = true;
2363 unsigned OrigOpNumber = 0;
2364 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
2365 OrigOpNumber++;
2366 assert((OrigOpNumber != Ops.size()) &&
2367 "expected to find TokenFactor Operand");
2368 // Re-mark worklist from OrigOpNumber to OpNumber
2369 for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
2370 if (Worklist[i].second == OrigOpNumber) {
2371 Worklist[i].second = OpNumber;
2372 }
2373 }
2374 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2375 OpWorkCount[OrigOpNumber] = 0;
2376 NumLeftToConsider--;
2377 }
2378 // Add if it's a new chain
2379 if (SeenChains.insert(Ptr: Op).second) {
2380 OpWorkCount[OpNumber]++;
2381 Worklist.push_back(Elt: std::make_pair(x&: Op, y&: OpNumber));
2382 }
2383 };
2384
2385 for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
2386 // We need at least be consider at least 2 Ops to prune.
2387 if (NumLeftToConsider <= 1)
2388 break;
2389 auto CurNode = Worklist[i].first;
2390 auto CurOpNumber = Worklist[i].second;
2391 assert((OpWorkCount[CurOpNumber] > 0) &&
2392 "Node should not appear in worklist");
2393 switch (CurNode->getOpcode()) {
2394 case ISD::EntryToken:
2395 // Hitting EntryToken is the only way for the search to terminate without
2396 // hitting
2397 // another operand's search. Prevent us from marking this operand
2398 // considered.
2399 NumLeftToConsider++;
2400 break;
2401 case ISD::TokenFactor:
2402 for (const SDValue &Op : CurNode->op_values())
2403 AddToWorklist(i, Op.getNode(), CurOpNumber);
2404 break;
2405 case ISD::LIFETIME_START:
2406 case ISD::LIFETIME_END:
2407 case ISD::CopyFromReg:
2408 case ISD::CopyToReg:
2409 AddToWorklist(i, CurNode->getOperand(Num: 0).getNode(), CurOpNumber);
2410 break;
2411 default:
2412 if (auto *MemNode = dyn_cast<MemSDNode>(Val: CurNode))
2413 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2414 break;
2415 }
2416 OpWorkCount[CurOpNumber]--;
2417 if (OpWorkCount[CurOpNumber] == 0)
2418 NumLeftToConsider--;
2419 }
2420
2421 // If we've changed things around then replace token factor.
2422 if (Changed) {
2423 SDValue Result;
2424 if (Ops.empty()) {
2425 // The entry token is the only possible outcome.
2426 Result = DAG.getEntryNode();
2427 } else {
2428 if (DidPruneOps) {
2429 SmallVector<SDValue, 8> PrunedOps;
2430 //
2431 for (const SDValue &Op : Ops) {
2432 if (SeenChains.count(Ptr: Op.getNode()) == 0)
2433 PrunedOps.push_back(Elt: Op);
2434 }
2435 Result = DAG.getTokenFactor(DL: SDLoc(N), Vals&: PrunedOps);
2436 } else {
2437 Result = DAG.getTokenFactor(DL: SDLoc(N), Vals&: Ops);
2438 }
2439 }
2440 return Result;
2441 }
2442 return SDValue();
2443}
2444
2445/// MERGE_VALUES can always be eliminated.
2446SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
2447 WorklistRemover DeadNodes(*this);
2448 // Replacing results may cause a different MERGE_VALUES to suddenly
2449 // be CSE'd with N, and carry its uses with it. Iterate until no
2450 // uses remain, to ensure that the node can be safely deleted.
2451 // First add the users of this node to the work list so that they
2452 // can be tried again once they have new operands.
2453 AddUsersToWorklist(N);
2454 do {
2455 // Do as a single replacement to avoid rewalking use lists.
2456 SmallVector<SDValue, 8> Ops(N->ops());
2457 DAG.ReplaceAllUsesWith(From: N, To: Ops.data());
2458 } while (!N->use_empty());
2459 deleteAndRecombine(N);
2460 return SDValue(N, 0); // Return N so it doesn't get rechecked!
2461}
2462
2463/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
2464/// ConstantSDNode pointer else nullptr.
2465static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
2466 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: N);
2467 return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
2468}
2469
2470// isTruncateOf - If N is a truncate of some other value, return true, record
2471// the value being truncated in Op and which of Op's bits are zero/one in Known.
2472// This function computes KnownBits to avoid a duplicated call to
2473// computeKnownBits in the caller.
2474static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
2475 KnownBits &Known) {
2476 if (N->getOpcode() == ISD::TRUNCATE) {
2477 Op = N->getOperand(Num: 0);
2478 Known = DAG.computeKnownBits(Op);
2479 if (N->getFlags().hasNoUnsignedWrap())
2480 Known.Zero.setBitsFrom(N.getScalarValueSizeInBits());
2481 return true;
2482 }
2483
2484 if (N.getValueType().getScalarType() != MVT::i1 ||
2485 !sd_match(
2486 N, P: m_c_SetCC(LHS: m_Value(N&: Op), RHS: m_Zero(), CC: m_SpecificCondCode(CC: ISD::SETNE))))
2487 return false;
2488
2489 Known = DAG.computeKnownBits(Op);
2490 return (Known.Zero | 1).isAllOnes();
2491}
2492
2493/// Return true if 'Use' is a load or a store that uses N as its base pointer
2494/// and that N may be folded in the load / store addressing mode.
2495static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG,
2496 const TargetLowering &TLI) {
2497 EVT VT;
2498 unsigned AS;
2499
2500 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Use)) {
2501 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2502 return false;
2503 VT = LD->getMemoryVT();
2504 AS = LD->getAddressSpace();
2505 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: Use)) {
2506 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2507 return false;
2508 VT = ST->getMemoryVT();
2509 AS = ST->getAddressSpace();
2510 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: Use)) {
2511 if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
2512 return false;
2513 VT = LD->getMemoryVT();
2514 AS = LD->getAddressSpace();
2515 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: Use)) {
2516 if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
2517 return false;
2518 VT = ST->getMemoryVT();
2519 AS = ST->getAddressSpace();
2520 } else {
2521 return false;
2522 }
2523
2524 TargetLowering::AddrMode AM;
2525 if (N->isAnyAdd()) {
2526 AM.HasBaseReg = true;
2527 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
2528 if (Offset)
2529 // [reg +/- imm]
2530 AM.BaseOffs = Offset->getSExtValue();
2531 else
2532 // [reg +/- reg]
2533 AM.Scale = 1;
2534 } else if (N->getOpcode() == ISD::SUB) {
2535 AM.HasBaseReg = true;
2536 ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
2537 if (Offset)
2538 // [reg +/- imm]
2539 AM.BaseOffs = -Offset->getSExtValue();
2540 else
2541 // [reg +/- reg]
2542 AM.Scale = 1;
2543 } else {
2544 return false;
2545 }
2546
2547 return TLI.isLegalAddressingMode(DL: DAG.getDataLayout(), AM,
2548 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()), AddrSpace: AS);
2549}
2550
2551/// This inverts a canonicalization in IR that replaces a variable select arm
2552/// with an identity constant. Codegen improves if we re-use the variable
2553/// operand rather than load a constant. This can also be converted into a
2554/// masked vector operation if the target supports it.
2555static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG,
2556 bool ShouldCommuteOperands) {
2557 SDValue N0 = N->getOperand(Num: 0);
2558 SDValue N1 = N->getOperand(Num: 1);
2559
2560 // Match a select as operand 1. The identity constant that we are looking for
2561 // is only valid as operand 1 of a non-commutative binop.
2562 if (ShouldCommuteOperands)
2563 std::swap(a&: N0, b&: N1);
2564
2565 SDValue Cond, TVal, FVal;
2566 if (!sd_match(N: N1, P: m_OneUse(P: m_SelectLike(Cond: m_Value(N&: Cond), T: m_Value(N&: TVal),
2567 F: m_Value(N&: FVal)))))
2568 return SDValue();
2569
2570 // We can't hoist all instructions because of immediate UB (not speculatable).
2571 // For example div/rem by zero.
2572 if (!DAG.isSafeToSpeculativelyExecuteNode(N))
2573 return SDValue();
2574
2575 unsigned SelOpcode = N1.getOpcode();
2576 unsigned Opcode = N->getOpcode();
2577 EVT VT = N->getValueType(ResNo: 0);
2578 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2579
2580 // This transform increases uses of N0, so freeze it to be safe.
2581 // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal)
2582 unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
2583 if (DAG.isIdentityElement(Opc: Opcode, Flags: N->getFlags(), V: TVal, OperandNo: OpNo) &&
2584 TLI.shouldFoldSelectWithIdentityConstant(BinOpcode: Opcode, VT, SelectOpcode: SelOpcode, X: N0,
2585 Y: FVal)) {
2586 SDValue F0 = DAG.getFreeze(V: N0);
2587 SDValue NewBO = DAG.getNode(Opcode, DL: SDLoc(N), VT, N1: F0, N2: FVal, Flags: N->getFlags());
2588 return DAG.getSelect(DL: SDLoc(N), VT, Cond, LHS: F0, RHS: NewBO);
2589 }
2590 // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0
2591 if (DAG.isIdentityElement(Opc: Opcode, Flags: N->getFlags(), V: FVal, OperandNo: OpNo) &&
2592 TLI.shouldFoldSelectWithIdentityConstant(BinOpcode: Opcode, VT, SelectOpcode: SelOpcode, X: N0,
2593 Y: TVal)) {
2594 SDValue F0 = DAG.getFreeze(V: N0);
2595 SDValue NewBO = DAG.getNode(Opcode, DL: SDLoc(N), VT, N1: F0, N2: TVal, Flags: N->getFlags());
2596 return DAG.getSelect(DL: SDLoc(N), VT, Cond, LHS: NewBO, RHS: F0);
2597 }
2598
2599 return SDValue();
2600}
2601
2602SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
2603 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2604 assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&
2605 "Unexpected binary operator");
2606
2607 if (SDValue Sel = foldSelectWithIdentityConstant(N: BO, DAG, ShouldCommuteOperands: false))
2608 return Sel;
2609
2610 if (TLI.isCommutativeBinOp(Opcode: BO->getOpcode()))
2611 if (SDValue Sel = foldSelectWithIdentityConstant(N: BO, DAG, ShouldCommuteOperands: true))
2612 return Sel;
2613
2614 // Don't do this unless the old select is going away. We want to eliminate the
2615 // binary operator, not replace a binop with a select.
2616 // TODO: Handle ISD::SELECT_CC.
2617 unsigned SelOpNo = 0;
2618 SDValue Sel = BO->getOperand(Num: 0);
2619 auto BinOpcode = BO->getOpcode();
2620 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
2621 SelOpNo = 1;
2622 Sel = BO->getOperand(Num: 1);
2623
2624 // Peek through trunc to shift amount type.
2625 if ((BinOpcode == ISD::SHL || BinOpcode == ISD::SRA ||
2626 BinOpcode == ISD::SRL) && Sel.hasOneUse()) {
2627 // This is valid when the truncated bits of x are already zero.
2628 SDValue Op;
2629 KnownBits Known;
2630 if (isTruncateOf(DAG, N: Sel, Op, Known) &&
2631 Known.countMaxActiveBits() < Sel.getScalarValueSizeInBits())
2632 Sel = Op;
2633 }
2634 }
2635
2636 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
2637 return SDValue();
2638
2639 SDValue CT = Sel.getOperand(i: 1);
2640 if (!isConstantOrConstantVector(N: CT, NoOpaques: true) &&
2641 !DAG.isConstantFPBuildVectorOrConstantFP(N: CT))
2642 return SDValue();
2643
2644 SDValue CF = Sel.getOperand(i: 2);
2645 if (!isConstantOrConstantVector(N: CF, NoOpaques: true) &&
2646 !DAG.isConstantFPBuildVectorOrConstantFP(N: CF))
2647 return SDValue();
2648
2649 // Bail out if any constants are opaque because we can't constant fold those.
2650 // The exception is "and" and "or" with either 0 or -1 in which case we can
2651 // propagate non constant operands into select. I.e.:
2652 // and (select Cond, 0, -1), X --> select Cond, 0, X
2653 // or X, (select Cond, -1, 0) --> select Cond, -1, X
2654 bool CanFoldNonConst =
2655 (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
2656 ((isNullOrNullSplat(V: CT) && isAllOnesOrAllOnesSplat(V: CF)) ||
2657 (isNullOrNullSplat(V: CF) && isAllOnesOrAllOnesSplat(V: CT)));
2658
2659 SDValue CBO = BO->getOperand(Num: SelOpNo ^ 1);
2660 if (!CanFoldNonConst &&
2661 !isConstantOrConstantVector(N: CBO, NoOpaques: true) &&
2662 !DAG.isConstantFPBuildVectorOrConstantFP(N: CBO))
2663 return SDValue();
2664
2665 SDLoc DL(Sel);
2666 SDValue NewCT, NewCF;
2667 EVT VT = BO->getValueType(ResNo: 0);
2668
2669 if (CanFoldNonConst) {
2670 // If CBO is an opaque constant, we can't rely on getNode to constant fold.
2671 if ((BinOpcode == ISD::AND && isNullOrNullSplat(V: CT)) ||
2672 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(V: CT)))
2673 NewCT = CT;
2674 else
2675 NewCT = CBO;
2676
2677 if ((BinOpcode == ISD::AND && isNullOrNullSplat(V: CF)) ||
2678 (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(V: CF)))
2679 NewCF = CF;
2680 else
2681 NewCF = CBO;
2682 } else {
2683 // We have a select-of-constants followed by a binary operator with a
2684 // constant. Eliminate the binop by pulling the constant math into the
2685 // select. Example: add (select Cond, CT, CF), CBO --> select Cond, CT +
2686 // CBO, CF + CBO
2687 NewCT = SelOpNo ? DAG.FoldConstantArithmetic(Opcode: BinOpcode, DL, VT, Ops: {CBO, CT})
2688 : DAG.FoldConstantArithmetic(Opcode: BinOpcode, DL, VT, Ops: {CT, CBO});
2689 if (!NewCT)
2690 return SDValue();
2691
2692 NewCF = SelOpNo ? DAG.FoldConstantArithmetic(Opcode: BinOpcode, DL, VT, Ops: {CBO, CF})
2693 : DAG.FoldConstantArithmetic(Opcode: BinOpcode, DL, VT, Ops: {CF, CBO});
2694 if (!NewCF)
2695 return SDValue();
2696 }
2697
2698 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewCT, RHS: NewCF, Flags: BO->getFlags());
2699}
2700
2701static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, const SDLoc &DL,
2702 SelectionDAG &DAG) {
2703 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
2704 "Expecting add or sub");
2705
2706 // Match a constant operand and a zext operand for the math instruction:
2707 // add Z, C
2708 // sub C, Z
2709 bool IsAdd = N->getOpcode() == ISD::ADD;
2710 SDValue C = IsAdd ? N->getOperand(Num: 1) : N->getOperand(Num: 0);
2711 SDValue Z = IsAdd ? N->getOperand(Num: 0) : N->getOperand(Num: 1);
2712 auto *CN = dyn_cast<ConstantSDNode>(Val&: C);
2713 if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
2714 return SDValue();
2715
2716 // Match the zext operand as a setcc of a boolean.
2717 if (Z.getOperand(i: 0).getValueType() != MVT::i1)
2718 return SDValue();
2719
2720 // Match the compare as: setcc (X & 1), 0, eq.
2721 if (!sd_match(N: Z.getOperand(i: 0), P: m_SetCC(LHS: m_And(L: m_Value(), R: m_One()), RHS: m_Zero(),
2722 CC: m_SpecificCondCode(CC: ISD::SETEQ))))
2723 return SDValue();
2724
2725 // We are adding/subtracting a constant and an inverted low bit. Turn that
2726 // into a subtract/add of the low bit with incremented/decremented constant:
2727 // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
2728 // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
2729 EVT VT = C.getValueType();
2730 SDValue LowBit = DAG.getZExtOrTrunc(Op: Z.getOperand(i: 0).getOperand(i: 0), DL, VT);
2731 SDValue C1 = IsAdd ? DAG.getConstant(Val: CN->getAPIntValue() + 1, DL, VT)
2732 : DAG.getConstant(Val: CN->getAPIntValue() - 1, DL, VT);
2733 return DAG.getNode(Opcode: IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N1: C1, N2: LowBit);
2734}
2735
2736// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1)
2737SDValue DAGCombiner::foldSubToAvg(SDNode *N, const SDLoc &DL) {
2738 SDValue N0 = N->getOperand(Num: 0);
2739 EVT VT = N0.getValueType();
2740 SDValue A, B;
2741
2742 if ((!LegalOperations || hasOperation(Opcode: ISD::AVGCEILU, VT)) &&
2743 sd_match(N, P: m_Sub(L: m_Or(L: m_Value(N&: A), R: m_Value(N&: B)),
2744 R: m_Srl(L: m_Xor(L: m_Deferred(V&: A), R: m_Deferred(V&: B)), R: m_One())))) {
2745 return DAG.getNode(Opcode: ISD::AVGCEILU, DL, VT, N1: A, N2: B);
2746 }
2747 if ((!LegalOperations || hasOperation(Opcode: ISD::AVGCEILS, VT)) &&
2748 sd_match(N, P: m_Sub(L: m_Or(L: m_Value(N&: A), R: m_Value(N&: B)),
2749 R: m_Sra(L: m_Xor(L: m_Deferred(V&: A), R: m_Deferred(V&: B)), R: m_One())))) {
2750 return DAG.getNode(Opcode: ISD::AVGCEILS, DL, VT, N1: A, N2: B);
2751 }
2752 return SDValue();
2753}
2754
2755/// Try to fold a pointer arithmetic node.
2756/// This needs to be done separately from normal addition, because pointer
2757/// addition is not commutative.
2758SDValue DAGCombiner::visitPTRADD(SDNode *N) {
2759 SDValue N0 = N->getOperand(Num: 0);
2760 SDValue N1 = N->getOperand(Num: 1);
2761 EVT PtrVT = N0.getValueType();
2762 EVT IntVT = N1.getValueType();
2763 SDLoc DL(N);
2764
2765 // This is already ensured by an assert in SelectionDAG::getNode(). Several
2766 // combines here depend on this assumption.
2767 assert(PtrVT == IntVT &&
2768 "PTRADD with different operand types is not supported");
2769
2770 // fold (ptradd x, 0) -> x
2771 if (isNullConstant(V: N1))
2772 return N0;
2773
2774 // fold (ptradd 0, x) -> x
2775 if (PtrVT == IntVT && isNullConstant(V: N0))
2776 return N1;
2777
2778 if (N0.getOpcode() == ISD::PTRADD &&
2779 !reassociationCanBreakAddressingModePattern(Opc: ISD::PTRADD, DL, N, N0, N1)) {
2780 SDValue X = N0.getOperand(i: 0);
2781 SDValue Y = N0.getOperand(i: 1);
2782 SDValue Z = N1;
2783 bool N0OneUse = N0.hasOneUse();
2784 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
2785 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
2786
2787 // (ptradd (ptradd x, y), z) -> (ptradd x, (add y, z)) if:
2788 // * y is a constant and (ptradd x, y) has one use; or
2789 // * y and z are both constants.
2790 if ((YIsConstant && N0OneUse) || (YIsConstant && ZIsConstant)) {
2791 // If both additions in the original were NUW, the new ones are as well.
2792 SDNodeFlags Flags =
2793 (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
2794 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: IntVT, Ops: {Y, Z}, Flags);
2795 AddToWorklist(N: Add.getNode());
2796 // We can't set InBounds even if both original ptradds were InBounds and
2797 // NUW: SDAG usually represents pointers as integers, therefore, the
2798 // matched pattern behaves as if it had implicit casts:
2799 // (ptradd inbounds (inttoptr (ptrtoint (ptradd inbounds x, y))), z)
2800 // The outer inbounds ptradd might therefore rely on a provenance that x
2801 // does not have.
2802 return DAG.getMemBasePlusOffset(Base: X, Offset: Add, DL, Flags);
2803 }
2804 }
2805
2806 // The following combines can turn in-bounds pointer arithmetic out of bounds.
2807 // That is problematic for settings like AArch64's CPA, which checks that
2808 // intermediate results of pointer arithmetic remain in bounds. The target
2809 // therefore needs to opt-in to enable them.
2810 if (!TLI.canTransformPtrArithOutOfBounds(
2811 F: DAG.getMachineFunction().getFunction(), PtrVT))
2812 return SDValue();
2813
2814 if (N0.getOpcode() == ISD::PTRADD && isa<ConstantSDNode>(Val: N1)) {
2815 // Fold (ptradd (ptradd GA, v), c) -> (ptradd (ptradd GA, c) v) with
2816 // global address GA and constant c, such that c can be folded into GA.
2817 // TODO: Support constant vector splats.
2818 SDValue GAValue = N0.getOperand(i: 0);
2819 if (const GlobalAddressSDNode *GA =
2820 dyn_cast<GlobalAddressSDNode>(Val&: GAValue)) {
2821 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2822 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
2823 // If both additions in the original were NUW, reassociation preserves
2824 // that.
2825 SDNodeFlags Flags =
2826 (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
2827 // We can't set InBounds even if both original ptradds were InBounds and
2828 // NUW: SDAG usually represents pointers as integers, therefore, the
2829 // matched pattern behaves as if it had implicit casts:
2830 // (ptradd inbounds (inttoptr (ptrtoint (ptradd inbounds GA, v))), c)
2831 // The outer inbounds ptradd might therefore rely on a provenance that
2832 // GA does not have.
2833 SDValue Inner = DAG.getMemBasePlusOffset(Base: GAValue, Offset: N1, DL, Flags);
2834 AddToWorklist(N: Inner.getNode());
2835 return DAG.getMemBasePlusOffset(Base: Inner, Offset: N0.getOperand(i: 1), DL, Flags);
2836 }
2837 }
2838 }
2839
2840 if (N1.getOpcode() == ISD::ADD && N1.hasOneUse()) {
2841 // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
2842 // y is not, and (add y, z) is used only once.
2843 // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
2844 // z is not, and (add y, z) is used only once.
2845 // The goal is to move constant offsets to the outermost ptradd, to create
2846 // more opportunities to fold offsets into memory instructions.
2847 // Together with the another combine above, this also implements
2848 // (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
2849 SDValue X = N0;
2850 SDValue Y = N1.getOperand(i: 0);
2851 SDValue Z = N1.getOperand(i: 1);
2852 bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Y);
2853 bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(N: Z);
2854
2855 // If both additions in the original were NUW, reassociation preserves that.
2856 SDNodeFlags CommonFlags = N->getFlags() & N1->getFlags();
2857 SDNodeFlags ReassocFlags = CommonFlags & SDNodeFlags::NoUnsignedWrap;
2858 if (CommonFlags.hasNoUnsignedWrap()) {
2859 // If both operations are NUW and the PTRADD is inbounds, the offests are
2860 // both non-negative, so the reassociated PTRADDs are also inbounds.
2861 ReassocFlags |= N->getFlags() & SDNodeFlags::InBounds;
2862 }
2863
2864 if (ZIsConstant != YIsConstant) {
2865 if (YIsConstant)
2866 std::swap(a&: Y, b&: Z);
2867 SDValue Inner = DAG.getMemBasePlusOffset(Base: X, Offset: Y, DL, Flags: ReassocFlags);
2868 AddToWorklist(N: Inner.getNode());
2869 return DAG.getMemBasePlusOffset(Base: Inner, Offset: Z, DL, Flags: ReassocFlags);
2870 }
2871 }
2872
2873 // Transform (ptradd a, b) -> (or disjoint a, b) if it is equivalent and if
2874 // that transformation can't block an offset folding at any use of the ptradd.
2875 // This should be done late, after legalization, so that it doesn't block
2876 // other ptradd combines that could enable more offset folding.
2877 if (LegalOperations && DAG.haveNoCommonBitsSet(A: N0, B: N1)) {
2878 bool TransformCannotBreakAddrMode = none_of(Range: N->users(), P: [&](SDNode *User) {
2879 return canFoldInAddressingMode(N, Use: User, DAG, TLI);
2880 });
2881
2882 if (TransformCannotBreakAddrMode)
2883 return DAG.getNode(Opcode: ISD::OR, DL, VT: PtrVT, N1: N0, N2: N1, Flags: SDNodeFlags::Disjoint);
2884 }
2885
2886 return SDValue();
2887}
2888
2889/// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
2890/// a shift and add with a different constant.
2891static SDValue foldAddSubOfSignBit(SDNode *N, const SDLoc &DL,
2892 SelectionDAG &DAG) {
2893 assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
2894 "Expecting add or sub");
2895
2896 // We need a constant operand for the add/sub, and the other operand is a
2897 // logical shift right: add (srl), C or sub C, (srl).
2898 bool IsAdd = N->getOpcode() == ISD::ADD;
2899 SDValue ConstantOp = IsAdd ? N->getOperand(Num: 1) : N->getOperand(Num: 0);
2900 SDValue ShiftOp = IsAdd ? N->getOperand(Num: 0) : N->getOperand(Num: 1);
2901 if (!DAG.isConstantIntBuildVectorOrConstantInt(N: ConstantOp) ||
2902 ShiftOp.getOpcode() != ISD::SRL)
2903 return SDValue();
2904
2905 // The shift must be of a 'not' value.
2906 SDValue Not = ShiftOp.getOperand(i: 0);
2907 if (!Not.hasOneUse() || !isBitwiseNot(V: Not))
2908 return SDValue();
2909
2910 // The shift must be moving the sign bit to the least-significant-bit.
2911 EVT VT = ShiftOp.getValueType();
2912 SDValue ShAmt = ShiftOp.getOperand(i: 1);
2913 ConstantSDNode *ShAmtC = isConstOrConstSplat(N: ShAmt);
2914 if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1))
2915 return SDValue();
2916
2917 // Eliminate the 'not' by adjusting the shift and add/sub constant:
2918 // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
2919 // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
2920 if (SDValue NewC = DAG.FoldConstantArithmetic(
2921 Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL, VT,
2922 Ops: {ConstantOp, DAG.getConstant(Val: 1, DL, VT)})) {
2923 SDValue NewShift = DAG.getNode(Opcode: IsAdd ? ISD::SRA : ISD::SRL, DL, VT,
2924 N1: Not.getOperand(i: 0), N2: ShAmt);
2925 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NewShift, N2: NewC);
2926 }
2927
2928 return SDValue();
2929}
2930
2931static bool
2932areBitwiseNotOfEachother(SDValue Op0, SDValue Op1) {
2933 return (isBitwiseNot(V: Op0) && Op0.getOperand(i: 0) == Op1) ||
2934 (isBitwiseNot(V: Op1) && Op1.getOperand(i: 0) == Op0);
2935}
2936
2937/// Try to fold a node that behaves like an ADD (note that N isn't necessarily
2938/// an ISD::ADD here, it could for example be an ISD::OR if we know that there
2939/// are no common bits set in the operands).
2940SDValue DAGCombiner::visitADDLike(SDNode *N) {
2941 SDValue N0 = N->getOperand(Num: 0);
2942 SDValue N1 = N->getOperand(Num: 1);
2943 EVT VT = N0.getValueType();
2944 SDLoc DL(N);
2945
2946 // fold (add x, undef) -> undef
2947 if (N0.isUndef())
2948 return N0;
2949 if (N1.isUndef())
2950 return N1;
2951
2952 // fold (add c1, c2) -> c1+c2
2953 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::ADD, DL, VT, Ops: {N0, N1}))
2954 return C;
2955
2956 // canonicalize constant to RHS
2957 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
2958 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
2959 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1, N2: N0);
2960
2961 if (areBitwiseNotOfEachother(Op0: N0, Op1: N1))
2962 return DAG.getConstant(Val: APInt::getAllOnes(numBits: VT.getScalarSizeInBits()), DL, VT);
2963
2964 // fold vector ops
2965 if (VT.isVector()) {
2966 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
2967 return FoldedVOp;
2968
2969 // fold (add x, 0) -> x, vector edition
2970 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
2971 return N0;
2972 }
2973
2974 // fold (add x, 0) -> x
2975 if (isNullConstant(V: N1))
2976 return N0;
2977
2978 if (N0.getOpcode() == ISD::SUB) {
2979 SDValue N00 = N0.getOperand(i: 0);
2980 SDValue N01 = N0.getOperand(i: 1);
2981
2982 // fold ((A-c1)+c2) -> (A+(c2-c1))
2983 if (SDValue Sub = DAG.FoldConstantArithmetic(Opcode: ISD::SUB, DL, VT, Ops: {N1, N01}))
2984 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 0), N2: Sub);
2985
2986 // fold ((c1-A)+c2) -> (c1+c2)-A
2987 if (SDValue Add = DAG.FoldConstantArithmetic(Opcode: ISD::ADD, DL, VT, Ops: {N1, N00}))
2988 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Add, N2: N0.getOperand(i: 1));
2989 }
2990
2991 // add (sext i1 X), 1 -> zext (not i1 X)
2992 // We don't transform this pattern:
2993 // add (zext i1 X), -1 -> sext (not i1 X)
2994 // because most (?) targets generate better code for the zext form.
2995 if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
2996 isOneOrOneSplat(V: N1)) {
2997 SDValue X = N0.getOperand(i: 0);
2998 if ((!LegalOperations ||
2999 (TLI.isOperationLegal(Op: ISD::XOR, VT: X.getValueType()) &&
3000 TLI.isOperationLegal(Op: ISD::ZERO_EXTEND, VT))) &&
3001 X.getScalarValueSizeInBits() == 1) {
3002 SDValue Not = DAG.getNOT(DL, Val: X, VT: X.getValueType());
3003 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Not);
3004 }
3005 }
3006
3007 // Fold (add (or x, c0), c1) -> (add x, (c0 + c1))
3008 // iff (or x, c0) is equivalent to (add x, c0).
3009 // Fold (add (xor x, c0), c1) -> (add x, (c0 + c1))
3010 // iff (xor x, c0) is equivalent to (add x, c0).
3011 if (DAG.isADDLike(Op: N0)) {
3012 SDValue N01 = N0.getOperand(i: 1);
3013 if (SDValue Add = DAG.FoldConstantArithmetic(Opcode: ISD::ADD, DL, VT, Ops: {N1, N01}))
3014 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 0), N2: Add);
3015 }
3016
3017 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
3018 return NewSel;
3019
3020 // reassociate add
3021 if (!reassociationCanBreakAddressingModePattern(Opc: ISD::ADD, DL, N, N0, N1)) {
3022 if (SDValue RADD = reassociateOps(Opc: ISD::ADD, DL, N0, N1, Flags: N->getFlags()))
3023 return RADD;
3024
3025 // (X + Y) + X --> Y + (X + X)
3026 SDValue X, Y, InnerAdd;
3027 if (sd_match(
3028 N, P: m_Add(L: m_OneUse(P: m_Value(N&: InnerAdd, P: m_Add(L: m_Value(N&: X), R: m_Value(N&: Y)))),
3029 R: m_Deferred(V&: X)))) {
3030 if (X != Y) {
3031 // Redistribute shared NUW flag.
3032 // TODO: If NSW+NUW occurs on both adds, that can be redistributed too.
3033 SDNodeFlags NewFlags =
3034 N->getFlags() & InnerAdd->getFlags() & SDNodeFlags::NoUnsignedWrap;
3035 SDValue X2 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: X, N2: X, Flags: NewFlags);
3036 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Y, N2: X2, Flags: NewFlags);
3037 }
3038 }
3039
3040 // Reassociate (add (or x, c), y) -> (add add(x, y), c)) if (or x, c) is
3041 // equivalent to (add x, c).
3042 // Reassociate (add (xor x, c), y) -> (add add(x, y), c)) if (xor x, c) is
3043 // equivalent to (add x, c).
3044 // Do this optimization only when adding c does not introduce instructions
3045 // for adding carries.
3046 auto ReassociateAddOr = [&](SDValue N0, SDValue N1) {
3047 if (DAG.isADDLike(Op: N0) && N0.hasOneUse() &&
3048 isConstantOrConstantVector(N: N0.getOperand(i: 1), /* NoOpaque */ NoOpaques: true)) {
3049 // If N0's type does not split or is a sign mask, it does not introduce
3050 // add carry.
3051 auto TyActn = TLI.getTypeAction(Context&: *DAG.getContext(), VT: N0.getValueType());
3052 bool NoAddCarry = TyActn == TargetLoweringBase::TypeLegal ||
3053 TyActn == TargetLoweringBase::TypePromoteInteger ||
3054 isMinSignedConstant(V: N0.getOperand(i: 1));
3055 if (NoAddCarry)
3056 return DAG.getNode(
3057 Opcode: ISD::ADD, DL, VT,
3058 N1: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1, N2: N0.getOperand(i: 0)),
3059 N2: N0.getOperand(i: 1));
3060 }
3061 return SDValue();
3062 };
3063 if (SDValue Add = ReassociateAddOr(N0, N1))
3064 return Add;
3065 if (SDValue Add = ReassociateAddOr(N1, N0))
3066 return Add;
3067
3068 // Fold add(vecreduce(x), vecreduce(y)) -> vecreduce(add(x, y))
3069 if (SDValue SD =
3070 reassociateReduction(RedOpc: ISD::VECREDUCE_ADD, Opc: ISD::ADD, DL, VT, N0, N1))
3071 return SD;
3072 }
3073
3074 SDValue A, B, C, D;
3075
3076 // fold ((0-A) + B) -> B-A
3077 if (sd_match(N: N0, P: m_Neg(V: m_Value(N&: A))))
3078 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1, N2: A);
3079
3080 // fold (A + (0-B)) -> A-B
3081 if (sd_match(N: N1, P: m_Neg(V: m_Value(N&: B))))
3082 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: B);
3083
3084 // fold (A+(B-A)) -> B
3085 if (sd_match(N: N1, P: m_Sub(L: m_Value(N&: B), R: m_Specific(N: N0))))
3086 return B;
3087
3088 // fold ((B-A)+A) -> B
3089 if (sd_match(N: N0, P: m_Sub(L: m_Value(N&: B), R: m_Specific(N: N1))))
3090 return B;
3091
3092 // fold ((A-B)+(C-A)) -> (C-B)
3093 if (sd_match(N: N0, P: m_Sub(L: m_Value(N&: A), R: m_Value(N&: B))) &&
3094 sd_match(N: N1, P: m_Sub(L: m_Value(N&: C), R: m_Specific(N: A))))
3095 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: C, N2: B);
3096
3097 // fold ((A-B)+(B-C)) -> (A-C)
3098 if (sd_match(N: N0, P: m_Sub(L: m_Value(N&: A), R: m_Value(N&: B))) &&
3099 sd_match(N: N1, P: m_Sub(L: m_Specific(N: B), R: m_Value(N&: C))))
3100 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: A, N2: C);
3101
3102 // fold (A+(B-(A+C))) to (B-C)
3103 // fold (A+(B-(C+A))) to (B-C)
3104 if (sd_match(N: N1, P: m_Sub(L: m_Value(N&: B), R: m_Add(L: m_Specific(N: N0), R: m_Value(N&: C)))))
3105 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: B, N2: C);
3106
3107 // fold (A+((B-A)+or-C)) to (B+or-C)
3108 if (sd_match(N: N1,
3109 P: m_AnyOf(preds: m_Add(L: m_Sub(L: m_Value(N&: B), R: m_Specific(N: N0)), R: m_Value(N&: C)),
3110 preds: m_Sub(L: m_Sub(L: m_Value(N&: B), R: m_Specific(N: N0)), R: m_Value(N&: C)))))
3111 return DAG.getNode(Opcode: N1.getOpcode(), DL, VT, N1: B, N2: C);
3112
3113 // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
3114 if (sd_match(N: N0, P: m_OneUse(P: m_Sub(L: m_Value(N&: A), R: m_Value(N&: B)))) &&
3115 sd_match(N: N1, P: m_OneUse(P: m_Sub(L: m_Value(N&: C), R: m_Value(N&: D)))) &&
3116 (isConstantOrConstantVector(N: A) || isConstantOrConstantVector(N: C)))
3117 return DAG.getNode(Opcode: ISD::SUB, DL, VT,
3118 N1: DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N0), VT, N1: A, N2: C),
3119 N2: DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N1), VT, N1: B, N2: D));
3120
3121 // fold (add (umax X, C), -C) --> (usubsat X, C)
3122 if (N0.getOpcode() == ISD::UMAX && hasOperation(Opcode: ISD::USUBSAT, VT)) {
3123 auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
3124 return (!Max && !Op) ||
3125 (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue()));
3126 };
3127 if (ISD::matchBinaryPredicate(LHS: N0.getOperand(i: 1), RHS: N1, Match: MatchUSUBSAT,
3128 /*AllowUndefs*/ true))
3129 return DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: N0.getOperand(i: 0),
3130 N2: N0.getOperand(i: 1));
3131 }
3132
3133 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
3134 return SDValue(N, 0);
3135
3136 if (isOneOrOneSplat(V: N1)) {
3137 // fold (add (xor a, -1), 1) -> (sub 0, a)
3138 if (isBitwiseNot(V: N0))
3139 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT),
3140 N2: N0.getOperand(i: 0));
3141
3142 // fold (add (add (xor a, -1), b), 1) -> (sub b, a)
3143 if (N0.getOpcode() == ISD::ADD) {
3144 SDValue A, Xor;
3145
3146 if (isBitwiseNot(V: N0.getOperand(i: 0))) {
3147 A = N0.getOperand(i: 1);
3148 Xor = N0.getOperand(i: 0);
3149 } else if (isBitwiseNot(V: N0.getOperand(i: 1))) {
3150 A = N0.getOperand(i: 0);
3151 Xor = N0.getOperand(i: 1);
3152 }
3153
3154 if (Xor)
3155 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: A, N2: Xor.getOperand(i: 0));
3156 }
3157
3158 // Look for:
3159 // add (add x, y), 1
3160 // And if the target does not like this form then turn into:
3161 // sub y, (xor x, -1)
3162 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
3163 N0.hasOneUse() &&
3164 // Limit this to after legalization if the add has wrap flags
3165 (Level >= AfterLegalizeDAG || (!N->getFlags().hasNoUnsignedWrap() &&
3166 !N->getFlags().hasNoSignedWrap()))) {
3167 SDValue Not = DAG.getNOT(DL, Val: N0.getOperand(i: 0), VT);
3168 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0.getOperand(i: 1), N2: Not);
3169 }
3170 }
3171
3172 // (x - y) + -1 -> add (xor y, -1), x
3173 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
3174 isAllOnesOrAllOnesSplat(V: N1, /*AllowUndefs=*/true)) {
3175 SDValue Not = DAG.getNOT(DL, Val: N0.getOperand(i: 1), VT);
3176 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Not, N2: N0.getOperand(i: 0));
3177 }
3178
3179 // Fold add(mul(add(A, CA), CM), CB) -> add(mul(A, CM), CM*CA+CB).
3180 // This can help if the inner add has multiple uses.
3181 APInt CM, CA;
3182 if (ConstantSDNode *CB = dyn_cast<ConstantSDNode>(Val&: N1)) {
3183 if (VT.getScalarSizeInBits() <= 64) {
3184 if (sd_match(N: N0, P: m_OneUse(P: m_Mul(L: m_Add(L: m_Value(N&: A), R: m_ConstInt(V&: CA)),
3185 R: m_ConstInt(V&: CM)))) &&
3186 TLI.isLegalAddImmediate(
3187 (CA * CM + CB->getAPIntValue()).getSExtValue())) {
3188 SDNodeFlags Flags;
3189 // If all the inputs are nuw, the outputs can be nuw. If all the input
3190 // are _also_ nsw the outputs can be too.
3191 if (N->getFlags().hasNoUnsignedWrap() &&
3192 N0->getFlags().hasNoUnsignedWrap() &&
3193 N0.getOperand(i: 0)->getFlags().hasNoUnsignedWrap()) {
3194 Flags |= SDNodeFlags::NoUnsignedWrap;
3195 if (N->getFlags().hasNoSignedWrap() &&
3196 N0->getFlags().hasNoSignedWrap() &&
3197 N0.getOperand(i: 0)->getFlags().hasNoSignedWrap())
3198 Flags |= SDNodeFlags::NoSignedWrap;
3199 }
3200 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N1), VT, N1: A,
3201 N2: DAG.getConstant(Val: CM, DL, VT), Flags);
3202 return DAG.getNode(
3203 Opcode: ISD::ADD, DL, VT, N1: Mul,
3204 N2: DAG.getConstant(Val: CA * CM + CB->getAPIntValue(), DL, VT), Flags);
3205 }
3206 // Also look in case there is an intermediate add.
3207 if (sd_match(N: N0, P: m_OneUse(P: m_Add(
3208 L: m_OneUse(P: m_Mul(L: m_Add(L: m_Value(N&: A), R: m_ConstInt(V&: CA)),
3209 R: m_ConstInt(V&: CM))),
3210 R: m_Value(N&: B)))) &&
3211 TLI.isLegalAddImmediate(
3212 (CA * CM + CB->getAPIntValue()).getSExtValue())) {
3213 SDNodeFlags Flags;
3214 // If all the inputs are nuw, the outputs can be nuw. If all the input
3215 // are _also_ nsw the outputs can be too.
3216 SDValue OMul =
3217 N0.getOperand(i: 0) == B ? N0.getOperand(i: 1) : N0.getOperand(i: 0);
3218 if (N->getFlags().hasNoUnsignedWrap() &&
3219 N0->getFlags().hasNoUnsignedWrap() &&
3220 OMul->getFlags().hasNoUnsignedWrap() &&
3221 OMul.getOperand(i: 0)->getFlags().hasNoUnsignedWrap()) {
3222 Flags |= SDNodeFlags::NoUnsignedWrap;
3223 if (N->getFlags().hasNoSignedWrap() &&
3224 N0->getFlags().hasNoSignedWrap() &&
3225 OMul->getFlags().hasNoSignedWrap() &&
3226 OMul.getOperand(i: 0)->getFlags().hasNoSignedWrap())
3227 Flags |= SDNodeFlags::NoSignedWrap;
3228 }
3229 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N1), VT, N1: A,
3230 N2: DAG.getConstant(Val: CM, DL, VT), Flags);
3231 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N1), VT, N1: Mul, N2: B, Flags);
3232 return DAG.getNode(
3233 Opcode: ISD::ADD, DL, VT, N1: Add,
3234 N2: DAG.getConstant(Val: CA * CM + CB->getAPIntValue(), DL, VT), Flags);
3235 }
3236 }
3237 }
3238
3239 if (SDValue Combined = visitADDLikeCommutative(N0, N1, DL))
3240 return Combined;
3241
3242 if (SDValue Combined = visitADDLikeCommutative(N0: N1, N1: N0, DL))
3243 return Combined;
3244
3245 return SDValue();
3246}
3247
3248// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1)
3249// Attempt to form avgfloor(A, B) from ((A >> 1) + (B >> 1)) + (A & B & 1)
3250// Attempt to form avgceil(A, B) from ((A >> 1) + (B >> 1)) + ((A | B) & 1)
3251SDValue DAGCombiner::foldAddToAvg(SDNode *N, const SDLoc &DL) {
3252 SDValue N0 = N->getOperand(Num: 0);
3253 EVT VT = N0.getValueType();
3254 SDValue A, B;
3255
3256 if ((!LegalOperations || hasOperation(Opcode: ISD::AVGFLOORU, VT)) &&
3257 (sd_match(N,
3258 P: m_Add(L: m_And(L: m_Value(N&: A), R: m_Value(N&: B)),
3259 R: m_Srl(L: m_Xor(L: m_Deferred(V&: A), R: m_Deferred(V&: B)), R: m_One()))) ||
3260 sd_match(N, P: m_ReassociatableAdd(
3261 Patterns: m_ReassociatableAnd(Patterns: m_Value(N&: A), Patterns: m_Value(N&: B), Patterns: m_One()),
3262 Patterns: m_Srl(L: m_Deferred(V&: A), R: m_One()),
3263 Patterns: m_Srl(L: m_Deferred(V&: B), R: m_One()))))) {
3264 return DAG.getNode(Opcode: ISD::AVGFLOORU, DL, VT, N1: A, N2: B);
3265 }
3266 if ((!LegalOperations || hasOperation(Opcode: ISD::AVGFLOORS, VT)) &&
3267 (sd_match(N,
3268 P: m_Add(L: m_And(L: m_Value(N&: A), R: m_Value(N&: B)),
3269 R: m_Sra(L: m_Xor(L: m_Deferred(V&: A), R: m_Deferred(V&: B)), R: m_One()))) ||
3270 sd_match(N, P: m_ReassociatableAdd(
3271 Patterns: m_ReassociatableAnd(Patterns: m_Value(N&: A), Patterns: m_Value(N&: B), Patterns: m_One()),
3272 Patterns: m_Sra(L: m_Deferred(V&: A), R: m_One()),
3273 Patterns: m_Sra(L: m_Deferred(V&: B), R: m_One()))))) {
3274 return DAG.getNode(Opcode: ISD::AVGFLOORS, DL, VT, N1: A, N2: B);
3275 }
3276
3277 if ((!LegalOperations || hasOperation(Opcode: ISD::AVGCEILU, VT)) &&
3278 sd_match(N,
3279 P: m_ReassociatableAdd(Patterns: m_And(L: m_Or(L: m_Value(N&: A), R: m_Value(N&: B)), R: m_One()),
3280 Patterns: m_Srl(L: m_Deferred(V&: A), R: m_One()),
3281 Patterns: m_Srl(L: m_Deferred(V&: B), R: m_One())))) {
3282 return DAG.getNode(Opcode: ISD::AVGCEILU, DL, VT, N1: A, N2: B);
3283 }
3284 if ((!LegalOperations || hasOperation(Opcode: ISD::AVGCEILS, VT)) &&
3285 sd_match(N,
3286 P: m_ReassociatableAdd(Patterns: m_And(L: m_Or(L: m_Value(N&: A), R: m_Value(N&: B)), R: m_One()),
3287 Patterns: m_Sra(L: m_Deferred(V&: A), R: m_One()),
3288 Patterns: m_Sra(L: m_Deferred(V&: B), R: m_One())))) {
3289 return DAG.getNode(Opcode: ISD::AVGCEILS, DL, VT, N1: A, N2: B);
3290 }
3291
3292 return SDValue();
3293}
3294
3295SDValue DAGCombiner::visitADD(SDNode *N) {
3296 SDValue N0 = N->getOperand(Num: 0);
3297 SDValue N1 = N->getOperand(Num: 1);
3298 EVT VT = N0.getValueType();
3299 SDLoc DL(N);
3300
3301 if (SDValue Combined = visitADDLike(N))
3302 return Combined;
3303
3304 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
3305 return V;
3306
3307 if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
3308 return V;
3309
3310 if (SDValue V = MatchRotate(LHS: N0, RHS: N1, DL: SDLoc(N), /*FromAdd=*/true))
3311 return V;
3312
3313 // Try to match AVGFLOOR fixedwidth pattern
3314 if (SDValue V = foldAddToAvg(N, DL))
3315 return V;
3316
3317 // fold (a+b) -> (a|b) iff a and b share no bits.
3318 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::OR, VT)) &&
3319 DAG.haveNoCommonBitsSet(A: N0, B: N1))
3320 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: N1, Flags: SDNodeFlags::Disjoint);
3321
3322 // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
3323 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
3324 const APInt &C0 = N0->getConstantOperandAPInt(Num: 0);
3325 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
3326 return DAG.getVScale(DL, VT, MulImm: C0 + C1);
3327 }
3328
3329 // fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
3330 if (N0.getOpcode() == ISD::ADD &&
3331 N0.getOperand(i: 1).getOpcode() == ISD::VSCALE &&
3332 N1.getOpcode() == ISD::VSCALE) {
3333 const APInt &VS0 = N0.getOperand(i: 1)->getConstantOperandAPInt(Num: 0);
3334 const APInt &VS1 = N1->getConstantOperandAPInt(Num: 0);
3335 SDValue VS = DAG.getVScale(DL, VT, MulImm: VS0 + VS1);
3336 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 0), N2: VS);
3337 }
3338
3339 // Fold (add step_vector(c1), step_vector(c2) to step_vector(c1+c2))
3340 if (N0.getOpcode() == ISD::STEP_VECTOR &&
3341 N1.getOpcode() == ISD::STEP_VECTOR) {
3342 const APInt &C0 = N0->getConstantOperandAPInt(Num: 0);
3343 const APInt &C1 = N1->getConstantOperandAPInt(Num: 0);
3344 APInt NewStep = C0 + C1;
3345 return DAG.getStepVector(DL, ResVT: VT, StepVal: NewStep);
3346 }
3347
3348 // Fold a + step_vector(c1) + step_vector(c2) to a + step_vector(c1+c2)
3349 if (N0.getOpcode() == ISD::ADD &&
3350 N0.getOperand(i: 1).getOpcode() == ISD::STEP_VECTOR &&
3351 N1.getOpcode() == ISD::STEP_VECTOR) {
3352 const APInt &SV0 = N0.getOperand(i: 1)->getConstantOperandAPInt(Num: 0);
3353 const APInt &SV1 = N1->getConstantOperandAPInt(Num: 0);
3354 APInt NewStep = SV0 + SV1;
3355 SDValue SV = DAG.getStepVector(DL, ResVT: VT, StepVal: NewStep);
3356 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 0), N2: SV);
3357 }
3358
3359 return SDValue();
3360}
3361
3362SDValue DAGCombiner::visitADDSAT(SDNode *N) {
3363 unsigned Opcode = N->getOpcode();
3364 SDValue N0 = N->getOperand(Num: 0);
3365 SDValue N1 = N->getOperand(Num: 1);
3366 EVT VT = N0.getValueType();
3367 bool IsSigned = Opcode == ISD::SADDSAT;
3368 SDLoc DL(N);
3369
3370 // fold (add_sat x, undef) -> -1
3371 if (N0.isUndef() || N1.isUndef())
3372 return DAG.getAllOnesConstant(DL, VT);
3373
3374 // fold (add_sat c1, c2) -> c3
3375 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, Ops: {N0, N1}))
3376 return C;
3377
3378 // canonicalize constant to RHS
3379 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
3380 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
3381 return DAG.getNode(Opcode, DL, VT, N1, N2: N0);
3382
3383 // fold vector ops
3384 if (VT.isVector()) {
3385 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
3386 return FoldedVOp;
3387
3388 // fold (add_sat x, 0) -> x, vector edition
3389 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
3390 return N0;
3391 }
3392
3393 // fold (add_sat x, 0) -> x
3394 if (isNullConstant(V: N1))
3395 return N0;
3396
3397 // If it cannot overflow, transform into an add.
3398 if (DAG.willNotOverflowAdd(IsSigned, N0, N1))
3399 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: N1);
3400
3401 return SDValue();
3402}
3403
3404static SDValue getAsCarry(const TargetLowering &TLI, SDValue V,
3405 bool ForceCarryReconstruction = false) {
3406 bool Masked = false;
3407
3408 // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
3409 while (true) {
3410 if (ForceCarryReconstruction && V.getValueType() == MVT::i1)
3411 return V;
3412
3413 if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
3414 V = V.getOperand(i: 0);
3415 continue;
3416 }
3417
3418 if (V.getOpcode() == ISD::AND && isOneConstant(V: V.getOperand(i: 1))) {
3419 if (ForceCarryReconstruction)
3420 return V;
3421
3422 Masked = true;
3423 V = V.getOperand(i: 0);
3424 continue;
3425 }
3426
3427 break;
3428 }
3429
3430 // If this is not a carry, return.
3431 if (V.getResNo() != 1)
3432 return SDValue();
3433
3434 if (V.getOpcode() != ISD::UADDO_CARRY && V.getOpcode() != ISD::USUBO_CARRY &&
3435 V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
3436 return SDValue();
3437
3438 EVT VT = V->getValueType(ResNo: 0);
3439 if (!TLI.isOperationLegalOrCustom(Op: V.getOpcode(), VT))
3440 return SDValue();
3441
3442 // If the result is masked, then no matter what kind of bool it is we can
3443 // return. If it isn't, then we need to make sure the bool type is either 0 or
3444 // 1 and not other values.
3445 if (Masked ||
3446 TLI.getBooleanContents(Type: V.getValueType()) ==
3447 TargetLoweringBase::ZeroOrOneBooleanContent)
3448 return V;
3449
3450 return SDValue();
3451}
3452
3453/// Given the operands of an add/sub operation, see if the 2nd operand is a
3454/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
3455/// the opcode and bypass the mask operation.
3456static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
3457 SelectionDAG &DAG, const SDLoc &DL) {
3458 if (N1.getOpcode() == ISD::ZERO_EXTEND)
3459 N1 = N1.getOperand(i: 0);
3460
3461 if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(V: N1->getOperand(Num: 1)))
3462 return SDValue();
3463
3464 EVT VT = N0.getValueType();
3465 SDValue N10 = N1.getOperand(i: 0);
3466 if (N10.getValueType() != VT && N10.getOpcode() == ISD::TRUNCATE)
3467 N10 = N10.getOperand(i: 0);
3468
3469 if (N10.getValueType() != VT)
3470 return SDValue();
3471
3472 if (DAG.ComputeNumSignBits(Op: N10) != VT.getScalarSizeInBits())
3473 return SDValue();
3474
3475 // add N0, (and (AssertSext X, i1), 1) --> sub N0, X
3476 // sub N0, (and (AssertSext X, i1), 1) --> add N0, X
3477 return DAG.getNode(Opcode: IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N1: N0, N2: N10);
3478}
3479
3480/// Helper for doing combines based on N0 and N1 being added to each other.
3481SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
3482 const SDLoc &DL) {
3483 EVT VT = N0.getValueType();
3484
3485 // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
3486 SDValue Y, N;
3487 if (sd_match(N: N1, P: m_Shl(L: m_Neg(V: m_Value(N&: Y)), R: m_Value(N))))
3488 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0,
3489 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Y, N2: N));
3490
3491 if (SDValue V = foldAddSubMasked1(IsAdd: true, N0, N1, DAG, DL))
3492 return V;
3493
3494 // Look for:
3495 // add (add x, 1), y
3496 // And if the target does not like this form then turn into:
3497 // sub y, (xor x, -1)
3498 if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
3499 N0.hasOneUse() && isOneOrOneSplat(V: N0.getOperand(i: 1)) &&
3500 // Limit this to after legalization if the add has wrap flags
3501 (Level >= AfterLegalizeDAG || (!N0->getFlags().hasNoUnsignedWrap() &&
3502 !N0->getFlags().hasNoSignedWrap()))) {
3503 SDValue Not = DAG.getNOT(DL, Val: N0.getOperand(i: 0), VT);
3504 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1, N2: Not);
3505 }
3506
3507 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) {
3508 // Hoist one-use subtraction by non-opaque constant:
3509 // (x - C) + y -> (x + y) - C
3510 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
3511 if (isConstantOrConstantVector(N: N0.getOperand(i: 1), /*NoOpaques=*/true)) {
3512 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 0), N2: N1);
3513 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Add, N2: N0.getOperand(i: 1));
3514 }
3515 // Hoist one-use subtraction from non-opaque constant:
3516 // (C - x) + y -> (y - x) + C
3517 if (isConstantOrConstantVector(N: N0.getOperand(i: 0), /*NoOpaques=*/true)) {
3518 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1, N2: N0.getOperand(i: 1));
3519 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Sub, N2: N0.getOperand(i: 0));
3520 }
3521 }
3522
3523 // add (mul x, C), x -> mul x, C+1
3524 if (N0.getOpcode() == ISD::MUL && N0.getOperand(i: 0) == N1 &&
3525 isConstantOrConstantVector(N: N0.getOperand(i: 1), /*NoOpaques=*/true) &&
3526 N0.hasOneUse()) {
3527 SDValue NewC = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 1),
3528 N2: DAG.getConstant(Val: 1, DL, VT));
3529 return DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N0.getOperand(i: 0), N2: NewC);
3530 }
3531
3532 // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1'
3533 // rather than 'add 0/-1' (the zext should get folded).
3534 // add (sext i1 Y), X --> sub X, (zext i1 Y)
3535 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
3536 N0.getOperand(i: 0).getScalarValueSizeInBits() == 1 &&
3537 TLI.getBooleanContents(Type: VT) == TargetLowering::ZeroOrOneBooleanContent) {
3538 SDValue ZExt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: N0.getOperand(i: 0));
3539 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1, N2: ZExt);
3540 }
3541
3542 // add X, (sextinreg Y i1) -> sub X, (and Y 1)
3543 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
3544 VTSDNode *TN = cast<VTSDNode>(Val: N1.getOperand(i: 1));
3545 if (TN->getVT() == MVT::i1) {
3546 SDValue ZExt = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N1.getOperand(i: 0),
3547 N2: DAG.getConstant(Val: 1, DL, VT));
3548 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: ZExt);
3549 }
3550 }
3551
3552 // (add X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
3553 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(V: N1.getOperand(i: 1)) &&
3554 N1.getResNo() == 0)
3555 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: N1->getVTList(),
3556 N1: N0, N2: N1.getOperand(i: 0), N3: N1.getOperand(i: 2));
3557
3558 // (add X, Carry) -> (uaddo_carry X, 0, Carry)
3559 if (TLI.isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT))
3560 if (SDValue Carry = getAsCarry(TLI, V: N1))
3561 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL,
3562 VTList: DAG.getVTList(VT1: VT, VT2: Carry.getValueType()), N1: N0,
3563 N2: DAG.getConstant(Val: 0, DL, VT), N3: Carry);
3564
3565 return SDValue();
3566}
3567
3568SDValue DAGCombiner::visitADDC(SDNode *N) {
3569 SDValue N0 = N->getOperand(Num: 0);
3570 SDValue N1 = N->getOperand(Num: 1);
3571 EVT VT = N0.getValueType();
3572 SDLoc DL(N);
3573
3574 // If the flag result is dead, turn this into an ADD.
3575 if (!N->hasAnyUseOfValue(Value: 1))
3576 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: N1),
3577 Res1: DAG.getNode(Opcode: ISD::CARRY_FALSE, DL, VT: MVT::Glue));
3578
3579 // canonicalize constant to RHS.
3580 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
3581 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
3582 if (N0C && !N1C)
3583 return DAG.getNode(Opcode: ISD::ADDC, DL, VTList: N->getVTList(), N1, N2: N0);
3584
3585 // fold (addc x, 0) -> x + no carry out
3586 if (isNullConstant(V: N1))
3587 return CombineTo(N, Res0: N0, Res1: DAG.getNode(Opcode: ISD::CARRY_FALSE,
3588 DL, VT: MVT::Glue));
3589
3590 // If it cannot overflow, transform into an add.
3591 if (DAG.computeOverflowForUnsignedAdd(N0, N1) == SelectionDAG::OFK_Never)
3592 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: N1),
3593 Res1: DAG.getNode(Opcode: ISD::CARRY_FALSE, DL, VT: MVT::Glue));
3594
3595 return SDValue();
3596}
3597
3598/**
3599 * Flips a boolean if it is cheaper to compute. If the Force parameters is set,
3600 * then the flip also occurs if computing the inverse is the same cost.
3601 * This function returns an empty SDValue in case it cannot flip the boolean
3602 * without increasing the cost of the computation. If you want to flip a boolean
3603 * no matter what, use DAG.getLogicalNOT.
3604 */
3605static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
3606 const TargetLowering &TLI,
3607 bool Force) {
3608 if (Force && isa<ConstantSDNode>(Val: V))
3609 return DAG.getLogicalNOT(DL: SDLoc(V), Val: V, VT: V.getValueType());
3610
3611 if (V.getOpcode() != ISD::XOR)
3612 return SDValue();
3613
3614 if (DAG.isBoolConstant(N: V.getOperand(i: 1)) == true)
3615 return V.getOperand(i: 0);
3616 if (Force && isConstOrConstSplat(N: V.getOperand(i: 1), AllowUndefs: false))
3617 return DAG.getLogicalNOT(DL: SDLoc(V), Val: V, VT: V.getValueType());
3618 return SDValue();
3619}
3620
3621SDValue DAGCombiner::visitADDO(SDNode *N) {
3622 SDValue N0 = N->getOperand(Num: 0);
3623 SDValue N1 = N->getOperand(Num: 1);
3624 EVT VT = N0.getValueType();
3625 bool IsSigned = (ISD::SADDO == N->getOpcode());
3626
3627 EVT CarryVT = N->getValueType(ResNo: 1);
3628 SDLoc DL(N);
3629
3630 // If the flag result is dead, turn this into an ADD.
3631 if (!N->hasAnyUseOfValue(Value: 1))
3632 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: N1),
3633 Res1: DAG.getUNDEF(VT: CarryVT));
3634
3635 // canonicalize constant to RHS.
3636 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
3637 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
3638 return DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), N1, N2: N0);
3639
3640 // fold (addo x, 0) -> x + no carry out
3641 if (isNullOrNullSplat(V: N1))
3642 return CombineTo(N, Res0: N0, Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
3643
3644 // If it cannot overflow, transform into an add.
3645 if (DAG.willNotOverflowAdd(IsSigned, N0, N1))
3646 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: N1),
3647 Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
3648
3649 if (IsSigned) {
3650 // fold (saddo (xor a, -1), 1) -> (ssub 0, a).
3651 if (isBitwiseNot(V: N0) && isOneOrOneSplat(V: N1))
3652 return DAG.getNode(Opcode: ISD::SSUBO, DL, VTList: N->getVTList(),
3653 N1: DAG.getConstant(Val: 0, DL, VT), N2: N0.getOperand(i: 0));
3654 } else {
3655 // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
3656 if (isBitwiseNot(V: N0) && isOneOrOneSplat(V: N1)) {
3657 SDValue Sub = DAG.getNode(Opcode: ISD::USUBO, DL, VTList: N->getVTList(),
3658 N1: DAG.getConstant(Val: 0, DL, VT), N2: N0.getOperand(i: 0));
3659 return CombineTo(
3660 N, Res0: Sub, Res1: DAG.getLogicalNOT(DL, Val: Sub.getValue(R: 1), VT: Sub->getValueType(ResNo: 1)));
3661 }
3662
3663 if (SDValue Combined = visitUADDOLike(N0, N1, N))
3664 return Combined;
3665
3666 if (SDValue Combined = visitUADDOLike(N0: N1, N1: N0, N))
3667 return Combined;
3668 }
3669
3670 return SDValue();
3671}
3672
3673SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
3674 EVT VT = N0.getValueType();
3675 if (VT.isVector())
3676 return SDValue();
3677
3678 // (uaddo X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
3679 // If Y + 1 cannot overflow.
3680 if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(V: N1.getOperand(i: 1))) {
3681 SDValue Y = N1.getOperand(i: 0);
3682 SDValue One = DAG.getConstant(Val: 1, DL: SDLoc(N), VT: Y.getValueType());
3683 if (DAG.computeOverflowForUnsignedAdd(N0: Y, N1: One) == SelectionDAG::OFK_Never)
3684 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: N->getVTList(), N1: N0, N2: Y,
3685 N3: N1.getOperand(i: 2));
3686 }
3687
3688 // (uaddo X, Carry) -> (uaddo_carry X, 0, Carry)
3689 if (TLI.isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT))
3690 if (SDValue Carry = getAsCarry(TLI, V: N1))
3691 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: N->getVTList(), N1: N0,
3692 N2: DAG.getConstant(Val: 0, DL: SDLoc(N), VT), N3: Carry);
3693
3694 return SDValue();
3695}
3696
3697SDValue DAGCombiner::visitADDE(SDNode *N) {
3698 SDValue N0 = N->getOperand(Num: 0);
3699 SDValue N1 = N->getOperand(Num: 1);
3700 SDValue CarryIn = N->getOperand(Num: 2);
3701
3702 // canonicalize constant to RHS
3703 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
3704 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
3705 if (N0C && !N1C)
3706 return DAG.getNode(Opcode: ISD::ADDE, DL: SDLoc(N), VTList: N->getVTList(),
3707 N1, N2: N0, N3: CarryIn);
3708
3709 // fold (adde x, y, false) -> (addc x, y)
3710 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
3711 return DAG.getNode(Opcode: ISD::ADDC, DL: SDLoc(N), VTList: N->getVTList(), N1: N0, N2: N1);
3712
3713 return SDValue();
3714}
3715
3716SDValue DAGCombiner::visitUADDO_CARRY(SDNode *N) {
3717 SDValue N0 = N->getOperand(Num: 0);
3718 SDValue N1 = N->getOperand(Num: 1);
3719 SDValue CarryIn = N->getOperand(Num: 2);
3720 SDLoc DL(N);
3721
3722 // canonicalize constant to RHS
3723 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
3724 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
3725 if (N0C && !N1C)
3726 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: N->getVTList(), N1, N2: N0, N3: CarryIn);
3727
3728 // fold (uaddo_carry x, y, false) -> (uaddo x, y)
3729 if (isNullConstant(V: CarryIn)) {
3730 if (!LegalOperations ||
3731 TLI.isOperationLegalOrCustom(Op: ISD::UADDO, VT: N->getValueType(ResNo: 0)))
3732 return DAG.getNode(Opcode: ISD::UADDO, DL, VTList: N->getVTList(), N1: N0, N2: N1);
3733 }
3734
3735 // fold (uaddo_carry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
3736 if (isNullConstant(V: N0) && isNullConstant(V: N1)) {
3737 EVT VT = N0.getValueType();
3738 EVT CarryVT = CarryIn.getValueType();
3739 SDValue CarryExt = DAG.getBoolExtOrTrunc(Op: CarryIn, SL: DL, VT, OpVT: CarryVT);
3740 AddToWorklist(N: CarryExt.getNode());
3741 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::AND, DL, VT, N1: CarryExt,
3742 N2: DAG.getConstant(Val: 1, DL, VT)),
3743 Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
3744 }
3745
3746 if (SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn, N))
3747 return Combined;
3748
3749 if (SDValue Combined = visitUADDO_CARRYLike(N0: N1, N1: N0, CarryIn, N))
3750 return Combined;
3751
3752 // We want to avoid useless duplication.
3753 // TODO: This is done automatically for binary operations. As UADDO_CARRY is
3754 // not a binary operation, this is not really possible to leverage this
3755 // existing mechanism for it. However, if more operations require the same
3756 // deduplication logic, then it may be worth generalize.
3757 SDValue Ops[] = {N1, N0, CarryIn};
3758 SDNode *CSENode =
3759 DAG.getNodeIfExists(Opcode: ISD::UADDO_CARRY, VTList: N->getVTList(), Ops, Flags: N->getFlags());
3760 if (CSENode)
3761 return SDValue(CSENode, 0);
3762
3763 return SDValue();
3764}
3765
3766/**
3767 * If we are facing some sort of diamond carry propagation pattern try to
3768 * break it up to generate something like:
3769 * (uaddo_carry X, 0, (uaddo_carry A, B, Z):Carry)
3770 *
3771 * The end result is usually an increase in operation required, but because the
3772 * carry is now linearized, other transforms can kick in and optimize the DAG.
3773 *
3774 * Patterns typically look something like
3775 * (uaddo A, B)
3776 * / \
3777 * Carry Sum
3778 * | \
3779 * | (uaddo_carry *, 0, Z)
3780 * | /
3781 * \ Carry
3782 * | /
3783 * (uaddo_carry X, *, *)
3784 *
3785 * But numerous variation exist. Our goal is to identify A, B, X and Z and
3786 * produce a combine with a single path for carry propagation.
3787 */
3788static SDValue combineUADDO_CARRYDiamond(DAGCombiner &Combiner,
3789 SelectionDAG &DAG, SDValue X,
3790 SDValue Carry0, SDValue Carry1,
3791 SDNode *N) {
3792 if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1)
3793 return SDValue();
3794 if (Carry1.getOpcode() != ISD::UADDO)
3795 return SDValue();
3796
3797 SDValue Z;
3798
3799 /**
3800 * First look for a suitable Z. It will present itself in the form of
3801 * (uaddo_carry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
3802 */
3803 if (Carry0.getOpcode() == ISD::UADDO_CARRY &&
3804 isNullConstant(V: Carry0.getOperand(i: 1))) {
3805 Z = Carry0.getOperand(i: 2);
3806 } else if (Carry0.getOpcode() == ISD::UADDO &&
3807 isOneConstant(V: Carry0.getOperand(i: 1))) {
3808 EVT VT = Carry0->getValueType(ResNo: 1);
3809 Z = DAG.getConstant(Val: 1, DL: SDLoc(Carry0.getOperand(i: 1)), VT);
3810 } else {
3811 // We couldn't find a suitable Z.
3812 return SDValue();
3813 }
3814
3815
3816 auto cancelDiamond = [&](SDValue A,SDValue B) {
3817 SDLoc DL(N);
3818 SDValue NewY =
3819 DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: Carry0->getVTList(), N1: A, N2: B, N3: Z);
3820 Combiner.AddToWorklist(N: NewY.getNode());
3821 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: N->getVTList(), N1: X,
3822 N2: DAG.getConstant(Val: 0, DL, VT: X.getValueType()),
3823 N3: NewY.getValue(R: 1));
3824 };
3825
3826 /**
3827 * (uaddo A, B)
3828 * |
3829 * Sum
3830 * |
3831 * (uaddo_carry *, 0, Z)
3832 */
3833 if (Carry0.getOperand(i: 0) == Carry1.getValue(R: 0)) {
3834 return cancelDiamond(Carry1.getOperand(i: 0), Carry1.getOperand(i: 1));
3835 }
3836
3837 /**
3838 * (uaddo_carry A, 0, Z)
3839 * |
3840 * Sum
3841 * |
3842 * (uaddo *, B)
3843 */
3844 if (Carry1.getOperand(i: 0) == Carry0.getValue(R: 0)) {
3845 return cancelDiamond(Carry0.getOperand(i: 0), Carry1.getOperand(i: 1));
3846 }
3847
3848 if (Carry1.getOperand(i: 1) == Carry0.getValue(R: 0)) {
3849 return cancelDiamond(Carry1.getOperand(i: 0), Carry0.getOperand(i: 0));
3850 }
3851
3852 return SDValue();
3853}
3854
3855// If we are facing some sort of diamond carry/borrow in/out pattern try to
3856// match patterns like:
3857//
3858// (uaddo A, B) CarryIn
3859// | \ |
3860// | \ |
3861// PartialSum PartialCarryOutX /
3862// | | /
3863// | ____|____________/
3864// | / |
3865// (uaddo *, *) \________
3866// | \ \
3867// | \ |
3868// | PartialCarryOutY |
3869// | \ |
3870// | \ /
3871// AddCarrySum | ______/
3872// | /
3873// CarryOut = (or *, *)
3874//
3875// And generate UADDO_CARRY (or USUBO_CARRY) with two result values:
3876//
3877// {AddCarrySum, CarryOut} = (uaddo_carry A, B, CarryIn)
3878//
3879// Our goal is to identify A, B, and CarryIn and produce UADDO_CARRY/USUBO_CARRY
3880// with a single path for carry/borrow out propagation.
3881static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI,
3882 SDValue N0, SDValue N1, SDNode *N) {
3883 SDValue Carry0 = getAsCarry(TLI, V: N0);
3884 if (!Carry0)
3885 return SDValue();
3886 SDValue Carry1 = getAsCarry(TLI, V: N1);
3887 if (!Carry1)
3888 return SDValue();
3889
3890 unsigned Opcode = Carry0.getOpcode();
3891 if (Opcode != Carry1.getOpcode())
3892 return SDValue();
3893 if (Opcode != ISD::UADDO && Opcode != ISD::USUBO)
3894 return SDValue();
3895 // Guarantee identical type of CarryOut
3896 EVT CarryOutType = N->getValueType(ResNo: 0);
3897 if (CarryOutType != Carry0.getValue(R: 1).getValueType() ||
3898 CarryOutType != Carry1.getValue(R: 1).getValueType())
3899 return SDValue();
3900
3901 // Canonicalize the add/sub of A and B (the top node in the above ASCII art)
3902 // as Carry0 and the add/sub of the carry in as Carry1 (the middle node).
3903 if (Carry1.getNode()->isOperandOf(N: Carry0.getNode()))
3904 std::swap(a&: Carry0, b&: Carry1);
3905
3906 // Check if nodes are connected in expected way.
3907 if (Carry1.getOperand(i: 0) != Carry0.getValue(R: 0) &&
3908 Carry1.getOperand(i: 1) != Carry0.getValue(R: 0))
3909 return SDValue();
3910
3911 // The carry in value must be on the righthand side for subtraction.
3912 unsigned CarryInOperandNum =
3913 Carry1.getOperand(i: 0) == Carry0.getValue(R: 0) ? 1 : 0;
3914 if (Opcode == ISD::USUBO && CarryInOperandNum != 1)
3915 return SDValue();
3916 SDValue CarryIn = Carry1.getOperand(i: CarryInOperandNum);
3917
3918 unsigned NewOp = Opcode == ISD::UADDO ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
3919 if (!TLI.isOperationLegalOrCustom(Op: NewOp, VT: Carry0.getValue(R: 0).getValueType()))
3920 return SDValue();
3921
3922 // Verify that the carry/borrow in is plausibly a carry/borrow bit.
3923 CarryIn = getAsCarry(TLI, V: CarryIn, ForceCarryReconstruction: true);
3924 if (!CarryIn)
3925 return SDValue();
3926
3927 SDLoc DL(N);
3928 CarryIn = DAG.getBoolExtOrTrunc(Op: CarryIn, SL: DL, VT: Carry1->getValueType(ResNo: 1),
3929 OpVT: Carry1->getValueType(ResNo: 0));
3930 SDValue Merged =
3931 DAG.getNode(Opcode: NewOp, DL, VTList: Carry1->getVTList(), N1: Carry0.getOperand(i: 0),
3932 N2: Carry0.getOperand(i: 1), N3: CarryIn);
3933
3934 // Please note that because we have proven that the result of the UADDO/USUBO
3935 // of A and B feeds into the UADDO/USUBO that does the carry/borrow in, we can
3936 // therefore prove that if the first UADDO/USUBO overflows, the second
3937 // UADDO/USUBO cannot. For example consider 8-bit numbers where 0xFF is the
3938 // maximum value.
3939 //
3940 // 0xFF + 0xFF == 0xFE with carry but 0xFE + 1 does not carry
3941 // 0x00 - 0xFF == 1 with a carry/borrow but 1 - 1 == 0 (no carry/borrow)
3942 //
3943 // This is important because it means that OR and XOR can be used to merge
3944 // carry flags; and that AND can return a constant zero.
3945 //
3946 // TODO: match other operations that can merge flags (ADD, etc)
3947 DAG.ReplaceAllUsesOfValueWith(From: Carry1.getValue(R: 0), To: Merged.getValue(R: 0));
3948 if (N->getOpcode() == ISD::AND)
3949 return DAG.getConstant(Val: 0, DL, VT: CarryOutType);
3950 return Merged.getValue(R: 1);
3951}
3952
3953// Reconstruct a subtract-with-borrow chain from its canonicalized icmp form:
3954// carry_out = or(icmp ult A, B, and(icmp eq A, B, carry_in))
3955// InstCombine folds usub.with.overflow chains into this, losing the
3956// USUBO_CARRY that lowers to sbb/sbcs.
3957static SDValue combineOrOfSetCCToUSUBOCarry(SDNode *N, SelectionDAG &DAG,
3958 const TargetLowering &TLI) {
3959 SDValue A, B, CarryIn;
3960 if (!sd_match(N, P: m_Or(L: m_SetCC(LHS: m_Value(N&: A), RHS: m_Value(N&: B),
3961 CC: m_SpecificCondCode(CC: ISD::SETULT)),
3962 R: m_And(L: m_c_SetCC(LHS: m_Deferred(V&: A), RHS: m_Deferred(V&: B),
3963 CC: m_SpecificCondCode(CC: ISD::SETEQ)),
3964 R: m_Value(N&: CarryIn)))))
3965 return SDValue();
3966
3967 EVT IntVT = A.getValueType();
3968 // Skip vectors: USUBO_CARRY on a vector type has no legalization path and
3969 // would crash.
3970 if (IntVT.isVector() || !TLI.isOperationLegalOrCustom(
3971 Op: ISD::USUBO_CARRY, VT: TLI.getLegalTypeToTransformTo(
3972 Context&: *DAG.getContext(), VT: IntVT)))
3973 return SDValue();
3974
3975 // USUBO_CARRY's carry-in must be 0 or 1, which the matched pattern does not
3976 // guarantee.
3977 if (!DAG.MaskedValueIsZero(
3978 Op: CarryIn,
3979 Mask: APInt::getBitsSetFrom(numBits: CarryIn.getScalarValueSizeInBits(), loBit: 1)))
3980 return SDValue();
3981
3982 SDLoc DL(N);
3983 SDVTList VTs = DAG.getVTList(VT1: IntVT, VT2: N->getValueType(ResNo: 0));
3984 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: VTs, N1: A, N2: B, N3: CarryIn).getValue(R: 1);
3985}
3986
3987SDValue DAGCombiner::visitUADDO_CARRYLike(SDValue N0, SDValue N1,
3988 SDValue CarryIn, SDNode *N) {
3989 // fold (uaddo_carry (xor a, -1), b, c) -> (usubo_carry b, a, !c) and flip
3990 // carry.
3991 if (isBitwiseNot(V: N0))
3992 if (SDValue NotC = extractBooleanFlip(V: CarryIn, DAG, TLI, Force: true)) {
3993 SDLoc DL(N);
3994 SDValue Sub = DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: N->getVTList(), N1,
3995 N2: N0.getOperand(i: 0), N3: NotC);
3996 return CombineTo(
3997 N, Res0: Sub, Res1: DAG.getLogicalNOT(DL, Val: Sub.getValue(R: 1), VT: Sub->getValueType(ResNo: 1)));
3998 }
3999
4000 // Iff the flag result is dead:
4001 // (uaddo_carry (add|uaddo X, Y), 0, Carry) -> (uaddo_carry X, Y, Carry)
4002 // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo
4003 // or the dependency between the instructions.
4004 if ((N0.getOpcode() == ISD::ADD ||
4005 (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 &&
4006 N0.getValue(R: 1) != CarryIn)) &&
4007 isNullConstant(V: N1) && !N->hasAnyUseOfValue(Value: 1))
4008 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL: SDLoc(N), VTList: N->getVTList(),
4009 N1: N0.getOperand(i: 0), N2: N0.getOperand(i: 1), N3: CarryIn);
4010
4011 /**
4012 * When one of the uaddo_carry argument is itself a carry, we may be facing
4013 * a diamond carry propagation. In which case we try to transform the DAG
4014 * to ensure linear carry propagation if that is possible.
4015 */
4016 if (auto Y = getAsCarry(TLI, V: N1)) {
4017 // Because both are carries, Y and Z can be swapped.
4018 if (auto R = combineUADDO_CARRYDiamond(Combiner&: *this, DAG, X: N0, Carry0: Y, Carry1: CarryIn, N))
4019 return R;
4020 if (auto R = combineUADDO_CARRYDiamond(Combiner&: *this, DAG, X: N0, Carry0: CarryIn, Carry1: Y, N))
4021 return R;
4022 }
4023
4024 return SDValue();
4025}
4026
4027SDValue DAGCombiner::visitSADDO_CARRYLike(SDValue N0, SDValue N1,
4028 SDValue CarryIn, SDNode *N) {
4029 // fold (saddo_carry (xor a, -1), b, c) -> (ssubo_carry b, a, !c)
4030 if (isBitwiseNot(V: N0)) {
4031 if (SDValue NotC = extractBooleanFlip(V: CarryIn, DAG, TLI, Force: true))
4032 return DAG.getNode(Opcode: ISD::SSUBO_CARRY, DL: SDLoc(N), VTList: N->getVTList(), N1,
4033 N2: N0.getOperand(i: 0), N3: NotC);
4034 }
4035
4036 return SDValue();
4037}
4038
4039SDValue DAGCombiner::visitSADDO_CARRY(SDNode *N) {
4040 SDValue N0 = N->getOperand(Num: 0);
4041 SDValue N1 = N->getOperand(Num: 1);
4042 SDValue CarryIn = N->getOperand(Num: 2);
4043 SDLoc DL(N);
4044
4045 // canonicalize constant to RHS
4046 ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
4047 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
4048 if (N0C && !N1C)
4049 return DAG.getNode(Opcode: ISD::SADDO_CARRY, DL, VTList: N->getVTList(), N1, N2: N0, N3: CarryIn);
4050
4051 // fold (saddo_carry x, y, false) -> (saddo x, y)
4052 if (isNullConstant(V: CarryIn)) {
4053 if (!LegalOperations ||
4054 TLI.isOperationLegalOrCustom(Op: ISD::SADDO, VT: N->getValueType(ResNo: 0)))
4055 return DAG.getNode(Opcode: ISD::SADDO, DL, VTList: N->getVTList(), N1: N0, N2: N1);
4056 }
4057
4058 if (SDValue Combined = visitSADDO_CARRYLike(N0, N1, CarryIn, N))
4059 return Combined;
4060
4061 if (SDValue Combined = visitSADDO_CARRYLike(N0: N1, N1: N0, CarryIn, N))
4062 return Combined;
4063
4064 return SDValue();
4065}
4066
4067// Attempt to create a USUBSAT(LHS, RHS) node with DstVT, performing a
4068// clamp/truncation if necessary.
4069static SDValue getTruncatedUSUBSAT(EVT DstVT, EVT SrcVT, SDValue LHS,
4070 SDValue RHS, SelectionDAG &DAG,
4071 const SDLoc &DL) {
4072 assert(DstVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&
4073 "Illegal truncation");
4074
4075 if (DstVT == SrcVT)
4076 return DAG.getNode(Opcode: ISD::USUBSAT, DL, VT: DstVT, N1: LHS, N2: RHS);
4077
4078 // If the LHS is zero-extended then we can perform the USUBSAT as DstVT by
4079 // clamping RHS.
4080 APInt UpperBits = APInt::getBitsSetFrom(numBits: SrcVT.getScalarSizeInBits(),
4081 loBit: DstVT.getScalarSizeInBits());
4082 if (!DAG.MaskedValueIsZero(Op: LHS, Mask: UpperBits))
4083 return SDValue();
4084
4085 SDValue SatLimit =
4086 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: SrcVT.getScalarSizeInBits(),
4087 loBitsSet: DstVT.getScalarSizeInBits()),
4088 DL, VT: SrcVT);
4089 RHS = DAG.getNode(Opcode: ISD::UMIN, DL, VT: SrcVT, N1: RHS, N2: SatLimit);
4090 RHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: RHS);
4091 LHS = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: DstVT, Operand: LHS);
4092 return DAG.getNode(Opcode: ISD::USUBSAT, DL, VT: DstVT, N1: LHS, N2: RHS);
4093}
4094
4095// Try to find umax(a,b) - b or a - umin(a,b) patterns that may be converted to
4096// usubsat(a,b), optionally as a truncated type.
4097SDValue DAGCombiner::foldSubToUSubSat(EVT DstVT, SDNode *N, const SDLoc &DL) {
4098 if (N->getOpcode() != ISD::SUB ||
4099 !(!LegalOperations || hasOperation(Opcode: ISD::USUBSAT, VT: DstVT)))
4100 return SDValue();
4101
4102 EVT SubVT = N->getValueType(ResNo: 0);
4103 SDValue Op0 = N->getOperand(Num: 0);
4104 SDValue Op1 = N->getOperand(Num: 1);
4105
4106 // Try to find umax(a,b) - b or a - umin(a,b) patterns
4107 // they may be converted to usubsat(a,b).
4108 if (Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
4109 SDValue MaxLHS = Op0.getOperand(i: 0);
4110 SDValue MaxRHS = Op0.getOperand(i: 1);
4111 if (MaxLHS == Op1)
4112 return getTruncatedUSUBSAT(DstVT, SrcVT: SubVT, LHS: MaxRHS, RHS: Op1, DAG, DL);
4113 if (MaxRHS == Op1)
4114 return getTruncatedUSUBSAT(DstVT, SrcVT: SubVT, LHS: MaxLHS, RHS: Op1, DAG, DL);
4115 }
4116
4117 if (Op1.getOpcode() == ISD::UMIN && Op1.hasOneUse()) {
4118 SDValue MinLHS = Op1.getOperand(i: 0);
4119 SDValue MinRHS = Op1.getOperand(i: 1);
4120 if (MinLHS == Op0)
4121 return getTruncatedUSUBSAT(DstVT, SrcVT: SubVT, LHS: Op0, RHS: MinRHS, DAG, DL);
4122 if (MinRHS == Op0)
4123 return getTruncatedUSUBSAT(DstVT, SrcVT: SubVT, LHS: Op0, RHS: MinLHS, DAG, DL);
4124 }
4125
4126 // sub(a,trunc(umin(zext(a),b))) -> usubsat(a,trunc(umin(b,SatLimit)))
4127 if (Op1.getOpcode() == ISD::TRUNCATE &&
4128 Op1.getOperand(i: 0).getOpcode() == ISD::UMIN &&
4129 Op1.getOperand(i: 0).hasOneUse()) {
4130 SDValue MinLHS = Op1.getOperand(i: 0).getOperand(i: 0);
4131 SDValue MinRHS = Op1.getOperand(i: 0).getOperand(i: 1);
4132 if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && MinLHS.getOperand(i: 0) == Op0)
4133 return getTruncatedUSUBSAT(DstVT, SrcVT: MinLHS.getValueType(), LHS: MinLHS, RHS: MinRHS,
4134 DAG, DL);
4135 if (MinRHS.getOpcode() == ISD::ZERO_EXTEND && MinRHS.getOperand(i: 0) == Op0)
4136 return getTruncatedUSUBSAT(DstVT, SrcVT: MinLHS.getValueType(), LHS: MinRHS, RHS: MinLHS,
4137 DAG, DL);
4138 }
4139
4140 return SDValue();
4141}
4142
4143// Refinement of DAG/Type Legalisation (promotion) when CTLZ is used for
4144// counting leading ones. Broadly, it replaces the substraction with a left
4145// shift.
4146//
4147// * DAG Legalisation Pattern:
4148//
4149// (sub (ctlz (zeroextend (not Src)))
4150// BitWidthDiff)
4151//
4152// if BitWidthDiff == BitWidth(Node) - BitWidth(Src)
4153// -->
4154//
4155// (ctlz_zero_poison (not (shl (anyextend Src)
4156// BitWidthDiff)))
4157//
4158// * Type Legalisation Pattern:
4159//
4160// (sub (ctlz (and (xor Src XorMask)
4161// AndMask))
4162// BitWidthDiff)
4163//
4164// if AndMask has only trailing ones
4165// and MaskBitWidth(AndMask) == BitWidth(Node) - BitWidthDiff
4166// and XorMask has more trailing ones than AndMask
4167// -->
4168//
4169// (ctlz_zero_poison (not (shl Src BitWidthDiff)))
4170static SDValue foldSubCtlzNot(SDNode *N, SelectionDAG &DAG) {
4171 const SDLoc DL(N);
4172 SDValue N0 = N->getOperand(Num: 0);
4173 EVT VT = N0.getValueType();
4174 unsigned BitWidth = VT.getScalarSizeInBits();
4175
4176 APInt AndMask;
4177 APInt XorMask;
4178 uint64_t BitWidthDiff;
4179
4180 SDValue CtlzOp;
4181 SDValue Src;
4182
4183 if (!sd_match(N, P: m_Sub(L: m_Ctlz(Op: m_Value(N&: CtlzOp)), R: m_ConstInt(V&: BitWidthDiff))))
4184 return SDValue();
4185
4186 if (sd_match(N: CtlzOp, P: m_ZExt(Op: m_Not(V: m_Value(N&: Src))))) {
4187 // DAG Legalisation Pattern:
4188 // (sub (ctlz (zero_extend (not Op)) BitWidthDiff))
4189 if ((BitWidth - Src.getValueType().getScalarSizeInBits()) != BitWidthDiff)
4190 return SDValue();
4191
4192 Src = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: Src);
4193 } else if (sd_match(N: CtlzOp, P: m_And(L: m_Xor(L: m_Value(N&: Src), R: m_ConstInt(V&: XorMask)),
4194 R: m_ConstInt(V&: AndMask)))) {
4195 // Type Legalisation Pattern:
4196 // (sub (ctlz (and (xor Op XorMask) AndMask)) BitWidthDiff)
4197 if (BitWidthDiff >= BitWidth)
4198 return SDValue();
4199 unsigned AndMaskWidth = BitWidth - BitWidthDiff;
4200 if (!(AndMask.isMask(numBits: AndMaskWidth) && XorMask.countr_one() >= AndMaskWidth))
4201 return SDValue();
4202 } else
4203 return SDValue();
4204
4205 SDValue ShiftConst = DAG.getShiftAmountConstant(Val: BitWidthDiff, VT, DL);
4206 SDValue LShift = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Src, N2: ShiftConst);
4207 SDValue Not =
4208 DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: LShift, N2: DAG.getAllOnesConstant(DL, VT));
4209
4210 return DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL, VT, Operand: Not);
4211}
4212
4213// Fold sub(x, mul(divrem(x,y)[0], y)) to divrem(x, y)[1]
4214static SDValue foldRemainderIdiom(SDNode *N, SelectionDAG &DAG,
4215 const SDLoc &DL) {
4216 assert(N->getOpcode() == ISD::SUB && "Node must be a SUB");
4217 SDValue Sub0 = N->getOperand(Num: 0);
4218 SDValue Sub1 = N->getOperand(Num: 1);
4219
4220 auto CheckAndFoldMulCase = [&](SDValue DivRem, SDValue MaybeY) -> SDValue {
4221 if ((DivRem.getOpcode() == ISD::SDIVREM ||
4222 DivRem.getOpcode() == ISD::UDIVREM) &&
4223 DivRem.getResNo() == 0 && DivRem.getOperand(i: 0) == Sub0 &&
4224 DivRem.getOperand(i: 1) == MaybeY) {
4225 return SDValue(DivRem.getNode(), 1);
4226 }
4227 return SDValue();
4228 };
4229
4230 if (Sub1.getOpcode() == ISD::MUL) {
4231 // (sub x, (mul divrem(x,y)[0], y))
4232 SDValue Mul0 = Sub1.getOperand(i: 0);
4233 SDValue Mul1 = Sub1.getOperand(i: 1);
4234
4235 if (SDValue Res = CheckAndFoldMulCase(Mul0, Mul1))
4236 return Res;
4237
4238 if (SDValue Res = CheckAndFoldMulCase(Mul1, Mul0))
4239 return Res;
4240
4241 } else if (Sub1.getOpcode() == ISD::SHL) {
4242 // Handle (sub x, (shl divrem(x,y)[0], C)) where y = 1 << C
4243 SDValue Shl0 = Sub1.getOperand(i: 0);
4244 SDValue Shl1 = Sub1.getOperand(i: 1);
4245 // Check if Shl0 is divrem(x, Y)[0]
4246 if ((Shl0.getOpcode() == ISD::SDIVREM ||
4247 Shl0.getOpcode() == ISD::UDIVREM) &&
4248 Shl0.getResNo() == 0 && Shl0.getOperand(i: 0) == Sub0) {
4249
4250 SDValue Divisor = Shl0.getOperand(i: 1);
4251
4252 ConstantSDNode *DivC = isConstOrConstSplat(N: Divisor);
4253 ConstantSDNode *ShC = isConstOrConstSplat(N: Shl1);
4254 if (!DivC || !ShC)
4255 return SDValue();
4256
4257 if (DivC->getAPIntValue().isPowerOf2() &&
4258 DivC->getAPIntValue().logBase2() == ShC->getAPIntValue())
4259 return SDValue(Shl0.getNode(), 1);
4260 }
4261 }
4262 return SDValue();
4263}
4264
4265// Since it may not be valid to emit a fold to zero for vector initializers
4266// check if we can before folding.
4267static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
4268 SelectionDAG &DAG, bool LegalOperations) {
4269 if (!VT.isVector())
4270 return DAG.getConstant(Val: 0, DL, VT);
4271 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::BUILD_VECTOR, VT))
4272 return DAG.getConstant(Val: 0, DL, VT);
4273 return SDValue();
4274}
4275
4276SDValue DAGCombiner::visitSUB(SDNode *N) {
4277 SDValue N0 = N->getOperand(Num: 0);
4278 SDValue N1 = N->getOperand(Num: 1);
4279 EVT VT = N0.getValueType();
4280 unsigned BitWidth = VT.getScalarSizeInBits();
4281 SDLoc DL(N);
4282
4283 if (SDValue V = foldSubCtlzNot(N, DAG))
4284 return V;
4285
4286 // fold (sub x, x) -> 0
4287 if (N0 == N1)
4288 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
4289
4290 // fold (sub c1, c2) -> c3
4291 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::SUB, DL, VT, Ops: {N0, N1}))
4292 return C;
4293
4294 // fold vector ops
4295 if (VT.isVector()) {
4296 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4297 return FoldedVOp;
4298
4299 // fold (sub x, 0) -> x, vector edition
4300 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
4301 return N0;
4302 }
4303
4304 // (sub x, ([v]select (ult x, y), 0, y)) -> (umin x, (sub x, y))
4305 // (sub x, ([v]select (uge x, y), y, 0)) -> (umin x, (sub x, y))
4306 if (N1.hasOneUse() && hasUMin(VT)) {
4307 SDValue Y;
4308 auto MS0 = m_Specific(N: N0);
4309 auto MVY = m_Value(N&: Y);
4310 auto MZ = m_Zero();
4311 auto MCC1 = m_SpecificCondCode(CC: ISD::SETULT);
4312 auto MCC2 = m_SpecificCondCode(CC: ISD::SETUGE);
4313
4314 if (sd_match(N: N1, P: m_SelectCCLike(L: MS0, R: MVY, T: MZ, F: m_Deferred(V&: Y), CC: MCC1)) ||
4315 sd_match(N: N1, P: m_SelectCCLike(L: MS0, R: MVY, T: m_Deferred(V&: Y), F: MZ, CC: MCC2)) ||
4316 sd_match(N: N1, P: m_VSelect(Cond: m_SetCC(LHS: MS0, RHS: MVY, CC: MCC1), T: MZ, F: m_Deferred(V&: Y))) ||
4317 sd_match(N: N1, P: m_VSelect(Cond: m_SetCC(LHS: MS0, RHS: MVY, CC: MCC2), T: m_Deferred(V&: Y), F: MZ)))
4318
4319 return DAG.getNode(Opcode: ISD::UMIN, DL, VT, N1: N0,
4320 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Y));
4321 }
4322
4323 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
4324 return NewSel;
4325
4326 // fold (sub x, c) -> (add x, -c)
4327 if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N: N1))
4328 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0,
4329 N2: DAG.getConstant(Val: -N1C->getAPIntValue(), DL, VT));
4330
4331 if (isNullOrNullSplat(V: N0)) {
4332 // Right-shifting everything out but the sign bit followed by negation is
4333 // the same as flipping arithmetic/logical shift type without the negation:
4334 // -(X >>u 31) -> (X >>s 31)
4335 // -(X >>s 31) -> (X >>u 31)
4336 if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
4337 ConstantSDNode *ShiftAmt = isConstOrConstSplat(N: N1.getOperand(i: 1));
4338 if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) {
4339 auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
4340 if (!LegalOperations || TLI.isOperationLegal(Op: NewSh, VT))
4341 return DAG.getNode(Opcode: NewSh, DL, VT, N1: N1.getOperand(i: 0), N2: N1.getOperand(i: 1));
4342 }
4343 }
4344
4345 // 0 - X --> 0 if the sub is NUW.
4346 if (N->getFlags().hasNoUnsignedWrap())
4347 return N0;
4348
4349 if (DAG.MaskedValueIsZero(Op: N1, Mask: ~APInt::getSignMask(BitWidth))) {
4350 // N1 is either 0 or the minimum signed value. If the sub is NSW, then
4351 // N1 must be 0 because negating the minimum signed value is undefined.
4352 if (N->getFlags().hasNoSignedWrap())
4353 return N0;
4354
4355 // 0 - X --> X if X is 0 or the minimum signed value.
4356 return N1;
4357 }
4358
4359 // Convert 0 - abs(x).
4360 if (ISD::isAbsOpcode(Opcode: N1.getOpcode()) && N1.hasOneUse() &&
4361 !TLI.isOperationLegalOrCustom(Op: N1.getOpcode(), VT))
4362 if (SDValue Result = TLI.expandABS(N: N1.getNode(), DAG, IsNegative: true))
4363 return Result;
4364
4365 // Similar to the previous rule, but this time targeting an expanded abs.
4366 // (sub 0, (max X, (sub 0, X))) --> (min X, (sub 0, X))
4367 // as well as
4368 // (sub 0, (min X, (sub 0, X))) --> (max X, (sub 0, X))
4369 // Note that these two are applicable to both signed and unsigned min/max.
4370 SDValue X;
4371 SDValue S0;
4372 auto NegPat = m_Value(N&: S0, P: m_Neg(V: m_Deferred(V&: X)));
4373 if (sd_match(N: N1, P: m_OneUse(P: m_AnyOf(preds: m_SMax(L: m_Value(N&: X), R: NegPat),
4374 preds: m_UMax(L: m_Value(N&: X), R: NegPat),
4375 preds: m_SMin(L: m_Value(N&: X), R: NegPat),
4376 preds: m_UMin(L: m_Value(N&: X), R: NegPat))))) {
4377 unsigned NewOpc = ISD::getInverseMinMaxOpcode(MinMaxOpc: N1->getOpcode());
4378 if (hasOperation(Opcode: NewOpc, VT))
4379 return DAG.getNode(Opcode: NewOpc, DL, VT, N1: X, N2: S0);
4380 }
4381
4382 // Fold neg(splat(neg(x)) -> splat(x)
4383 if (VT.isVector()) {
4384 SDValue N1S = DAG.getSplatValue(V: N1, LegalTypes: true);
4385 if (N1S && N1S.getOpcode() == ISD::SUB &&
4386 isNullConstant(V: N1S.getOperand(i: 0)))
4387 return DAG.getSplat(VT, DL, Op: N1S.getOperand(i: 1));
4388 }
4389
4390 // sub 0, (and x, 1) --> SIGN_EXTEND_INREG x, i1
4391 if (N1.getOpcode() == ISD::AND && N1.hasOneUse() &&
4392 isOneOrOneSplat(V: N1->getOperand(Num: 1))) {
4393 EVT ExtVT = VT.changeElementType(Context&: *DAG.getContext(), EltVT: MVT::i1);
4394 if (TLI.getOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: ExtVT) ==
4395 TargetLowering::Legal) {
4396 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: N1->getOperand(Num: 0),
4397 N2: DAG.getValueType(ExtVT));
4398 }
4399 }
4400 }
4401
4402 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
4403 if (isAllOnesOrAllOnesSplat(V: N0))
4404 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1, N2: N0);
4405
4406 // fold (A - (0-B)) -> A+B
4407 if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(V: N1.getOperand(i: 0)))
4408 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: N1.getOperand(i: 1));
4409
4410 // fold A-(A-B) -> B
4411 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(i: 0))
4412 return N1.getOperand(i: 1);
4413
4414 // fold (A+B)-A -> B
4415 if (N0.getOpcode() == ISD::ADD && N0.getOperand(i: 0) == N1)
4416 return N0.getOperand(i: 1);
4417
4418 // fold (A+B)-B -> A
4419 if (N0.getOpcode() == ISD::ADD && N0.getOperand(i: 1) == N1)
4420 return N0.getOperand(i: 0);
4421
4422 // fold (A+C1)-C2 -> A+(C1-C2)
4423 if (N0.getOpcode() == ISD::ADD) {
4424 SDValue N01 = N0.getOperand(i: 1);
4425 if (SDValue NewC = DAG.FoldConstantArithmetic(Opcode: ISD::SUB, DL, VT, Ops: {N01, N1}))
4426 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 0), N2: NewC);
4427 }
4428
4429 // fold C2-(A+C1) -> (C2-C1)-A
4430 if (N1.getOpcode() == ISD::ADD) {
4431 SDValue N11 = N1.getOperand(i: 1);
4432 if (SDValue NewC = DAG.FoldConstantArithmetic(Opcode: ISD::SUB, DL, VT, Ops: {N0, N11}))
4433 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: NewC, N2: N1.getOperand(i: 0));
4434 }
4435
4436 // fold (A-C1)-C2 -> A-(C1+C2)
4437 if (N0.getOpcode() == ISD::SUB) {
4438 SDValue N01 = N0.getOperand(i: 1);
4439 if (SDValue NewC = DAG.FoldConstantArithmetic(Opcode: ISD::ADD, DL, VT, Ops: {N01, N1}))
4440 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0.getOperand(i: 0), N2: NewC);
4441 }
4442
4443 // fold (c1-A)-c2 -> (c1-c2)-A
4444 if (N0.getOpcode() == ISD::SUB) {
4445 SDValue N00 = N0.getOperand(i: 0);
4446 if (SDValue NewC = DAG.FoldConstantArithmetic(Opcode: ISD::SUB, DL, VT, Ops: {N00, N1}))
4447 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: NewC, N2: N0.getOperand(i: 1));
4448 }
4449
4450 SDValue A, B, C;
4451
4452 // fold ((A+(B+C))-B) -> A+C
4453 if (sd_match(N: N0, P: m_Add(L: m_Value(N&: A), R: m_Add(L: m_Specific(N: N1), R: m_Value(N&: C)))))
4454 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: C);
4455
4456 // fold ((A+(B-C))-B) -> A-C
4457 if (sd_match(N: N0, P: m_Add(L: m_Value(N&: A), R: m_Sub(L: m_Specific(N: N1), R: m_Value(N&: C)))))
4458 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: A, N2: C);
4459
4460 // fold ((A-(B-C))-C) -> A-B
4461 if (sd_match(N: N0, P: m_Sub(L: m_Value(N&: A), R: m_Sub(L: m_Value(N&: B), R: m_Specific(N: N1)))))
4462 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: A, N2: B);
4463
4464 // fold (A-(B-C)) -> A+(C-B)
4465 if (sd_match(N: N1, P: m_OneUse(P: m_Sub(L: m_Value(N&: B), R: m_Value(N&: C)))))
4466 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0,
4467 N2: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: C, N2: B));
4468
4469 // A - (A & B) -> A & (~B)
4470 if (sd_match(N: N1, P: m_And(L: m_Specific(N: N0), R: m_Value(N&: B))) &&
4471 (N1.hasOneUse() || isConstantOrConstantVector(N: B, /*NoOpaques=*/true)))
4472 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N0, N2: DAG.getNOT(DL, Val: B, VT));
4473
4474 // fold (A - (-B * C)) -> (A + (B * C))
4475 if (sd_match(N: N1, P: m_OneUse(P: m_Mul(L: m_Neg(V: m_Value(N&: B)), R: m_Value(N&: C)))))
4476 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0,
4477 N2: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: B, N2: C));
4478
4479 // If either operand of a sub is undef, the result is undef
4480 if (N0.isUndef())
4481 return N0;
4482 if (N1.isUndef())
4483 return N1;
4484
4485 if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
4486 return V;
4487
4488 if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
4489 return V;
4490
4491 // Try to match AVGCEIL fixedwidth pattern
4492 if (SDValue V = foldSubToAvg(N, DL))
4493 return V;
4494
4495 if (SDValue V = foldAddSubMasked1(IsAdd: false, N0, N1, DAG, DL))
4496 return V;
4497
4498 if (SDValue V = foldSubToUSubSat(DstVT: VT, N, DL))
4499 return V;
4500
4501 if (SDValue V = foldRemainderIdiom(N, DAG, DL))
4502 return V;
4503
4504 // (A - B) - 1 -> add (xor B, -1), A
4505 if (sd_match(N, P: m_Sub(L: m_OneUse(P: m_Sub(L: m_Value(N&: A), R: m_Value(N&: B))),
4506 R: m_One(/*AllowUndefs=*/true))))
4507 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: DAG.getNOT(DL, Val: B, VT));
4508
4509 // Look for:
4510 // sub y, (xor x, -1)
4511 // And if the target does not like this form then turn into:
4512 // add (add x, y), 1
4513 if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(V: N1)) {
4514 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: N1.getOperand(i: 0));
4515 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Add, N2: DAG.getConstant(Val: 1, DL, VT));
4516 }
4517
4518 // Hoist one-use addition by non-opaque constant:
4519 // (x + C) - y -> (x - y) + C
4520 if (!reassociationCanBreakAddressingModePattern(Opc: ISD::SUB, DL, N, N0, N1) &&
4521 N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
4522 isConstantOrConstantVector(N: N0.getOperand(i: 1), /*NoOpaques=*/true)) {
4523 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0.getOperand(i: 0), N2: N1);
4524 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Sub, N2: N0.getOperand(i: 1));
4525 }
4526 // y - (x + C) -> (y - x) - C
4527 if (N1.getOpcode() == ISD::ADD && N1.hasOneUse() &&
4528 isConstantOrConstantVector(N: N1.getOperand(i: 1), /*NoOpaques=*/true)) {
4529 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: N1.getOperand(i: 0));
4530 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Sub, N2: N1.getOperand(i: 1));
4531 }
4532 // (x - C) - y -> (x - y) - C
4533 // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
4534 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
4535 isConstantOrConstantVector(N: N0.getOperand(i: 1), /*NoOpaques=*/true)) {
4536 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0.getOperand(i: 0), N2: N1);
4537 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Sub, N2: N0.getOperand(i: 1));
4538 }
4539 // (C - x) - y -> C - (x + y)
4540 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
4541 isConstantOrConstantVector(N: N0.getOperand(i: 0), /*NoOpaques=*/true)) {
4542 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0.getOperand(i: 1), N2: N1);
4543 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0.getOperand(i: 0), N2: Add);
4544 }
4545
4546 // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1'
4547 // rather than 'sub 0/1' (the sext should get folded).
4548 // sub X, (zext i1 Y) --> add X, (sext i1 Y)
4549 if (N1.getOpcode() == ISD::ZERO_EXTEND &&
4550 N1.getOperand(i: 0).getScalarValueSizeInBits() == 1 &&
4551 TLI.getBooleanContents(Type: VT) ==
4552 TargetLowering::ZeroOrNegativeOneBooleanContent) {
4553 SDValue SExt = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: N1.getOperand(i: 0));
4554 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: SExt);
4555 }
4556
4557 // fold B = sra (A, size(A)-1); sub (xor (A, B), B) -> (abs A)
4558 if ((!LegalOperations || hasOperation(Opcode: ISD::ABS, VT)) &&
4559 sd_match(N: N1, P: m_Sra(L: m_Value(N&: A), R: m_SpecificInt(V: BitWidth - 1))) &&
4560 sd_match(N: N0, P: m_Xor(L: m_Specific(N: A), R: m_Specific(N: N1))))
4561 return DAG.getNode(Opcode: ISD::ABS, DL, VT, Operand: A);
4562
4563 // If the relocation model supports it, consider symbol offsets.
4564 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: N0))
4565 if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
4566 // fold (sub Sym+c1, Sym+c2) -> c1-c2
4567 if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(Val&: N1))
4568 if (GA->getGlobal() == GB->getGlobal())
4569 return DAG.getConstant(
4570 Val: APInt(VT.getScalarSizeInBits(), GA->getOffset() - GB->getOffset(),
4571 /*isSigned=*/false, /*implicitTrunc=*/true),
4572 DL, VT);
4573 }
4574
4575 // sub X, (sextinreg Y i1) -> add X, (and Y 1)
4576 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
4577 VTSDNode *TN = cast<VTSDNode>(Val: N1.getOperand(i: 1));
4578 if (TN->getVT() == MVT::i1) {
4579 SDValue ZExt = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N1.getOperand(i: 0),
4580 N2: DAG.getConstant(Val: 1, DL, VT));
4581 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: ZExt);
4582 }
4583 }
4584
4585 // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C)) if this is the
4586 // only use of the vscale value or if (vscale * -C) is a valid add immediate.
4587 // avoid if ISD::MUL handling is poor and ISD::SHL isn't an option.
4588 if (N1.getOpcode() == ISD::VSCALE) {
4589 const APInt &IntVal = N1.getConstantOperandAPInt(i: 0);
4590 if ((N1.hasOneUse() ||
4591 TLI.isLegalAddScalableImmediate(-IntVal.getSExtValue())) &&
4592 (!IntVal.isPowerOf2() ||
4593 hasOperation(Opcode: ISD::MUL, VT: N1.getOperand(i: 0).getValueType())))
4594 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: DAG.getVScale(DL, VT, MulImm: -IntVal));
4595 }
4596
4597 // canonicalize (sub X, step_vector(C)) to (add X, step_vector(-C))
4598 if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) {
4599 APInt NewStep = -N1.getConstantOperandAPInt(i: 0);
4600 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0,
4601 N2: DAG.getStepVector(DL, ResVT: VT, StepVal: NewStep));
4602 }
4603
4604 // Prefer an add for more folding potential and possibly better codegen:
4605 // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
4606 if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
4607 SDValue ShAmt = N1.getOperand(i: 1);
4608 ConstantSDNode *ShAmtC = isConstOrConstSplat(N: ShAmt);
4609 if (ShAmtC && ShAmtC->getAPIntValue() == (BitWidth - 1)) {
4610 SDValue SRA = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N1.getOperand(i: 0), N2: ShAmt);
4611 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: SRA);
4612 }
4613 }
4614
4615 // As with the previous fold, prefer add for more folding potential.
4616 // Subtracting SMIN/0 is the same as adding SMIN/0:
4617 // N0 - (X << BW-1) --> N0 + (X << BW-1)
4618 if (N1.getOpcode() == ISD::SHL) {
4619 ConstantSDNode *ShlC = isConstOrConstSplat(N: N1.getOperand(i: 1));
4620 if (ShlC && ShlC->getAPIntValue() == (BitWidth - 1))
4621 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1, N2: N0);
4622 }
4623
4624 // (sub (usubo_carry X, 0, Carry), Y) -> (usubo_carry X, Y, Carry)
4625 if (N0.getOpcode() == ISD::USUBO_CARRY && isNullConstant(V: N0.getOperand(i: 1)) &&
4626 N0.getResNo() == 0 && N0.hasOneUse())
4627 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: N0->getVTList(),
4628 N1: N0.getOperand(i: 0), N2: N1, N3: N0.getOperand(i: 2));
4629
4630 if (TLI.isOperationLegalOrCustom(Op: ISD::UADDO_CARRY, VT)) {
4631 // (sub Carry, X) -> (uaddo_carry (sub 0, X), 0, Carry)
4632 if (SDValue Carry = getAsCarry(TLI, V: N0)) {
4633 SDValue X = N1;
4634 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
4635 SDValue NegX = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: X);
4636 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL,
4637 VTList: DAG.getVTList(VT1: VT, VT2: Carry.getValueType()), N1: NegX, N2: Zero,
4638 N3: Carry);
4639 }
4640 }
4641
4642 if (ConstantSDNode *C0 = isConstOrConstSplat(N: N0)) {
4643 const APInt &C0Val = C0->getAPIntValue();
4644
4645 // sub nuw C, x --> xor x, C when C is a mask (2^k - 1)
4646 if (N->getFlags().hasNoUnsignedWrap() && C0Val.isMask())
4647 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1, N2: N0);
4648
4649 // If there's no chance of borrowing from adjacent bits, then sub is xor:
4650 // sub C0, X --> xor X, C0
4651 if (!C0->isOpaque()) {
4652 const APInt &MaybeOnes = ~DAG.computeKnownBits(Op: N1).Zero;
4653 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
4654 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1, N2: N0);
4655 }
4656 }
4657
4658 // smax(a,b) - smin(a,b) --> abds(a,b)
4659 if ((!LegalOperations || hasOperation(Opcode: ISD::ABDS, VT)) &&
4660 sd_match(N: N0, DAG: &DAG, P: m_SMaxLike(L: m_Value(N&: A), R: m_Value(N&: B))) &&
4661 sd_match(N: N1, DAG: &DAG, P: m_SMinLike(L: m_Specific(N: A), R: m_Specific(N: B))))
4662 return DAG.getNode(Opcode: ISD::ABDS, DL, VT, N1: A, N2: B);
4663
4664 // smin(a,b) - smax(a,b) --> neg(abds(a,b))
4665 if ((!LegalOperations || hasOperation(Opcode: ISD::ABDS, VT)) &&
4666 sd_match(N: N0, DAG: &DAG, P: m_SMinLike(L: m_Value(N&: A), R: m_Value(N&: B))) &&
4667 sd_match(N: N1, DAG: &DAG, P: m_SMaxLike(L: m_Specific(N: A), R: m_Specific(N: B))))
4668 return DAG.getNegative(Val: DAG.getNode(Opcode: ISD::ABDS, DL, VT, N1: A, N2: B), DL, VT);
4669
4670 // umax(a,b) - umin(a,b) --> abdu(a,b)
4671 if ((!LegalOperations || hasOperation(Opcode: ISD::ABDU, VT)) &&
4672 sd_match(N: N0, DAG: &DAG, P: m_UMaxLike(L: m_Value(N&: A), R: m_Value(N&: B))) &&
4673 sd_match(N: N1, DAG: &DAG, P: m_UMinLike(L: m_Specific(N: A), R: m_Specific(N: B))))
4674 return DAG.getNode(Opcode: ISD::ABDU, DL, VT, N1: A, N2: B);
4675
4676 // umin(a,b) - umax(a,b) --> neg(abdu(a,b))
4677 if ((!LegalOperations || hasOperation(Opcode: ISD::ABDU, VT)) &&
4678 sd_match(N: N0, DAG: &DAG, P: m_UMinLike(L: m_Value(N&: A), R: m_Value(N&: B))) &&
4679 sd_match(N: N1, DAG: &DAG, P: m_UMaxLike(L: m_Specific(N: A), R: m_Specific(N: B))))
4680 return DAG.getNegative(Val: DAG.getNode(Opcode: ISD::ABDU, DL, VT, N1: A, N2: B), DL, VT);
4681
4682 return SDValue();
4683}
4684
4685SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
4686 unsigned Opcode = N->getOpcode();
4687 SDValue N0 = N->getOperand(Num: 0);
4688 SDValue N1 = N->getOperand(Num: 1);
4689 EVT VT = N0.getValueType();
4690 bool IsSigned = Opcode == ISD::SSUBSAT;
4691 SDLoc DL(N);
4692
4693 // fold (sub_sat x, undef) -> 0
4694 if (N0.isUndef() || N1.isUndef())
4695 return DAG.getConstant(Val: 0, DL, VT);
4696
4697 // fold (sub_sat x, x) -> 0
4698 if (N0 == N1)
4699 return DAG.getConstant(Val: 0, DL, VT);
4700
4701 // fold (sub_sat c1, c2) -> c3
4702 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, Ops: {N0, N1}))
4703 return C;
4704
4705 // fold vector ops
4706 if (VT.isVector()) {
4707 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4708 return FoldedVOp;
4709
4710 // fold (sub_sat x, 0) -> x, vector edition
4711 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
4712 return N0;
4713 }
4714
4715 // fold (sub_sat x, 0) -> x
4716 if (isNullConstant(V: N1))
4717 return N0;
4718
4719 // If it cannot overflow, transform into an sub.
4720 if (DAG.willNotOverflowSub(IsSigned, N0, N1))
4721 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: N1);
4722
4723 // Narrow a vXiN USUBSAT to a smaller type when both operands are known
4724 // to fit in fewer bits. This allows targets with native narrow USUBSAT
4725 // (e.g. vpsubusb/vpsubusw) to avoid emulation with vpmaxu + vsub.
4726 if (!IsSigned && VT.isVector() && VT.isSimple()) {
4727 unsigned ScalarBits = VT.getScalarSizeInBits();
4728 if (ScalarBits > 8 && isPowerOf2_32(Value: ScalarBits) &&
4729 !TLI.isOperationLegal(Op: ISD::USUBSAT, VT)) {
4730 KnownBits Known0 = DAG.computeKnownBits(Op: N0);
4731 unsigned ActiveBits = Known0.countMaxActiveBits();
4732 for (unsigned NarrowBits = PowerOf2Ceil(A: ActiveBits);
4733 NarrowBits != 0 && NarrowBits < ScalarBits; NarrowBits *= 2) {
4734 unsigned Scale = ScalarBits / NarrowBits;
4735 ElementCount ScaledEC = VT.getVectorElementCount() * Scale;
4736 MVT NarrowSVT = MVT::getIntegerVT(BitWidth: NarrowBits);
4737 EVT NarrowVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NarrowSVT, EC: ScaledEC);
4738
4739 if (!TLI.isOperationLegalOrCustom(Op: ISD::USUBSAT, VT: NarrowVT))
4740 continue;
4741 KnownBits Known1 = DAG.computeKnownBits(Op: N1);
4742 if (Known1.countMaxActiveBits() <= NarrowBits) {
4743 SDValue NarrowN0 = DAG.getBitcast(VT: NarrowVT, V: N0);
4744 SDValue NarrowN1 = DAG.getBitcast(VT: NarrowVT, V: N1);
4745 SDValue NarrowSub =
4746 DAG.getNode(Opcode: ISD::USUBSAT, DL, VT: NarrowVT, N1: NarrowN0, N2: NarrowN1);
4747 return DAG.getBitcast(VT, V: NarrowSub);
4748 }
4749 // TODO: If N1 doesn't fit in NarrowBits, we could OR the upper bits
4750 // of N1 with 1s to force saturation in those lanes, allowing the
4751 // narrow USUBSAT to still be used. This requires a TLI hook to check
4752 // whether the constant can be folded as a broadcast memory operand
4753 // (profitable on AVX512, not on SSE/AVX), to avoid introducing an
4754 // extra register and instruction on non-AVX512 targets.
4755 break;
4756 }
4757 }
4758 }
4759 return SDValue();
4760}
4761
4762SDValue DAGCombiner::visitSUBC(SDNode *N) {
4763 SDValue N0 = N->getOperand(Num: 0);
4764 SDValue N1 = N->getOperand(Num: 1);
4765 EVT VT = N0.getValueType();
4766 SDLoc DL(N);
4767
4768 // If the flag result is dead, turn this into an SUB.
4769 if (!N->hasAnyUseOfValue(Value: 1))
4770 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: N1),
4771 Res1: DAG.getNode(Opcode: ISD::CARRY_FALSE, DL, VT: MVT::Glue));
4772
4773 // fold (subc x, x) -> 0 + no borrow
4774 if (N0 == N1)
4775 return CombineTo(N, Res0: DAG.getConstant(Val: 0, DL, VT),
4776 Res1: DAG.getNode(Opcode: ISD::CARRY_FALSE, DL, VT: MVT::Glue));
4777
4778 // fold (subc x, 0) -> x + no borrow
4779 if (isNullConstant(V: N1))
4780 return CombineTo(N, Res0: N0, Res1: DAG.getNode(Opcode: ISD::CARRY_FALSE, DL, VT: MVT::Glue));
4781
4782 // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
4783 if (isAllOnesConstant(V: N0))
4784 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1, N2: N0),
4785 Res1: DAG.getNode(Opcode: ISD::CARRY_FALSE, DL, VT: MVT::Glue));
4786
4787 return SDValue();
4788}
4789
4790SDValue DAGCombiner::visitSUBO(SDNode *N) {
4791 SDValue N0 = N->getOperand(Num: 0);
4792 SDValue N1 = N->getOperand(Num: 1);
4793 EVT VT = N0.getValueType();
4794 bool IsSigned = (ISD::SSUBO == N->getOpcode());
4795
4796 EVT CarryVT = N->getValueType(ResNo: 1);
4797 SDLoc DL(N);
4798
4799 // If the flag result is dead, turn this into an SUB.
4800 if (!N->hasAnyUseOfValue(Value: 1))
4801 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: N1),
4802 Res1: DAG.getUNDEF(VT: CarryVT));
4803
4804 // fold (subo x, x) -> 0 + no borrow
4805 if (N0 == N1)
4806 return CombineTo(N, Res0: DAG.getConstant(Val: 0, DL, VT),
4807 Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
4808
4809 // fold (subox, c) -> (addo x, -c)
4810 if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N: N1))
4811 if (IsSigned && !N1C->isMinSignedValue())
4812 return DAG.getNode(Opcode: ISD::SADDO, DL, VTList: N->getVTList(), N1: N0,
4813 N2: DAG.getConstant(Val: -N1C->getAPIntValue(), DL, VT));
4814
4815 // fold (subo x, 0) -> x + no borrow
4816 if (isNullOrNullSplat(V: N1))
4817 return CombineTo(N, Res0: N0, Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
4818
4819 // If it cannot overflow, transform into an sub.
4820 if (DAG.willNotOverflowSub(IsSigned, N0, N1))
4821 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: N1),
4822 Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
4823
4824 // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
4825 if (!IsSigned && isAllOnesOrAllOnesSplat(V: N0))
4826 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::XOR, DL, VT, N1, N2: N0),
4827 Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
4828
4829 return SDValue();
4830}
4831
4832SDValue DAGCombiner::visitSUBE(SDNode *N) {
4833 SDValue N0 = N->getOperand(Num: 0);
4834 SDValue N1 = N->getOperand(Num: 1);
4835 SDValue CarryIn = N->getOperand(Num: 2);
4836
4837 // fold (sube x, y, false) -> (subc x, y)
4838 if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
4839 return DAG.getNode(Opcode: ISD::SUBC, DL: SDLoc(N), VTList: N->getVTList(), N1: N0, N2: N1);
4840
4841 return SDValue();
4842}
4843
4844SDValue DAGCombiner::visitUSUBO_CARRY(SDNode *N) {
4845 SDValue N0 = N->getOperand(Num: 0);
4846 SDValue N1 = N->getOperand(Num: 1);
4847 SDValue CarryIn = N->getOperand(Num: 2);
4848
4849 // fold (usubo_carry x, y, false) -> (usubo x, y)
4850 if (isNullConstant(V: CarryIn)) {
4851 if (!LegalOperations ||
4852 TLI.isOperationLegalOrCustom(Op: ISD::USUBO, VT: N->getValueType(ResNo: 0)))
4853 return DAG.getNode(Opcode: ISD::USUBO, DL: SDLoc(N), VTList: N->getVTList(), N1: N0, N2: N1);
4854 }
4855
4856 // Iff the flag result is dead:
4857 // (usubo_carry (sub X, Y), 0, Carry) -> (usubo_carry X, Y, Carry)
4858 if (N0.getOpcode() == ISD::SUB && isNullConstant(V: N1) &&
4859 !N->hasAnyUseOfValue(Value: 1))
4860 return DAG.getNode(Opcode: ISD::USUBO_CARRY, DL: SDLoc(N), VTList: N->getVTList(),
4861 N1: N0.getOperand(i: 0), N2: N0.getOperand(i: 1), N3: CarryIn);
4862
4863 return SDValue();
4864}
4865
4866SDValue DAGCombiner::visitSSUBO_CARRY(SDNode *N) {
4867 SDValue N0 = N->getOperand(Num: 0);
4868 SDValue N1 = N->getOperand(Num: 1);
4869 SDValue CarryIn = N->getOperand(Num: 2);
4870
4871 // fold (ssubo_carry x, y, false) -> (ssubo x, y)
4872 if (isNullConstant(V: CarryIn)) {
4873 if (!LegalOperations ||
4874 TLI.isOperationLegalOrCustom(Op: ISD::SSUBO, VT: N->getValueType(ResNo: 0)))
4875 return DAG.getNode(Opcode: ISD::SSUBO, DL: SDLoc(N), VTList: N->getVTList(), N1: N0, N2: N1);
4876 }
4877
4878 return SDValue();
4879}
4880
4881// Notice that "mulfix" can be any of SMULFIX, SMULFIXSAT, UMULFIX and
4882// UMULFIXSAT here.
4883SDValue DAGCombiner::visitMULFIX(SDNode *N) {
4884 SDValue N0 = N->getOperand(Num: 0);
4885 SDValue N1 = N->getOperand(Num: 1);
4886 SDValue Scale = N->getOperand(Num: 2);
4887 EVT VT = N0.getValueType();
4888
4889 // fold (mulfix x, undef, scale) -> 0
4890 if (N0.isUndef() || N1.isUndef())
4891 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
4892
4893 // Canonicalize constant to RHS (vector doesn't have to splat)
4894 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
4895 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
4896 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, N1, N2: N0, N3: Scale);
4897
4898 // fold (mulfix x, 0, scale) -> 0
4899 if (isNullConstant(V: N1))
4900 return DAG.getConstant(Val: 0, DL: SDLoc(N), VT);
4901
4902 return SDValue();
4903}
4904
4905SDValue DAGCombiner::visitMUL(SDNode *N) {
4906 SDValue N0 = N->getOperand(Num: 0);
4907 SDValue N1 = N->getOperand(Num: 1);
4908 EVT VT = N0.getValueType();
4909 unsigned BitWidth = VT.getScalarSizeInBits();
4910 SDLoc DL(N);
4911
4912 // fold (mul x, undef) -> 0
4913 if (N0.isUndef() || N1.isUndef())
4914 return DAG.getConstant(Val: 0, DL, VT);
4915
4916 // fold (mul c1, c2) -> c1*c2
4917 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::MUL, DL, VT, Ops: {N0, N1}))
4918 return C;
4919
4920 // canonicalize constant to RHS (vector doesn't have to splat). An opaque
4921 // constant on the RHS is treated as non-constant so that a foldable constant
4922 // still ends up on the RHS.
4923 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0, /*AllowOpaques=*/false) &&
4924 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1, /*AllowOpaques=*/false))
4925 return DAG.getNode(Opcode: ISD::MUL, DL, VT, N1, N2: N0);
4926
4927 bool N1IsConst = false;
4928 bool N1IsOpaqueConst = false;
4929 APInt ConstValue1;
4930
4931 // fold vector ops
4932 if (VT.isVector()) {
4933 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
4934 return FoldedVOp;
4935
4936 N1IsConst = ISD::isConstantSplatVector(N: N1.getNode(), SplatValue&: ConstValue1);
4937 assert((!N1IsConst || ConstValue1.getBitWidth() == BitWidth) &&
4938 "Splat APInt should be element width");
4939 } else {
4940 N1IsConst = isa<ConstantSDNode>(Val: N1);
4941 if (N1IsConst) {
4942 ConstValue1 = N1->getAsAPIntVal();
4943 N1IsOpaqueConst = cast<ConstantSDNode>(Val&: N1)->isOpaque();
4944 }
4945 }
4946
4947 // fold (mul x, 0) -> 0
4948 if (N1IsConst && ConstValue1.isZero())
4949 return N1;
4950
4951 // fold (mul x, 1) -> x
4952 if (N1IsConst && ConstValue1.isOne())
4953 return N0;
4954
4955 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
4956 return NewSel;
4957
4958 // fold (mul x, -1) -> 0-x
4959 if (N1IsConst && ConstValue1.isAllOnes())
4960 return DAG.getNegative(Val: N0, DL, VT);
4961
4962 // fold (mul x, (1 << c)) -> x << c
4963 if (isConstantOrConstantVector(N: N1, /*NoOpaques*/ true) &&
4964 (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
4965 if (SDValue LogBase2 = BuildLogBase2(V: N1, DL)) {
4966 EVT ShiftVT = getShiftAmountTy(LHSTy: N0.getValueType());
4967 SDValue Trunc = DAG.getZExtOrTrunc(Op: LogBase2, DL, VT: ShiftVT);
4968 SDNodeFlags Flags;
4969 Flags.setNoUnsignedWrap(N->getFlags().hasNoUnsignedWrap());
4970 // Preserve nsw when the shift amount is strictly less than BitWidth - 1,
4971 // i.e. the multiplier is not the signed minimum value.
4972 if (N->getFlags().hasNoSignedWrap() && N1IsConst &&
4973 ConstValue1.logBase2() < BitWidth - 1)
4974 Flags.setNoSignedWrap(true);
4975 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0, N2: Trunc, Flags);
4976 }
4977 }
4978
4979 // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
4980 if (N1IsConst && !N1IsOpaqueConst && ConstValue1.isNegatedPowerOf2()) {
4981 unsigned Log2Val = (-ConstValue1).logBase2();
4982
4983 // FIXME: If the input is something that is easily negated (e.g. a
4984 // single-use add), we should put the negate there.
4985 return DAG.getNode(
4986 Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT),
4987 N2: DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0,
4988 N2: DAG.getShiftAmountConstant(Val: Log2Val, VT, DL)));
4989 }
4990
4991 // Attempt to reuse an existing umul_lohi/smul_lohi node, but only if the
4992 // hi result is in use in case we hit this mid-legalization.
4993 for (unsigned LoHiOpc : {ISD::UMUL_LOHI, ISD::SMUL_LOHI}) {
4994 if (!LegalOperations || TLI.isOperationLegalOrCustom(Op: LoHiOpc, VT)) {
4995 SDVTList LoHiVT = DAG.getVTList(VT1: VT, VT2: VT);
4996 // TODO: Can we match commutable operands with getNodeIfExists?
4997 if (SDNode *LoHi = DAG.getNodeIfExists(Opcode: LoHiOpc, VTList: LoHiVT, Ops: {N0, N1}))
4998 if (LoHi->hasAnyUseOfValue(Value: 1))
4999 return SDValue(LoHi, 0);
5000 if (SDNode *LoHi = DAG.getNodeIfExists(Opcode: LoHiOpc, VTList: LoHiVT, Ops: {N1, N0}))
5001 if (LoHi->hasAnyUseOfValue(Value: 1))
5002 return SDValue(LoHi, 0);
5003 }
5004 }
5005
5006 // Try to transform:
5007 // (1) multiply-by-(power-of-2 +/- 1) into shift and add/sub.
5008 // mul x, (2^N + 1) --> add (shl x, N), x
5009 // mul x, (2^N - 1) --> sub (shl x, N), x
5010 // Examples: x * 33 --> (x << 5) + x
5011 // x * 15 --> (x << 4) - x
5012 // x * -33 --> -((x << 5) + x)
5013 // x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
5014 // (2) multiply-by-(power-of-2 +/- power-of-2) into shifts and add/sub.
5015 // mul x, (2^N + 2^M) --> (add (shl x, N), (shl x, M))
5016 // mul x, (2^N - 2^M) --> (sub (shl x, N), (shl x, M))
5017 // Examples: x * 0x8800 --> (x << 15) + (x << 11)
5018 // x * 0xf800 --> (x << 16) - (x << 11)
5019 // x * -0x8800 --> -((x << 15) + (x << 11))
5020 // x * -0xf800 --> -((x << 16) - (x << 11)) ; (x << 11) - (x << 16)
5021 if (N1IsConst && TLI.decomposeMulByConstant(Context&: *DAG.getContext(), VT, C: N1)) {
5022 // TODO: We could handle more general decomposition of any constant by
5023 // having the target set a limit on number of ops and making a
5024 // callback to determine that sequence (similar to sqrt expansion).
5025 unsigned MathOp = ISD::DELETED_NODE;
5026 APInt MulC = ConstValue1.abs();
5027 // The constant `2` should be treated as (2^0 + 1).
5028 unsigned TZeros = MulC == 2 ? 0 : MulC.countr_zero();
5029 MulC.lshrInPlace(ShiftAmt: TZeros);
5030 if ((MulC - 1).isPowerOf2())
5031 MathOp = ISD::ADD;
5032 else if ((MulC + 1).isPowerOf2())
5033 MathOp = ISD::SUB;
5034
5035 if (MathOp != ISD::DELETED_NODE) {
5036 unsigned ShAmt =
5037 MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
5038 ShAmt += TZeros;
5039 assert(ShAmt < BitWidth &&
5040 "multiply-by-constant generated out of bounds shift");
5041 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0,
5042 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT, DL));
5043 SDValue R = N0;
5044 if (TZeros)
5045 R = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0,
5046 N2: DAG.getShiftAmountConstant(Val: TZeros, VT, DL));
5047 R = DAG.getNode(Opcode: MathOp, DL, VT, N1: Shl, N2: R);
5048 if (ConstValue1.isNegative())
5049 R = DAG.getNegative(Val: R, DL, VT);
5050 return R;
5051 }
5052 }
5053
5054 // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
5055 {
5056 SDValue X, C1;
5057 if (sd_match(N: N0, P: m_Shl(L: m_Value(N&: X), R: m_Value(N&: C1))))
5058 if (SDValue C3 = DAG.FoldConstantArithmetic(Opcode: ISD::SHL, DL, VT, Ops: {N1, C1}))
5059 return DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: C3);
5060 }
5061
5062 // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
5063 // use.
5064 {
5065 SDValue X, C, Y;
5066 if (sd_match(N,
5067 P: m_Mul(L: m_OneUse(P: m_Shl(L: m_Value(N&: X), R: m_Value(N&: C))), R: m_Value(N&: Y))) &&
5068 isConstantOrConstantVector(N: C)) {
5069 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: Y);
5070 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Mul, N2: C);
5071 }
5072 }
5073
5074 // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
5075 if (sd_match(N: N0, P: m_SpecificOpc(Opcode: ISD::ADD)) && isConstantOrConstantVector(N: N1) &&
5076 isConstantOrConstantVector(N: N0.getOperand(i: 1)) &&
5077 isMulAddWithConstProfitable(MulNode: N, AddNode: N0, ConstNode: N1))
5078 return DAG.getNode(
5079 Opcode: ISD::ADD, DL, VT,
5080 N1: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N0), VT, N1: N0.getOperand(i: 0), N2: N1),
5081 N2: DAG.getNode(Opcode: ISD::MUL, DL: SDLoc(N1), VT, N1: N0.getOperand(i: 1), N2: N1));
5082
5083 // Fold (mul (vscale * C0), C1) to (vscale * (C0 * C1)).
5084 // avoid if ISD::MUL handling is poor and ISD::SHL isn't an option.
5085 ConstantSDNode *NC1 = isConstOrConstSplat(N: N1);
5086 if (N0.getOpcode() == ISD::VSCALE && NC1) {
5087 const APInt &C0 = N0.getConstantOperandAPInt(i: 0);
5088 const APInt &C1 = NC1->getAPIntValue();
5089 if (!C0.isPowerOf2() || C1.isPowerOf2() ||
5090 hasOperation(Opcode: ISD::MUL, VT: NC1->getValueType(ResNo: 0)))
5091 return DAG.getVScale(DL, VT, MulImm: C0 * C1);
5092 }
5093
5094 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
5095 APInt MulVal;
5096 if (N0.getOpcode() == ISD::STEP_VECTOR &&
5097 ISD::isConstantSplatVector(N: N1.getNode(), SplatValue&: MulVal)) {
5098 const APInt &C0 = N0.getConstantOperandAPInt(i: 0);
5099 APInt NewStep = C0 * MulVal;
5100 return DAG.getStepVector(DL, ResVT: VT, StepVal: NewStep);
5101 }
5102
5103 // Fold Y = sra (X, size(X)-1); mul (or (Y, 1), X) -> (abs X)
5104 SDValue X;
5105 if ((!LegalOperations || hasOperation(Opcode: ISD::ABS, VT)) &&
5106 sd_match(N, P: m_Mul(L: m_Or(L: m_Sra(L: m_Value(N&: X), R: m_SpecificInt(V: BitWidth - 1)),
5107 R: m_One()),
5108 R: m_Deferred(V&: X)))) {
5109 return DAG.getNode(Opcode: ISD::ABS, DL, VT, Operand: X);
5110 }
5111
5112 // Fold ((mul x, 0/undef) -> 0,
5113 // (mul x, 1) -> x) -> x)
5114 // -> and(x, mask)
5115 // We can replace vectors with '0' and '1' factors with a clearing mask.
5116 if (VT.isFixedLengthVector()) {
5117 unsigned NumElts = VT.getVectorNumElements();
5118 SmallBitVector ClearMask;
5119 ClearMask.reserve(N: NumElts);
5120 auto IsClearMask = [&ClearMask](ConstantSDNode *V) {
5121 if (!V || V->isZero()) {
5122 ClearMask.push_back(Val: true);
5123 return true;
5124 }
5125 ClearMask.push_back(Val: false);
5126 return V->isOne();
5127 };
5128 if ((!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::AND, VT)) &&
5129 ISD::matchUnaryPredicate(Op: N1, Match: IsClearMask, /*AllowUndefs*/ true)) {
5130 assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector");
5131 EVT LegalSVT = N1.getOperand(i: 0).getValueType();
5132 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: LegalSVT);
5133 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: LegalSVT);
5134 SmallVector<SDValue, 16> Mask(NumElts, AllOnes);
5135 for (unsigned I = 0; I != NumElts; ++I)
5136 if (ClearMask[I])
5137 Mask[I] = Zero;
5138 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N0, N2: DAG.getBuildVector(VT, DL, Ops: Mask));
5139 }
5140 }
5141
5142 // reassociate mul
5143 if (SDValue RMUL = reassociateOps(Opc: ISD::MUL, DL, N0, N1, Flags: N->getFlags()))
5144 return RMUL;
5145
5146 // Fold mul(vecreduce(x), vecreduce(y)) -> vecreduce(mul(x, y))
5147 if (SDValue SD =
5148 reassociateReduction(RedOpc: ISD::VECREDUCE_MUL, Opc: ISD::MUL, DL, VT, N0, N1))
5149 return SD;
5150
5151 // Simplify the operands using demanded-bits information.
5152 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
5153 return SDValue(N, 0);
5154
5155 return SDValue();
5156}
5157
5158/// Return true if divmod libcall is available.
5159static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
5160 const SelectionDAG &DAG) {
5161 RTLIB::Libcall LC;
5162 EVT NodeType = Node->getValueType(ResNo: 0);
5163 if (!NodeType.isSimple())
5164 return false;
5165 switch (NodeType.getSimpleVT().SimpleTy) {
5166 default: return false; // No libcall for vector types.
5167 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
5168 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
5169 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
5170 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
5171 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
5172 }
5173
5174 return DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported;
5175}
5176
5177/// Issue divrem if both quotient and remainder are needed.
5178SDValue DAGCombiner::useDivRem(SDNode *Node) {
5179 if (Node->use_empty())
5180 return SDValue(); // This is a dead node, leave it alone.
5181
5182 unsigned Opcode = Node->getOpcode();
5183 bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
5184 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
5185
5186 // DivMod lib calls can still work on non-legal types if using lib-calls.
5187 EVT VT = Node->getValueType(ResNo: 0);
5188 if (VT.isVector() || !VT.isInteger())
5189 return SDValue();
5190
5191 if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(Op: DivRemOpc, VT))
5192 return SDValue();
5193
5194 // If DIVREM is going to get expanded into a libcall,
5195 // but there is no libcall available, then don't combine.
5196 if (!TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT) &&
5197 !isDivRemLibcallAvailable(Node, isSigned, DAG))
5198 return SDValue();
5199
5200 // If div is legal, it's better to do the normal expansion
5201 unsigned OtherOpcode = 0;
5202 if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
5203 OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
5204 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT))
5205 return SDValue();
5206 } else {
5207 OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
5208 if (TLI.isOperationLegalOrCustom(Op: OtherOpcode, VT))
5209 return SDValue();
5210 }
5211
5212 SDValue Op0 = Node->getOperand(Num: 0);
5213 SDValue Op1 = Node->getOperand(Num: 1);
5214 SDValue combined;
5215 for (SDNode *User : Op0->users()) {
5216 if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
5217 User->use_empty())
5218 continue;
5219 // Convert the other matching node(s), too;
5220 // otherwise, the DIVREM may get target-legalized into something
5221 // target-specific that we won't be able to recognize.
5222 unsigned UserOpc = User->getOpcode();
5223 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
5224 User->getOperand(Num: 0) == Op0 &&
5225 User->getOperand(Num: 1) == Op1) {
5226 if (!combined) {
5227 if (UserOpc == OtherOpcode) {
5228 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT);
5229 combined = DAG.getNode(Opcode: DivRemOpc, DL: SDLoc(Node), VTList: VTs, N1: Op0, N2: Op1);
5230 } else if (UserOpc == DivRemOpc) {
5231 combined = SDValue(User, 0);
5232 } else {
5233 assert(UserOpc == Opcode);
5234 continue;
5235 }
5236 }
5237 if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
5238 CombineTo(N: User, Res: combined);
5239 else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
5240 CombineTo(N: User, Res: combined.getValue(R: 1));
5241 }
5242 }
5243 return combined;
5244}
5245
5246static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
5247 SDValue N0 = N->getOperand(Num: 0);
5248 SDValue N1 = N->getOperand(Num: 1);
5249 EVT VT = N->getValueType(ResNo: 0);
5250 SDLoc DL(N);
5251
5252 unsigned Opc = N->getOpcode();
5253 bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
5254
5255 // X / undef -> undef
5256 // X % undef -> undef
5257 // X / 0 -> undef
5258 // X % 0 -> undef
5259 // NOTE: This includes vectors where any divisor element is zero/undef.
5260 if (DAG.isUndef(Opcode: Opc, Ops: {N0, N1}))
5261 return DAG.getUNDEF(VT);
5262
5263 // undef / X -> 0
5264 // undef % X -> 0
5265 if (N0.isUndef())
5266 return DAG.getConstant(Val: 0, DL, VT);
5267
5268 // 0 / X -> 0
5269 // 0 % X -> 0
5270 ConstantSDNode *N0C = isConstOrConstSplat(N: N0);
5271 if (N0C && N0C->isZero())
5272 return N0;
5273
5274 // X / X -> 1
5275 // X % X -> 0
5276 if (N0 == N1)
5277 return DAG.getConstant(Val: IsDiv ? 1 : 0, DL, VT);
5278
5279 // X / 1 -> X
5280 // X % 1 -> 0
5281 // If this is a boolean op (single-bit element type), we can't have
5282 // division-by-zero or remainder-by-zero, so assume the divisor is 1.
5283 // TODO: Similarly, if we're zero-extending a boolean divisor, then assume
5284 // it's a 1.
5285 if (isOneOrOneSplat(V: N1) || (VT.getScalarType() == MVT::i1))
5286 return IsDiv ? N0 : DAG.getConstant(Val: 0, DL, VT);
5287
5288 return SDValue();
5289}
5290
5291SDValue DAGCombiner::visitSDIV(SDNode *N) {
5292 SDValue N0 = N->getOperand(Num: 0);
5293 SDValue N1 = N->getOperand(Num: 1);
5294 EVT VT = N->getValueType(ResNo: 0);
5295 EVT CCVT = getSetCCResultType(VT);
5296 SDLoc DL(N);
5297
5298 // fold (sdiv c1, c2) -> c1/c2
5299 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::SDIV, DL, VT, Ops: {N0, N1}))
5300 return C;
5301
5302 // fold vector ops
5303 if (VT.isVector())
5304 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5305 return FoldedVOp;
5306
5307 // fold (sdiv X, -1) -> 0-X
5308 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
5309 if (N1C && N1C->isAllOnes())
5310 return DAG.getNegative(Val: N0, DL, VT);
5311
5312 // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
5313 if (N1C && N1C->isMinSignedValue())
5314 return DAG.getSelect(DL, VT, Cond: DAG.getSetCC(DL, VT: CCVT, LHS: N0, RHS: N1, Cond: ISD::SETEQ),
5315 LHS: DAG.getConstant(Val: 1, DL, VT),
5316 RHS: DAG.getConstant(Val: 0, DL, VT));
5317
5318 if (SDValue V = simplifyDivRem(N, DAG))
5319 return V;
5320
5321 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
5322 return NewSel;
5323
5324 // If we know the sign bits of both operands are zero, strength reduce to a
5325 // udiv instead. Handles (X&15) /s 4 -> X&15 >> 2
5326 if (DAG.SignBitIsZero(Op: N1) && DAG.SignBitIsZero(Op: N0))
5327 return DAG.getNode(Opcode: ISD::UDIV, DL, VT: N1.getValueType(), N1: N0, N2: N1);
5328
5329 if (SDValue V = visitSDIVLike(N0, N1, N)) {
5330 // If the corresponding remainder node exists, update its users with
5331 // (Dividend - (Quotient * Divisor).
5332 if (SDNode *RemNode = DAG.getNodeIfExists(Opcode: ISD::SREM, VTList: N->getVTList(),
5333 Ops: { N0, N1 })) {
5334 // If the sdiv has the exact flag we shouldn't propagate it to the
5335 // remainder node.
5336 if (!N->getFlags().hasExact()) {
5337 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: V, N2: N1);
5338 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Mul);
5339 AddToWorklist(N: Mul.getNode());
5340 AddToWorklist(N: Sub.getNode());
5341 CombineTo(N: RemNode, Res: Sub);
5342 }
5343 }
5344 return V;
5345 }
5346
5347 // sdiv, srem -> sdivrem
5348 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
5349 // true. Otherwise, we break the simplification logic in visitREM().
5350 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5351 if (!N1C || TLI.isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
5352 if (SDValue DivRem = useDivRem(Node: N))
5353 return DivRem;
5354
5355 return SDValue();
5356}
5357
5358static bool isDivisorPowerOfTwo(SDValue Divisor) {
5359 // Helper for determining whether a value is a power-2 constant scalar or a
5360 // vector of such elements.
5361 auto IsPowerOfTwo = [](ConstantSDNode *C) {
5362 if (C->isZero() || C->isOpaque())
5363 return false;
5364 if (C->getAPIntValue().isPowerOf2())
5365 return true;
5366 if (C->getAPIntValue().isNegatedPowerOf2())
5367 return true;
5368 return false;
5369 };
5370
5371 return ISD::matchUnaryPredicate(Op: Divisor, Match: IsPowerOfTwo, /*AllowUndefs=*/false,
5372 /*AllowTruncation=*/true);
5373}
5374
5375SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
5376 SDLoc DL(N);
5377 EVT VT = N->getValueType(ResNo: 0);
5378 EVT CCVT = getSetCCResultType(VT);
5379 unsigned BitWidth = VT.getScalarSizeInBits();
5380 unsigned MaxLegalDivRemBitWidth = TLI.getMaxDivRemBitWidthSupported();
5381
5382 // fold (sdiv X, pow2) -> simple ops after legalize
5383 // FIXME: We check for the exact bit here because the generic lowering gives
5384 // better results in that case. The target-specific lowering should learn how
5385 // to handle exact sdivs efficiently. An exception is made for large bitwidths
5386 // exceeding what the target can natively support, as division expansion was
5387 // skipped in favor of this optimization.
5388 if ((!N->getFlags().hasExact() || BitWidth > MaxLegalDivRemBitWidth) &&
5389 isDivisorPowerOfTwo(Divisor: N1)) {
5390 // Target-specific implementation of sdiv x, pow2.
5391 if (SDValue Res = BuildSDIVPow2(N))
5392 return Res;
5393
5394 // Create constants that are functions of the shift amount value.
5395 EVT ShiftAmtTy = getShiftAmountTy(LHSTy: N0.getValueType());
5396 SDValue Bits = DAG.getConstant(Val: BitWidth, DL, VT: ShiftAmtTy);
5397 SDValue C1 = DAG.getNode(Opcode: ISD::CTTZ, DL, VT, Operand: N1);
5398 C1 = DAG.getZExtOrTrunc(Op: C1, DL, VT: ShiftAmtTy);
5399 SDValue Inexact = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShiftAmtTy, N1: Bits, N2: C1);
5400 if (!isConstantOrConstantVector(N: Inexact))
5401 return SDValue();
5402
5403 // Splat the sign bit into the register
5404 SDValue Sign = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N0,
5405 N2: DAG.getConstant(Val: BitWidth - 1, DL, VT: ShiftAmtTy));
5406 AddToWorklist(N: Sign.getNode());
5407
5408 // Add (N0 < 0) ? abs2 - 1 : 0;
5409 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Sign, N2: Inexact);
5410 AddToWorklist(N: Srl.getNode());
5411 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: Srl);
5412 AddToWorklist(N: Add.getNode());
5413 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Add, N2: C1);
5414 AddToWorklist(N: Sra.getNode());
5415
5416 // Special case: (sdiv X, 1) -> X
5417 // Special Case: (sdiv X, -1) -> 0-X
5418 SDValue One = DAG.getConstant(Val: 1, DL, VT);
5419 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
5420 SDValue IsOne = DAG.getSetCC(DL, VT: CCVT, LHS: N1, RHS: One, Cond: ISD::SETEQ);
5421 SDValue IsAllOnes = DAG.getSetCC(DL, VT: CCVT, LHS: N1, RHS: AllOnes, Cond: ISD::SETEQ);
5422 SDValue IsOneOrAllOnes = DAG.getNode(Opcode: ISD::OR, DL, VT: CCVT, N1: IsOne, N2: IsAllOnes);
5423 Sra = DAG.getSelect(DL, VT, Cond: IsOneOrAllOnes, LHS: N0, RHS: Sra);
5424
5425 // If dividing by a positive value, we're done. Otherwise, the result must
5426 // be negated.
5427 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
5428 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Zero, N2: Sra);
5429
5430 // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
5431 SDValue IsNeg = DAG.getSetCC(DL, VT: CCVT, LHS: N1, RHS: Zero, Cond: ISD::SETLT);
5432 SDValue Res = DAG.getSelect(DL, VT, Cond: IsNeg, LHS: Sub, RHS: Sra);
5433 return Res;
5434 }
5435
5436 // If integer divide is expensive and we satisfy the requirements, emit an
5437 // alternate sequence. Targets may check function attributes for size/speed
5438 // trade-offs.
5439 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5440 if (isConstantOrConstantVector(N: N1, /*NoOpaques=*/false,
5441 /*AllowTruncation=*/true) &&
5442 !TLI.isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
5443 if (SDValue Op = BuildSDIV(N))
5444 return Op;
5445
5446 return SDValue();
5447}
5448
5449SDValue DAGCombiner::visitUDIV(SDNode *N) {
5450 SDValue N0 = N->getOperand(Num: 0);
5451 SDValue N1 = N->getOperand(Num: 1);
5452 EVT VT = N->getValueType(ResNo: 0);
5453 EVT CCVT = getSetCCResultType(VT);
5454 SDLoc DL(N);
5455
5456 // fold (udiv c1, c2) -> c1/c2
5457 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::UDIV, DL, VT, Ops: {N0, N1}))
5458 return C;
5459
5460 // fold vector ops
5461 if (VT.isVector())
5462 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5463 return FoldedVOp;
5464
5465 // fold (udiv X, -1) -> select(X == -1, 1, 0)
5466 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
5467 if (N1C && N1C->isAllOnes() && CCVT.isVector() == VT.isVector()) {
5468 return DAG.getSelect(DL, VT, Cond: DAG.getSetCC(DL, VT: CCVT, LHS: N0, RHS: N1, Cond: ISD::SETEQ),
5469 LHS: DAG.getConstant(Val: 1, DL, VT),
5470 RHS: DAG.getConstant(Val: 0, DL, VT));
5471 }
5472
5473 if (SDValue V = simplifyDivRem(N, DAG))
5474 return V;
5475
5476 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
5477 return NewSel;
5478
5479 if (SDValue V = visitUDIVLike(N0, N1, N)) {
5480 // If the corresponding remainder node exists, update its users with
5481 // (Dividend - (Quotient * Divisor).
5482 if (SDNode *RemNode = DAG.getNodeIfExists(Opcode: ISD::UREM, VTList: N->getVTList(),
5483 Ops: { N0, N1 })) {
5484 // If the udiv has the exact flag we shouldn't propagate it to the
5485 // remainder node.
5486 if (!N->getFlags().hasExact()) {
5487 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: V, N2: N1);
5488 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Mul);
5489 AddToWorklist(N: Mul.getNode());
5490 AddToWorklist(N: Sub.getNode());
5491 CombineTo(N: RemNode, Res: Sub);
5492 }
5493 }
5494 return V;
5495 }
5496
5497 // sdiv, srem -> sdivrem
5498 // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
5499 // true. Otherwise, we break the simplification logic in visitREM().
5500 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5501 if (!N1C || TLI.isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
5502 if (SDValue DivRem = useDivRem(Node: N))
5503 return DivRem;
5504
5505 // Simplify the operands using demanded-bits information.
5506 // We don't have demanded bits support for UDIV so this just enables constant
5507 // folding based on known bits.
5508 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
5509 return SDValue(N, 0);
5510
5511 return SDValue();
5512}
5513
5514SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
5515 SDLoc DL(N);
5516 EVT VT = N->getValueType(ResNo: 0);
5517
5518 // fold (udiv x, (1 << c)) -> x >>u c
5519 if (isConstantOrConstantVector(N: N1, /*NoOpaques=*/true,
5520 /*AllowTruncation=*/true)) {
5521 if (SDValue LogBase2 = BuildLogBase2(V: N1, DL)) {
5522 AddToWorklist(N: LogBase2.getNode());
5523
5524 EVT ShiftVT = getShiftAmountTy(LHSTy: N0.getValueType());
5525 SDValue Trunc = DAG.getZExtOrTrunc(Op: LogBase2, DL, VT: ShiftVT);
5526 AddToWorklist(N: Trunc.getNode());
5527 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0, N2: Trunc);
5528 }
5529 }
5530
5531 // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
5532 if (N1.getOpcode() == ISD::SHL) {
5533 SDValue N10 = N1.getOperand(i: 0);
5534 if (isConstantOrConstantVector(N: N10, /*NoOpaques=*/true,
5535 /*AllowTruncation=*/true)) {
5536 if (SDValue LogBase2 = BuildLogBase2(V: N10, DL)) {
5537 AddToWorklist(N: LogBase2.getNode());
5538
5539 EVT ADDVT = N1.getOperand(i: 1).getValueType();
5540 SDValue Trunc = DAG.getZExtOrTrunc(Op: LogBase2, DL, VT: ADDVT);
5541 AddToWorklist(N: Trunc.getNode());
5542 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: ADDVT, N1: N1.getOperand(i: 1), N2: Trunc);
5543 AddToWorklist(N: Add.getNode());
5544 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0, N2: Add);
5545 }
5546 }
5547 }
5548
5549 // fold (udiv x, c) -> alternate
5550 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5551 if (isConstantOrConstantVector(N: N1, /*NoOpaques=*/false,
5552 /*AllowTruncation=*/true) &&
5553 !TLI.isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
5554 if (SDValue Op = BuildUDIV(N))
5555 return Op;
5556
5557 return SDValue();
5558}
5559
5560SDValue DAGCombiner::buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N) {
5561 if (!N->getFlags().hasExact() && isDivisorPowerOfTwo(Divisor: N1) &&
5562 !DAG.doesNodeExist(Opcode: ISD::SDIV, VTList: N->getVTList(), Ops: {N0, N1})) {
5563 // Target-specific implementation of srem x, pow2.
5564 if (SDValue Res = BuildSREMPow2(N))
5565 return Res;
5566 }
5567 return SDValue();
5568}
5569
5570// handles ISD::SREM and ISD::UREM
5571SDValue DAGCombiner::visitREM(SDNode *N) {
5572 unsigned Opcode = N->getOpcode();
5573 SDValue N0 = N->getOperand(Num: 0);
5574 SDValue N1 = N->getOperand(Num: 1);
5575 EVT VT = N->getValueType(ResNo: 0);
5576 EVT CCVT = getSetCCResultType(VT);
5577
5578 bool isSigned = (Opcode == ISD::SREM);
5579 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
5580 SDLoc DL(N);
5581
5582 // fold (rem c1, c2) -> c1%c2
5583 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, Ops: {N0, N1}))
5584 return C;
5585
5586 // fold (urem X, -1) -> select(FX == -1, 0, FX)
5587 // Freeze the numerator to avoid a miscompile with an undefined value.
5588 if (!isSigned && llvm::isAllOnesOrAllOnesSplat(V: N1, /*AllowUndefs*/ false) &&
5589 CCVT.isVector() == VT.isVector()) {
5590 SDValue F0 = DAG.getFreeze(V: N0);
5591 SDValue EqualsNeg1 = DAG.getSetCC(DL, VT: CCVT, LHS: F0, RHS: N1, Cond: ISD::SETEQ);
5592 return DAG.getSelect(DL, VT, Cond: EqualsNeg1, LHS: DAG.getConstant(Val: 0, DL, VT), RHS: F0);
5593 }
5594
5595 if (SDValue V = simplifyDivRem(N, DAG))
5596 return V;
5597
5598 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
5599 return NewSel;
5600
5601 if (isSigned) {
5602 // If we know the sign bits of both operands are zero, strength reduce to a
5603 // urem instead. Handles (X & 0x0FFFFFFF) %s 16 -> X&15
5604 if (DAG.SignBitIsZero(Op: N1) && DAG.SignBitIsZero(Op: N0))
5605 return DAG.getNode(Opcode: ISD::UREM, DL, VT, N1: N0, N2: N1);
5606 } else {
5607 if (DAG.isKnownToBeAPowerOfTwo(Val: N1, /*OrZero=*/true)) {
5608 // fold (urem x, pow2) -> (and x, pow2-1)
5609 SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
5610 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1, N2: NegOne);
5611 AddToWorklist(N: Add.getNode());
5612 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N0, N2: Add);
5613 }
5614 }
5615
5616 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5617
5618 // If X/C can be simplified by the division-by-constant logic, lower
5619 // X%C to the equivalent of X-X/C*C.
5620 // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
5621 // speculative DIV must not cause a DIVREM conversion. We guard against this
5622 // by skipping the simplification if isIntDivCheap(). When div is not cheap,
5623 // combine will not return a DIVREM. Regardless, checking cheapness here
5624 // makes sense since the simplification results in fatter code.
5625 if (DAG.isKnownNeverZero(Op: N1) && !TLI.isIntDivCheap(VT, Attr)) {
5626 if (isSigned) {
5627 // check if we can build faster implementation for srem
5628 if (SDValue OptimizedRem = buildOptimizedSREM(N0, N1, N))
5629 return OptimizedRem;
5630 }
5631
5632 SDValue OptimizedDiv =
5633 isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
5634 if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != N) {
5635 // If the equivalent Div node also exists, update its users.
5636 if (SDNode *DivNode = DAG.getNodeIfExists(Opcode: DivOpcode, VTList: N->getVTList(),
5637 Ops: { N0, N1 }))
5638 CombineTo(N: DivNode, Res: OptimizedDiv);
5639 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: OptimizedDiv, N2: N1);
5640 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Mul);
5641 AddToWorklist(N: OptimizedDiv.getNode());
5642 AddToWorklist(N: Mul.getNode());
5643 return Sub;
5644 }
5645 }
5646
5647 // Fold Num % Den -> Num - (Num / Den) * Den, if (Num / Den) is already
5648 // computed. Defer for types that will be promoted and do not fold if DIVREM
5649 // is available
5650 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
5651 if (!ForCodeSize &&
5652 !TLI.isOperationLegalOrCustom(Op: DivRemOpc, VT: VT.getScalarType()) &&
5653 !isDivRemLibcallAvailable(Node: N, isSigned, DAG) &&
5654 TLI.getTypeAction(Context&: *DAG.getContext(), VT) !=
5655 TargetLowering::TypePromoteInteger) {
5656 if (SDNode *Div =
5657 DAG.getNodeIfExists(Opcode: DivOpcode, VTList: N->getVTList(), Ops: {N0, N1})) {
5658 SDValue Mul = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: SDValue(Div, 0), N2: N1);
5659 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: N0, N2: Mul);
5660 }
5661 }
5662
5663 // sdiv, srem -> sdivrem
5664 if (SDValue DivRem = useDivRem(Node: N))
5665 return DivRem.getValue(R: 1);
5666
5667 // fold urem(urem(A, BCst), Op1Cst) -> urem(A, Op1Cst)
5668 // iff urem(BCst, Op1Cst) == 0
5669 SDValue A;
5670 APInt Op1Cst, BCst;
5671 if (sd_match(N, P: m_URem(L: m_URem(L: m_Value(N&: A), R: m_ConstInt(V&: BCst)),
5672 R: m_ConstInt(V&: Op1Cst))) &&
5673 BCst.urem(RHS: Op1Cst).isZero()) {
5674 return DAG.getNode(Opcode: ISD::UREM, DL, VT, N1: A, N2: DAG.getConstant(Val: Op1Cst, DL, VT));
5675 }
5676
5677 // fold srem(srem(A, BCst), Op1Cst) -> srem(A, Op1Cst)
5678 // iff srem(BCst, Op1Cst) == 0 && Op1Cst != 1
5679 if (sd_match(N, P: m_SRem(L: m_SRem(L: m_Value(N&: A), R: m_ConstInt(V&: BCst)),
5680 R: m_ConstInt(V&: Op1Cst))) &&
5681 BCst.srem(RHS: Op1Cst).isZero() && !Op1Cst.isAllOnes()) {
5682 return DAG.getNode(Opcode: ISD::SREM, DL, VT, N1: A, N2: DAG.getConstant(Val: Op1Cst, DL, VT));
5683 }
5684
5685 return SDValue();
5686}
5687
5688SDValue DAGCombiner::visitMULHS(SDNode *N) {
5689 SDValue N0 = N->getOperand(Num: 0);
5690 SDValue N1 = N->getOperand(Num: 1);
5691 EVT VT = N->getValueType(ResNo: 0);
5692 SDLoc DL(N);
5693
5694 // fold (mulhs c1, c2)
5695 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::MULHS, DL, VT, Ops: {N0, N1}))
5696 return C;
5697
5698 // canonicalize constant to RHS.
5699 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
5700 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
5701 return DAG.getNode(Opcode: ISD::MULHS, DL, VTList: N->getVTList(), N1, N2: N0);
5702
5703 if (VT.isVector()) {
5704 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5705 return FoldedVOp;
5706
5707 // fold (mulhs x, 0) -> 0
5708 // do not return N1, because undef node may exist.
5709 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
5710 return DAG.getConstant(Val: 0, DL, VT);
5711 }
5712
5713 // fold (mulhs x, 0) -> 0
5714 if (isNullConstant(V: N1))
5715 return N1;
5716
5717 // fold (mulhs x, 1) -> (sra x, size(x)-1)
5718 if (isOneConstant(V: N1))
5719 return DAG.getNode(
5720 Opcode: ISD::SRA, DL, VT, N1: N0,
5721 N2: DAG.getShiftAmountConstant(Val: N0.getScalarValueSizeInBits() - 1, VT, DL));
5722
5723 // fold (mulhs x, undef) -> 0
5724 if (N0.isUndef() || N1.isUndef())
5725 return DAG.getConstant(Val: 0, DL, VT);
5726
5727 // If the type twice as wide is legal, transform the mulhs to a wider multiply
5728 // plus a shift.
5729 if (!TLI.isOperationLegalOrCustom(Op: ISD::MULHS, VT) && VT.isSimple() &&
5730 !VT.isVector()) {
5731 MVT Simple = VT.getSimpleVT();
5732 unsigned SimpleSize = Simple.getSizeInBits();
5733 EVT NewVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SimpleSize*2);
5734 if (TLI.isOperationLegal(Op: ISD::MUL, VT: NewVT)) {
5735 N0 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewVT, Operand: N0);
5736 N1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewVT, Operand: N1);
5737 N1 = DAG.getNode(Opcode: ISD::MUL, DL, VT: NewVT, N1: N0, N2: N1);
5738 N1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: NewVT, N1,
5739 N2: DAG.getShiftAmountConstant(Val: SimpleSize, VT: NewVT, DL));
5740 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N1);
5741 }
5742 }
5743
5744 return SDValue();
5745}
5746
5747SDValue DAGCombiner::visitMULHU(SDNode *N) {
5748 SDValue N0 = N->getOperand(Num: 0);
5749 SDValue N1 = N->getOperand(Num: 1);
5750 EVT VT = N->getValueType(ResNo: 0);
5751 SDLoc DL(N);
5752
5753 // fold (mulhu c1, c2)
5754 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::MULHU, DL, VT, Ops: {N0, N1}))
5755 return C;
5756
5757 // canonicalize constant to RHS.
5758 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
5759 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
5760 return DAG.getNode(Opcode: ISD::MULHU, DL, VTList: N->getVTList(), N1, N2: N0);
5761
5762 if (VT.isVector()) {
5763 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5764 return FoldedVOp;
5765
5766 // fold (mulhu x, 0) -> 0
5767 // do not return N1, because undef node may exist.
5768 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
5769 return DAG.getConstant(Val: 0, DL, VT);
5770 }
5771
5772 // fold (mulhu x, 0) -> 0
5773 if (isNullConstant(V: N1))
5774 return N1;
5775
5776 // fold (mulhu x, 1) -> 0
5777 if (isOneConstant(V: N1))
5778 return DAG.getConstant(Val: 0, DL, VT);
5779
5780 // fold (mulhu x, undef) -> 0
5781 if (N0.isUndef() || N1.isUndef())
5782 return DAG.getConstant(Val: 0, DL, VT);
5783
5784 // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
5785 if (isConstantOrConstantVector(N: N1, /*NoOpaques=*/true,
5786 /*AllowTruncation=*/true) &&
5787 (!LegalOperations || hasOperation(Opcode: ISD::SRL, VT))) {
5788 if (SDValue LogBase2 = BuildLogBase2(V: N1, DL)) {
5789 unsigned NumEltBits = VT.getScalarSizeInBits();
5790 SDValue SRLAmt = DAG.getNode(
5791 Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: NumEltBits, DL, VT), N2: LogBase2);
5792 EVT ShiftVT = getShiftAmountTy(LHSTy: N0.getValueType());
5793 SDValue Trunc = DAG.getZExtOrTrunc(Op: SRLAmt, DL, VT: ShiftVT);
5794 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0, N2: Trunc);
5795 }
5796 }
5797
5798 // If the type twice as wide is legal, transform the mulhu to a wider multiply
5799 // plus a shift.
5800 if (!TLI.isOperationLegalOrCustom(Op: ISD::MULHU, VT) && VT.isSimple() &&
5801 !VT.isVector()) {
5802 MVT Simple = VT.getSimpleVT();
5803 unsigned SimpleSize = Simple.getSizeInBits();
5804 EVT NewVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SimpleSize*2);
5805 if (TLI.isOperationLegal(Op: ISD::MUL, VT: NewVT)) {
5806 N0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: NewVT, Operand: N0);
5807 N1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: NewVT, Operand: N1);
5808 N1 = DAG.getNode(Opcode: ISD::MUL, DL, VT: NewVT, N1: N0, N2: N1);
5809 N1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: NewVT, N1,
5810 N2: DAG.getShiftAmountConstant(Val: SimpleSize, VT: NewVT, DL));
5811 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N1);
5812 }
5813 }
5814
5815 // Simplify the operands using demanded-bits information.
5816 // We don't have demanded bits support for MULHU so this just enables constant
5817 // folding based on known bits.
5818 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
5819 return SDValue(N, 0);
5820
5821 return SDValue();
5822}
5823
5824SDValue DAGCombiner::visitAVG(SDNode *N) {
5825 unsigned Opcode = N->getOpcode();
5826 SDValue N0 = N->getOperand(Num: 0);
5827 SDValue N1 = N->getOperand(Num: 1);
5828 EVT VT = N->getValueType(ResNo: 0);
5829 SDLoc DL(N);
5830 bool IsSigned = Opcode == ISD::AVGCEILS || Opcode == ISD::AVGFLOORS;
5831
5832 // fold (avg c1, c2)
5833 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, Ops: {N0, N1}))
5834 return C;
5835
5836 // canonicalize constant to RHS.
5837 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
5838 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
5839 return DAG.getNode(Opcode, DL, VTList: N->getVTList(), N1, N2: N0);
5840
5841 if (VT.isVector())
5842 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5843 return FoldedVOp;
5844
5845 // fold (avg x, undef) -> x
5846 if (N0.isUndef())
5847 return N1;
5848 if (N1.isUndef())
5849 return N0;
5850
5851 // fold (avg x, x) --> x
5852 if (N0 == N1 && Level >= AfterLegalizeTypes)
5853 return N0;
5854
5855 // fold (avgfloor x, 0) -> x >> 1
5856 SDValue X, Y;
5857 if (sd_match(N, P: m_c_BinOp(Opc: ISD::AVGFLOORS, L: m_Value(N&: X), R: m_Zero())))
5858 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: X,
5859 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
5860 if (sd_match(N, P: m_c_BinOp(Opc: ISD::AVGFLOORU, L: m_Value(N&: X), R: m_Zero())))
5861 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X,
5862 N2: DAG.getShiftAmountConstant(Val: 1, VT, DL));
5863
5864 // fold avgu(zext(x), zext(y)) -> zext(avgu(x, y))
5865 // fold avgs(sext(x), sext(y)) -> sext(avgs(x, y))
5866 if (!IsSigned &&
5867 sd_match(N, P: m_BinOp(Opc: Opcode, L: m_ZExt(Op: m_Value(N&: X)), R: m_ZExt(Op: m_Value(N&: Y)))) &&
5868 X.getValueType() == Y.getValueType() &&
5869 hasOperation(Opcode, VT: X.getValueType())) {
5870 SDValue AvgU = DAG.getNode(Opcode, DL, VT: X.getValueType(), N1: X, N2: Y);
5871 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: AvgU);
5872 }
5873 if (IsSigned &&
5874 sd_match(N, P: m_BinOp(Opc: Opcode, L: m_SExt(Op: m_Value(N&: X)), R: m_SExt(Op: m_Value(N&: Y)))) &&
5875 X.getValueType() == Y.getValueType() &&
5876 hasOperation(Opcode, VT: X.getValueType())) {
5877 SDValue AvgS = DAG.getNode(Opcode, DL, VT: X.getValueType(), N1: X, N2: Y);
5878 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: AvgS);
5879 }
5880
5881 // Fold avgflooru(x,y) -> avgceilu(x,y-1) iff y != 0
5882 // Fold avgflooru(x,y) -> avgceilu(x-1,y) iff x != 0
5883 // Check if avgflooru isn't legal/custom but avgceilu is.
5884 if (Opcode == ISD::AVGFLOORU && !hasOperation(Opcode: ISD::AVGFLOORU, VT) &&
5885 (!LegalOperations || hasOperation(Opcode: ISD::AVGCEILU, VT))) {
5886 if (DAG.isKnownNeverZero(Op: N1))
5887 return DAG.getNode(
5888 Opcode: ISD::AVGCEILU, DL, VT, N1: N0,
5889 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1, N2: DAG.getAllOnesConstant(DL, VT)));
5890 if (DAG.isKnownNeverZero(Op: N0))
5891 return DAG.getNode(
5892 Opcode: ISD::AVGCEILU, DL, VT, N1,
5893 N2: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0, N2: DAG.getAllOnesConstant(DL, VT)));
5894 }
5895
5896 // Fold avgfloor((add nw x,y), 1) -> avgceil(x,y)
5897 // Fold avgfloor((add nw x,1), y) -> avgceil(x,y)
5898 if ((Opcode == ISD::AVGFLOORU && hasOperation(Opcode: ISD::AVGCEILU, VT)) ||
5899 (Opcode == ISD::AVGFLOORS && hasOperation(Opcode: ISD::AVGCEILS, VT))) {
5900 SDValue Add;
5901 if (sd_match(N,
5902 P: m_c_BinOp(Opc: Opcode, L: m_Value(N&: Add, P: m_Add(L: m_Value(N&: X), R: m_Value(N&: Y))),
5903 R: m_One())) ||
5904 sd_match(N, P: m_c_BinOp(Opc: Opcode, L: m_Value(N&: Add, P: m_Add(L: m_Value(N&: X), R: m_One())),
5905 R: m_Value(N&: Y)))) {
5906
5907 if (IsSigned && Add->getFlags().hasNoSignedWrap())
5908 return DAG.getNode(Opcode: ISD::AVGCEILS, DL, VT, N1: X, N2: Y);
5909
5910 if (!IsSigned && Add->getFlags().hasNoUnsignedWrap())
5911 return DAG.getNode(Opcode: ISD::AVGCEILU, DL, VT, N1: X, N2: Y);
5912 }
5913 }
5914
5915 // Fold avgfloors(x,y) -> avgflooru(x,y) if both x and y are non-negative
5916 if (Opcode == ISD::AVGFLOORS && hasOperation(Opcode: ISD::AVGFLOORU, VT)) {
5917 if (DAG.SignBitIsZero(Op: N0) && DAG.SignBitIsZero(Op: N1))
5918 return DAG.getNode(Opcode: ISD::AVGFLOORU, DL, VT, N1: N0, N2: N1);
5919 }
5920
5921 return SDValue();
5922}
5923
5924SDValue DAGCombiner::visitABD(SDNode *N) {
5925 unsigned Opcode = N->getOpcode();
5926 SDValue N0 = N->getOperand(Num: 0);
5927 SDValue N1 = N->getOperand(Num: 1);
5928 EVT VT = N->getValueType(ResNo: 0);
5929 SDLoc DL(N);
5930
5931 // fold (abd c1, c2)
5932 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, Ops: {N0, N1}))
5933 return C;
5934
5935 // canonicalize constant to RHS.
5936 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
5937 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
5938 return DAG.getNode(Opcode, DL, VTList: N->getVTList(), N1, N2: N0);
5939
5940 if (VT.isVector())
5941 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
5942 return FoldedVOp;
5943
5944 // fold (abd x, undef) -> 0
5945 if (N0.isUndef() || N1.isUndef())
5946 return DAG.getConstant(Val: 0, DL, VT);
5947
5948 // fold (abd x, x) -> 0
5949 if (N0 == N1)
5950 return DAG.getConstant(Val: 0, DL, VT);
5951
5952 SDValue X, Y;
5953
5954 // fold (abds x, 0) -> abs x
5955 if (sd_match(N, P: m_c_BinOp(Opc: ISD::ABDS, L: m_Value(N&: X), R: m_Zero())) &&
5956 (!LegalOperations || hasOperation(Opcode: ISD::ABS, VT)))
5957 return DAG.getNode(Opcode: ISD::ABS, DL, VT, Operand: X);
5958
5959 // fold (abdu x, 0) -> x
5960 if (sd_match(N, P: m_c_BinOp(Opc: ISD::ABDU, L: m_Value(N&: X), R: m_Zero())))
5961 return X;
5962
5963 // fold (abds x, y) -> (abdu x, y) iff both args are known positive
5964 if (Opcode == ISD::ABDS && hasOperation(Opcode: ISD::ABDU, VT) &&
5965 DAG.SignBitIsZero(Op: N0) && DAG.SignBitIsZero(Op: N1))
5966 return DAG.getNode(Opcode: ISD::ABDU, DL, VT, N1, N2: N0);
5967
5968 // fold (abd? (?ext x), (?ext y)) -> (zext (abd? x, y))
5969 if (sd_match(N, P: m_BinOp(Opc: ISD::ABDU, L: m_ZExt(Op: m_Value(N&: X)), R: m_ZExt(Op: m_Value(N&: Y)))) ||
5970 sd_match(N, P: m_BinOp(Opc: ISD::ABDS, L: m_SExt(Op: m_Value(N&: X)), R: m_SExt(Op: m_Value(N&: Y))))) {
5971 EVT SmallVT = X.getScalarValueSizeInBits() > Y.getScalarValueSizeInBits()
5972 ? X.getValueType()
5973 : Y.getValueType();
5974 if (!LegalOperations || hasOperation(Opcode, VT: SmallVT)) {
5975 SDValue ExtedX = DAG.getExtOrTrunc(Op: X, DL: SDLoc(X), VT: SmallVT, Opcode: N0->getOpcode());
5976 SDValue ExtedY = DAG.getExtOrTrunc(Op: Y, DL: SDLoc(Y), VT: SmallVT, Opcode: N0->getOpcode());
5977 SDValue SmallABD = DAG.getNode(Opcode, DL, VT: SmallVT, Ops: {ExtedX, ExtedY});
5978 SDValue ZExted = DAG.getZExtOrTrunc(Op: SmallABD, DL, VT);
5979 return ZExted;
5980 }
5981 }
5982
5983 // fold (abd? (?ext ty:x), small_const:c) -> (zext (abd? x, c))
5984 if (sd_match(N, P: m_c_BinOp(Opc: ISD::ABDU, L: m_ZExt(Op: m_Value(N&: X)), R: m_Value(N&: Y))) ||
5985 sd_match(N, P: m_c_BinOp(Opc: ISD::ABDS, L: m_SExt(Op: m_Value(N&: X)), R: m_Value(N&: Y)))) {
5986 EVT SmallVT = X.getValueType();
5987 if (!LegalOperations || hasOperation(Opcode, VT: SmallVT)) {
5988 uint64_t Bits = SmallVT.getScalarSizeInBits();
5989 unsigned RelevantBits =
5990 (Opcode == ISD::ABDS) ? DAG.ComputeMaxSignificantBits(Op: Y)
5991 : DAG.computeKnownBits(Op: Y).countMaxActiveBits();
5992 bool TruncatingYIsCheap = TLI.isTruncateFree(Val: Y, VT2: SmallVT) ||
5993 ISD::matchUnaryPredicate(
5994 Op: Y,
5995 Match: [&](auto *C) {
5996 if (!C)
5997 return true;
5998 const APInt &YConst = C->getAsAPIntVal();
5999 return (Opcode == ISD::ABDS)
6000 ? YConst.isSignedIntN(N: Bits)
6001 : YConst.isIntN(N: Bits);
6002 },
6003 /*AllowUndefs=*/true);
6004
6005 if (RelevantBits <= Bits && TruncatingYIsCheap) {
6006 SDValue NewY = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Y), VT: SmallVT, Operand: Y);
6007 SDValue SmallABD = DAG.getNode(Opcode, DL, VT: SmallVT, Ops: {X, NewY});
6008 return DAG.getZExtOrTrunc(Op: SmallABD, DL, VT);
6009 }
6010 }
6011 }
6012
6013 return SDValue();
6014}
6015
6016/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
6017/// give the opcodes for the two computations that are being performed. Return
6018/// true if a simplification was made.
6019SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
6020 unsigned HiOp) {
6021 // If the high half is not needed, just compute the low half.
6022 bool HiExists = N->hasAnyUseOfValue(Value: 1);
6023 if (!HiExists && (!LegalOperations ||
6024 TLI.isOperationLegalOrCustom(Op: LoOp, VT: N->getValueType(ResNo: 0)))) {
6025 SDValue Res = DAG.getNode(Opcode: LoOp, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Ops: N->ops());
6026 return CombineTo(N, Res0: Res, Res1: Res);
6027 }
6028
6029 // If the low half is not needed, just compute the high half.
6030 bool LoExists = N->hasAnyUseOfValue(Value: 0);
6031 if (!LoExists && (!LegalOperations ||
6032 TLI.isOperationLegalOrCustom(Op: HiOp, VT: N->getValueType(ResNo: 1)))) {
6033 SDValue Res = DAG.getNode(Opcode: HiOp, DL: SDLoc(N), VT: N->getValueType(ResNo: 1), Ops: N->ops());
6034 return CombineTo(N, Res0: Res, Res1: Res);
6035 }
6036
6037 // If both halves are used, return as it is.
6038 if (LoExists && HiExists)
6039 return SDValue();
6040
6041 // If the two computed results can be simplified separately, separate them.
6042 if (LoExists) {
6043 SDValue Lo = DAG.getNode(Opcode: LoOp, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Ops: N->ops());
6044 AddToWorklist(N: Lo.getNode());
6045 SDValue LoOpt = combine(N: Lo.getNode());
6046 if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
6047 (!LegalOperations ||
6048 TLI.isOperationLegalOrCustom(Op: LoOpt.getOpcode(), VT: LoOpt.getValueType())))
6049 return CombineTo(N, Res0: LoOpt, Res1: LoOpt);
6050 }
6051
6052 if (HiExists) {
6053 SDValue Hi = DAG.getNode(Opcode: HiOp, DL: SDLoc(N), VT: N->getValueType(ResNo: 1), Ops: N->ops());
6054 AddToWorklist(N: Hi.getNode());
6055 SDValue HiOpt = combine(N: Hi.getNode());
6056 if (HiOpt.getNode() && HiOpt != Hi &&
6057 (!LegalOperations ||
6058 TLI.isOperationLegalOrCustom(Op: HiOpt.getOpcode(), VT: HiOpt.getValueType())))
6059 return CombineTo(N, Res0: HiOpt, Res1: HiOpt);
6060 }
6061
6062 return SDValue();
6063}
6064
6065SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
6066 if (SDValue Res = SimplifyNodeWithTwoResults(N, LoOp: ISD::MUL, HiOp: ISD::MULHS))
6067 return Res;
6068
6069 SDValue N0 = N->getOperand(Num: 0);
6070 SDValue N1 = N->getOperand(Num: 1);
6071 EVT VT = N->getValueType(ResNo: 0);
6072 SDLoc DL(N);
6073
6074 // Constant fold.
6075 if (isa<ConstantSDNode>(Val: N0) && isa<ConstantSDNode>(Val: N1))
6076 return DAG.getNode(Opcode: ISD::SMUL_LOHI, DL, VTList: N->getVTList(), N1: N0, N2: N1);
6077
6078 // canonicalize constant to RHS (vector doesn't have to splat)
6079 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
6080 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
6081 return DAG.getNode(Opcode: ISD::SMUL_LOHI, DL, VTList: N->getVTList(), N1, N2: N0);
6082
6083 // If the type is twice as wide is legal, transform the mulhu to a wider
6084 // multiply plus a shift.
6085 if (VT.isSimple() && !VT.isVector()) {
6086 MVT Simple = VT.getSimpleVT();
6087 unsigned SimpleSize = Simple.getSizeInBits();
6088 EVT NewVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SimpleSize*2);
6089 if (TLI.isOperationLegal(Op: ISD::MUL, VT: NewVT)) {
6090 SDValue Lo = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewVT, Operand: N0);
6091 SDValue Hi = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: NewVT, Operand: N1);
6092 Lo = DAG.getNode(Opcode: ISD::MUL, DL, VT: NewVT, N1: Lo, N2: Hi);
6093 // Compute the high part as N1.
6094 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: NewVT, N1: Lo,
6095 N2: DAG.getShiftAmountConstant(Val: SimpleSize, VT: NewVT, DL));
6096 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Hi);
6097 // Compute the low part as N0.
6098 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Lo);
6099 return CombineTo(N, Res0: Lo, Res1: Hi);
6100 }
6101 }
6102
6103 return SDValue();
6104}
6105
6106SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
6107 if (SDValue Res = SimplifyNodeWithTwoResults(N, LoOp: ISD::MUL, HiOp: ISD::MULHU))
6108 return Res;
6109
6110 SDValue N0 = N->getOperand(Num: 0);
6111 SDValue N1 = N->getOperand(Num: 1);
6112 EVT VT = N->getValueType(ResNo: 0);
6113 SDLoc DL(N);
6114
6115 // Constant fold.
6116 if (isa<ConstantSDNode>(Val: N0) && isa<ConstantSDNode>(Val: N1))
6117 return DAG.getNode(Opcode: ISD::UMUL_LOHI, DL, VTList: N->getVTList(), N1: N0, N2: N1);
6118
6119 // canonicalize constant to RHS (vector doesn't have to splat)
6120 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
6121 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
6122 return DAG.getNode(Opcode: ISD::UMUL_LOHI, DL, VTList: N->getVTList(), N1, N2: N0);
6123
6124 // (umul_lohi N0, 0) -> (0, 0)
6125 if (isNullConstant(V: N1)) {
6126 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
6127 return CombineTo(N, Res0: Zero, Res1: Zero);
6128 }
6129
6130 // (umul_lohi N0, 1) -> (N0, 0)
6131 if (isOneConstant(V: N1)) {
6132 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
6133 return CombineTo(N, Res0: N0, Res1: Zero);
6134 }
6135
6136 // If the type is twice as wide is legal, transform the mulhu to a wider
6137 // multiply plus a shift.
6138 if (VT.isSimple() && !VT.isVector()) {
6139 MVT Simple = VT.getSimpleVT();
6140 unsigned SimpleSize = Simple.getSizeInBits();
6141 EVT NewVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SimpleSize*2);
6142 if (TLI.isOperationLegal(Op: ISD::MUL, VT: NewVT)) {
6143 SDValue Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: NewVT, Operand: N0);
6144 SDValue Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: NewVT, Operand: N1);
6145 Lo = DAG.getNode(Opcode: ISD::MUL, DL, VT: NewVT, N1: Lo, N2: Hi);
6146 // Compute the high part as N1.
6147 Hi = DAG.getNode(Opcode: ISD::SRL, DL, VT: NewVT, N1: Lo,
6148 N2: DAG.getShiftAmountConstant(Val: SimpleSize, VT: NewVT, DL));
6149 Hi = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Hi);
6150 // Compute the low part as N0.
6151 Lo = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Lo);
6152 return CombineTo(N, Res0: Lo, Res1: Hi);
6153 }
6154 }
6155
6156 return SDValue();
6157}
6158
6159SDValue DAGCombiner::visitMULO(SDNode *N) {
6160 SDValue N0 = N->getOperand(Num: 0);
6161 SDValue N1 = N->getOperand(Num: 1);
6162 EVT VT = N0.getValueType();
6163 bool IsSigned = (ISD::SMULO == N->getOpcode());
6164
6165 EVT CarryVT = N->getValueType(ResNo: 1);
6166 SDLoc DL(N);
6167
6168 ConstantSDNode *N0C = isConstOrConstSplat(N: N0);
6169 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
6170
6171 // fold operation with constant operands.
6172 // TODO: Move this to FoldConstantArithmetic when it supports nodes with
6173 // multiple results.
6174 if (N0C && N1C && !N0C->isOpaque() && !N1C->isOpaque()) {
6175 bool Overflow;
6176 APInt Result =
6177 IsSigned ? N0C->getAPIntValue().smul_ov(RHS: N1C->getAPIntValue(), Overflow)
6178 : N0C->getAPIntValue().umul_ov(RHS: N1C->getAPIntValue(), Overflow);
6179 return CombineTo(N, Res0: DAG.getConstant(Val: Result, DL, VT),
6180 Res1: DAG.getBoolConstant(V: Overflow, DL, VT: CarryVT, OpVT: CarryVT));
6181 }
6182
6183 // canonicalize constant to RHS.
6184 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
6185 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
6186 return DAG.getNode(Opcode: N->getOpcode(), DL, VTList: N->getVTList(), N1, N2: N0);
6187
6188 // fold (mulo x, 0) -> 0 + no carry out
6189 if (isNullOrNullSplat(V: N1))
6190 return CombineTo(N, Res0: DAG.getConstant(Val: 0, DL, VT),
6191 Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
6192
6193 // (mulo x, 2) -> (addo x, x)
6194 // FIXME: This needs a freeze.
6195 if (N1C && N1C->getAPIntValue() == 2 &&
6196 (!IsSigned || VT.getScalarSizeInBits() > 2))
6197 return DAG.getNode(Opcode: IsSigned ? ISD::SADDO : ISD::UADDO, DL,
6198 VTList: N->getVTList(), N1: N0, N2: N0);
6199
6200 // A 1 bit SMULO overflows if both inputs are 1.
6201 if (IsSigned && VT.getScalarSizeInBits() == 1) {
6202 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N0, N2: N1);
6203 SDValue Cmp = DAG.getSetCC(DL, VT: CarryVT, LHS: And,
6204 RHS: DAG.getConstant(Val: 0, DL, VT), Cond: ISD::SETNE);
6205 return CombineTo(N, Res0: And, Res1: Cmp);
6206 }
6207
6208 // If it cannot overflow, transform into a mul.
6209 if (DAG.willNotOverflowMul(IsSigned, N0, N1))
6210 return CombineTo(N, Res0: DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N0, N2: N1),
6211 Res1: DAG.getConstant(Val: 0, DL, VT: CarryVT));
6212 return SDValue();
6213}
6214
6215// Function to calculate whether the Min/Max pair of SDNodes (potentially
6216// swapped around) make a signed saturate pattern, clamping to between a signed
6217// saturate of -2^(BW-1) and 2^(BW-1)-1, or an unsigned saturate of 0 and 2^BW.
6218// Returns the node being clamped and the bitwidth of the clamp in BW. Should
6219// work with both SMIN/SMAX nodes and setcc/select combo. The operands are the
6220// same as SimplifySelectCC. N0<N1 ? N2 : N3.
6221static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
6222 SDValue N3, ISD::CondCode CC, unsigned &BW,
6223 bool &Unsigned, SelectionDAG &DAG) {
6224 auto isSignedMinMax = [&](SDValue N0, SDValue N1, SDValue N2, SDValue N3,
6225 ISD::CondCode CC) {
6226 // The compare and select operand should be the same or the select operands
6227 // should be truncated versions of the comparison.
6228 if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(i: 0)))
6229 return 0;
6230 // The constants need to be the same or a truncated version of each other.
6231 ConstantSDNode *N1C = isConstOrConstSplat(N: peekThroughTruncates(V: N1));
6232 ConstantSDNode *N3C = isConstOrConstSplat(N: peekThroughTruncates(V: N3));
6233 if (!N1C || !N3C)
6234 return 0;
6235 const APInt &C1 = N1C->getAPIntValue().trunc(width: N1.getScalarValueSizeInBits());
6236 const APInt &C2 = N3C->getAPIntValue().trunc(width: N3.getScalarValueSizeInBits());
6237 if (C1.getBitWidth() < C2.getBitWidth() || C1 != C2.sext(width: C1.getBitWidth()))
6238 return 0;
6239 return CC == ISD::SETLT ? ISD::SMIN : (CC == ISD::SETGT ? ISD::SMAX : 0);
6240 };
6241
6242 // Check the initial value is a SMIN/SMAX equivalent.
6243 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3, CC);
6244 if (!Opcode0)
6245 return SDValue();
6246
6247 // We could only need one range check, if the fptosi could never produce
6248 // the upper value.
6249 if (N0.getOpcode() == ISD::FP_TO_SINT && Opcode0 == ISD::SMAX) {
6250 if (isNullOrNullSplat(V: N3)) {
6251 EVT IntVT = N0.getValueType().getScalarType();
6252 EVT FPVT = N0.getOperand(i: 0).getValueType().getScalarType();
6253 if (FPVT.isSimple()) {
6254 Type *InputTy = FPVT.getTypeForEVT(Context&: *DAG.getContext());
6255 const fltSemantics &Semantics = InputTy->getFltSemantics();
6256 uint32_t MinBitWidth =
6257 APFloatBase::semanticsIntSizeInBits(Semantics, /*isSigned*/ true);
6258 if (IntVT.getSizeInBits() >= MinBitWidth) {
6259 Unsigned = true;
6260 BW = PowerOf2Ceil(A: MinBitWidth);
6261 return N0;
6262 }
6263 }
6264 }
6265 }
6266
6267 SDValue N00, N01, N02, N03;
6268 ISD::CondCode N0CC;
6269 switch (N0.getOpcode()) {
6270 case ISD::SMIN:
6271 case ISD::SMAX:
6272 N00 = N02 = N0.getOperand(i: 0);
6273 N01 = N03 = N0.getOperand(i: 1);
6274 N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT;
6275 break;
6276 case ISD::SELECT_CC:
6277 N00 = N0.getOperand(i: 0);
6278 N01 = N0.getOperand(i: 1);
6279 N02 = N0.getOperand(i: 2);
6280 N03 = N0.getOperand(i: 3);
6281 N0CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 4))->get();
6282 break;
6283 case ISD::SELECT:
6284 case ISD::VSELECT:
6285 if (N0.getOperand(i: 0).getOpcode() != ISD::SETCC)
6286 return SDValue();
6287 N00 = N0.getOperand(i: 0).getOperand(i: 0);
6288 N01 = N0.getOperand(i: 0).getOperand(i: 1);
6289 N02 = N0.getOperand(i: 1);
6290 N03 = N0.getOperand(i: 2);
6291 N0CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 0).getOperand(i: 2))->get();
6292 break;
6293 default:
6294 return SDValue();
6295 }
6296
6297 unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
6298 if (!Opcode1 || Opcode0 == Opcode1)
6299 return SDValue();
6300
6301 ConstantSDNode *MinCOp = isConstOrConstSplat(N: Opcode0 == ISD::SMIN ? N1 : N01);
6302 ConstantSDNode *MaxCOp = isConstOrConstSplat(N: Opcode0 == ISD::SMIN ? N01 : N1);
6303 if (!MinCOp || !MaxCOp || MinCOp->getValueType(ResNo: 0) != MaxCOp->getValueType(ResNo: 0))
6304 return SDValue();
6305
6306 const APInt &MinC = MinCOp->getAPIntValue();
6307 const APInt &MaxC = MaxCOp->getAPIntValue();
6308 APInt MinCPlus1 = MinC + 1;
6309 if (-MaxC == MinCPlus1 && MinCPlus1.isPowerOf2()) {
6310 BW = MinCPlus1.exactLogBase2() + 1;
6311 Unsigned = false;
6312 return N02;
6313 }
6314
6315 if (MaxC == 0 && MinC != 0 && MinCPlus1.isPowerOf2()) {
6316 BW = MinCPlus1.exactLogBase2();
6317 Unsigned = true;
6318 return N02;
6319 }
6320
6321 return SDValue();
6322}
6323
6324static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
6325 SDValue N3, ISD::CondCode CC,
6326 SelectionDAG &DAG) {
6327 unsigned BW;
6328 bool Unsigned;
6329 SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW, Unsigned, DAG);
6330 if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT)
6331 return SDValue();
6332 EVT FPVT = Fp.getOperand(i: 0).getValueType();
6333 EVT NewVT = FPVT.changeElementType(Context&: *DAG.getContext(),
6334 EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BW));
6335 unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT;
6336 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(Op: NewOpc, FPVT, VT: NewVT))
6337 return SDValue();
6338 SDLoc DL(Fp);
6339 SDValue Sat = DAG.getNode(Opcode: NewOpc, DL, VT: NewVT, N1: Fp.getOperand(i: 0),
6340 N2: DAG.getValueType(NewVT.getScalarType()));
6341 return DAG.getExtOrTrunc(IsSigned: !Unsigned, Op: Sat, DL, VT: N2->getValueType(ResNo: 0));
6342}
6343
6344static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
6345 SDValue N3, ISD::CondCode CC,
6346 SelectionDAG &DAG) {
6347 // We are looking for UMIN(FPTOUI(X), (2^n)-1), which may have come via a
6348 // select/vselect/select_cc. The two operands pairs for the select (N2/N3) may
6349 // be truncated versions of the setcc (N0/N1).
6350 if ((N0 != N2 &&
6351 (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(i: 0))) ||
6352 N0.getOpcode() != ISD::FP_TO_UINT || CC != ISD::SETULT)
6353 return SDValue();
6354 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
6355 ConstantSDNode *N3C = isConstOrConstSplat(N: N3);
6356 if (!N1C || !N3C)
6357 return SDValue();
6358 const APInt &C1 = N1C->getAPIntValue();
6359 const APInt &C3 = N3C->getAPIntValue();
6360 if (!(C1 + 1).isPowerOf2() || C1.getBitWidth() < C3.getBitWidth() ||
6361 C1 != C3.zext(width: C1.getBitWidth()))
6362 return SDValue();
6363
6364 unsigned BW = (C1 + 1).exactLogBase2();
6365 EVT FPVT = N0.getOperand(i: 0).getValueType();
6366 EVT NewVT = FPVT.changeElementType(Context&: *DAG.getContext(),
6367 EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BW));
6368 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(Op: ISD::FP_TO_UINT_SAT,
6369 FPVT, VT: NewVT))
6370 return SDValue();
6371
6372 SDValue Sat =
6373 DAG.getNode(Opcode: ISD::FP_TO_UINT_SAT, DL: SDLoc(N0), VT: NewVT, N1: N0.getOperand(i: 0),
6374 N2: DAG.getValueType(NewVT.getScalarType()));
6375 return DAG.getZExtOrTrunc(Op: Sat, DL: SDLoc(N0), VT: N3.getValueType());
6376}
6377
6378// Fold a NaN-guard select of fp_to_sint/fp_to_uint into the saturating
6379// variant, which returns 0 for NaN.
6380static SDValue performNanGuardFpToSatCombine(SDNode *N, SelectionDAG &DAG) {
6381 EVT VT = N->getValueType(ResNo: 0);
6382 SDLoc DL(N);
6383
6384 // Match an isnan-guarded select, requiring the compare to be single-use.
6385 // The guarded value is fp_to_sint/fp_to_uint of X, optionally masked by an
6386 // AND:
6387 // select (setcc X, 0.0, uno), 0, (fp_to_sint/uint X)
6388 // select (setcc X, 0.0, ord), (fp_to_sint/uint X), 0
6389 // select (setcc X, 0.0, uno), 0, (and (fp_to_sint/uint X), M)
6390 // select (setcc X, 0.0, ord), (and (fp_to_sint/uint X), M), 0
6391 SDValue X, GuardedVal;
6392 if (!sd_match(N,
6393 P: m_SelectLike(Cond: m_OneUse(P: m_SetCC(LHS: m_Value(N&: X), RHS: m_AnyZeroFP(),
6394 CC: m_SpecificCondCode(CC: ISD::SETUO))),
6395 T: m_Zero(), F: m_Value(N&: GuardedVal))) &&
6396 !sd_match(N,
6397 P: m_SelectLike(Cond: m_OneUse(P: m_SetCC(LHS: m_Value(N&: X), RHS: m_AnyZeroFP(),
6398 CC: m_SpecificCondCode(CC: ISD::SETO))),
6399 T: m_Value(N&: GuardedVal), F: m_Zero())))
6400 return SDValue();
6401
6402 // The guarded value must be fp_to_sint/fp_to_uint of the same X, optionally
6403 // masked by a (commutative) AND.
6404 SDValue Mask;
6405 unsigned NewOpc;
6406 if (sd_match(N: GuardedVal, P: m_FPToSI(Op: m_Specific(N: X))) ||
6407 sd_match(N: GuardedVal, P: m_And(L: m_FPToSI(Op: m_Specific(N: X)), R: m_Value(N&: Mask))))
6408 NewOpc = ISD::FP_TO_SINT_SAT;
6409 else if (sd_match(N: GuardedVal, P: m_FPToUI(Op: m_Specific(N: X))) ||
6410 sd_match(N: GuardedVal, P: m_And(L: m_FPToUI(Op: m_Specific(N: X)), R: m_Value(N&: Mask))))
6411 NewOpc = ISD::FP_TO_UINT_SAT;
6412 else
6413 return SDValue();
6414
6415 if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(Op: NewOpc,
6416 FPVT: X.getValueType(), VT))
6417 return SDValue();
6418
6419 SDValue Sat =
6420 DAG.getNode(Opcode: NewOpc, DL, VT, N1: X, N2: DAG.getValueType(VT.getScalarType()));
6421 if (Mask) {
6422 // For NaN inputs the saturating conversion yields 0, so (and 0, Mask) must
6423 // stay 0 to match the original select. A poison Mask would make it poison,
6424 // so freeze Mask to guarantee a defined value.
6425 Sat = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Sat, N2: DAG.getFreeze(V: Mask));
6426 }
6427 return Sat;
6428}
6429
6430SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
6431 SDValue N0 = N->getOperand(Num: 0);
6432 SDValue N1 = N->getOperand(Num: 1);
6433 EVT VT = N0.getValueType();
6434 unsigned Opcode = N->getOpcode();
6435 SDLoc DL(N);
6436
6437 // fold operation with constant operands.
6438 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, Ops: {N0, N1}))
6439 return C;
6440
6441 // If the operands are the same, this is a no-op.
6442 if (N0 == N1)
6443 return N0;
6444
6445 // canonicalize constant to RHS
6446 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
6447 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
6448 return DAG.getNode(Opcode, DL, VT, N1, N2: N0);
6449
6450 // fold vector ops
6451 if (VT.isVector())
6452 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
6453 return FoldedVOp;
6454
6455 // reassociate minmax
6456 if (SDValue RMINMAX = reassociateOps(Opc: Opcode, DL, N0, N1, Flags: N->getFlags()))
6457 return RMINMAX;
6458
6459 // Fold sign-extension masks using arithmetic shift:
6460 // smax(X, -1) -> or(X, ashr(X, BW-1))
6461 // smin(X, 0) -> and(X, ashr(X, BW-1))
6462 // ashr(X, BW-1) sign-extends the sign bit: 0 for X>=0, -1 for X<0.
6463 // OR with X yields X (non-negative) or -1 (negative) = smax(X,-1).
6464 // AND with X yields 0 (non-negative) or X (negative) = smin(X, 0).
6465 // Both reduce to two instructions vs. a compare+cmov on x86-64.
6466 // Only fold when the target has no native SMAX/SMIN instruction for this
6467 // type (isOperationExpand), the type is legal (not needing splitting),
6468 // the operand is not a min/max chain (preserving target combine patterns
6469 // that fold smax(smin(x,C),D) into a single saturation instruction), and
6470 // for smax(X,-1) the operand is not a sign extension (doubling its use
6471 // count can cause the target to lower the extension less efficiently).
6472 APInt C;
6473 if (TLI.isTypeLegal(VT) &&
6474 !TLI.shouldAvoidTransformToShift(VT, Amount: VT.getScalarSizeInBits() - 1) &&
6475 sd_match(N: N1, P: m_ConstInt(V&: C))) {
6476 if (Opcode == ISD::SMAX && TLI.isOperationExpand(Op: ISD::SMAX, VT) &&
6477 N0.getOpcode() != ISD::SMIN && N0.getOpcode() != ISD::SIGN_EXTEND &&
6478 C.isAllOnes()) {
6479 SDValue ShiftAmt =
6480 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL);
6481 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N0, N2: ShiftAmt);
6482 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: Shift);
6483 }
6484 if (Opcode == ISD::SMIN && TLI.isOperationExpand(Op: ISD::SMIN, VT) &&
6485 N0.getOpcode() != ISD::SMAX && C.isZero()) {
6486 SDValue ShiftAmt =
6487 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL);
6488 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N0, N2: ShiftAmt);
6489 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N0, N2: Shift);
6490 }
6491 }
6492
6493 // If both operands are known to have the same sign (both non-negative or both
6494 // negative), flip between UMIN/UMAX and SMIN/SMAX.
6495 // Only do this if:
6496 // 1. The current op isn't legal and the flipped is.
6497 // 2. The saturation pattern is broken by canonicalization in InstCombine.
6498 bool IsOpIllegal = !TLI.isOperationLegal(Op: Opcode, VT);
6499 bool IsSatBroken = Opcode == ISD::UMIN && N0.getOpcode() == ISD::SMAX;
6500
6501 if (IsSatBroken || IsOpIllegal) {
6502 auto HasKnownSameSign = [&](SDValue A, SDValue B) {
6503 if (A.isUndef() || B.isUndef())
6504 return true;
6505
6506 KnownBits KA = DAG.computeKnownBits(Op: A);
6507 if (!KA.isNonNegative() && !KA.isNegative())
6508 return false;
6509
6510 KnownBits KB = DAG.computeKnownBits(Op: B);
6511 if (KA.isNonNegative())
6512 return KB.isNonNegative();
6513 return KB.isNegative();
6514 };
6515
6516 if (HasKnownSameSign(N0, N1)) {
6517 unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(MinMaxOpc: Opcode);
6518 if ((IsSatBroken && IsOpIllegal) || TLI.isOperationLegal(Op: AltOpcode, VT))
6519 return DAG.getNode(Opcode: AltOpcode, DL, VT, N1: N0, N2: N1);
6520 }
6521 }
6522
6523 if (Opcode == ISD::SMIN || Opcode == ISD::SMAX)
6524 if (SDValue S = PerformMinMaxFpToSatCombine(
6525 N0, N1, N2: N0, N3: N1, CC: Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG))
6526 return S;
6527 if (Opcode == ISD::UMIN)
6528 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N2: N0, N3: N1, CC: ISD::SETULT, DAG))
6529 return S;
6530
6531 // Fold min/max(vecreduce(x), vecreduce(y)) -> vecreduce(min/max(x, y))
6532 auto ReductionOpcode = [](unsigned Opcode) {
6533 switch (Opcode) {
6534 case ISD::SMIN:
6535 return ISD::VECREDUCE_SMIN;
6536 case ISD::SMAX:
6537 return ISD::VECREDUCE_SMAX;
6538 case ISD::UMIN:
6539 return ISD::VECREDUCE_UMIN;
6540 case ISD::UMAX:
6541 return ISD::VECREDUCE_UMAX;
6542 default:
6543 llvm_unreachable("Unexpected opcode");
6544 }
6545 };
6546 if (SDValue SD = reassociateReduction(RedOpc: ReductionOpcode(Opcode), Opc: Opcode,
6547 DL: SDLoc(N), VT, N0, N1))
6548 return SD;
6549
6550 // Fold operation with vscale operands.
6551 if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
6552 uint64_t C0 = N0->getConstantOperandVal(Num: 0);
6553 uint64_t C1 = N1->getConstantOperandVal(Num: 0);
6554 if (Opcode == ISD::UMAX)
6555 return C0 > C1 ? N0 : N1;
6556 else if (Opcode == ISD::UMIN)
6557 return C0 > C1 ? N1 : N0;
6558 }
6559
6560 // If we know the range of vscale, see if we can fold it given a constant.
6561 if (N0.getOpcode() == ISD::VSCALE) {
6562 if (auto *C1 = dyn_cast<ConstantSDNode>(Val&: N1)) {
6563 bool ForSigned = (Opcode == ISD::SMAX || Opcode == ISD::SMIN);
6564 ConstantRange Range = DAG.computeConstantRange(Op: N0, ForSigned);
6565
6566 const APInt &C1V = C1->getAPIntValue();
6567 if ((Opcode == ISD::UMAX && Range.getUnsignedMax().ule(RHS: C1V)) ||
6568 (Opcode == ISD::UMIN && Range.getUnsignedMin().uge(RHS: C1V)) ||
6569 (Opcode == ISD::SMAX && Range.getSignedMax().sle(RHS: C1V)) ||
6570 (Opcode == ISD::SMIN && Range.getSignedMin().sge(RHS: C1V))) {
6571 return N1;
6572 }
6573 }
6574 }
6575
6576 // Simplify the operands using demanded-bits information.
6577 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
6578 return SDValue(N, 0);
6579
6580 return SDValue();
6581}
6582
6583/// If this is a bitwise logic instruction and both operands have the same
6584/// opcode, try to sink the other opcode after the logic instruction.
6585SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
6586 SDValue N0 = N->getOperand(Num: 0), N1 = N->getOperand(Num: 1);
6587 EVT VT = N0.getValueType();
6588 unsigned LogicOpcode = N->getOpcode();
6589 unsigned HandOpcode = N0.getOpcode();
6590 assert(ISD::isBitwiseLogicOp(LogicOpcode) && "Expected logic opcode");
6591 assert(HandOpcode == N1.getOpcode() && "Bad input!");
6592
6593 // Bail early if none of these transforms apply.
6594 if (N0.getNumOperands() == 0)
6595 return SDValue();
6596
6597 // FIXME: We should check number of uses of the operands to not increase
6598 // the instruction count for all transforms.
6599
6600 // Handle size-changing casts (or sign_extend_inreg).
6601 SDValue X = N0.getOperand(i: 0);
6602 SDValue Y = N1.getOperand(i: 0);
6603 EVT XVT = X.getValueType();
6604 SDLoc DL(N);
6605 if (ISD::isExtOpcode(Opcode: HandOpcode) || ISD::isExtVecInRegOpcode(Opcode: HandOpcode) ||
6606 (HandOpcode == ISD::SIGN_EXTEND_INREG &&
6607 N0.getOperand(i: 1) == N1.getOperand(i: 1))) {
6608 // If both operands have other uses, this transform would create extra
6609 // instructions without eliminating anything.
6610 if (!N0.hasOneUse() && !N1.hasOneUse())
6611 return SDValue();
6612 // We need matching integer source types.
6613 if (XVT != Y.getValueType())
6614 return SDValue();
6615 // Don't create an illegal op during or after legalization. Don't ever
6616 // create an unsupported vector op.
6617 if ((VT.isVector() || LegalOperations) &&
6618 !TLI.isOperationLegalOrCustom(Op: LogicOpcode, VT: XVT))
6619 return SDValue();
6620 // Avoid infinite looping with PromoteIntBinOp.
6621 // TODO: Should we apply desirable/legal constraints to all opcodes?
6622 if ((HandOpcode == ISD::ANY_EXTEND ||
6623 HandOpcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
6624 LegalTypes && !TLI.isTypeDesirableForOp(LogicOpcode, VT: XVT))
6625 return SDValue();
6626 // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y)
6627 SDNodeFlags LogicFlags;
6628 LogicFlags.setDisjoint(N->getFlags().hasDisjoint() &&
6629 ISD::isExtOpcode(Opcode: HandOpcode));
6630 SDValue Logic = DAG.getNode(Opcode: LogicOpcode, DL, VT: XVT, N1: X, N2: Y, Flags: LogicFlags);
6631 if (HandOpcode == ISD::SIGN_EXTEND_INREG)
6632 return DAG.getNode(Opcode: HandOpcode, DL, VT, N1: Logic, N2: N0.getOperand(i: 1));
6633 return DAG.getNode(Opcode: HandOpcode, DL, VT, Operand: Logic);
6634 }
6635
6636 // logic_op (truncate x), (truncate y) --> truncate (logic_op x, y)
6637 if (HandOpcode == ISD::TRUNCATE) {
6638 // If both operands have other uses, this transform would create extra
6639 // instructions without eliminating anything.
6640 if (!N0.hasOneUse() && !N1.hasOneUse())
6641 return SDValue();
6642 // We need matching source types.
6643 if (XVT != Y.getValueType())
6644 return SDValue();
6645 // Don't create an illegal op during or after legalization.
6646 if (LegalOperations && !TLI.isOperationLegal(Op: LogicOpcode, VT: XVT))
6647 return SDValue();
6648 // Be extra careful sinking truncate. If it's free, there's no benefit in
6649 // widening a binop. Also, don't create a logic op on an illegal type.
6650 if (TLI.isZExtFree(FromTy: VT, ToTy: XVT) && TLI.isTruncateFree(FromVT: XVT, ToVT: VT))
6651 return SDValue();
6652 if (!TLI.isTypeLegal(VT: XVT))
6653 return SDValue();
6654 SDValue Logic = DAG.getNode(Opcode: LogicOpcode, DL, VT: XVT, N1: X, N2: Y);
6655 return DAG.getNode(Opcode: HandOpcode, DL, VT, Operand: Logic);
6656 }
6657
6658 // For binops SHL/SRL/SRA/AND:
6659 // logic_op (OP x, z), (OP y, z) --> OP (logic_op x, y), z
6660 if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL ||
6661 HandOpcode == ISD::SRA || HandOpcode == ISD::AND) &&
6662 N0.getOperand(i: 1) == N1.getOperand(i: 1)) {
6663 // If either operand has other uses, this transform is not an improvement.
6664 if (!N0.hasOneUse() || !N1.hasOneUse())
6665 return SDValue();
6666 SDValue Logic = DAG.getNode(Opcode: LogicOpcode, DL, VT: XVT, N1: X, N2: Y);
6667 return DAG.getNode(Opcode: HandOpcode, DL, VT, N1: Logic, N2: N0.getOperand(i: 1));
6668 }
6669
6670 // Unary ops: logic_op (bswap x), (bswap y) --> bswap (logic_op x, y)
6671 if (HandOpcode == ISD::BSWAP) {
6672 // If either operand has other uses, this transform is not an improvement.
6673 if (!N0.hasOneUse() || !N1.hasOneUse())
6674 return SDValue();
6675 SDValue Logic = DAG.getNode(Opcode: LogicOpcode, DL, VT: XVT, N1: X, N2: Y);
6676 return DAG.getNode(Opcode: HandOpcode, DL, VT, Operand: Logic);
6677 }
6678
6679 // For funnel shifts FSHL/FSHR:
6680 // logic_op (OP x, x1, s), (OP y, y1, s) -->
6681 // --> OP (logic_op x, y), (logic_op, x1, y1), s
6682 if ((HandOpcode == ISD::FSHL || HandOpcode == ISD::FSHR) &&
6683 N0.getOperand(i: 2) == N1.getOperand(i: 2)) {
6684 if (!N0.hasOneUse() || !N1.hasOneUse())
6685 return SDValue();
6686 SDValue X1 = N0.getOperand(i: 1);
6687 SDValue Y1 = N1.getOperand(i: 1);
6688 SDValue S = N0.getOperand(i: 2);
6689 SDValue Logic0 = DAG.getNode(Opcode: LogicOpcode, DL, VT, N1: X, N2: Y);
6690 SDValue Logic1 = DAG.getNode(Opcode: LogicOpcode, DL, VT, N1: X1, N2: Y1);
6691 return DAG.getNode(Opcode: HandOpcode, DL, VT, N1: Logic0, N2: Logic1, N3: S);
6692 }
6693
6694 // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
6695 // Only perform this optimization up until type legalization, before
6696 // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
6697 // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
6698 // we don't want to undo this promotion.
6699 // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
6700 // on scalars.
6701 if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) &&
6702 Level <= AfterLegalizeTypes) {
6703 // Input types must be integer and the same.
6704 if (XVT.isInteger() && XVT == Y.getValueType() &&
6705 !(VT.isVector() && TLI.isTypeLegal(VT) &&
6706 !XVT.isVector() && !TLI.isTypeLegal(VT: XVT))) {
6707 SDValue Logic = DAG.getNode(Opcode: LogicOpcode, DL, VT: XVT, N1: X, N2: Y);
6708 return DAG.getNode(Opcode: HandOpcode, DL, VT, Operand: Logic);
6709 }
6710 }
6711
6712 // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
6713 // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
6714 // If both shuffles use the same mask, and both shuffle within a single
6715 // vector, then it is worthwhile to move the swizzle after the operation.
6716 // The type-legalizer generates this pattern when loading illegal
6717 // vector types from memory. In many cases this allows additional shuffle
6718 // optimizations.
6719 // There are other cases where moving the shuffle after the xor/and/or
6720 // is profitable even if shuffles don't perform a swizzle.
6721 // If both shuffles use the same mask, and both shuffles have the same first
6722 // or second operand, then it might still be profitable to move the shuffle
6723 // after the xor/and/or operation.
6724 if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
6725 auto *SVN0 = cast<ShuffleVectorSDNode>(Val&: N0);
6726 auto *SVN1 = cast<ShuffleVectorSDNode>(Val&: N1);
6727 assert(X.getValueType() == Y.getValueType() &&
6728 "Inputs to shuffles are not the same type");
6729
6730 // Check that both shuffles use the same mask. The masks are known to be of
6731 // the same length because the result vector type is the same.
6732 // Check also that shuffles have only one use to avoid introducing extra
6733 // instructions.
6734 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
6735 !SVN0->getMask().equals(RHS: SVN1->getMask()))
6736 return SDValue();
6737
6738 // Don't try to fold this node if it requires introducing a
6739 // build vector of all zeros that might be illegal at this stage.
6740 SDValue ShOp = N0.getOperand(i: 1);
6741 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
6742 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
6743
6744 // (logic_op (shuf (A, C), shuf (B, C))) --> shuf (logic_op (A, B), C)
6745 if (N0.getOperand(i: 1) == N1.getOperand(i: 1) && ShOp.getNode()) {
6746 SDValue Logic = DAG.getNode(Opcode: LogicOpcode, DL, VT,
6747 N1: N0.getOperand(i: 0), N2: N1.getOperand(i: 0));
6748 return DAG.getVectorShuffle(VT, dl: DL, N1: Logic, N2: ShOp, Mask: SVN0->getMask());
6749 }
6750
6751 // Don't try to fold this node if it requires introducing a
6752 // build vector of all zeros that might be illegal at this stage.
6753 ShOp = N0.getOperand(i: 0);
6754 if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
6755 ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
6756
6757 // (logic_op (shuf (C, A), shuf (C, B))) --> shuf (C, logic_op (A, B))
6758 if (N0.getOperand(i: 0) == N1.getOperand(i: 0) && ShOp.getNode()) {
6759 SDValue Logic = DAG.getNode(Opcode: LogicOpcode, DL, VT, N1: N0.getOperand(i: 1),
6760 N2: N1.getOperand(i: 1));
6761 return DAG.getVectorShuffle(VT, dl: DL, N1: ShOp, N2: Logic, Mask: SVN0->getMask());
6762 }
6763 }
6764
6765 return SDValue();
6766}
6767
6768/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
6769SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
6770 const SDLoc &DL) {
6771 SDValue LL, LR, RL, RR, N0CC, N1CC;
6772 if (!isSetCCEquivalent(N: N0, LHS&: LL, RHS&: LR, CC&: N0CC) ||
6773 !isSetCCEquivalent(N: N1, LHS&: RL, RHS&: RR, CC&: N1CC))
6774 return SDValue();
6775
6776 assert(N0.getValueType() == N1.getValueType() &&
6777 "Unexpected operand types for bitwise logic op");
6778 assert(LL.getValueType() == LR.getValueType() &&
6779 RL.getValueType() == RR.getValueType() &&
6780 "Unexpected operand types for setcc");
6781
6782 // If we're here post-legalization or the logic op type is not i1, the logic
6783 // op type must match a setcc result type. Also, all folds require new
6784 // operations on the left and right operands, so those types must match.
6785 EVT VT = N0.getValueType();
6786 EVT OpVT = LL.getValueType();
6787 if (LegalOperations || VT.getScalarType() != MVT::i1)
6788 if (VT != getSetCCResultType(VT: OpVT))
6789 return SDValue();
6790 if (OpVT != RL.getValueType())
6791 return SDValue();
6792
6793 ISD::CondCode CC0 = cast<CondCodeSDNode>(Val&: N0CC)->get();
6794 ISD::CondCode CC1 = cast<CondCodeSDNode>(Val&: N1CC)->get();
6795 bool IsInteger = OpVT.isInteger();
6796 if (LR == RR && CC0 == CC1 && IsInteger) {
6797 bool IsZero = isNullOrNullSplat(V: LR);
6798 bool IsNeg1 = isAllOnesOrAllOnesSplat(V: LR);
6799
6800 // All bits clear?
6801 bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
6802 // All sign bits clear?
6803 bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
6804 // Any bits set?
6805 bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
6806 // Any sign bits set?
6807 bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
6808
6809 // (and (seteq X, 0), (seteq Y, 0)) --> (seteq (or X, Y), 0)
6810 // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
6811 // (or (setne X, 0), (setne Y, 0)) --> (setne (or X, Y), 0)
6812 // (or (setlt X, 0), (setlt Y, 0)) --> (setlt (or X, Y), 0)
6813 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
6814 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: SDLoc(N0), VT: OpVT, N1: LL, N2: RL);
6815 AddToWorklist(N: Or.getNode());
6816 return DAG.getSetCC(DL, VT, LHS: Or, RHS: LR, Cond: CC1);
6817 }
6818
6819 // All bits set?
6820 bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
6821 // All sign bits set?
6822 bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
6823 // Any bits clear?
6824 bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
6825 // Any sign bits clear?
6826 bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
6827
6828 // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
6829 // (and (setlt X, 0), (setlt Y, 0)) --> (setlt (and X, Y), 0)
6830 // (or (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
6831 // (or (setgt X, -1), (setgt Y -1)) --> (setgt (and X, Y), -1)
6832 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
6833 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: LL, N2: RL);
6834 AddToWorklist(N: And.getNode());
6835 return DAG.getSetCC(DL, VT, LHS: And, RHS: LR, Cond: CC1);
6836 }
6837 }
6838
6839 // (and (setne (and X, LL1), 0), (setne (and X, RL1), 0))
6840 // --> (seteq (and X, (LL1|RL1)), (LL1|RL1))
6841 // (or (seteq (and X, LL1), 0), (seteq (and X, RL1), 0))
6842 // --> (setne (and X, (LL1|RL1)), (LL1|RL1))
6843 if (LL.getOpcode() == ISD::AND && RL.getOpcode() == ISD::AND &&
6844 isNullConstant(V: LR) && isNullConstant(V: RR) && CC0 == CC1 &&
6845 (CC0 == ISD::SETNE || CC0 == ISD::SETEQ)) {
6846 SDValue LL0, LL1, RL0, RL1;
6847 LL0 = LL.getOperand(i: 0);
6848 RL0 = RL.getOperand(i: 0);
6849 LL1 = LL.getOperand(i: 1);
6850 RL1 = RL.getOperand(i: 1);
6851 if (LL0 == RL0 && DAG.isKnownToBeAPowerOfTwo(Val: LL1) &&
6852 DAG.isKnownToBeAPowerOfTwo(Val: RL1)) {
6853 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL: SDLoc(N0), VT: OpVT, N1: LL1, N2: RL1);
6854 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: OpVT, N1: LL0, N2: Or);
6855 return DAG.getSetCC(DL, VT, LHS: And, RHS: Or, Cond: IsAnd ? ISD::SETEQ : ISD::SETNE);
6856 }
6857 }
6858
6859 // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
6860 // (or (seteq X, 0), (seteq X, -1)) --> (setult (add X, 1), 2)
6861 if (LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 && IsInteger &&
6862 ((IsAnd && CC0 == ISD::SETNE) || (!IsAnd && CC0 == ISD::SETEQ)) &&
6863 ((isNullConstant(V: LR) && isAllOnesConstant(V: RR)) ||
6864 (isAllOnesConstant(V: LR) && isNullConstant(V: RR)))) {
6865 SDValue One = DAG.getConstant(Val: 1, DL, VT: OpVT);
6866 SDValue Two = DAG.getConstant(Val: 2, DL, VT: OpVT);
6867 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N0), VT: OpVT, N1: LL, N2: One);
6868 AddToWorklist(N: Add.getNode());
6869 return DAG.getSetCC(DL, VT, LHS: Add, RHS: Two, Cond: IsAnd ? ISD::SETUGE : ISD::SETULT);
6870 }
6871
6872 // Try more general transforms if the predicates match and the only user of
6873 // the compares is the 'and' or 'or'.
6874 if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(VT: OpVT) && CC0 == CC1 &&
6875 N0.hasOneUse() && N1.hasOneUse()) {
6876 // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
6877 // or (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
6878 if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
6879 SDValue XorL = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT: OpVT, N1: LL, N2: LR);
6880 SDValue XorR = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N1), VT: OpVT, N1: RL, N2: RR);
6881 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT: OpVT, N1: XorL, N2: XorR);
6882 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: OpVT);
6883 return DAG.getSetCC(DL, VT, LHS: Or, RHS: Zero, Cond: CC1);
6884 }
6885
6886 // Turn compare of constants whose difference is 1 bit into add+and+setcc.
6887 if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) {
6888 // Match a shared variable operand and 2 non-opaque constant operands.
6889 auto MatchDiffPow2 = [&](ConstantSDNode *C0, ConstantSDNode *C1) {
6890 // The difference of the constants must be a single bit.
6891 const APInt &CMax =
6892 APIntOps::umax(A: C0->getAPIntValue(), B: C1->getAPIntValue());
6893 const APInt &CMin =
6894 APIntOps::umin(A: C0->getAPIntValue(), B: C1->getAPIntValue());
6895 return !C0->isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2();
6896 };
6897 if (LL == RL && ISD::matchBinaryPredicate(LHS: LR, RHS: RR, Match: MatchDiffPow2)) {
6898 // and/or (setcc X, CMax, ne), (setcc X, CMin, ne/eq) -->
6899 // setcc ((sub X, CMin), ~(CMax - CMin)), 0, ne/eq
6900 SDValue Max = DAG.getNode(Opcode: ISD::UMAX, DL, VT: OpVT, N1: LR, N2: RR);
6901 SDValue Min = DAG.getNode(Opcode: ISD::UMIN, DL, VT: OpVT, N1: LR, N2: RR);
6902 SDValue Offset = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: LL, N2: Min);
6903 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: Max, N2: Min);
6904 SDValue Mask = DAG.getNOT(DL, Val: Diff, VT: OpVT);
6905 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: Offset, N2: Mask);
6906 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: OpVT);
6907 return DAG.getSetCC(DL, VT, LHS: And, RHS: Zero, Cond: CC0);
6908 }
6909 }
6910 }
6911
6912 // Canonicalize equivalent operands to LL == RL.
6913 if (LL == RR && LR == RL) {
6914 CC1 = ISD::getSetCCSwappedOperands(Operation: CC1);
6915 std::swap(a&: RL, b&: RR);
6916 }
6917
6918 // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
6919 // (or (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
6920 if (LL == RL && LR == RR) {
6921 ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(Op1: CC0, Op2: CC1, Type: OpVT)
6922 : ISD::getSetCCOrOperation(Op1: CC0, Op2: CC1, Type: OpVT);
6923 if (NewCC != ISD::SETCC_INVALID &&
6924 (!LegalOperations ||
6925 (TLI.isCondCodeLegal(CC: NewCC, VT: LL.getSimpleValueType()) &&
6926 TLI.isOperationLegal(Op: ISD::SETCC, VT: OpVT))))
6927 return DAG.getSetCC(DL, VT, LHS: LL, RHS: LR, Cond: NewCC);
6928 }
6929
6930 return SDValue();
6931}
6932
6933static bool arebothOperandsNotSNan(SDValue Operand1, SDValue Operand2,
6934 SelectionDAG &DAG) {
6935 return DAG.isKnownNeverSNaN(Op: Operand2) && DAG.isKnownNeverSNaN(Op: Operand1);
6936}
6937
6938static bool arebothOperandsNotNan(SDValue Operand1, SDValue Operand2,
6939 SelectionDAG &DAG) {
6940 return DAG.isKnownNeverNaN(Op: Operand2) && DAG.isKnownNeverNaN(Op: Operand1);
6941}
6942
6943/// Returns an appropriate FP min/max opcode for clamping operations.
6944static unsigned getMinMaxOpcodeForClamp(bool IsMin, SDValue Operand1,
6945 SDValue Operand2, SelectionDAG &DAG,
6946 const TargetLowering &TLI) {
6947 EVT VT = Operand1.getValueType();
6948 unsigned IEEEOp = IsMin ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
6949 if (TLI.isOperationLegalOrCustom(Op: IEEEOp, VT) &&
6950 arebothOperandsNotNan(Operand1, Operand2, DAG))
6951 return IEEEOp;
6952 unsigned PreferredOp = IsMin ? ISD::FMINNUM : ISD::FMAXNUM;
6953 if (TLI.isOperationLegalOrCustom(Op: PreferredOp, VT))
6954 return PreferredOp;
6955 return ISD::DELETED_NODE;
6956}
6957
6958// FIXME: use FMINIMUMNUM if possible, such as for RISC-V.
6959static unsigned getMinMaxOpcodeForCompareFold(
6960 SDValue Operand1, SDValue Operand2, bool SetCCNoNaNs, ISD::CondCode CC,
6961 unsigned OrAndOpcode, SelectionDAG &DAG, bool isFMAXNUMFMINNUM_IEEE,
6962 bool isFMAXNUMFMINNUM) {
6963 // The optimization cannot be applied for all the predicates because
6964 // of the way FMINNUM/FMAXNUM and FMINNUM_IEEE/FMAXNUM_IEEE handle
6965 // NaNs. For FMINNUM_IEEE/FMAXNUM_IEEE, the optimization cannot be
6966 // applied at all if one of the operands is a signaling NaN.
6967
6968 // It is safe to use FMINNUM_IEEE/FMAXNUM_IEEE if all the operands
6969 // are non NaN values.
6970 if (((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::OR)) ||
6971 ((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::AND))) {
6972 return (SetCCNoNaNs || arebothOperandsNotNan(Operand1, Operand2, DAG)) &&
6973 isFMAXNUMFMINNUM_IEEE
6974 ? ISD::FMINNUM_IEEE
6975 : ISD::DELETED_NODE;
6976 }
6977
6978 if (((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::OR)) ||
6979 ((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::AND))) {
6980 return (SetCCNoNaNs || arebothOperandsNotNan(Operand1, Operand2, DAG)) &&
6981 isFMAXNUMFMINNUM_IEEE
6982 ? ISD::FMAXNUM_IEEE
6983 : ISD::DELETED_NODE;
6984 }
6985
6986 // Both FMINNUM/FMAXNUM and FMINNUM_IEEE/FMAXNUM_IEEE handle quiet
6987 // NaNs in the same way. But, FMINNUM/FMAXNUM and FMINNUM_IEEE/
6988 // FMAXNUM_IEEE handle signaling NaNs differently. If we cannot prove
6989 // that there are not any sNaNs, then the optimization is not valid
6990 // for FMINNUM_IEEE/FMAXNUM_IEEE. In the presence of sNaNs, we apply
6991 // the optimization using FMINNUM/FMAXNUM for the following cases. If
6992 // we can prove that we do not have any sNaNs, then we can do the
6993 // optimization using FMINNUM_IEEE/FMAXNUM_IEEE for the following
6994 // cases.
6995 if (((CC == ISD::SETOLT || CC == ISD::SETOLE) && (OrAndOpcode == ISD::OR)) ||
6996 ((CC == ISD::SETUGT || CC == ISD::SETUGE) && (OrAndOpcode == ISD::AND))) {
6997 return isFMAXNUMFMINNUM ? ISD::FMINNUM
6998 : arebothOperandsNotSNan(Operand1, Operand2, DAG) &&
6999 isFMAXNUMFMINNUM_IEEE
7000 ? ISD::FMINNUM_IEEE
7001 : ISD::DELETED_NODE;
7002 }
7003
7004 if (((CC == ISD::SETOGT || CC == ISD::SETOGE) && (OrAndOpcode == ISD::OR)) ||
7005 ((CC == ISD::SETULT || CC == ISD::SETULE) && (OrAndOpcode == ISD::AND))) {
7006 return isFMAXNUMFMINNUM ? ISD::FMAXNUM
7007 : arebothOperandsNotSNan(Operand1, Operand2, DAG) &&
7008 isFMAXNUMFMINNUM_IEEE
7009 ? ISD::FMAXNUM_IEEE
7010 : ISD::DELETED_NODE;
7011 }
7012
7013 return ISD::DELETED_NODE;
7014}
7015
7016static SDValue foldAndOrOfSETCC(SDNode *LogicOp, SelectionDAG &DAG) {
7017 using AndOrSETCCFoldKind = TargetLowering::AndOrSETCCFoldKind;
7018 assert(
7019 (LogicOp->getOpcode() == ISD::AND || LogicOp->getOpcode() == ISD::OR) &&
7020 "Invalid Op to combine SETCC with");
7021
7022 // TODO: Search past casts/truncates.
7023 SDValue LHS = LogicOp->getOperand(Num: 0);
7024 SDValue RHS = LogicOp->getOperand(Num: 1);
7025 if (LHS->getOpcode() != ISD::SETCC || RHS->getOpcode() != ISD::SETCC ||
7026 !LHS->hasOneUse() || !RHS->hasOneUse())
7027 return SDValue();
7028
7029 SDNodeFlags LHSSetCCFlags = LHS->getFlags();
7030 SDNodeFlags RHSSetCCFlags = RHS->getFlags();
7031 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7032 AndOrSETCCFoldKind TargetPreference = TLI.isDesirableToCombineLogicOpOfSETCC(
7033 LogicOp, SETCC0: LHS.getNode(), SETCC1: RHS.getNode());
7034
7035 SDValue LHS0 = LHS->getOperand(Num: 0);
7036 SDValue RHS0 = RHS->getOperand(Num: 0);
7037 SDValue LHS1 = LHS->getOperand(Num: 1);
7038 SDValue RHS1 = RHS->getOperand(Num: 1);
7039 // TODO: We don't actually need a splat here, for vectors we just need the
7040 // invariants to hold for each element.
7041 auto *LHS1C = isConstOrConstSplat(N: LHS1);
7042 auto *RHS1C = isConstOrConstSplat(N: RHS1);
7043 ISD::CondCode CCL = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
7044 ISD::CondCode CCR = cast<CondCodeSDNode>(Val: RHS.getOperand(i: 2))->get();
7045 EVT VT = LogicOp->getValueType(ResNo: 0);
7046 EVT OpVT = LHS0.getValueType();
7047 SDLoc DL(LogicOp);
7048
7049 // Check if the operands of an and/or operation are comparisons and if they
7050 // compare against the same value. Replace the and/or-cmp-cmp sequence with
7051 // min/max cmp sequence. If LHS1 is equal to RHS1, then the or-cmp-cmp
7052 // sequence will be replaced with min-cmp sequence:
7053 // (LHS0 < LHS1) | (RHS0 < RHS1) -> min(LHS0, RHS0) < LHS1
7054 // and and-cmp-cmp will be replaced with max-cmp sequence:
7055 // (LHS0 < LHS1) & (RHS0 < RHS1) -> max(LHS0, RHS0) < LHS1
7056 // The optimization does not work for `==` or `!=` .
7057 // The two comparisons should have either the same predicate or the
7058 // predicate of one of the comparisons is the opposite of the other one.
7059 bool isFMAXNUMFMINNUM_IEEE = TLI.isOperationLegal(Op: ISD::FMAXNUM_IEEE, VT: OpVT) &&
7060 TLI.isOperationLegal(Op: ISD::FMINNUM_IEEE, VT: OpVT);
7061 bool isFMAXNUMFMINNUM = TLI.isOperationLegalOrCustom(Op: ISD::FMAXNUM, VT: OpVT) &&
7062 TLI.isOperationLegalOrCustom(Op: ISD::FMINNUM, VT: OpVT);
7063 if (((OpVT.isInteger() && TLI.isOperationLegal(Op: ISD::UMAX, VT: OpVT) &&
7064 TLI.isOperationLegal(Op: ISD::SMAX, VT: OpVT) &&
7065 TLI.isOperationLegal(Op: ISD::UMIN, VT: OpVT) &&
7066 TLI.isOperationLegal(Op: ISD::SMIN, VT: OpVT)) ||
7067 (OpVT.isFloatingPoint() &&
7068 (isFMAXNUMFMINNUM_IEEE || isFMAXNUMFMINNUM))) &&
7069 !ISD::isIntEqualitySetCC(Code: CCL) && !ISD::isFPEqualitySetCC(Code: CCL) &&
7070 CCL != ISD::SETFALSE && CCL != ISD::SETO && CCL != ISD::SETUO &&
7071 CCL != ISD::SETTRUE &&
7072 (CCL == CCR || CCL == ISD::getSetCCSwappedOperands(Operation: CCR))) {
7073
7074 SDValue CommonValue, Operand1, Operand2;
7075 ISD::CondCode CC = ISD::SETCC_INVALID;
7076 if (CCL == CCR) {
7077 if (LHS0 == RHS0) {
7078 CommonValue = LHS0;
7079 Operand1 = LHS1;
7080 Operand2 = RHS1;
7081 CC = ISD::getSetCCSwappedOperands(Operation: CCL);
7082 } else if (LHS1 == RHS1) {
7083 CommonValue = LHS1;
7084 Operand1 = LHS0;
7085 Operand2 = RHS0;
7086 CC = CCL;
7087 }
7088 } else {
7089 assert(CCL == ISD::getSetCCSwappedOperands(CCR) && "Unexpected CC");
7090 if (LHS0 == RHS1) {
7091 CommonValue = LHS0;
7092 Operand1 = LHS1;
7093 Operand2 = RHS0;
7094 CC = CCR;
7095 } else if (RHS0 == LHS1) {
7096 CommonValue = LHS1;
7097 Operand1 = LHS0;
7098 Operand2 = RHS1;
7099 CC = CCL;
7100 }
7101 }
7102
7103 // Don't do this transform for sign bit tests. Let foldLogicOfSetCCs
7104 // handle it using OR/AND.
7105 if (CC == ISD::SETLT && isNullOrNullSplat(V: CommonValue))
7106 CC = ISD::SETCC_INVALID;
7107 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(V: CommonValue))
7108 CC = ISD::SETCC_INVALID;
7109
7110 if (CC != ISD::SETCC_INVALID) {
7111 unsigned NewOpcode = ISD::DELETED_NODE;
7112 bool IsSigned = isSignedIntSetCC(Code: CC);
7113 if (OpVT.isInteger()) {
7114 bool IsLess = (CC == ISD::SETLE || CC == ISD::SETULE ||
7115 CC == ISD::SETLT || CC == ISD::SETULT);
7116 bool IsOr = (LogicOp->getOpcode() == ISD::OR);
7117 if (IsLess == IsOr)
7118 NewOpcode = IsSigned ? ISD::SMIN : ISD::UMIN;
7119 else
7120 NewOpcode = IsSigned ? ISD::SMAX : ISD::UMAX;
7121 } else if (OpVT.isFloatingPoint())
7122 NewOpcode = getMinMaxOpcodeForCompareFold(
7123 Operand1, Operand2,
7124 SetCCNoNaNs: LHSSetCCFlags.hasNoNaNs() && RHSSetCCFlags.hasNoNaNs(), CC,
7125 OrAndOpcode: LogicOp->getOpcode(), DAG, isFMAXNUMFMINNUM_IEEE, isFMAXNUMFMINNUM);
7126
7127 if (NewOpcode != ISD::DELETED_NODE) {
7128 // Propagate fast-math flags from setcc.
7129 SDNodeFlags Flags = LHS->getFlags() & RHS->getFlags();
7130 SDValue MinMaxValue =
7131 DAG.getNode(Opcode: NewOpcode, DL, VT: OpVT, N1: Operand1, N2: Operand2, Flags);
7132 return DAG.getSetCC(DL, VT, LHS: MinMaxValue, RHS: CommonValue, Cond: CC, /*Chain=*/{},
7133 /*IsSignaling=*/false, Flags);
7134 }
7135 }
7136 }
7137
7138 if (LHS0 == LHS1 && RHS0 == RHS1 && CCL == CCR &&
7139 LHS0.getValueType() == RHS0.getValueType() &&
7140 ((LogicOp->getOpcode() == ISD::AND && CCL == ISD::SETO) ||
7141 (LogicOp->getOpcode() == ISD::OR && CCL == ISD::SETUO)))
7142 return DAG.getSetCC(DL, VT, LHS: LHS0, RHS: RHS0, Cond: CCL);
7143
7144 if (TargetPreference == AndOrSETCCFoldKind::None)
7145 return SDValue();
7146
7147 if (CCL == CCR &&
7148 CCL == (LogicOp->getOpcode() == ISD::AND ? ISD::SETNE : ISD::SETEQ) &&
7149 LHS0 == RHS0 && LHS1C && RHS1C && OpVT.isInteger()) {
7150 const APInt &APLhs = LHS1C->getAPIntValue();
7151 const APInt &APRhs = RHS1C->getAPIntValue();
7152
7153 // Preference is to use ISD::ABS or we already have an ISD::ABS (in which
7154 // case this is just a compare).
7155 if (APLhs == (-APRhs) &&
7156 ((TargetPreference & AndOrSETCCFoldKind::ABS) ||
7157 DAG.doesNodeExist(Opcode: ISD::ABS, VTList: DAG.getVTList(VT: OpVT), Ops: {LHS0}))) {
7158 const APInt &C = APLhs.isNegative() ? APRhs : APLhs;
7159 // (icmp eq A, C) | (icmp eq A, -C)
7160 // -> (icmp eq Abs(A), C)
7161 // (icmp ne A, C) & (icmp ne A, -C)
7162 // -> (icmp ne Abs(A), C)
7163 SDValue AbsOp = DAG.getNode(Opcode: ISD::ABS, DL, VT: OpVT, Operand: LHS0);
7164 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: AbsOp,
7165 N2: DAG.getConstant(Val: C, DL, VT: OpVT), N3: LHS.getOperand(i: 2));
7166 } else if (TargetPreference &
7167 (AndOrSETCCFoldKind::AddAnd | AndOrSETCCFoldKind::NotAnd)) {
7168
7169 // AndOrSETCCFoldKind::AddAnd:
7170 // A == C0 | A == C1
7171 // IF IsPow2(smax(C0, C1)-smin(C0, C1))
7172 // -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) == 0
7173 // A != C0 & A != C1
7174 // IF IsPow2(smax(C0, C1)-smin(C0, C1))
7175 // -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) != 0
7176
7177 // AndOrSETCCFoldKind::NotAnd:
7178 // A == C0 | A == C1
7179 // IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1))
7180 // -> ~A & smin(C0, C1) == 0
7181 // A != C0 & A != C1
7182 // IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1))
7183 // -> ~A & smin(C0, C1) != 0
7184
7185 const APInt &MaxC = APIntOps::smax(A: APRhs, B: APLhs);
7186 const APInt &MinC = APIntOps::smin(A: APRhs, B: APLhs);
7187 APInt Dif = MaxC - MinC;
7188 if (!Dif.isZero() && Dif.isPowerOf2()) {
7189 if (MaxC.isAllOnes() &&
7190 (TargetPreference & AndOrSETCCFoldKind::NotAnd)) {
7191 SDValue NotOp = DAG.getNOT(DL, Val: LHS0, VT: OpVT);
7192 SDValue AndOp = DAG.getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: NotOp,
7193 N2: DAG.getConstant(Val: MinC, DL, VT: OpVT));
7194 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: AndOp,
7195 N2: DAG.getConstant(Val: 0, DL, VT: OpVT), N3: LHS.getOperand(i: 2));
7196 } else if (TargetPreference & AndOrSETCCFoldKind::AddAnd) {
7197
7198 SDValue AddOp = DAG.getNode(Opcode: ISD::ADD, DL, VT: OpVT, N1: LHS0,
7199 N2: DAG.getConstant(Val: -MinC, DL, VT: OpVT));
7200 SDValue AndOp = DAG.getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: AddOp,
7201 N2: DAG.getConstant(Val: ~Dif, DL, VT: OpVT));
7202 return DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: AndOp,
7203 N2: DAG.getConstant(Val: 0, DL, VT: OpVT), N3: LHS.getOperand(i: 2));
7204 }
7205 }
7206 }
7207 }
7208
7209 return SDValue();
7210}
7211
7212// Combine `(select c, (X & 1), 0)` -> `(and (zext c), X)`.
7213// We canonicalize to the `select` form in the middle end, but the `and` form
7214// gets better codegen and all tested targets (arm, x86, riscv)
7215static SDValue combineSelectAsExtAnd(SDValue Cond, SDValue T, SDValue F,
7216 const SDLoc &DL, SelectionDAG &DAG) {
7217 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7218 if (!isNullConstant(V: F))
7219 return SDValue();
7220
7221 EVT CondVT = Cond.getValueType();
7222 if (TLI.getBooleanContents(Type: CondVT) !=
7223 TargetLoweringBase::ZeroOrOneBooleanContent)
7224 return SDValue();
7225
7226 if (T.getOpcode() != ISD::AND)
7227 return SDValue();
7228
7229 if (!isOneConstant(V: T.getOperand(i: 1)))
7230 return SDValue();
7231
7232 EVT OpVT = T.getValueType();
7233
7234 SDValue CondMask =
7235 OpVT == CondVT ? Cond : DAG.getBoolExtOrTrunc(Op: Cond, SL: DL, VT: OpVT, OpVT: CondVT);
7236 return DAG.getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: CondMask, N2: T.getOperand(i: 0));
7237}
7238
7239/// This contains all DAGCombine rules which reduce two values combined by
7240/// an And operation to a single value. This makes them reusable in the context
7241/// of visitSELECT(). Rules involving constants are not included as
7242/// visitSELECT() already handles those cases.
7243SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
7244 EVT VT = N1.getValueType();
7245 SDLoc DL(N);
7246
7247 // fold (and x, undef) -> 0
7248 if (N0.isUndef() || N1.isUndef())
7249 return DAG.getConstant(Val: 0, DL, VT);
7250
7251 if (SDValue V = foldLogicOfSetCCs(IsAnd: true, N0, N1, DL))
7252 return V;
7253
7254 // Canonicalize:
7255 // and(x, add) -> and(add, x)
7256 if (N1.getOpcode() == ISD::ADD)
7257 std::swap(a&: N0, b&: N1);
7258
7259 // TODO: Rewrite this to return a new 'AND' instead of using CombineTo.
7260 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
7261 VT.isScalarInteger() && VT.getSizeInBits() <= 64 && N0->hasOneUse()) {
7262 if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
7263 if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1))) {
7264 // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
7265 // immediate for an add, but it is legal if its top c2 bits are set,
7266 // transform the ADD so the immediate doesn't need to be materialized
7267 // in a register.
7268 APInt ADDC = ADDI->getAPIntValue();
7269 APInt SRLC = SRLI->getAPIntValue();
7270 if (ADDC.getSignificantBits() <= 64 && SRLC.ult(RHS: VT.getSizeInBits()) &&
7271 !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
7272 APInt Mask = APInt::getHighBitsSet(numBits: VT.getSizeInBits(),
7273 hiBitsSet: SRLC.getZExtValue());
7274 if (DAG.MaskedValueIsZero(Op: N0.getOperand(i: 1), Mask)) {
7275 ADDC |= Mask;
7276 if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
7277 SDLoc DL0(N0);
7278 SDValue NewAdd =
7279 DAG.getNode(Opcode: ISD::ADD, DL: DL0, VT,
7280 N1: N0.getOperand(i: 0), N2: DAG.getConstant(Val: ADDC, DL, VT));
7281 CombineTo(N: N0.getNode(), Res: NewAdd);
7282 // Return N so it doesn't get rechecked!
7283 return SDValue(N, 0);
7284 }
7285 }
7286 }
7287 }
7288 }
7289 }
7290
7291 return SDValue();
7292}
7293
7294bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
7295 EVT LoadResultTy, EVT &ExtVT) {
7296 if (!AndC->getAPIntValue().isMask())
7297 return false;
7298
7299 unsigned ActiveBits = AndC->getAPIntValue().countr_one();
7300
7301 ExtVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ActiveBits);
7302 EVT LoadedVT = LoadN->getMemoryVT();
7303
7304 if (ExtVT == LoadedVT &&
7305 (!LegalOperations ||
7306 TLI.isLoadLegal(ValVT: LoadResultTy, MemVT: ExtVT, Alignment: LoadN->getAlign(),
7307 AddrSpace: LoadN->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false))) {
7308 // ZEXTLOAD will match without needing to change the size of the value being
7309 // loaded.
7310 return true;
7311 }
7312
7313 // Do not change the width of a volatile or atomic loads.
7314 if (!LoadN->isSimple())
7315 return false;
7316
7317 // Do not generate loads of non-round integer types since these can
7318 // be expensive (and would be wrong if the type is not byte sized).
7319 if (!LoadedVT.bitsGT(VT: ExtVT) || !ExtVT.isRound())
7320 return false;
7321
7322 if (LegalOperations &&
7323 !TLI.isLoadLegal(ValVT: LoadResultTy, MemVT: ExtVT, Alignment: LoadN->getAlign(),
7324 AddrSpace: LoadN->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false))
7325 return false;
7326
7327 if (!TLI.shouldReduceLoadWidth(Load: LoadN, ExtTy: ISD::ZEXTLOAD, NewVT: ExtVT, /*ByteOffset=*/0))
7328 return false;
7329
7330 return true;
7331}
7332
7333bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
7334 ISD::LoadExtType ExtType, EVT &MemVT,
7335 unsigned ShAmt) {
7336 if (!LDST)
7337 return false;
7338
7339 // Only allow byte offsets.
7340 if (ShAmt % 8)
7341 return false;
7342 const unsigned ByteShAmt = ShAmt / 8;
7343
7344 // Do not generate loads of non-round integer types since these can
7345 // be expensive (and would be wrong if the type is not byte sized).
7346 if (!MemVT.isRound())
7347 return false;
7348
7349 // Don't change the width of a volatile or atomic loads.
7350 if (!LDST->isSimple())
7351 return false;
7352
7353 EVT LdStMemVT = LDST->getMemoryVT();
7354
7355 // Bail out when changing the scalable property, since we can't be sure that
7356 // we're actually narrowing here.
7357 if (LdStMemVT.isScalableVector() != MemVT.isScalableVector())
7358 return false;
7359
7360 // Verify that we are actually reducing a load width here.
7361 if (LdStMemVT.bitsLT(VT: MemVT))
7362 return false;
7363
7364 // Ensure that this isn't going to produce an unsupported memory access.
7365 if (ShAmt) {
7366 const Align LDSTAlign = LDST->getAlign();
7367 const Align NarrowAlign = commonAlignment(A: LDSTAlign, Offset: ByteShAmt);
7368 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: MemVT,
7369 AddrSpace: LDST->getAddressSpace(), Alignment: NarrowAlign,
7370 Flags: LDST->getMemOperand()->getFlags()))
7371 return false;
7372 }
7373
7374 // It's not possible to generate a constant of extended or untyped type.
7375 EVT PtrType = LDST->getBasePtr().getValueType();
7376 if (PtrType == MVT::Untyped || PtrType.isExtended())
7377 return false;
7378
7379 if (isa<LoadSDNode>(Val: LDST)) {
7380 LoadSDNode *Load = cast<LoadSDNode>(Val: LDST);
7381 // Don't transform one with multiple uses, this would require adding a new
7382 // load.
7383 if (!SDValue(Load, 0).hasOneUse())
7384 return false;
7385
7386 if (LegalOperations &&
7387 !TLI.isLoadLegal(ValVT: Load->getValueType(ResNo: 0), MemVT, Alignment: Load->getAlign(),
7388 AddrSpace: Load->getAddressSpace(), ExtType, Atomic: false))
7389 return false;
7390
7391 // For the transform to be legal, the load must produce only two values
7392 // (the value loaded and the chain). Don't transform a pre-increment
7393 // load, for example, which produces an extra value. Otherwise the
7394 // transformation is not equivalent, and the downstream logic to replace
7395 // uses gets things wrong.
7396 if (Load->getNumValues() > 2)
7397 return false;
7398
7399 // If the load that we're shrinking is an extload and we're not just
7400 // discarding the extension we can't simply shrink the load. Bail.
7401 // TODO: It would be possible to merge the extensions in some cases.
7402 if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
7403 Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
7404 return false;
7405
7406 if (!TLI.shouldReduceLoadWidth(Load, ExtTy: ExtType, NewVT: MemVT, ByteOffset: ByteShAmt))
7407 return false;
7408 } else {
7409 assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode");
7410 StoreSDNode *Store = cast<StoreSDNode>(Val: LDST);
7411 // Can't write outside the original store
7412 if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
7413 return false;
7414
7415 if (LegalOperations &&
7416 !TLI.isTruncStoreLegal(ValVT: Store->getValue().getValueType(), MemVT,
7417 Alignment: Store->getAlign(), AddrSpace: Store->getAddressSpace()))
7418 return false;
7419 }
7420 return true;
7421}
7422
7423bool DAGCombiner::SearchForAndLoads(SDNode *N,
7424 SmallVectorImpl<LoadSDNode*> &Loads,
7425 SmallPtrSetImpl<SDNode*> &NodesWithConsts,
7426 ConstantSDNode *Mask,
7427 SDNode *&NodeToMask) {
7428 // Recursively search for the operands, looking for loads which can be
7429 // narrowed.
7430 for (SDValue Op : N->op_values()) {
7431 if (Op.getValueType().isVector())
7432 return false;
7433
7434 // Some constants may need fixing up later if they are too large.
7435 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
7436 assert(ISD::isBitwiseLogicOp(N->getOpcode()) &&
7437 "Expected bitwise logic operation");
7438 if (!C->getAPIntValue().isSubsetOf(RHS: Mask->getAPIntValue()))
7439 NodesWithConsts.insert(Ptr: N);
7440 continue;
7441 }
7442
7443 if (!Op.hasOneUse())
7444 return false;
7445
7446 switch(Op.getOpcode()) {
7447 case ISD::LOAD: {
7448 auto *Load = cast<LoadSDNode>(Val&: Op);
7449 EVT ExtVT;
7450 if (isAndLoadExtLoad(AndC: Mask, LoadN: Load, LoadResultTy: Load->getValueType(ResNo: 0), ExtVT) &&
7451 isLegalNarrowLdSt(LDST: Load, ExtType: ISD::ZEXTLOAD, MemVT&: ExtVT)) {
7452
7453 // ZEXTLOAD is already small enough.
7454 if (Load->getExtensionType() == ISD::ZEXTLOAD &&
7455 ExtVT.bitsGE(VT: Load->getMemoryVT()))
7456 continue;
7457
7458 // Use LE to convert equal sized loads to zext.
7459 if (ExtVT.bitsLE(VT: Load->getMemoryVT()))
7460 Loads.push_back(Elt: Load);
7461
7462 continue;
7463 }
7464 return false;
7465 }
7466 case ISD::ZERO_EXTEND:
7467 case ISD::AssertZext: {
7468 unsigned ActiveBits = Mask->getAPIntValue().countr_one();
7469 EVT ExtVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ActiveBits);
7470 EVT VT = Op.getOpcode() == ISD::AssertZext ?
7471 cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT() :
7472 Op.getOperand(i: 0).getValueType();
7473
7474 // We can accept extending nodes if the mask is wider or an equal
7475 // width to the original type.
7476 if (ExtVT.bitsGE(VT))
7477 continue;
7478 break;
7479 }
7480 case ISD::OR:
7481 case ISD::XOR:
7482 case ISD::AND:
7483 if (!SearchForAndLoads(N: Op.getNode(), Loads, NodesWithConsts, Mask,
7484 NodeToMask))
7485 return false;
7486 continue;
7487 }
7488
7489 // Allow one node which will masked along with any loads found.
7490 if (NodeToMask)
7491 return false;
7492
7493 // Also ensure that the node to be masked only produces one data result.
7494 NodeToMask = Op.getNode();
7495 if (NodeToMask->getNumValues() > 1) {
7496 bool HasValue = false;
7497 for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
7498 MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
7499 if (VT != MVT::Glue && VT != MVT::Other) {
7500 if (HasValue) {
7501 NodeToMask = nullptr;
7502 return false;
7503 }
7504 HasValue = true;
7505 }
7506 }
7507 assert(HasValue && "Node to be masked has no data result?");
7508 }
7509 }
7510 return true;
7511}
7512
7513bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
7514 auto *Mask = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
7515 if (!Mask)
7516 return false;
7517
7518 if (!Mask->getAPIntValue().isMask())
7519 return false;
7520
7521 // No need to do anything if the and directly uses a load.
7522 if (isa<LoadSDNode>(Val: N->getOperand(Num: 0)))
7523 return false;
7524
7525 SmallVector<LoadSDNode*, 8> Loads;
7526 SmallPtrSet<SDNode*, 2> NodesWithConsts;
7527 SDNode *FixupNode = nullptr;
7528 if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, NodeToMask&: FixupNode)) {
7529 if (Loads.empty())
7530 return false;
7531
7532 LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump());
7533 SDValue MaskOp = N->getOperand(Num: 1);
7534
7535 // If it exists, fixup the single node we allow in the tree that needs
7536 // masking.
7537 if (FixupNode) {
7538 LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump());
7539 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(FixupNode),
7540 VT: FixupNode->getValueType(ResNo: 0),
7541 N1: SDValue(FixupNode, 0), N2: MaskOp);
7542 DAG.ReplaceAllUsesOfValueWith(From: SDValue(FixupNode, 0), To: And);
7543 if (And.getOpcode() == ISD ::AND)
7544 DAG.UpdateNodeOperands(N: And.getNode(), Op1: SDValue(FixupNode, 0), Op2: MaskOp);
7545 }
7546
7547 // Narrow any constants that need it.
7548 for (auto *LogicN : NodesWithConsts) {
7549 SDValue Op0 = LogicN->getOperand(Num: 0);
7550 SDValue Op1 = LogicN->getOperand(Num: 1);
7551
7552 // We only need to fix AND if both inputs are constants. And we only need
7553 // to fix one of the constants.
7554 if (LogicN->getOpcode() == ISD::AND &&
7555 (!isa<ConstantSDNode>(Val: Op0) || !isa<ConstantSDNode>(Val: Op1)))
7556 continue;
7557
7558 if (isa<ConstantSDNode>(Val: Op0) && LogicN->getOpcode() != ISD::AND)
7559 Op0 =
7560 DAG.getNode(Opcode: ISD::AND, DL: SDLoc(Op0), VT: Op0.getValueType(), N1: Op0, N2: MaskOp);
7561
7562 if (isa<ConstantSDNode>(Val: Op1))
7563 Op1 =
7564 DAG.getNode(Opcode: ISD::AND, DL: SDLoc(Op1), VT: Op1.getValueType(), N1: Op1, N2: MaskOp);
7565
7566 if (isa<ConstantSDNode>(Val: Op0) && !isa<ConstantSDNode>(Val: Op1))
7567 std::swap(a&: Op0, b&: Op1);
7568
7569 DAG.UpdateNodeOperands(N: LogicN, Op1: Op0, Op2: Op1);
7570 }
7571
7572 // Create narrow loads.
7573 for (auto *Load : Loads) {
7574 LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump());
7575 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(Load), VT: Load->getValueType(ResNo: 0),
7576 N1: SDValue(Load, 0), N2: MaskOp);
7577 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 0), To: And);
7578 if (And.getOpcode() == ISD ::AND)
7579 And = SDValue(
7580 DAG.UpdateNodeOperands(N: And.getNode(), Op1: SDValue(Load, 0), Op2: MaskOp), 0);
7581 SDValue NewLoad = reduceLoadWidth(N: And.getNode());
7582 assert(NewLoad &&
7583 "Shouldn't be masking the load if it can't be narrowed");
7584 CombineTo(N: Load, Res0: NewLoad, Res1: NewLoad.getValue(R: 1));
7585 }
7586 DAG.ReplaceAllUsesWith(From: N, To: N->getOperand(Num: 0).getNode());
7587 return true;
7588 }
7589 return false;
7590}
7591
7592// Unfold
7593// x & (-1 'logical shift' y)
7594// To
7595// (x 'opposite logical shift' y) 'logical shift' y
7596// if it is better for performance.
7597SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
7598 assert(N->getOpcode() == ISD::AND);
7599
7600 SDValue N0 = N->getOperand(Num: 0);
7601 SDValue N1 = N->getOperand(Num: 1);
7602
7603 // Do we actually prefer shifts over mask?
7604 if (!TLI.shouldFoldMaskToVariableShiftPair(X: N0))
7605 return SDValue();
7606
7607 // Try to match (-1 '[outer] logical shift' y)
7608 unsigned OuterShift;
7609 unsigned InnerShift; // The opposite direction to the OuterShift.
7610 SDValue Y; // Shift amount.
7611 auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
7612 if (!M.hasOneUse())
7613 return false;
7614 OuterShift = M->getOpcode();
7615 if (OuterShift == ISD::SHL)
7616 InnerShift = ISD::SRL;
7617 else if (OuterShift == ISD::SRL)
7618 InnerShift = ISD::SHL;
7619 else
7620 return false;
7621 if (!isAllOnesConstant(V: M->getOperand(Num: 0)))
7622 return false;
7623 Y = M->getOperand(Num: 1);
7624 return true;
7625 };
7626
7627 SDValue X;
7628 if (matchMask(N1))
7629 X = N0;
7630 else if (matchMask(N0))
7631 X = N1;
7632 else
7633 return SDValue();
7634
7635 SDLoc DL(N);
7636 EVT VT = N->getValueType(ResNo: 0);
7637
7638 // tmp = x 'opposite logical shift' y
7639 SDValue T0 = DAG.getNode(Opcode: InnerShift, DL, VT, N1: X, N2: Y);
7640 // ret = tmp 'logical shift' y
7641 SDValue T1 = DAG.getNode(Opcode: OuterShift, DL, VT, N1: T0, N2: Y);
7642
7643 return T1;
7644}
7645
7646/// Try to replace shift/logic that tests if a bit is clear with mask + setcc.
7647/// For a target with a bit test, this is expected to become test + set and save
7648/// at least 1 instruction.
7649static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
7650 assert(And->getOpcode() == ISD::AND && "Expected an 'and' op");
7651
7652 // Look through an optional extension.
7653 SDValue And0 = And->getOperand(Num: 0), And1 = And->getOperand(Num: 1);
7654 if (And0.getOpcode() == ISD::ANY_EXTEND && And0.hasOneUse())
7655 And0 = And0.getOperand(i: 0);
7656 if (!isOneConstant(V: And1) || !And0.hasOneUse())
7657 return SDValue();
7658
7659 SDValue Src = And0;
7660
7661 // Attempt to find a 'not' op.
7662 // TODO: Should we favor test+set even without the 'not' op?
7663 bool FoundNot = false;
7664 if (isBitwiseNot(V: Src)) {
7665 FoundNot = true;
7666 Src = Src.getOperand(i: 0);
7667
7668 // Look though an optional truncation. The source operand may not be the
7669 // same type as the original 'and', but that is ok because we are masking
7670 // off everything but the low bit.
7671 if (Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse())
7672 Src = Src.getOperand(i: 0);
7673 }
7674
7675 // Match a shift-right by constant.
7676 if (Src.getOpcode() != ISD::SRL || !Src.hasOneUse())
7677 return SDValue();
7678
7679 // This is probably not worthwhile without a supported type.
7680 EVT SrcVT = Src.getValueType();
7681 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7682 if (!TLI.isTypeLegal(VT: SrcVT))
7683 return SDValue();
7684
7685 // We might have looked through casts that make this transform invalid.
7686 unsigned BitWidth = SrcVT.getScalarSizeInBits();
7687 SDValue ShiftAmt = Src.getOperand(i: 1);
7688 auto *ShiftAmtC = dyn_cast<ConstantSDNode>(Val&: ShiftAmt);
7689 if (!ShiftAmtC || !ShiftAmtC->getAPIntValue().ult(RHS: BitWidth))
7690 return SDValue();
7691
7692 // Set source to shift source.
7693 Src = Src.getOperand(i: 0);
7694
7695 // Try again to find a 'not' op.
7696 // TODO: Should we favor test+set even with two 'not' ops?
7697 if (!FoundNot) {
7698 if (!isBitwiseNot(V: Src))
7699 return SDValue();
7700 Src = Src.getOperand(i: 0);
7701 }
7702
7703 if (!TLI.hasBitTest(X: Src, Y: ShiftAmt))
7704 return SDValue();
7705
7706 // Turn this into a bit-test pattern using mask op + setcc:
7707 // and (not (srl X, C)), 1 --> (and X, 1<<C) == 0
7708 // and (srl (not X), C)), 1 --> (and X, 1<<C) == 0
7709 SDLoc DL(And);
7710 SDValue X = DAG.getZExtOrTrunc(Op: Src, DL, VT: SrcVT);
7711 EVT CCVT =
7712 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: SrcVT);
7713 SDValue Mask = DAG.getConstant(
7714 Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmtC->getZExtValue()), DL, VT: SrcVT);
7715 SDValue NewAnd = DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: X, N2: Mask);
7716 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
7717 SDValue Setcc = DAG.getSetCC(DL, VT: CCVT, LHS: NewAnd, RHS: Zero, Cond: ISD::SETEQ);
7718 return DAG.getZExtOrTrunc(Op: Setcc, DL, VT: And->getValueType(ResNo: 0));
7719}
7720
7721/// For targets that support usubsat, match a bit-hack form of that operation
7722/// that ends in 'and' and convert it.
7723static SDValue foldAndToUsubsat(SDNode *N, SelectionDAG &DAG, const SDLoc &DL) {
7724 EVT VT = N->getValueType(ResNo: 0);
7725 unsigned BitWidth = VT.getScalarSizeInBits();
7726 APInt SignMask = APInt::getSignMask(BitWidth);
7727
7728 // (i8 X ^ 128) & (i8 X s>> 7) --> usubsat X, 128
7729 // (i8 X + 128) & (i8 X s>> 7) --> usubsat X, 128
7730 // xor/add with SMIN (signmask) are logically equivalent.
7731 SDValue X;
7732 if (!sd_match(N, P: m_And(L: m_OneUse(P: m_Xor(L: m_Value(N&: X), R: m_SpecificInt(V: SignMask))),
7733 R: m_OneUse(P: m_Sra(L: m_Deferred(V&: X),
7734 R: m_SpecificInt(V: BitWidth - 1))))) &&
7735 !sd_match(N, P: m_And(L: m_OneUse(P: m_Add(L: m_Value(N&: X), R: m_SpecificInt(V: SignMask))),
7736 R: m_OneUse(P: m_Sra(L: m_Deferred(V&: X),
7737 R: m_SpecificInt(V: BitWidth - 1))))))
7738 return SDValue();
7739
7740 return DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: X,
7741 N2: DAG.getConstant(Val: SignMask, DL, VT));
7742}
7743
7744/// Given a bitwise logic operation N with a matching bitwise logic operand,
7745/// fold a pattern where 2 of the source operands are identically shifted
7746/// values. For example:
7747/// ((X0 << Y) | Z) | (X1 << Y) --> ((X0 | X1) << Y) | Z
7748static SDValue foldLogicOfShifts(SDNode *N, SDValue LogicOp, SDValue ShiftOp,
7749 SelectionDAG &DAG) {
7750 unsigned LogicOpcode = N->getOpcode();
7751 assert(ISD::isBitwiseLogicOp(LogicOpcode) &&
7752 "Expected bitwise logic operation");
7753
7754 if (!LogicOp.hasOneUse() || !ShiftOp.hasOneUse())
7755 return SDValue();
7756
7757 // Match another bitwise logic op and a shift.
7758 unsigned ShiftOpcode = ShiftOp.getOpcode();
7759 if (LogicOp.getOpcode() != LogicOpcode ||
7760 !(ShiftOpcode == ISD::SHL || ShiftOpcode == ISD::SRL ||
7761 ShiftOpcode == ISD::SRA))
7762 return SDValue();
7763
7764 // Match another shift op inside the first logic operand. Handle both commuted
7765 // possibilities.
7766 // LOGIC (LOGIC (SH X0, Y), Z), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z
7767 // LOGIC (LOGIC Z, (SH X0, Y)), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z
7768 SDValue X1 = ShiftOp.getOperand(i: 0);
7769 SDValue Y = ShiftOp.getOperand(i: 1);
7770 SDValue X0, Z;
7771 if (LogicOp.getOperand(i: 0).getOpcode() == ShiftOpcode &&
7772 LogicOp.getOperand(i: 0).getOperand(i: 1) == Y) {
7773 X0 = LogicOp.getOperand(i: 0).getOperand(i: 0);
7774 Z = LogicOp.getOperand(i: 1);
7775 } else if (LogicOp.getOperand(i: 1).getOpcode() == ShiftOpcode &&
7776 LogicOp.getOperand(i: 1).getOperand(i: 1) == Y) {
7777 X0 = LogicOp.getOperand(i: 1).getOperand(i: 0);
7778 Z = LogicOp.getOperand(i: 0);
7779 } else {
7780 return SDValue();
7781 }
7782
7783 EVT VT = N->getValueType(ResNo: 0);
7784 SDLoc DL(N);
7785 SDValue LogicX = DAG.getNode(Opcode: LogicOpcode, DL, VT, N1: X0, N2: X1);
7786 SDValue NewShift = DAG.getNode(Opcode: ShiftOpcode, DL, VT, N1: LogicX, N2: Y);
7787 return DAG.getNode(Opcode: LogicOpcode, DL, VT, N1: NewShift, N2: Z);
7788}
7789
7790/// Given a tree of logic operations with shape like
7791/// (LOGIC (LOGIC (X, Y), LOGIC (Z, Y)))
7792/// try to match and fold shift operations with the same shift amount.
7793/// For example:
7794/// LOGIC (LOGIC (SH X0, Y), Z), (LOGIC (SH X1, Y), W) -->
7795/// --> LOGIC (SH (LOGIC X0, X1), Y), (LOGIC Z, W)
7796static SDValue foldLogicTreeOfShifts(SDNode *N, SDValue LeftHand,
7797 SDValue RightHand, SelectionDAG &DAG) {
7798 unsigned LogicOpcode = N->getOpcode();
7799 assert(ISD::isBitwiseLogicOp(LogicOpcode) &&
7800 "Expected bitwise logic operation");
7801 if (LeftHand.getOpcode() != LogicOpcode ||
7802 RightHand.getOpcode() != LogicOpcode)
7803 return SDValue();
7804 if (!LeftHand.hasOneUse() || !RightHand.hasOneUse())
7805 return SDValue();
7806
7807 // Try to match one of following patterns:
7808 // LOGIC (LOGIC (SH X0, Y), Z), (LOGIC (SH X1, Y), W)
7809 // LOGIC (LOGIC (SH X0, Y), Z), (LOGIC W, (SH X1, Y))
7810 // Note that foldLogicOfShifts will handle commuted versions of the left hand
7811 // itself.
7812 SDValue CombinedShifts, W;
7813 SDValue R0 = RightHand.getOperand(i: 0);
7814 SDValue R1 = RightHand.getOperand(i: 1);
7815 if ((CombinedShifts = foldLogicOfShifts(N, LogicOp: LeftHand, ShiftOp: R0, DAG)))
7816 W = R1;
7817 else if ((CombinedShifts = foldLogicOfShifts(N, LogicOp: LeftHand, ShiftOp: R1, DAG)))
7818 W = R0;
7819 else
7820 return SDValue();
7821
7822 EVT VT = N->getValueType(ResNo: 0);
7823 SDLoc DL(N);
7824 return DAG.getNode(Opcode: LogicOpcode, DL, VT, N1: CombinedShifts, N2: W);
7825}
7826
7827/// Fold "masked merge" expressions like `(m & x) | (~m & y)` and its DeMorgan
7828/// variant `(~m | x) & (m | y)` into the equivalent `((x ^ y) & m) ^ y)`
7829/// pattern. This is typically a better representation for targets without a
7830/// fused "and-not" operation.
7831static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG,
7832 const TargetLowering &TLI, const SDLoc &DL) {
7833 // Note that masked-merge variants using XOR or ADD expressions are
7834 // normalized to OR by InstCombine so we only check for OR or AND.
7835 assert((Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::AND) &&
7836 "Must be called with ISD::OR or ISD::AND node");
7837
7838 // If the target supports and-not, don't fold this.
7839 if (TLI.hasAndNot(X: SDValue(Node, 0)))
7840 return SDValue();
7841
7842 SDValue M, X, Y;
7843
7844 if (sd_match(N: Node,
7845 P: m_Or(L: m_OneUse(P: m_And(L: m_OneUse(P: m_Not(V: m_Value(N&: M))), R: m_Value(N&: Y))),
7846 R: m_OneUse(P: m_And(L: m_Deferred(V&: M), R: m_Value(N&: X))))) ||
7847 sd_match(N: Node,
7848 P: m_And(L: m_OneUse(P: m_Or(L: m_OneUse(P: m_Not(V: m_Value(N&: M))), R: m_Value(N&: X))),
7849 R: m_OneUse(P: m_Or(L: m_Deferred(V&: M), R: m_Value(N&: Y)))))) {
7850 EVT VT = M.getValueType();
7851 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: X, N2: Y);
7852 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Xor, N2: M);
7853 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: And, N2: Y);
7854 }
7855 return SDValue();
7856}
7857
7858SDValue DAGCombiner::visitAND(SDNode *N) {
7859 SDValue N0 = N->getOperand(Num: 0);
7860 SDValue N1 = N->getOperand(Num: 1);
7861 EVT VT = N1.getValueType();
7862 SDLoc DL(N);
7863
7864 // x & x --> x
7865 if (N0 == N1)
7866 return N0;
7867
7868 // fold (and c1, c2) -> c1&c2
7869 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::AND, DL, VT, Ops: {N0, N1}))
7870 return C;
7871
7872 // canonicalize constant to RHS
7873 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
7874 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
7875 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1, N2: N0);
7876
7877 if (areBitwiseNotOfEachother(Op0: N0, Op1: N1))
7878 return DAG.getConstant(Val: APInt::getZero(numBits: VT.getScalarSizeInBits()), DL, VT);
7879
7880 // fold vector ops
7881 if (VT.isVector()) {
7882 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
7883 return FoldedVOp;
7884
7885 // fold (and x, 0) -> 0, vector edition
7886 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
7887 // do not return N1, because undef node may exist in N1
7888 return DAG.getConstant(Val: APInt::getZero(numBits: N1.getScalarValueSizeInBits()), DL,
7889 VT: N1.getValueType());
7890
7891 // fold (and x, -1) -> x, vector edition
7892 if (ISD::isConstantSplatVectorAllOnes(N: N1.getNode()))
7893 return N0;
7894
7895 // fold (and buildvector(x,0,-1,w), buildvector(0,y,z,w))
7896 // --> buildvector(0,0,z,w)
7897 auto *BV0 = dyn_cast<BuildVectorSDNode>(Val&: N0);
7898 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
7899 if (BV0 && BV1 && !BV0->getSplatValue() && !BV1->getSplatValue() &&
7900 N0.hasOneUse() && N1.hasOneUse() &&
7901 BV0->getOperand(Num: 0).getValueType() ==
7902 BV1->getOperand(Num: 0).getValueType()) {
7903 SmallVector<SDValue> MergedOps;
7904 unsigned NumElts = VT.getVectorNumElements();
7905 EVT EltVT = BV0->getOperand(Num: 0).getValueType();
7906 for (unsigned I = 0; I != NumElts; ++I) {
7907 auto *C0 = dyn_cast<ConstantSDNode>(Val: BV0->getOperand(Num: I));
7908 auto *C1 = dyn_cast<ConstantSDNode>(Val: BV1->getOperand(Num: I));
7909 if (C0 && C1)
7910 MergedOps.push_back(Elt: DAG.getConstant(
7911 Val: C0->getAPIntValue() & C1->getAPIntValue(), DL, VT: EltVT));
7912 else if (C0 && C0->isZero())
7913 MergedOps.push_back(Elt: BV0->getOperand(Num: I));
7914 else if (C1 && C1->isZero())
7915 MergedOps.push_back(Elt: BV1->getOperand(Num: I));
7916 else if (C0 && C0->isAllOnes())
7917 MergedOps.push_back(Elt: BV1->getOperand(Num: I));
7918 else if (C1 && C1->isAllOnes())
7919 MergedOps.push_back(Elt: BV0->getOperand(Num: I));
7920 else if (BV0->getOperand(Num: I) == BV1->getOperand(Num: I))
7921 MergedOps.push_back(Elt: BV0->getOperand(Num: I));
7922 else
7923 break;
7924 }
7925 if (MergedOps.size() == NumElts)
7926 return DAG.getBuildVector(VT, DL, Ops: MergedOps);
7927 }
7928
7929 // fold (and (masked_load) (splat_vec (x, ...))) to zext_masked_load
7930 bool Frozen = N0.getOpcode() == ISD::FREEZE;
7931 auto *MLoad = dyn_cast<MaskedLoadSDNode>(Val: Frozen ? N0.getOperand(i: 0) : N0);
7932 ConstantSDNode *Splat = isConstOrConstSplat(N: N1, AllowUndefs: true, AllowTruncation: true);
7933 if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat) {
7934 EVT MemVT = MLoad->getMemoryVT();
7935 if (TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: MLoad->getAlign(),
7936 AddrSpace: MLoad->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false)) {
7937 // For this AND to be a zero extension of the masked load the elements
7938 // of the BuildVec must mask the bottom bits of the extended element
7939 // type
7940 if (Splat->getAPIntValue().isMask(numBits: MemVT.getScalarSizeInBits())) {
7941 SDValue NewLoad = DAG.getMaskedLoad(
7942 VT, dl: DL, Chain: MLoad->getChain(), Base: MLoad->getBasePtr(),
7943 Offset: MLoad->getOffset(), Mask: MLoad->getMask(), Src0: MLoad->getPassThru(), MemVT,
7944 MMO: MLoad->getMemOperand(), AM: MLoad->getAddressingMode(), ISD::ZEXTLOAD,
7945 IsExpanding: MLoad->isExpandingLoad());
7946 CombineTo(N, Res: Frozen ? N0 : NewLoad);
7947 CombineTo(N: MLoad, Res0: NewLoad, Res1: NewLoad.getValue(R: 1));
7948 return SDValue(N, 0);
7949 }
7950 }
7951 }
7952 }
7953
7954 // fold (and x, -1) -> x
7955 if (isAllOnesConstant(V: N1))
7956 return N0;
7957
7958 // if (and x, c) is known to be zero, return 0
7959 unsigned BitWidth = VT.getScalarSizeInBits();
7960 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
7961 if (N1C && DAG.MaskedValueIsZero(Op: SDValue(N, 0), Mask: APInt::getAllOnes(numBits: BitWidth)))
7962 return DAG.getConstant(Val: 0, DL, VT);
7963
7964 if (SDValue R = foldAndOrOfSETCC(LogicOp: N, DAG))
7965 return R;
7966
7967 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
7968 return NewSel;
7969
7970 // reassociate and
7971 if (SDValue RAND = reassociateOps(Opc: ISD::AND, DL, N0, N1, Flags: N->getFlags()))
7972 return RAND;
7973
7974 // Fold and(vecreduce(x), vecreduce(y)) -> vecreduce(and(x, y))
7975 if (SDValue SD =
7976 reassociateReduction(RedOpc: ISD::VECREDUCE_AND, Opc: ISD::AND, DL, VT, N0, N1))
7977 return SD;
7978
7979 // fold (and (or x, C), D) -> D if (C & D) == D
7980 auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
7981 return RHS->getAPIntValue().isSubsetOf(RHS: LHS->getAPIntValue());
7982 };
7983 if (N0.getOpcode() == ISD::OR &&
7984 ISD::matchBinaryPredicate(LHS: N0.getOperand(i: 1), RHS: N1, Match: MatchSubset))
7985 return N1;
7986
7987 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
7988 SDValue N0Op0 = N0.getOperand(i: 0);
7989 EVT SrcVT = N0Op0.getValueType();
7990 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
7991 APInt Mask = ~N1C->getAPIntValue();
7992 Mask = Mask.trunc(width: SrcBitWidth);
7993
7994 // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
7995 if (DAG.MaskedValueIsZero(Op: N0Op0, Mask))
7996 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: N0Op0);
7997
7998 // fold (and (any_ext V), c) -> (zero_ext (and V, c)) if profitable, when
7999 // the zext is free or the anyext costs the same as the zext.
8000 if (N1C->getAPIntValue().countLeadingZeros() >= (BitWidth - SrcBitWidth) &&
8001 (TLI.isZExtFree(FromTy: SrcVT, ToTy: VT) || !TLI.isAnyExtFree(FromTy: SrcVT, ToTy: VT)) &&
8002 TLI.isTypeDesirableForOp(ISD::AND, VT: SrcVT) &&
8003 TLI.isNarrowingProfitable(N, SrcVT: VT, DestVT: SrcVT))
8004 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT,
8005 Operand: DAG.getNode(Opcode: ISD::AND, DL, VT: SrcVT, N1: N0Op0,
8006 N2: DAG.getZExtOrTrunc(Op: N1, DL, VT: SrcVT)));
8007 }
8008
8009 // fold (and (ext (and V, c1)), c2) -> (and (ext V), (and c1, (ext c2)))
8010 if (ISD::isExtOpcode(Opcode: N0.getOpcode())) {
8011 unsigned ExtOpc = N0.getOpcode();
8012 SDValue N0Op0 = N0.getOperand(i: 0);
8013 if (N0Op0.getOpcode() == ISD::AND &&
8014 (ExtOpc != ISD::ZERO_EXTEND || !TLI.isZExtFree(Val: N0Op0, VT2: VT)) &&
8015 N0->hasOneUse() && N0Op0->hasOneUse()) {
8016 if (SDValue NewExt = DAG.FoldConstantArithmetic(Opcode: ExtOpc, DL, VT,
8017 Ops: {N0Op0.getOperand(i: 1)})) {
8018 if (SDValue NewMask =
8019 DAG.FoldConstantArithmetic(Opcode: ISD::AND, DL, VT, Ops: {N1, NewExt})) {
8020 return DAG.getNode(Opcode: ISD::AND, DL, VT,
8021 N1: DAG.getNode(Opcode: ExtOpc, DL, VT, Operand: N0Op0.getOperand(i: 0)),
8022 N2: NewMask);
8023 }
8024 }
8025 }
8026 }
8027
8028 // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
8029 // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
8030 // already be zero by virtue of the width of the base type of the load.
8031 //
8032 // the 'X' node here can either be nothing or an extract_vector_elt to catch
8033 // more cases.
8034 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8035 N0.getValueSizeInBits() == N0.getOperand(i: 0).getScalarValueSizeInBits() &&
8036 N0.getOperand(i: 0).getOpcode() == ISD::LOAD &&
8037 N0.getOperand(i: 0).getResNo() == 0) ||
8038 (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
8039 auto *Load =
8040 cast<LoadSDNode>(Val: (N0.getOpcode() == ISD::LOAD) ? N0 : N0.getOperand(i: 0));
8041
8042 // Get the constant (if applicable) the zero'th operand is being ANDed with.
8043 // This can be a pure constant or a vector splat, in which case we treat the
8044 // vector as a scalar and use the splat value.
8045 APInt Constant = APInt::getZero(numBits: 1);
8046 if (const ConstantSDNode *C = isConstOrConstSplat(
8047 N: N1, /*AllowUndefs=*/false, /*AllowTruncation=*/true)) {
8048 Constant = C->getAPIntValue();
8049 } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(Val&: N1)) {
8050 unsigned EltBitWidth = Vector->getValueType(ResNo: 0).getScalarSizeInBits();
8051 APInt SplatValue, SplatUndef;
8052 unsigned SplatBitSize;
8053 bool HasAnyUndefs;
8054 // Endianness should not matter here. Code below makes sure that we only
8055 // use the result if the SplatBitSize is a multiple of the vector element
8056 // size. And after that we AND all element sized parts of the splat
8057 // together. So the end result should be the same regardless of in which
8058 // order we do those operations.
8059 const bool IsBigEndian = false;
8060 bool IsSplat =
8061 Vector->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
8062 HasAnyUndefs, MinSplatBits: EltBitWidth, isBigEndian: IsBigEndian);
8063
8064 // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
8065 // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
8066 if (IsSplat && (SplatBitSize % EltBitWidth) == 0) {
8067 // Undef bits can contribute to a possible optimisation if set, so
8068 // set them.
8069 SplatValue |= SplatUndef;
8070
8071 // The splat value may be something like "0x00FFFFFF", which means 0 for
8072 // the first vector value and FF for the rest, repeating. We need a mask
8073 // that will apply equally to all members of the vector, so AND all the
8074 // lanes of the constant together.
8075 Constant = APInt::getAllOnes(numBits: EltBitWidth);
8076 for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
8077 Constant &= SplatValue.extractBits(numBits: EltBitWidth, bitPosition: i * EltBitWidth);
8078 }
8079 }
8080
8081 // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
8082 // actually legal and isn't going to get expanded, else this is a false
8083 // optimisation.
8084 bool CanZextLoadProfitably = TLI.isLoadLegal(
8085 ValVT: Load->getValueType(ResNo: 0), MemVT: Load->getMemoryVT(), Alignment: Load->getAlign(),
8086 AddrSpace: Load->getAddressSpace(), ExtType: ISD::ZEXTLOAD, Atomic: false);
8087
8088 // Resize the constant to the same size as the original memory access before
8089 // extension. If it is still the AllOnesValue then this AND is completely
8090 // unneeded.
8091 Constant = Constant.zextOrTrunc(width: Load->getMemoryVT().getScalarSizeInBits());
8092
8093 bool B;
8094 switch (Load->getExtensionType()) {
8095 default: B = false; break;
8096 case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
8097 case ISD::ZEXTLOAD:
8098 case ISD::NON_EXTLOAD: B = true; break;
8099 }
8100
8101 if (B && Constant.isAllOnes()) {
8102 // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
8103 // preserve semantics once we get rid of the AND.
8104 SDValue NewLoad(Load, 0);
8105
8106 // Fold the AND away. NewLoad may get replaced immediately.
8107 CombineTo(N, Res: (N0.getNode() == Load) ? NewLoad : N0);
8108
8109 if (Load->getExtensionType() == ISD::EXTLOAD) {
8110 NewLoad = DAG.getLoad(AM: Load->getAddressingMode(), ExtType: ISD::ZEXTLOAD,
8111 VT: Load->getValueType(ResNo: 0), dl: SDLoc(Load),
8112 Chain: Load->getChain(), Ptr: Load->getBasePtr(),
8113 Offset: Load->getOffset(), MemVT: Load->getMemoryVT(),
8114 MMO: Load->getMemOperand());
8115 // Replace uses of the EXTLOAD with the new ZEXTLOAD.
8116 if (Load->getNumValues() == 3) {
8117 // PRE/POST_INC loads have 3 values.
8118 SDValue To[] = { NewLoad.getValue(R: 0), NewLoad.getValue(R: 1),
8119 NewLoad.getValue(R: 2) };
8120 CombineTo(N: Load, To, NumTo: 3, AddTo: true);
8121 } else {
8122 CombineTo(N: Load, Res0: NewLoad.getValue(R: 0), Res1: NewLoad.getValue(R: 1));
8123 }
8124 }
8125
8126 return SDValue(N, 0); // Return N so it doesn't get rechecked!
8127 }
8128 }
8129
8130 // Try to convert a constant mask AND into a shuffle clear mask.
8131 if (VT.isVector())
8132 if (SDValue Shuffle = XformToShuffleWithZero(N))
8133 return Shuffle;
8134
8135 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
8136 return Combined;
8137
8138 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C &&
8139 ISD::isExtOpcode(Opcode: N0.getOperand(i: 0).getOpcode())) {
8140 SDValue Ext = N0.getOperand(i: 0);
8141 EVT ExtVT = Ext->getValueType(ResNo: 0);
8142 SDValue Extendee = Ext->getOperand(Num: 0);
8143
8144 unsigned ScalarWidth = Extendee.getValueType().getScalarSizeInBits();
8145 if (N1C->getAPIntValue().isMask(numBits: ScalarWidth) &&
8146 (!LegalOperations || TLI.isOperationLegal(Op: ISD::ZERO_EXTEND, VT: ExtVT))) {
8147 // (and (extract_subvector (zext|anyext|sext v) _) iN_mask)
8148 // => (extract_subvector (iN_zeroext v))
8149 SDValue ZeroExtExtendee =
8150 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVT, Operand: Extendee);
8151
8152 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: ZeroExtExtendee,
8153 N2: N0.getOperand(i: 1));
8154 }
8155 }
8156
8157 // fold (and (masked_gather x)) -> (zext_masked_gather x)
8158 if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(Val&: N0)) {
8159 EVT MemVT = GN0->getMemoryVT();
8160 EVT ScalarVT = MemVT.getScalarType();
8161
8162 if (SDValue(GN0, 0).hasOneUse() &&
8163 isConstantSplatVectorMaskForType(N: N1.getNode(), ScalarTy: ScalarVT) &&
8164 TLI.isVectorLoadExtDesirable(ExtVal: SDValue(N, 0))) {
8165 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
8166 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
8167
8168 SDValue ZExtLoad = DAG.getMaskedGather(
8169 VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), MemVT, dl: DL, Ops, MMO: GN0->getMemOperand(),
8170 IndexType: GN0->getIndexType(), ExtTy: ISD::ZEXTLOAD);
8171
8172 CombineTo(N, Res: ZExtLoad);
8173 AddToWorklist(N: ZExtLoad.getNode());
8174 // Avoid recheck of N.
8175 return SDValue(N, 0);
8176 }
8177 }
8178
8179 // fold (and (load x), 255) -> (zextload x, i8)
8180 // fold (and (extload x, i16), 255) -> (zextload x, i8)
8181 // fold (and (freeze (load x)), 255) -> (freeze (zextload x, i8))
8182 // fold (and (freeze (extload x, i16)), 255) -> (freeze (zextload x, i8))
8183 if (N1C && !VT.isVector()) {
8184 SDValue Inner = peekThroughFreeze(V: N0);
8185 if (Inner.getOpcode() == ISD::LOAD)
8186 if (SDValue Res = reduceLoadWidth(N))
8187 return Res;
8188 }
8189
8190 if (LegalTypes) {
8191 // Attempt to propagate the AND back up to the leaves which, if they're
8192 // loads, can be combined to narrow loads and the AND node can be removed.
8193 // Perform after legalization so that extend nodes will already be
8194 // combined into the loads.
8195 if (BackwardsPropagateMask(N))
8196 return SDValue(N, 0);
8197 }
8198
8199 if (SDValue Combined = visitANDLike(N0, N1, N))
8200 return Combined;
8201
8202 // Simplify: (and (op x...), (op y...)) -> (op (and x, y))
8203 if (N0.getOpcode() == N1.getOpcode())
8204 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
8205 return V;
8206
8207 if (SDValue R = foldLogicOfShifts(N, LogicOp: N0, ShiftOp: N1, DAG))
8208 return R;
8209 if (SDValue R = foldLogicOfShifts(N, LogicOp: N1, ShiftOp: N0, DAG))
8210 return R;
8211
8212 // Fold (and X, (bswap (not Y))) -> (and X, (not (bswap Y)))
8213 // Fold (and X, (bitreverse (not Y))) -> (and X, (not (bitreverse Y)))
8214 SDValue X, Y, Z, NotY;
8215 for (unsigned Opc : {ISD::BSWAP, ISD::BITREVERSE})
8216 if (sd_match(N,
8217 P: m_And(L: m_Value(N&: X), R: m_OneUse(P: m_UnaryOp(Opc, Op: m_Value(N&: NotY))))) &&
8218 sd_match(N: NotY, P: m_Not(V: m_Value(N&: Y))) &&
8219 (TLI.hasAndNot(X: SDValue(N, 0)) || NotY->hasOneUse()))
8220 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X,
8221 N2: DAG.getNOT(DL, Val: DAG.getNode(Opcode: Opc, DL, VT, Operand: Y), VT));
8222
8223 // Fold (and X, (rot (not Y), Z)) -> (and X, (not (rot Y, Z)))
8224 for (unsigned Opc : {ISD::ROTL, ISD::ROTR})
8225 if (sd_match(N, P: m_And(L: m_Value(N&: X),
8226 R: m_OneUse(P: m_BinOp(Opc, L: m_Value(N&: NotY), R: m_Value(N&: Z))))) &&
8227 sd_match(N: NotY, P: m_Not(V: m_Value(N&: Y))) &&
8228 (TLI.hasAndNot(X: SDValue(N, 0)) || NotY->hasOneUse()))
8229 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X,
8230 N2: DAG.getNOT(DL, Val: DAG.getNode(Opcode: Opc, DL, VT, N1: Y, N2: Z), VT));
8231
8232 // Fold (and X, (add (not Y), Z)) -> (and X, (not (sub Y, Z)))
8233 // Fold (and X, (sub (not Y), Z)) -> (and X, (not (add Y, Z)))
8234 if (TLI.hasAndNot(X: SDValue(N, 0)))
8235 if (SDValue Folded = foldBitwiseOpWithNeg(N, DL, VT))
8236 return Folded;
8237
8238 // Fold (and (srl X, C), 1) -> (srl X, BW-1) for signbit extraction
8239 // If we are shifting down an extended sign bit, see if we can simplify
8240 // this to shifting the MSB directly to expose further simplifications.
8241 // This pattern often appears after sext_inreg legalization.
8242 APInt Amt;
8243 if (sd_match(N, P: m_And(L: m_Srl(L: m_Value(N&: X), R: m_ConstInt(V&: Amt)), R: m_One())) &&
8244 Amt.ult(RHS: BitWidth - 1) && Amt.uge(RHS: BitWidth - DAG.ComputeNumSignBits(Op: X)))
8245 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X,
8246 N2: DAG.getShiftAmountConstant(Val: BitWidth - 1, VT, DL));
8247
8248 // Masking the negated extension of a boolean is just the zero-extended
8249 // boolean:
8250 // and (sub 0, zext(bool X)), 1 --> zext(bool X)
8251 // and (sub 0, sext(bool X)), 1 --> zext(bool X)
8252 //
8253 // Note: the SimplifyDemandedBits fold below can make an information-losing
8254 // transform, and then we have no way to find this better fold.
8255 if (sd_match(N, P: m_And(L: m_Sub(L: m_Zero(), R: m_Value(N&: X)), R: m_One()))) {
8256 if (X.getOpcode() == ISD::ZERO_EXTEND &&
8257 X.getOperand(i: 0).getScalarValueSizeInBits() == 1)
8258 return X;
8259 if (X.getOpcode() == ISD::SIGN_EXTEND &&
8260 X.getOperand(i: 0).getScalarValueSizeInBits() == 1)
8261 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: X.getOperand(i: 0));
8262 }
8263
8264 // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
8265 // fold (and (sra)) -> (and (srl)) when possible.
8266 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
8267 return SDValue(N, 0);
8268
8269 // fold (zext_inreg (extload x)) -> (zextload x)
8270 // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
8271 if (ISD::isUNINDEXEDLoad(N: N0.getNode()) &&
8272 (ISD::isEXTLoad(N: N0.getNode()) ||
8273 (ISD::isSEXTLoad(N: N0.getNode()) && N0.hasOneUse()))) {
8274 auto *LN0 = cast<LoadSDNode>(Val&: N0);
8275 EVT MemVT = LN0->getMemoryVT();
8276 // If we zero all the possible extended bits, then we can turn this into
8277 // a zextload if we are running before legalize or the operation is legal.
8278 unsigned ExtBitSize = N1.getScalarValueSizeInBits();
8279 unsigned MemBitSize = MemVT.getScalarSizeInBits();
8280 APInt ExtBits = APInt::getHighBitsSet(numBits: ExtBitSize, hiBitsSet: ExtBitSize - MemBitSize);
8281 if (DAG.MaskedValueIsZero(Op: N1, Mask: ExtBits) &&
8282 ((!LegalOperations && LN0->isSimple()) ||
8283 TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: LN0->getAlign(), AddrSpace: LN0->getAddressSpace(),
8284 ExtType: ISD::ZEXTLOAD, Atomic: false))) {
8285 SDValue ExtLoad =
8286 DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(N0), VT, Chain: LN0->getChain(),
8287 Ptr: LN0->getBasePtr(), MemVT, MMO: LN0->getMemOperand());
8288 AddToWorklist(N);
8289 CombineTo(N: N0.getNode(), Res0: ExtLoad, Res1: ExtLoad.getValue(R: 1));
8290 return SDValue(N, 0); // Return N so it doesn't get rechecked!
8291 }
8292 }
8293
8294 // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
8295 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
8296 if (SDValue BSwap = MatchBSwapHWordLow(N: N0.getNode(), N0: N0.getOperand(i: 0),
8297 N1: N0.getOperand(i: 1), DemandHighBits: false))
8298 return BSwap;
8299 }
8300
8301 if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
8302 return Shifts;
8303
8304 if (SDValue V = combineShiftAnd1ToBitTest(And: N, DAG))
8305 return V;
8306
8307 // Recognize the following pattern:
8308 //
8309 // AndVT = (and (sign_extend NarrowVT to AndVT) #bitmask)
8310 //
8311 // where bitmask is a mask that clears the upper bits of AndVT. The
8312 // number of bits in bitmask must be a power of two.
8313 auto IsAndZeroExtMask = [](SDValue LHS, SDValue RHS) {
8314 if (LHS->getOpcode() != ISD::SIGN_EXTEND)
8315 return false;
8316
8317 auto *C = isConstOrConstSplat(N: RHS, AllowUndefs: false, AllowTruncation: true);
8318 if (!C)
8319 return false;
8320
8321 if (!C->getAPIntValue().isMask(
8322 numBits: LHS.getOperand(i: 0).getValueType().getScalarSizeInBits()))
8323 return false;
8324
8325 return true;
8326 };
8327
8328 // Replace (and (sign_extend ...) #bitmask) with (zero_extend ...).
8329 if (IsAndZeroExtMask(N0, N1) &&
8330 (!LegalOperations || TLI.isOperationLegal(Op: ISD::ZERO_EXTEND, VT)))
8331 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: N0.getOperand(i: 0));
8332
8333 if (hasOperation(Opcode: ISD::USUBSAT, VT))
8334 if (SDValue V = foldAndToUsubsat(N, DAG, DL))
8335 return V;
8336
8337 // Postpone until legalization completed to avoid interference with bswap
8338 // folding
8339 if (LegalOperations || VT.isVector())
8340 if (SDValue R = foldLogicTreeOfShifts(N, LeftHand: N0, RightHand: N1, DAG))
8341 return R;
8342
8343 if (VT.isScalarInteger() && VT != MVT::i1)
8344 if (SDValue R = foldMaskedMerge(Node: N, DAG, TLI, DL))
8345 return R;
8346
8347 return SDValue();
8348}
8349
8350/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
8351SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
8352 bool DemandHighBits) {
8353 if (!LegalOperations)
8354 return SDValue();
8355
8356 EVT VT = N->getValueType(ResNo: 0);
8357 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
8358 return SDValue();
8359 if (!TLI.isOperationLegalOrCustom(Op: ISD::BSWAP, VT))
8360 return SDValue();
8361
8362 // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
8363 bool LookPassAnd0 = false;
8364 bool LookPassAnd1 = false;
8365 if (N0.getOpcode() == ISD::AND && N0.getOperand(i: 0).getOpcode() == ISD::SRL)
8366 std::swap(a&: N0, b&: N1);
8367 if (N1.getOpcode() == ISD::AND && N1.getOperand(i: 0).getOpcode() == ISD::SHL)
8368 std::swap(a&: N0, b&: N1);
8369 if (N0.getOpcode() == ISD::AND) {
8370 if (!N0->hasOneUse())
8371 return SDValue();
8372 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
8373 // Also handle 0xffff since the LHS is guaranteed to have zeros there.
8374 // This is needed for X86.
8375 if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
8376 N01C->getZExtValue() != 0xFFFF))
8377 return SDValue();
8378 N0 = N0.getOperand(i: 0);
8379 LookPassAnd0 = true;
8380 }
8381
8382 if (N1.getOpcode() == ISD::AND) {
8383 if (!N1->hasOneUse())
8384 return SDValue();
8385 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
8386 if (!N11C || N11C->getZExtValue() != 0xFF)
8387 return SDValue();
8388 N1 = N1.getOperand(i: 0);
8389 LookPassAnd1 = true;
8390 }
8391
8392 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
8393 std::swap(a&: N0, b&: N1);
8394 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
8395 return SDValue();
8396 if (!N0->hasOneUse() || !N1->hasOneUse())
8397 return SDValue();
8398
8399 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
8400 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
8401 if (!N01C || !N11C)
8402 return SDValue();
8403 if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
8404 return SDValue();
8405
8406 // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
8407 SDValue N00 = N0->getOperand(Num: 0);
8408 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
8409 if (!N00->hasOneUse())
8410 return SDValue();
8411 ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(Val: N00.getOperand(i: 1));
8412 if (!N001C || N001C->getZExtValue() != 0xFF)
8413 return SDValue();
8414 N00 = N00.getOperand(i: 0);
8415 LookPassAnd0 = true;
8416 }
8417
8418 SDValue N10 = N1->getOperand(Num: 0);
8419 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
8420 if (!N10->hasOneUse())
8421 return SDValue();
8422 ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(Val: N10.getOperand(i: 1));
8423 // Also allow 0xFFFF since the bits will be shifted out. This is needed
8424 // for X86.
8425 if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
8426 N101C->getZExtValue() != 0xFFFF))
8427 return SDValue();
8428 N10 = N10.getOperand(i: 0);
8429 LookPassAnd1 = true;
8430 }
8431
8432 if (N00 != N10)
8433 return SDValue();
8434
8435 // Make sure everything beyond the low halfword gets set to zero since the SRL
8436 // 16 will clear the top bits.
8437 unsigned OpSizeInBits = VT.getSizeInBits();
8438 if (OpSizeInBits > 16) {
8439 // If the left-shift isn't masked out then the only way this is a bswap is
8440 // if all bits beyond the low 8 are 0. In that case the entire pattern
8441 // reduces to a left shift anyway: leave it for other parts of the combiner.
8442 if (DemandHighBits && !LookPassAnd0)
8443 return SDValue();
8444
8445 // However, if the right shift isn't masked out then it might be because
8446 // it's not needed. See if we can spot that too. If the high bits aren't
8447 // demanded, we only need bits 23:16 to be zero. Otherwise, we need all
8448 // upper bits to be zero.
8449 if (!LookPassAnd1) {
8450 unsigned HighBit = DemandHighBits ? OpSizeInBits : 24;
8451 if (!DAG.MaskedValueIsZero(Op: N10,
8452 Mask: APInt::getBitsSet(numBits: OpSizeInBits, loBit: 16, hiBit: HighBit)))
8453 return SDValue();
8454 }
8455 }
8456
8457 SDValue Res = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT, Operand: N00);
8458 if (OpSizeInBits > 16) {
8459 SDLoc DL(N);
8460 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Res,
8461 N2: DAG.getShiftAmountConstant(Val: OpSizeInBits - 16, VT, DL));
8462 }
8463 return Res;
8464}
8465
8466/// Return true if the specified node is an element that makes up a 32-bit
8467/// packed halfword byteswap.
8468/// ((x & 0x000000ff) << 8) |
8469/// ((x & 0x0000ff00) >> 8) |
8470/// ((x & 0x00ff0000) << 8) |
8471/// ((x & 0xff000000) >> 8)
8472static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
8473 if (!N->hasOneUse())
8474 return false;
8475
8476 unsigned Opc = N.getOpcode();
8477 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
8478 return false;
8479
8480 SDValue N0 = N.getOperand(i: 0);
8481 unsigned Opc0 = N0.getOpcode();
8482 if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
8483 return false;
8484
8485 ConstantSDNode *N1C = nullptr;
8486 // SHL or SRL: look upstream for AND mask operand
8487 if (Opc == ISD::AND)
8488 N1C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
8489 else if (Opc0 == ISD::AND)
8490 N1C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
8491 if (!N1C)
8492 return false;
8493
8494 unsigned MaskByteOffset;
8495 switch (N1C->getZExtValue()) {
8496 default:
8497 return false;
8498 case 0xFF: MaskByteOffset = 0; break;
8499 case 0xFF00: MaskByteOffset = 1; break;
8500 case 0xFFFF:
8501 // In case demanded bits didn't clear the bits that will be shifted out.
8502 // This is needed for X86.
8503 if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
8504 MaskByteOffset = 1;
8505 break;
8506 }
8507 return false;
8508 case 0xFF0000: MaskByteOffset = 2; break;
8509 case 0xFF000000: MaskByteOffset = 3; break;
8510 }
8511
8512 // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
8513 if (Opc == ISD::AND) {
8514 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
8515 // (x >> 8) & 0xff
8516 // (x >> 8) & 0xff0000
8517 if (Opc0 != ISD::SRL)
8518 return false;
8519 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
8520 if (!C || C->getZExtValue() != 8)
8521 return false;
8522 } else {
8523 // (x << 8) & 0xff00
8524 // (x << 8) & 0xff000000
8525 if (Opc0 != ISD::SHL)
8526 return false;
8527 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
8528 if (!C || C->getZExtValue() != 8)
8529 return false;
8530 }
8531 } else if (Opc == ISD::SHL) {
8532 // (x & 0xff) << 8
8533 // (x & 0xff0000) << 8
8534 if (MaskByteOffset != 0 && MaskByteOffset != 2)
8535 return false;
8536 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
8537 if (!C || C->getZExtValue() != 8)
8538 return false;
8539 } else { // Opc == ISD::SRL
8540 // (x & 0xff00) >> 8
8541 // (x & 0xff000000) >> 8
8542 if (MaskByteOffset != 1 && MaskByteOffset != 3)
8543 return false;
8544 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
8545 if (!C || C->getZExtValue() != 8)
8546 return false;
8547 }
8548
8549 if (Parts[MaskByteOffset])
8550 return false;
8551
8552 Parts[MaskByteOffset] = N0.getOperand(i: 0).getNode();
8553 return true;
8554}
8555
8556// Match 2 elements of a packed halfword bswap.
8557static bool isBSwapHWordPair(SDValue N, MutableArrayRef<SDNode *> Parts) {
8558 if (N.getOpcode() == ISD::OR)
8559 return isBSwapHWordElement(N: N.getOperand(i: 0), Parts) &&
8560 isBSwapHWordElement(N: N.getOperand(i: 1), Parts);
8561
8562 if (N.getOpcode() == ISD::SRL && N.getOperand(i: 0).getOpcode() == ISD::BSWAP) {
8563 ConstantSDNode *C = isConstOrConstSplat(N: N.getOperand(i: 1));
8564 if (!C || C->getAPIntValue() != 16)
8565 return false;
8566 Parts[0] = Parts[1] = N.getOperand(i: 0).getOperand(i: 0).getNode();
8567 return true;
8568 }
8569
8570 return false;
8571}
8572
8573// Match this pattern:
8574// (or (and (shl (A, 8)), 0xff00ff00), (and (srl (A, 8)), 0x00ff00ff))
8575// And rewrite this to:
8576// (rotr (bswap A), 16)
8577static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI,
8578 SelectionDAG &DAG, SDNode *N, SDValue N0,
8579 SDValue N1, EVT VT) {
8580 assert(N->getOpcode() == ISD::OR && VT == MVT::i32 &&
8581 "MatchBSwapHWordOrAndAnd: expecting i32");
8582 if (!TLI.isOperationLegalOrCustom(Op: ISD::ROTR, VT))
8583 return SDValue();
8584 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
8585 return SDValue();
8586 // TODO: this is too restrictive; lifting this restriction requires more tests
8587 if (!N0->hasOneUse() || !N1->hasOneUse())
8588 return SDValue();
8589 ConstantSDNode *Mask0 = isConstOrConstSplat(N: N0.getOperand(i: 1));
8590 ConstantSDNode *Mask1 = isConstOrConstSplat(N: N1.getOperand(i: 1));
8591 if (!Mask0 || !Mask1)
8592 return SDValue();
8593 if (Mask0->getAPIntValue() != 0xff00ff00 ||
8594 Mask1->getAPIntValue() != 0x00ff00ff)
8595 return SDValue();
8596 SDValue Shift0 = N0.getOperand(i: 0);
8597 SDValue Shift1 = N1.getOperand(i: 0);
8598 if (Shift0.getOpcode() != ISD::SHL || Shift1.getOpcode() != ISD::SRL)
8599 return SDValue();
8600 ConstantSDNode *ShiftAmt0 = isConstOrConstSplat(N: Shift0.getOperand(i: 1));
8601 ConstantSDNode *ShiftAmt1 = isConstOrConstSplat(N: Shift1.getOperand(i: 1));
8602 if (!ShiftAmt0 || !ShiftAmt1)
8603 return SDValue();
8604 if (ShiftAmt0->getAPIntValue() != 8 || ShiftAmt1->getAPIntValue() != 8)
8605 return SDValue();
8606 if (Shift0.getOperand(i: 0) != Shift1.getOperand(i: 0))
8607 return SDValue();
8608
8609 SDLoc DL(N);
8610 SDValue BSwap = DAG.getNode(Opcode: ISD::BSWAP, DL, VT, Operand: Shift0.getOperand(i: 0));
8611 SDValue ShAmt = DAG.getShiftAmountConstant(Val: 16, VT, DL);
8612 return DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: BSwap, N2: ShAmt);
8613}
8614
8615/// Match a 32-bit packed halfword bswap. That is
8616/// ((x & 0x000000ff) << 8) |
8617/// ((x & 0x0000ff00) >> 8) |
8618/// ((x & 0x00ff0000) << 8) |
8619/// ((x & 0xff000000) >> 8)
8620/// => (rotl (bswap x), 16)
8621SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
8622 if (!LegalOperations)
8623 return SDValue();
8624
8625 EVT VT = N->getValueType(ResNo: 0);
8626 if (VT != MVT::i32)
8627 return SDValue();
8628 if (!TLI.isOperationLegalOrCustom(Op: ISD::BSWAP, VT))
8629 return SDValue();
8630
8631 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT))
8632 return BSwap;
8633
8634 // Try again with commuted operands.
8635 if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0: N1, N1: N0, VT))
8636 return BSwap;
8637
8638
8639 // Look for either
8640 // (or (bswaphpair), (bswaphpair))
8641 // (or (or (bswaphpair), (and)), (and))
8642 // (or (or (and), (bswaphpair)), (and))
8643 SDNode *Parts[4] = {};
8644
8645 if (isBSwapHWordPair(N: N0, Parts)) {
8646 // (or (or (and), (and)), (or (and), (and)))
8647 if (!isBSwapHWordPair(N: N1, Parts))
8648 return SDValue();
8649 } else if (N0.getOpcode() == ISD::OR) {
8650 // (or (or (or (and), (and)), (and)), (and))
8651 if (!isBSwapHWordElement(N: N1, Parts))
8652 return SDValue();
8653 SDValue N00 = N0.getOperand(i: 0);
8654 SDValue N01 = N0.getOperand(i: 1);
8655 if (!(isBSwapHWordElement(N: N01, Parts) && isBSwapHWordPair(N: N00, Parts)) &&
8656 !(isBSwapHWordElement(N: N00, Parts) && isBSwapHWordPair(N: N01, Parts)))
8657 return SDValue();
8658 } else {
8659 return SDValue();
8660 }
8661
8662 // Make sure the parts are all coming from the same node.
8663 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
8664 return SDValue();
8665
8666 SDLoc DL(N);
8667 SDValue BSwap = DAG.getNode(Opcode: ISD::BSWAP, DL, VT,
8668 Operand: SDValue(Parts[0], 0));
8669
8670 // Result of the bswap should be rotated by 16. If it's not legal, then
8671 // do (x << 16) | (x >> 16).
8672 SDValue ShAmt = DAG.getShiftAmountConstant(Val: 16, VT, DL);
8673 if (TLI.isOperationLegalOrCustom(Op: ISD::ROTL, VT))
8674 return DAG.getNode(Opcode: ISD::ROTL, DL, VT, N1: BSwap, N2: ShAmt);
8675 if (TLI.isOperationLegalOrCustom(Op: ISD::ROTR, VT))
8676 return DAG.getNode(Opcode: ISD::ROTR, DL, VT, N1: BSwap, N2: ShAmt);
8677 return DAG.getNode(Opcode: ISD::OR, DL, VT,
8678 N1: DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: BSwap, N2: ShAmt),
8679 N2: DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: BSwap, N2: ShAmt));
8680}
8681
8682/// This contains all DAGCombine rules which reduce two values combined by
8683/// an Or operation to a single value \see visitANDLike().
8684SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, const SDLoc &DL) {
8685 EVT VT = N1.getValueType();
8686
8687 // fold (or x, undef) -> -1
8688 if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
8689 return DAG.getAllOnesConstant(DL, VT);
8690
8691 if (SDValue V = foldLogicOfSetCCs(IsAnd: false, N0, N1, DL))
8692 return V;
8693
8694 // (or (and X, C1), (and Y, C2)) -> (and (or X, Y), C3) if possible.
8695 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
8696 // Don't increase # computations.
8697 (N0->hasOneUse() || N1->hasOneUse())) {
8698 // We can only do this xform if we know that bits from X that are set in C2
8699 // but not in C1 are already zero. Likewise for Y.
8700 if (const ConstantSDNode *N0O1C =
8701 getAsNonOpaqueConstant(N: N0.getOperand(i: 1))) {
8702 if (const ConstantSDNode *N1O1C =
8703 getAsNonOpaqueConstant(N: N1.getOperand(i: 1))) {
8704 // We can only do this xform if we know that bits from X that are set in
8705 // C2 but not in C1 are already zero. Likewise for Y.
8706 const APInt &LHSMask = N0O1C->getAPIntValue();
8707 const APInt &RHSMask = N1O1C->getAPIntValue();
8708
8709 if (DAG.MaskedValueIsZero(Op: N0.getOperand(i: 0), Mask: RHSMask&~LHSMask) &&
8710 DAG.MaskedValueIsZero(Op: N1.getOperand(i: 0), Mask: LHSMask&~RHSMask)) {
8711 SDValue X = DAG.getNode(Opcode: ISD::OR, DL: SDLoc(N0), VT,
8712 N1: N0.getOperand(i: 0), N2: N1.getOperand(i: 0));
8713 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X,
8714 N2: DAG.getConstant(Val: LHSMask | RHSMask, DL, VT));
8715 }
8716 }
8717 }
8718 }
8719
8720 // (or (and X, M), (and X, N)) -> (and X, (or M, N))
8721 if (N0.getOpcode() == ISD::AND &&
8722 N1.getOpcode() == ISD::AND &&
8723 N0.getOperand(i: 0) == N1.getOperand(i: 0) &&
8724 // Don't increase # computations.
8725 (N0->hasOneUse() || N1->hasOneUse())) {
8726 SDValue X = DAG.getNode(Opcode: ISD::OR, DL: SDLoc(N0), VT,
8727 N1: N0.getOperand(i: 1), N2: N1.getOperand(i: 1));
8728 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N0.getOperand(i: 0), N2: X);
8729 }
8730
8731 return SDValue();
8732}
8733
8734/// OR combines for which the commuted variant will be tried as well.
8735static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1,
8736 SDNode *N) {
8737 EVT VT = N0.getValueType();
8738 unsigned BW = VT.getScalarSizeInBits();
8739 SDLoc DL(N);
8740
8741 auto peekThroughResize = [](SDValue V) {
8742 if (V->getOpcode() == ISD::ZERO_EXTEND || V->getOpcode() == ISD::TRUNCATE)
8743 return V->getOperand(Num: 0);
8744 return V;
8745 };
8746
8747 SDValue N0Resized = peekThroughResize(N0);
8748 if (N0Resized.getOpcode() == ISD::AND) {
8749 SDValue N1Resized = peekThroughResize(N1);
8750 SDValue N00 = N0Resized.getOperand(i: 0);
8751 SDValue N01 = N0Resized.getOperand(i: 1);
8752
8753 // fold or (and x, y), x --> x
8754 if (N00 == N1Resized || N01 == N1Resized)
8755 return N1;
8756
8757 // fold (or (and X, (xor Y, -1)), Y) -> (or X, Y)
8758 // TODO: Set AllowUndefs = true.
8759 if (SDValue NotOperand = getBitwiseNotOperand(V: N01, Mask: N00,
8760 /* AllowUndefs */ false)) {
8761 if (peekThroughResize(NotOperand) == N1Resized)
8762 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getZExtOrTrunc(Op: N00, DL, VT),
8763 N2: N1);
8764 }
8765
8766 // fold (or (and (xor Y, -1), X), Y) -> (or X, Y)
8767 if (SDValue NotOperand = getBitwiseNotOperand(V: N00, Mask: N01,
8768 /* AllowUndefs */ false)) {
8769 if (peekThroughResize(NotOperand) == N1Resized)
8770 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getZExtOrTrunc(Op: N01, DL, VT),
8771 N2: N1);
8772 }
8773 }
8774
8775 SDValue X, Y;
8776
8777 // fold or (xor X, N1), N1 --> or X, N1
8778 if (sd_match(N: N0, P: m_Xor(L: m_Value(N&: X), R: m_Specific(N: N1))))
8779 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: X, N2: N1);
8780
8781 // fold or (xor x, y), (x and/or y) --> or x, y
8782 if (sd_match(N: N0, P: m_Xor(L: m_Value(N&: X), R: m_Value(N&: Y))) &&
8783 (sd_match(N: N1, P: m_And(L: m_Specific(N: X), R: m_Specific(N: Y))) ||
8784 sd_match(N: N1, P: m_Or(L: m_Specific(N: X), R: m_Specific(N: Y)))))
8785 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: X, N2: Y);
8786
8787 if (SDValue R = foldLogicOfShifts(N, LogicOp: N0, ShiftOp: N1, DAG))
8788 return R;
8789
8790 auto peekThroughZext = [](SDValue V) {
8791 if (V->getOpcode() == ISD::ZERO_EXTEND)
8792 return V->getOperand(Num: 0);
8793 return V;
8794 };
8795
8796 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::SHL &&
8797 peekThroughZext(N0.getOperand(i: 2)) == peekThroughZext(N1.getOperand(i: 1))) {
8798 // (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y
8799 if (N0.getOperand(i: 0) == N1.getOperand(i: 0))
8800 return N0;
8801 // (fshl A, X, Y) | (shl X, Y) --> fshl (A|X), X, Y
8802 if (N0.getOperand(i: 1) == N1.getOperand(i: 0) && N0.hasOneUse() &&
8803 N1.hasOneUse()) {
8804 SDValue A = N0.getOperand(i: 0);
8805 SDValue X = N1.getOperand(i: 0);
8806 SDValue NewLHS = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: A, N2: X);
8807 return DAG.getNode(Opcode: ISD::FSHL, DL, VT, N1: NewLHS, N2: X, N3: N0.getOperand(i: 2));
8808 }
8809 }
8810
8811 if (N0.getOpcode() == ISD::FSHR && N1.getOpcode() == ISD::SRL &&
8812 peekThroughZext(N0.getOperand(i: 2)) == peekThroughZext(N1.getOperand(i: 1))) {
8813 // (fshr ?, X, Y) | (srl X, Y) --> fshr ?, X, Y
8814 if (N0.getOperand(i: 1) == N1.getOperand(i: 0))
8815 return N0;
8816 // (fshr X, B, Y) | (srl X, Y) --> fshr X, (X|B), Y
8817 if (N0.getOperand(i: 0) == N1.getOperand(i: 0) && N0.hasOneUse() &&
8818 N1.hasOneUse()) {
8819 SDValue X = N1.getOperand(i: 0);
8820 SDValue B = N0.getOperand(i: 1);
8821 SDValue NewRHS = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: X, N2: B);
8822 return DAG.getNode(Opcode: ISD::FSHR, DL, VT, N1: X, N2: NewRHS, N3: N0.getOperand(i: 2));
8823 }
8824 }
8825
8826 // (fshl A, B, S0) | (fshr C, D, S1) --> fshl (A|C), (B|D), S0
8827 // iff S0 + S1 == bitwidth(S1)
8828 if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::FSHR &&
8829 N0.hasOneUse() && N1.hasOneUse()) {
8830 auto *S0 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 2));
8831 auto *S1 = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 2));
8832 if (S0 && S1 && S0->getZExtValue() < BW && S1->getZExtValue() < BW &&
8833 S0->getZExtValue() == (BW - S1->getZExtValue())) {
8834 SDValue A = N0.getOperand(i: 0);
8835 SDValue B = N0.getOperand(i: 1);
8836 SDValue C = N1.getOperand(i: 0);
8837 SDValue D = N1.getOperand(i: 1);
8838 SDValue NewLHS = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: A, N2: C);
8839 SDValue NewRHS = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: B, N2: D);
8840 return DAG.getNode(Opcode: ISD::FSHL, DL, VT, N1: NewLHS, N2: NewRHS, N3: N0.getOperand(i: 2));
8841 }
8842 }
8843
8844 // Attempt to match a legalized build_pair-esque pattern:
8845 // or(shl(aext(Hi),BW/2),zext(Lo))
8846 SDValue Lo, Hi;
8847 if (sd_match(N: N0,
8848 P: m_OneUse(P: m_Shl(L: m_AnyExt(Op: m_Value(N&: Hi)), R: m_SpecificInt(V: BW / 2)))) &&
8849 sd_match(N: N1, P: m_ZExt(Op: m_Value(N&: Lo))) &&
8850 Lo.getScalarValueSizeInBits() == (BW / 2) &&
8851 Lo.getValueType() == Hi.getValueType()) {
8852 // Fold build_pair(not(Lo),not(Hi)) -> not(build_pair(Lo,Hi)).
8853 SDValue NotLo, NotHi;
8854 if (sd_match(N: Lo, P: m_OneUse(P: m_Not(V: m_Value(N&: NotLo)))) &&
8855 sd_match(N: Hi, P: m_OneUse(P: m_Not(V: m_Value(N&: NotHi))))) {
8856 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: NotLo);
8857 Hi = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: NotHi);
8858 Hi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi,
8859 N2: DAG.getShiftAmountConstant(Val: BW / 2, VT, DL));
8860 return DAG.getNOT(DL, Val: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Lo, N2: Hi), VT);
8861 }
8862 }
8863
8864 return SDValue();
8865}
8866
8867SDValue DAGCombiner::visitOR(SDNode *N) {
8868 SDValue N0 = N->getOperand(Num: 0);
8869 SDValue N1 = N->getOperand(Num: 1);
8870 EVT VT = N1.getValueType();
8871 SDLoc DL(N);
8872
8873 // x | x --> x
8874 if (N0 == N1)
8875 return N0;
8876
8877 // fold (or c1, c2) -> c1|c2
8878 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::OR, DL, VT, Ops: {N0, N1}))
8879 return C;
8880
8881 // canonicalize constant to RHS
8882 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
8883 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
8884 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1, N2: N0);
8885
8886 // fold vector ops
8887 if (VT.isVector()) {
8888 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
8889 return FoldedVOp;
8890
8891 // fold (or x, 0) -> x, vector edition
8892 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
8893 return N0;
8894
8895 // fold (or x, -1) -> -1, vector edition
8896 if (ISD::isConstantSplatVectorAllOnes(N: N1.getNode()))
8897 // do not return N1, because undef node may exist in N1
8898 return DAG.getAllOnesConstant(DL, VT: N1.getValueType());
8899
8900 // fold (or buildvector(x,0,-1,w), buildvector(0,y,z,w))
8901 // --> buildvector(x,y,-1,w)
8902 auto *BV0 = dyn_cast<BuildVectorSDNode>(Val&: N0);
8903 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
8904 if (BV0 && BV1 && !BV0->getSplatValue() && !BV1->getSplatValue() &&
8905 N0.hasOneUse() && N1.hasOneUse() &&
8906 BV0->getOperand(Num: 0).getValueType() ==
8907 BV1->getOperand(Num: 0).getValueType()) {
8908 SmallVector<SDValue> MergedOps;
8909 unsigned NumElts = VT.getVectorNumElements();
8910 EVT EltVT = BV0->getOperand(Num: 0).getValueType();
8911 for (unsigned I = 0; I != NumElts; ++I) {
8912 auto *C0 = dyn_cast<ConstantSDNode>(Val: BV0->getOperand(Num: I));
8913 auto *C1 = dyn_cast<ConstantSDNode>(Val: BV1->getOperand(Num: I));
8914 if (C0 && C1)
8915 MergedOps.push_back(Elt: DAG.getConstant(
8916 Val: C0->getAPIntValue() | C1->getAPIntValue(), DL, VT: EltVT));
8917 else if (C0 && C0->isZero())
8918 MergedOps.push_back(Elt: BV1->getOperand(Num: I));
8919 else if (C1 && C1->isZero())
8920 MergedOps.push_back(Elt: BV0->getOperand(Num: I));
8921 else if (C0 && C0->isAllOnes())
8922 MergedOps.push_back(Elt: BV0->getOperand(Num: I));
8923 else if (C1 && C1->isAllOnes())
8924 MergedOps.push_back(Elt: BV1->getOperand(Num: I));
8925 else if (BV0->getOperand(Num: I) == BV1->getOperand(Num: I))
8926 MergedOps.push_back(Elt: BV0->getOperand(Num: I));
8927 else
8928 break;
8929 }
8930 if (MergedOps.size() == NumElts)
8931 return DAG.getBuildVector(VT, DL, Ops: MergedOps);
8932 }
8933
8934 // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
8935 // Do this only if the resulting type / shuffle is legal.
8936 auto *SV0 = dyn_cast<ShuffleVectorSDNode>(Val&: N0);
8937 auto *SV1 = dyn_cast<ShuffleVectorSDNode>(Val&: N1);
8938 if (SV0 && SV1 && TLI.isTypeLegal(VT)) {
8939 bool ZeroN00 = ISD::isBuildVectorAllZeros(N: N0.getOperand(i: 0).getNode());
8940 bool ZeroN01 = ISD::isBuildVectorAllZeros(N: N0.getOperand(i: 1).getNode());
8941 bool ZeroN10 = ISD::isBuildVectorAllZeros(N: N1.getOperand(i: 0).getNode());
8942 bool ZeroN11 = ISD::isBuildVectorAllZeros(N: N1.getOperand(i: 1).getNode());
8943 // Ensure both shuffles have a zero input.
8944 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
8945 assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!");
8946 assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!");
8947 bool CanFold = true;
8948 int NumElts = VT.getVectorNumElements();
8949 SmallVector<int, 4> Mask(NumElts, -1);
8950
8951 for (int i = 0; i != NumElts; ++i) {
8952 int M0 = SV0->getMaskElt(Idx: i);
8953 int M1 = SV1->getMaskElt(Idx: i);
8954
8955 // Determine if either index is pointing to a zero vector.
8956 bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
8957 bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
8958
8959 // If one element is zero and the otherside is undef, keep undef.
8960 // This also handles the case that both are undef.
8961 if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0))
8962 continue;
8963
8964 // Make sure only one of the elements is zero.
8965 if (M0Zero == M1Zero) {
8966 CanFold = false;
8967 break;
8968 }
8969
8970 assert((M0 >= 0 || M1 >= 0) && "Undef index!");
8971
8972 // We have a zero and non-zero element. If the non-zero came from
8973 // SV0 make the index a LHS index. If it came from SV1, make it
8974 // a RHS index. We need to mod by NumElts because we don't care
8975 // which operand it came from in the original shuffles.
8976 Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
8977 }
8978
8979 if (CanFold) {
8980 SDValue NewLHS = ZeroN00 ? N0.getOperand(i: 1) : N0.getOperand(i: 0);
8981 SDValue NewRHS = ZeroN10 ? N1.getOperand(i: 1) : N1.getOperand(i: 0);
8982 SDValue LegalShuffle =
8983 TLI.buildLegalVectorShuffle(VT, DL, N0: NewLHS, N1: NewRHS, Mask, DAG);
8984 if (LegalShuffle)
8985 return LegalShuffle;
8986 }
8987 }
8988 }
8989 }
8990
8991 // fold (or x, 0) -> x
8992 if (isNullConstant(V: N1))
8993 return N0;
8994
8995 // fold (or x, -1) -> -1
8996 if (isAllOnesConstant(V: N1))
8997 return N1;
8998
8999 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
9000 return NewSel;
9001
9002 // fold (or x, c) -> c iff (x & ~c) == 0
9003 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
9004 if (N1C && DAG.MaskedValueIsZero(Op: N0, Mask: ~N1C->getAPIntValue()))
9005 return N1;
9006
9007 if (SDValue R = foldAndOrOfSETCC(LogicOp: N, DAG))
9008 return R;
9009
9010 if (SDValue Combined = visitORLike(N0, N1, DL))
9011 return Combined;
9012
9013 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
9014 return Combined;
9015
9016 if (SDValue Combined = combineOrOfSetCCToUSUBOCarry(N, DAG, TLI))
9017 return Combined;
9018
9019 // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
9020 if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
9021 return BSwap;
9022 if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
9023 return BSwap;
9024
9025 // reassociate or
9026 if (SDValue ROR = reassociateOps(Opc: ISD::OR, DL, N0, N1, Flags: N->getFlags()))
9027 return ROR;
9028
9029 // Fold or(vecreduce(x), vecreduce(y)) -> vecreduce(or(x, y))
9030 if (SDValue SD =
9031 reassociateReduction(RedOpc: ISD::VECREDUCE_OR, Opc: ISD::OR, DL, VT, N0, N1))
9032 return SD;
9033
9034 // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
9035 // iff (c1 & c2) != 0 or c1/c2 are undef.
9036 auto MatchIntersect = [](ConstantSDNode *C1, ConstantSDNode *C2) {
9037 return !C1 || !C2 || C1->getAPIntValue().intersects(RHS: C2->getAPIntValue());
9038 };
9039 if (N0.getOpcode() == ISD::AND && N0->hasOneUse() &&
9040 ISD::matchBinaryPredicate(LHS: N0.getOperand(i: 1), RHS: N1, Match: MatchIntersect, AllowUndefs: true)) {
9041 if (SDValue COR = DAG.FoldConstantArithmetic(Opcode: ISD::OR, DL: SDLoc(N1), VT,
9042 Ops: {N1, N0.getOperand(i: 1)})) {
9043 SDValue IOR = DAG.getNode(Opcode: ISD::OR, DL: SDLoc(N0), VT, N1: N0.getOperand(i: 0), N2: N1);
9044 AddToWorklist(N: IOR.getNode());
9045 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: COR, N2: IOR);
9046 }
9047 }
9048
9049 if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
9050 return Combined;
9051 if (SDValue Combined = visitORCommutative(DAG, N0: N1, N1: N0, N))
9052 return Combined;
9053
9054 // Simplify: (or (op x...), (op y...)) -> (op (or x, y))
9055 if (N0.getOpcode() == N1.getOpcode())
9056 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
9057 return V;
9058
9059 // See if this is some rotate idiom.
9060 if (SDValue Rot = MatchRotate(LHS: N0, RHS: N1, DL, /*FromAdd=*/false))
9061 return Rot;
9062
9063 if (SDValue Load = MatchLoadCombine(N))
9064 return Load;
9065
9066 // Simplify the operands using demanded-bits information.
9067 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
9068 return SDValue(N, 0);
9069
9070 // If OR can be rewritten into ADD, try combines based on ADD.
9071 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::ADD, VT)) &&
9072 DAG.isADDLike(Op: SDValue(N, 0)))
9073 if (SDValue Combined = visitADDLike(N))
9074 return Combined;
9075
9076 // Postpone until legalization completed to avoid interference with bswap
9077 // folding
9078 if (LegalOperations || VT.isVector())
9079 if (SDValue R = foldLogicTreeOfShifts(N, LeftHand: N0, RightHand: N1, DAG))
9080 return R;
9081
9082 if (VT.isScalarInteger() && VT != MVT::i1)
9083 if (SDValue R = foldMaskedMerge(Node: N, DAG, TLI, DL))
9084 return R;
9085
9086 return SDValue();
9087}
9088
9089static SDValue stripConstantMask(const SelectionDAG &DAG, SDValue Op,
9090 SDValue &Mask) {
9091 if (Op.getOpcode() == ISD::AND &&
9092 DAG.isConstantIntBuildVectorOrConstantInt(N: Op.getOperand(i: 1))) {
9093 Mask = Op.getOperand(i: 1);
9094 return Op.getOperand(i: 0);
9095 }
9096 return Op;
9097}
9098
9099/// Match "(X shl/srl V1) & V2" where V2 may not be present.
9100static bool matchRotateHalf(const SelectionDAG &DAG, SDValue Op, SDValue &Shift,
9101 SDValue &Mask) {
9102 Op = stripConstantMask(DAG, Op, Mask);
9103 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
9104 Shift = Op;
9105 return true;
9106 }
9107 return false;
9108}
9109
9110/// Helper function for visitOR to extract the needed side of a rotate idiom
9111/// from a shl/srl/mul/udiv. This is meant to handle cases where
9112/// InstCombine merged some outside op with one of the shifts from
9113/// the rotate pattern.
9114/// \returns An empty \c SDValue if the needed shift couldn't be extracted.
9115/// Otherwise, returns an expansion of \p ExtractFrom based on the following
9116/// patterns:
9117///
9118/// (or (add v v) (shrl v bitwidth-1)):
9119/// expands (add v v) -> (shl v 1)
9120///
9121/// (or (mul v c0) (shrl (mul v c1) c2)):
9122/// expands (mul v c0) -> (shl (mul v c1) c3)
9123///
9124/// (or (udiv v c0) (shl (udiv v c1) c2)):
9125/// expands (udiv v c0) -> (shrl (udiv v c1) c3)
9126///
9127/// (or (shl v c0) (shrl (shl v c1) c2)):
9128/// expands (shl v c0) -> (shl (shl v c1) c3)
9129///
9130/// (or (shrl v c0) (shl (shrl v c1) c2)):
9131/// expands (shrl v c0) -> (shrl (shrl v c1) c3)
9132///
9133/// Such that in all cases, c3+c2==bitwidth(op v c1).
9134static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
9135 SDValue ExtractFrom, SDValue &Mask,
9136 const SDLoc &DL) {
9137 assert(OppShift && ExtractFrom && "Empty SDValue");
9138 if (OppShift.getOpcode() != ISD::SHL && OppShift.getOpcode() != ISD::SRL)
9139 return SDValue();
9140
9141 ExtractFrom = stripConstantMask(DAG, Op: ExtractFrom, Mask);
9142
9143 // Value and Type of the shift.
9144 SDValue OppShiftLHS = OppShift.getOperand(i: 0);
9145 EVT ShiftedVT = OppShiftLHS.getValueType();
9146
9147 // Amount of the existing shift.
9148 ConstantSDNode *OppShiftCst = isConstOrConstSplat(N: OppShift.getOperand(i: 1));
9149
9150 // (add v v) -> (shl v 1)
9151 // TODO: Should this be a general DAG canonicalization?
9152 if (OppShift.getOpcode() == ISD::SRL && OppShiftCst &&
9153 ExtractFrom.getOpcode() == ISD::ADD &&
9154 ExtractFrom.getOperand(i: 0) == ExtractFrom.getOperand(i: 1) &&
9155 ExtractFrom.getOperand(i: 0) == OppShiftLHS &&
9156 OppShiftCst->getAPIntValue() == ShiftedVT.getScalarSizeInBits() - 1)
9157 return DAG.getNode(Opcode: ISD::SHL, DL, VT: ShiftedVT, N1: OppShiftLHS,
9158 N2: DAG.getShiftAmountConstant(Val: 1, VT: ShiftedVT, DL));
9159
9160 // Preconditions:
9161 // (or (op0 v c0) (shiftl/r (op0 v c1) c2))
9162 //
9163 // Find opcode of the needed shift to be extracted from (op0 v c0).
9164 unsigned Opcode = ISD::DELETED_NODE;
9165 bool IsMulOrDiv = false;
9166 // Set Opcode and IsMulOrDiv if the extract opcode matches the needed shift
9167 // opcode or its arithmetic (mul or udiv) variant.
9168 auto SelectOpcode = [&](unsigned NeededShift, unsigned MulOrDivVariant) {
9169 IsMulOrDiv = ExtractFrom.getOpcode() == MulOrDivVariant;
9170 if (!IsMulOrDiv && ExtractFrom.getOpcode() != NeededShift)
9171 return false;
9172 Opcode = NeededShift;
9173 return true;
9174 };
9175 // op0 must be either the needed shift opcode or the mul/udiv equivalent
9176 // that the needed shift can be extracted from.
9177 if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) &&
9178 (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV)))
9179 return SDValue();
9180
9181 // op0 must be the same opcode on both sides, have the same LHS argument,
9182 // and produce the same value type.
9183 if (OppShiftLHS.getOpcode() != ExtractFrom.getOpcode() ||
9184 OppShiftLHS.getOperand(i: 0) != ExtractFrom.getOperand(i: 0) ||
9185 ShiftedVT != ExtractFrom.getValueType())
9186 return SDValue();
9187
9188 // Constant mul/udiv/shift amount from the RHS of the shift's LHS op.
9189 ConstantSDNode *OppLHSCst = isConstOrConstSplat(N: OppShiftLHS.getOperand(i: 1));
9190 // Constant mul/udiv/shift amount from the RHS of the ExtractFrom op.
9191 ConstantSDNode *ExtractFromCst =
9192 isConstOrConstSplat(N: ExtractFrom.getOperand(i: 1));
9193 // TODO: We should be able to handle non-uniform constant vectors for these values
9194 // Check that we have constant values.
9195 if (!OppShiftCst || !OppShiftCst->getAPIntValue() ||
9196 !OppLHSCst || !OppLHSCst->getAPIntValue() ||
9197 !ExtractFromCst || !ExtractFromCst->getAPIntValue())
9198 return SDValue();
9199
9200 // Compute the shift amount we need to extract to complete the rotate.
9201 const unsigned VTWidth = ShiftedVT.getScalarSizeInBits();
9202 if (OppShiftCst->getAPIntValue().ugt(RHS: VTWidth))
9203 return SDValue();
9204 APInt NeededShiftAmt = VTWidth - OppShiftCst->getAPIntValue();
9205 // Normalize the bitwidth of the two mul/udiv/shift constant operands.
9206 APInt ExtractFromAmt = ExtractFromCst->getAPIntValue();
9207 APInt OppLHSAmt = OppLHSCst->getAPIntValue();
9208 zeroExtendToMatch(LHS&: ExtractFromAmt, RHS&: OppLHSAmt);
9209
9210 // Now try extract the needed shift from the ExtractFrom op and see if the
9211 // result matches up with the existing shift's LHS op.
9212 if (IsMulOrDiv) {
9213 // Op to extract from is a mul or udiv by a constant.
9214 // Check:
9215 // c2 / (1 << (bitwidth(op0 v c0) - c1)) == c0
9216 // c2 % (1 << (bitwidth(op0 v c0) - c1)) == 0
9217 const APInt ExtractDiv = APInt::getOneBitSet(numBits: ExtractFromAmt.getBitWidth(),
9218 BitNo: NeededShiftAmt.getZExtValue());
9219 APInt ResultAmt;
9220 APInt Rem;
9221 APInt::udivrem(LHS: ExtractFromAmt, RHS: ExtractDiv, Quotient&: ResultAmt, Remainder&: Rem);
9222 if (Rem != 0 || ResultAmt != OppLHSAmt)
9223 return SDValue();
9224 } else {
9225 // Op to extract from is a shift by a constant.
9226 // Check:
9227 // c2 - (bitwidth(op0 v c0) - c1) == c0
9228 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.zextOrTrunc(
9229 width: ExtractFromAmt.getBitWidth()))
9230 return SDValue();
9231 }
9232
9233 // Return the expanded shift op that should allow a rotate to be formed.
9234 EVT ShiftVT = OppShift.getOperand(i: 1).getValueType();
9235 EVT ResVT = ExtractFrom.getValueType();
9236 SDValue NewShiftNode = DAG.getConstant(Val: NeededShiftAmt, DL, VT: ShiftVT);
9237 return DAG.getNode(Opcode, DL, VT: ResVT, N1: OppShiftLHS, N2: NewShiftNode);
9238}
9239
9240// Return true if we can prove that, whenever Neg and Pos are both in the
9241// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos). This means that
9242// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
9243//
9244// (or (shift1 X, Neg), (shift2 X, Pos))
9245//
9246// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
9247// in direction shift1 by Neg. The range [0, EltSize) means that we only need
9248// to consider shift amounts with defined behavior.
9249//
9250// The IsRotate flag should be set when the LHS of both shifts is the same.
9251// Otherwise if matching a general funnel shift, it should be clear.
9252static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
9253 SelectionDAG &DAG, bool IsRotate, bool FromAdd) {
9254 const auto &TLI = DAG.getTargetLoweringInfo();
9255 // If EltSize is a power of 2 then:
9256 //
9257 // (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
9258 // (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
9259 //
9260 // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
9261 // for the stronger condition:
9262 //
9263 // Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1) [A]
9264 //
9265 // for all Neg and Pos. Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
9266 // we can just replace Neg with Neg' for the rest of the function.
9267 //
9268 // In other cases we check for the even stronger condition:
9269 //
9270 // Neg == EltSize - Pos [B]
9271 //
9272 // for all Neg and Pos. Note that the (or ...) then invokes undefined
9273 // behavior if Pos == 0 (and consequently Neg == EltSize).
9274 //
9275 // We could actually use [A] whenever EltSize is a power of 2, but the
9276 // only extra cases that it would match are those uninteresting ones
9277 // where Neg and Pos are never in range at the same time. E.g. for
9278 // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
9279 // as well as (sub 32, Pos), but:
9280 //
9281 // (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
9282 //
9283 // always invokes undefined behavior for 32-bit X.
9284 //
9285 // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
9286 // This allows us to peek through any operations that only affect Mask's
9287 // un-demanded bits.
9288 //
9289 // NOTE: We can only do this when matching operations which won't modify the
9290 // least Log2(EltSize) significant bits and not a general funnel shift.
9291 unsigned MaskLoBits = 0;
9292 if (IsRotate && !FromAdd && isPowerOf2_64(Value: EltSize)) {
9293 unsigned Bits = Log2_64(Value: EltSize);
9294 unsigned NegBits = Neg.getScalarValueSizeInBits();
9295 if (NegBits >= Bits) {
9296 APInt DemandedBits = APInt::getLowBitsSet(numBits: NegBits, loBitsSet: Bits);
9297 if (SDValue Inner =
9298 TLI.SimplifyMultipleUseDemandedBits(Op: Neg, DemandedBits, DAG)) {
9299 Neg = Inner;
9300 MaskLoBits = Bits;
9301 }
9302 }
9303 }
9304
9305 // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
9306 if (Neg.getOpcode() != ISD::SUB)
9307 return false;
9308 ConstantSDNode *NegC = isConstOrConstSplat(N: Neg.getOperand(i: 0));
9309 if (!NegC)
9310 return false;
9311 SDValue NegOp1 = Neg.getOperand(i: 1);
9312
9313 // On the RHS of [A], if Pos is the result of operation on Pos' that won't
9314 // affect Mask's demanded bits, just replace Pos with Pos'. These operations
9315 // are redundant for the purpose of the equality.
9316 if (MaskLoBits) {
9317 unsigned PosBits = Pos.getScalarValueSizeInBits();
9318 if (PosBits >= MaskLoBits) {
9319 APInt DemandedBits = APInt::getLowBitsSet(numBits: PosBits, loBitsSet: MaskLoBits);
9320 if (SDValue Inner =
9321 TLI.SimplifyMultipleUseDemandedBits(Op: Pos, DemandedBits, DAG)) {
9322 Pos = Inner;
9323 }
9324 }
9325 }
9326
9327 // The condition we need is now:
9328 //
9329 // (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
9330 //
9331 // If NegOp1 == Pos then we need:
9332 //
9333 // EltSize & Mask == NegC & Mask
9334 //
9335 // (because "x & Mask" is a truncation and distributes through subtraction).
9336 //
9337 // We also need to account for a potential truncation of NegOp1 if the amount
9338 // has already been legalized to a shift amount type.
9339 APInt Width;
9340 if ((Pos == NegOp1) ||
9341 (NegOp1.getOpcode() == ISD::TRUNCATE && Pos == NegOp1.getOperand(i: 0)))
9342 Width = NegC->getAPIntValue();
9343
9344 // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
9345 // Then the condition we want to prove becomes:
9346 //
9347 // (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
9348 //
9349 // which, again because "x & Mask" is a truncation, becomes:
9350 //
9351 // NegC & Mask == (EltSize - PosC) & Mask
9352 // EltSize & Mask == (NegC + PosC) & Mask
9353 else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(i: 0) == NegOp1) {
9354 if (ConstantSDNode *PosC = isConstOrConstSplat(N: Pos.getOperand(i: 1)))
9355 Width = PosC->getAPIntValue() + NegC->getAPIntValue();
9356 else
9357 return false;
9358 } else
9359 return false;
9360
9361 // Now we just need to check that EltSize & Mask == Width & Mask.
9362 if (MaskLoBits)
9363 // EltSize & Mask is 0 since Mask is EltSize - 1.
9364 return Width.getLoBits(numBits: MaskLoBits) == 0;
9365 return Width == EltSize;
9366}
9367
9368// A subroutine of MatchRotate used once we have found an OR of two opposite
9369// shifts of Shifted. If Neg == <operand size> - Pos then the OR reduces
9370// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
9371// former being preferred if supported. InnerPos and InnerNeg are Pos and
9372// Neg with outer conversions stripped away.
9373SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
9374 SDValue Neg, SDValue InnerPos,
9375 SDValue InnerNeg, bool FromAdd,
9376 bool HasPos, unsigned PosOpcode,
9377 unsigned NegOpcode, const SDLoc &DL) {
9378 // fold (or/add (shl x, (*ext y)),
9379 // (srl x, (*ext (sub 32, y)))) ->
9380 // (rotl x, y) or (rotr x, (sub 32, y))
9381 //
9382 // fold (or/add (shl x, (*ext (sub 32, y))),
9383 // (srl x, (*ext y))) ->
9384 // (rotr x, y) or (rotl x, (sub 32, y))
9385 EVT VT = Shifted.getValueType();
9386 if (matchRotateSub(Pos: InnerPos, Neg: InnerNeg, EltSize: VT.getScalarSizeInBits(), DAG,
9387 /*IsRotate*/ true, FromAdd))
9388 return DAG.getNode(Opcode: HasPos ? PosOpcode : NegOpcode, DL, VT, N1: Shifted,
9389 N2: HasPos ? Pos : Neg);
9390
9391 return SDValue();
9392}
9393
9394// A subroutine of MatchRotate used once we have found an OR of two opposite
9395// shifts of N0 + N1. If Neg == <operand size> - Pos then the OR reduces
9396// to both (PosOpcode N0, N1, Pos) and (NegOpcode N0, N1, Neg), with the
9397// former being preferred if supported. InnerPos and InnerNeg are Pos and
9398// Neg with outer conversions stripped away.
9399// TODO: Merge with MatchRotatePosNeg.
9400SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos,
9401 SDValue Neg, SDValue InnerPos,
9402 SDValue InnerNeg, bool FromAdd,
9403 bool HasPos, unsigned PosOpcode,
9404 unsigned NegOpcode, const SDLoc &DL) {
9405 EVT VT = N0.getValueType();
9406 unsigned EltBits = VT.getScalarSizeInBits();
9407
9408 // fold (or/add (shl x0, (*ext y)),
9409 // (srl x1, (*ext (sub 32, y)))) ->
9410 // (fshl x0, x1, y) or (fshr x0, x1, (sub 32, y))
9411 //
9412 // fold (or/add (shl x0, (*ext (sub 32, y))),
9413 // (srl x1, (*ext y))) ->
9414 // (fshr x0, x1, y) or (fshl x0, x1, (sub 32, y))
9415 if (matchRotateSub(Pos: InnerPos, Neg: InnerNeg, EltSize: EltBits, DAG, /*IsRotate*/ N0 == N1,
9416 FromAdd))
9417 return DAG.getNode(Opcode: HasPos ? PosOpcode : NegOpcode, DL, VT, N1: N0, N2: N1,
9418 N3: HasPos ? Pos : Neg);
9419
9420 // Matching the shift+xor cases, we can't easily use the xor'd shift amount
9421 // so for now just use the PosOpcode case if its legal.
9422 // TODO: When can we use the NegOpcode case?
9423 if (PosOpcode == ISD::FSHL && isPowerOf2_32(Value: EltBits)) {
9424 SDValue X;
9425 // fold (or/add (shl x0, y), (srl (srl x1, 1), (xor y, 31)))
9426 // -> (fshl x0, x1, y)
9427 if (sd_match(N: N1, P: m_Srl(L: m_Value(N&: X), R: m_One())) &&
9428 sd_match(N: InnerNeg,
9429 P: m_Xor(L: m_Specific(N: InnerPos), R: m_SpecificInt(V: EltBits - 1))) &&
9430 TLI.isOperationLegalOrCustom(Op: ISD::FSHL, VT)) {
9431 return DAG.getNode(Opcode: ISD::FSHL, DL, VT, N1: N0, N2: X, N3: Pos);
9432 }
9433
9434 // fold (or/add (shl (shl x0, 1), (xor y, 31)), (srl x1, y))
9435 // -> (fshr x0, x1, y)
9436 if (sd_match(N: N0, P: m_Shl(L: m_Value(N&: X), R: m_One())) &&
9437 sd_match(N: InnerPos,
9438 P: m_Xor(L: m_Specific(N: InnerNeg), R: m_SpecificInt(V: EltBits - 1))) &&
9439 TLI.isOperationLegalOrCustom(Op: ISD::FSHR, VT)) {
9440 return DAG.getNode(Opcode: ISD::FSHR, DL, VT, N1: X, N2: N1, N3: Neg);
9441 }
9442
9443 // fold (or/add (shl (add x0, x0), (xor y, 31)), (srl x1, y))
9444 // -> (fshr x0, x1, y)
9445 // TODO: Should add(x,x) -> shl(x,1) be a general DAG canonicalization?
9446 if (sd_match(N: N0, P: m_Add(L: m_Value(N&: X), R: m_Deferred(V&: X))) &&
9447 sd_match(N: InnerPos,
9448 P: m_Xor(L: m_Specific(N: InnerNeg), R: m_SpecificInt(V: EltBits - 1))) &&
9449 TLI.isOperationLegalOrCustom(Op: ISD::FSHR, VT)) {
9450 return DAG.getNode(Opcode: ISD::FSHR, DL, VT, N1: X, N2: N1, N3: Neg);
9451 }
9452 }
9453
9454 return SDValue();
9455}
9456
9457// MatchRotate - Handle an 'or' or 'add' of two operands. If this is one of the
9458// many idioms for rotate, and if the target supports rotation instructions,
9459// generate a rot[lr]. This also matches funnel shift patterns, similar to
9460// rotation but with different shifted sources.
9461SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
9462 bool FromAdd) {
9463 EVT VT = LHS.getValueType();
9464
9465 // The target must have at least one rotate/funnel flavor.
9466 // We still try to match rotate by constant pre-legalization.
9467 // TODO: Support pre-legalization funnel-shift by constant.
9468 bool HasROTL = hasOperation(Opcode: ISD::ROTL, VT);
9469 bool HasROTR = hasOperation(Opcode: ISD::ROTR, VT);
9470 bool HasFSHL = hasOperation(Opcode: ISD::FSHL, VT);
9471 bool HasFSHR = hasOperation(Opcode: ISD::FSHR, VT);
9472
9473 // If the type is going to be promoted and the target has enabled custom
9474 // lowering for rotate, allow matching rotate by non-constants. Only allow
9475 // this for scalar types.
9476 if (VT.isScalarInteger() && TLI.getTypeAction(Context&: *DAG.getContext(), VT) ==
9477 TargetLowering::TypePromoteInteger) {
9478 HasROTL |= TLI.getOperationAction(Op: ISD::ROTL, VT) == TargetLowering::Custom;
9479 HasROTR |= TLI.getOperationAction(Op: ISD::ROTR, VT) == TargetLowering::Custom;
9480 }
9481
9482 if (LegalOperations && !HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
9483 return SDValue();
9484
9485 // Check for truncated rotate.
9486 if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
9487 LHS.getOperand(i: 0).getValueType() == RHS.getOperand(i: 0).getValueType()) {
9488 assert(LHS.getValueType() == RHS.getValueType());
9489 if (SDValue Rot =
9490 MatchRotate(LHS: LHS.getOperand(i: 0), RHS: RHS.getOperand(i: 0), DL, FromAdd))
9491 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(LHS), VT: LHS.getValueType(), Operand: Rot);
9492 }
9493
9494 // Match "(X shl/srl V1) & V2" where V2 may not be present.
9495 SDValue LHSShift; // The shift.
9496 SDValue LHSMask; // AND value if any.
9497 matchRotateHalf(DAG, Op: LHS, Shift&: LHSShift, Mask&: LHSMask);
9498
9499 SDValue RHSShift; // The shift.
9500 SDValue RHSMask; // AND value if any.
9501 matchRotateHalf(DAG, Op: RHS, Shift&: RHSShift, Mask&: RHSMask);
9502
9503 // If neither side matched a rotate half, bail
9504 if (!LHSShift && !RHSShift)
9505 return SDValue();
9506
9507 // InstCombine may have combined a constant shl, srl, mul, or udiv with one
9508 // side of the rotate, so try to handle that here. In all cases we need to
9509 // pass the matched shift from the opposite side to compute the opcode and
9510 // needed shift amount to extract. We still want to do this if both sides
9511 // matched a rotate half because one half may be a potential overshift that
9512 // can be broken down (ie if InstCombine merged two shl or srl ops into a
9513 // single one).
9514
9515 // Have LHS side of the rotate, try to extract the needed shift from the RHS.
9516 if (LHSShift)
9517 if (SDValue NewRHSShift =
9518 extractShiftForRotate(DAG, OppShift: LHSShift, ExtractFrom: RHS, Mask&: RHSMask, DL))
9519 RHSShift = NewRHSShift;
9520 // Have RHS side of the rotate, try to extract the needed shift from the LHS.
9521 if (RHSShift)
9522 if (SDValue NewLHSShift =
9523 extractShiftForRotate(DAG, OppShift: RHSShift, ExtractFrom: LHS, Mask&: LHSMask, DL))
9524 LHSShift = NewLHSShift;
9525
9526 // If a side is still missing, nothing else we can do.
9527 if (!RHSShift || !LHSShift)
9528 return SDValue();
9529
9530 // At this point we've matched or extracted a shift op on each side.
9531
9532 if (LHSShift.getOpcode() == RHSShift.getOpcode())
9533 return SDValue(); // Shifts must disagree.
9534
9535 // Canonicalize shl to left side in a shl/srl pair.
9536 if (RHSShift.getOpcode() == ISD::SHL) {
9537 std::swap(a&: LHS, b&: RHS);
9538 std::swap(a&: LHSShift, b&: RHSShift);
9539 std::swap(a&: LHSMask, b&: RHSMask);
9540 }
9541
9542 // Something has gone wrong - we've lost the shl/srl pair - bail.
9543 if (LHSShift.getOpcode() != ISD::SHL || RHSShift.getOpcode() != ISD::SRL)
9544 return SDValue();
9545
9546 unsigned EltSizeInBits = VT.getScalarSizeInBits();
9547 SDValue LHSShiftArg = LHSShift.getOperand(i: 0);
9548 SDValue LHSShiftAmt = LHSShift.getOperand(i: 1);
9549 SDValue RHSShiftArg = RHSShift.getOperand(i: 0);
9550 SDValue RHSShiftAmt = RHSShift.getOperand(i: 1);
9551
9552 auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
9553 ConstantSDNode *RHS) {
9554 return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
9555 };
9556
9557 auto ApplyMasks = [&](SDValue Res) {
9558 // If there is an AND of either shifted operand, apply it to the result.
9559 if (LHSMask.getNode() || RHSMask.getNode()) {
9560 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
9561 SDValue Mask = AllOnes;
9562
9563 if (LHSMask.getNode()) {
9564 SDValue RHSBits = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: AllOnes, N2: RHSShiftAmt);
9565 Mask = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mask,
9566 N2: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LHSMask, N2: RHSBits));
9567 }
9568 if (RHSMask.getNode()) {
9569 SDValue LHSBits = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: AllOnes, N2: LHSShiftAmt);
9570 Mask = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mask,
9571 N2: DAG.getNode(Opcode: ISD::OR, DL, VT, N1: RHSMask, N2: LHSBits));
9572 }
9573
9574 Res = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Res, N2: Mask);
9575 }
9576
9577 return Res;
9578 };
9579
9580 // TODO: Support pre-legalization funnel-shift by constant.
9581 bool IsRotate = LHSShiftArg == RHSShiftArg;
9582 if (!IsRotate && !(HasFSHL || HasFSHR)) {
9583 if (TLI.isTypeLegal(VT) && LHS.hasOneUse() && RHS.hasOneUse() &&
9584 ISD::matchBinaryPredicate(LHS: LHSShiftAmt, RHS: RHSShiftAmt, Match: MatchRotateSum)) {
9585 // Look for a disguised rotate by constant.
9586 // The common shifted operand X may be hidden inside another 'or'.
9587 SDValue X, Y;
9588 auto matchOr = [&X, &Y](SDValue Or, SDValue CommonOp) {
9589 if (!Or.hasOneUse() || Or.getOpcode() != ISD::OR)
9590 return false;
9591 if (CommonOp == Or.getOperand(i: 0)) {
9592 X = CommonOp;
9593 Y = Or.getOperand(i: 1);
9594 return true;
9595 }
9596 if (CommonOp == Or.getOperand(i: 1)) {
9597 X = CommonOp;
9598 Y = Or.getOperand(i: 0);
9599 return true;
9600 }
9601 return false;
9602 };
9603
9604 SDValue Res;
9605 if (matchOr(LHSShiftArg, RHSShiftArg)) {
9606 // (shl (X | Y), C1) | (srl X, C2) --> (rotl X, C1) | (shl Y, C1)
9607 SDValue RotX = DAG.getNode(Opcode: ISD::ROTL, DL, VT, N1: X, N2: LHSShiftAmt);
9608 SDValue ShlY = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Y, N2: LHSShiftAmt);
9609 Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: RotX, N2: ShlY);
9610 } else if (matchOr(RHSShiftArg, LHSShiftArg)) {
9611 // (shl X, C1) | (srl (X | Y), C2) --> (rotl X, C1) | (srl Y, C2)
9612 SDValue RotX = DAG.getNode(Opcode: ISD::ROTL, DL, VT, N1: X, N2: LHSShiftAmt);
9613 SDValue SrlY = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Y, N2: RHSShiftAmt);
9614 Res = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: RotX, N2: SrlY);
9615 } else {
9616 return SDValue();
9617 }
9618
9619 return ApplyMasks(Res);
9620 }
9621
9622 return SDValue(); // Requires funnel shift support.
9623 }
9624
9625 // fold (or/add (shl x, C1), (srl x, C2)) -> (rotl x, C1)
9626 // fold (or/add (shl x, C1), (srl x, C2)) -> (rotr x, C2)
9627 // fold (or/add (shl x, C1), (srl y, C2)) -> (fshl x, y, C1)
9628 // fold (or/add (shl x, C1), (srl y, C2)) -> (fshr x, y, C2)
9629 // iff C1+C2 == EltSizeInBits
9630 if (ISD::matchBinaryPredicate(LHS: LHSShiftAmt, RHS: RHSShiftAmt, Match: MatchRotateSum)) {
9631 SDValue Res;
9632 if (IsRotate && (HasROTL || HasROTR || !(HasFSHL || HasFSHR))) {
9633 bool UseROTL = !LegalOperations || HasROTL;
9634 Res = DAG.getNode(Opcode: UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, N1: LHSShiftArg,
9635 N2: UseROTL ? LHSShiftAmt : RHSShiftAmt);
9636 } else {
9637 bool UseFSHL = !LegalOperations || HasFSHL;
9638 Res = DAG.getNode(Opcode: UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, N1: LHSShiftArg,
9639 N2: RHSShiftArg, N3: UseFSHL ? LHSShiftAmt : RHSShiftAmt);
9640 }
9641
9642 return ApplyMasks(Res);
9643 }
9644
9645 // Even pre-legalization, we can't easily rotate/funnel-shift by a variable
9646 // shift.
9647 if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
9648 return SDValue();
9649
9650 // If there is a mask here, and we have a variable shift, we can't be sure
9651 // that we're masking out the right stuff.
9652 if (LHSMask.getNode() || RHSMask.getNode())
9653 return SDValue();
9654
9655 // If the shift amount is sign/zext/any-extended just peel it off.
9656 SDValue LExtOp0 = LHSShiftAmt;
9657 SDValue RExtOp0 = RHSShiftAmt;
9658 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
9659 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
9660 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
9661 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
9662 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
9663 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
9664 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
9665 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
9666 LExtOp0 = LHSShiftAmt.getOperand(i: 0);
9667 RExtOp0 = RHSShiftAmt.getOperand(i: 0);
9668 }
9669
9670 if (IsRotate && (HasROTL || HasROTR)) {
9671 if (SDValue TryL = MatchRotatePosNeg(Shifted: LHSShiftArg, Pos: LHSShiftAmt, Neg: RHSShiftAmt,
9672 InnerPos: LExtOp0, InnerNeg: RExtOp0, FromAdd, HasPos: HasROTL,
9673 PosOpcode: ISD::ROTL, NegOpcode: ISD::ROTR, DL))
9674 return TryL;
9675
9676 if (SDValue TryR = MatchRotatePosNeg(Shifted: RHSShiftArg, Pos: RHSShiftAmt, Neg: LHSShiftAmt,
9677 InnerPos: RExtOp0, InnerNeg: LExtOp0, FromAdd, HasPos: HasROTR,
9678 PosOpcode: ISD::ROTR, NegOpcode: ISD::ROTL, DL))
9679 return TryR;
9680 }
9681
9682 if (SDValue TryL = MatchFunnelPosNeg(N0: LHSShiftArg, N1: RHSShiftArg, Pos: LHSShiftAmt,
9683 Neg: RHSShiftAmt, InnerPos: LExtOp0, InnerNeg: RExtOp0, FromAdd,
9684 HasPos: HasFSHL, PosOpcode: ISD::FSHL, NegOpcode: ISD::FSHR, DL))
9685 return TryL;
9686
9687 if (SDValue TryR = MatchFunnelPosNeg(N0: LHSShiftArg, N1: RHSShiftArg, Pos: RHSShiftAmt,
9688 Neg: LHSShiftAmt, InnerPos: RExtOp0, InnerNeg: LExtOp0, FromAdd,
9689 HasPos: HasFSHR, PosOpcode: ISD::FSHR, NegOpcode: ISD::FSHL, DL))
9690 return TryR;
9691
9692 return SDValue();
9693}
9694
9695/// Recursively traverses the expression calculating the origin of the requested
9696/// byte of the given value. Returns std::nullopt if the provider can't be
9697/// calculated.
9698///
9699/// For all the values except the root of the expression, we verify that the
9700/// value has exactly one use and if not then return std::nullopt. This way if
9701/// the origin of the byte is returned it's guaranteed that the values which
9702/// contribute to the byte are not used outside of this expression.
9703
9704/// However, there is a special case when dealing with vector loads -- we allow
9705/// more than one use if the load is a vector type. Since the values that
9706/// contribute to the byte ultimately come from the ExtractVectorElements of the
9707/// Load, we don't care if the Load has uses other than ExtractVectorElements,
9708/// because those operations are independent from the pattern to be combined.
9709/// For vector loads, we simply care that the ByteProviders are adjacent
9710/// positions of the same vector, and their index matches the byte that is being
9711/// provided. This is captured by the \p VectorIndex algorithm. \p VectorIndex
9712/// is the index used in an ExtractVectorElement, and \p StartingIndex is the
9713/// byte position we are trying to provide for the LoadCombine. If these do
9714/// not match, then we can not combine the vector loads. \p Index uses the
9715/// byte position we are trying to provide for and is matched against the
9716/// shl and load size. The \p Index algorithm ensures the requested byte is
9717/// provided for by the pattern, and the pattern does not over provide bytes.
9718///
9719///
9720/// The supported LoadCombine pattern for vector loads is as follows
9721/// or
9722/// / \
9723/// or shl
9724/// / \ |
9725/// or shl zext
9726/// / \ | |
9727/// shl zext zext EVE*
9728/// | | | |
9729/// zext EVE* EVE* LOAD
9730/// | | |
9731/// EVE* LOAD LOAD
9732/// |
9733/// LOAD
9734///
9735/// *ExtractVectorElement
9736using SDByteProvider = ByteProvider<SDNode *>;
9737
9738static std::optional<SDByteProvider>
9739calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
9740 std::optional<uint64_t> VectorIndex,
9741 unsigned StartingIndex = 0,
9742 MutableArrayRef<uint8_t> ByteMask = {}) {
9743
9744 // Typical i64 by i8 pattern requires recursion up to 8 calls depth
9745 if (Depth == 10)
9746 return std::nullopt;
9747
9748 // Only allow multiple uses if the instruction is a vector load (in which
9749 // case we will use the load for every ExtractVectorElement)
9750 if (Depth && !Op.hasOneUse() &&
9751 (Op.getOpcode() != ISD::LOAD || !Op.getValueType().isVector()))
9752 return std::nullopt;
9753
9754 // Fail to combine if we have encountered anything but a LOAD after handling
9755 // an ExtractVectorElement.
9756 if (Op.getOpcode() != ISD::LOAD && VectorIndex.has_value())
9757 return std::nullopt;
9758
9759 unsigned BitWidth = Op.getScalarValueSizeInBits();
9760 if (BitWidth % 8 != 0)
9761 return std::nullopt;
9762 unsigned ByteWidth = BitWidth / 8;
9763 assert(Index < ByteWidth && "invalid index requested");
9764 (void) ByteWidth;
9765
9766 switch (Op.getOpcode()) {
9767 case ISD::OR: {
9768 auto LHS = calculateByteProvider(Op: Op->getOperand(Num: 0), Index, Depth: Depth + 1,
9769 VectorIndex, StartingIndex, ByteMask);
9770 if (!LHS)
9771 return std::nullopt;
9772 auto RHS = calculateByteProvider(Op: Op->getOperand(Num: 1), Index, Depth: Depth + 1,
9773 VectorIndex, StartingIndex, ByteMask);
9774 if (!RHS)
9775 return std::nullopt;
9776
9777 if (LHS->isConstantZero())
9778 return RHS;
9779 if (RHS->isConstantZero())
9780 return LHS;
9781 return std::nullopt;
9782 }
9783 case ISD::SHL: {
9784 auto ShiftOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
9785 if (!ShiftOp)
9786 return std::nullopt;
9787
9788 uint64_t BitShift = ShiftOp->getZExtValue();
9789
9790 if (BitShift % 8 != 0)
9791 return std::nullopt;
9792 uint64_t ByteShift = BitShift / 8;
9793
9794 // If we are shifting by an amount greater than the index we are trying to
9795 // provide, then do not provide anything. Otherwise, subtract the index by
9796 // the amount we shifted by.
9797 return Index < ByteShift
9798 ? SDByteProvider::getConstantZero()
9799 : calculateByteProvider(Op: Op->getOperand(Num: 0), Index: Index - ByteShift,
9800 Depth: Depth + 1, VectorIndex, StartingIndex: Index, ByteMask);
9801 }
9802 case ISD::ANY_EXTEND:
9803 case ISD::SIGN_EXTEND:
9804 case ISD::ZERO_EXTEND: {
9805 SDValue NarrowOp = Op->getOperand(Num: 0);
9806 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
9807 if (NarrowBitWidth % 8 != 0)
9808 return std::nullopt;
9809 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
9810
9811 if (Index >= NarrowByteWidth)
9812 return Op.getOpcode() == ISD::ZERO_EXTEND
9813 ? std::optional<SDByteProvider>(
9814 SDByteProvider::getConstantZero())
9815 : std::nullopt;
9816 return calculateByteProvider(Op: NarrowOp, Index, Depth: Depth + 1, VectorIndex,
9817 StartingIndex, ByteMask);
9818 }
9819 case ISD::BSWAP:
9820 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index: ByteWidth - Index - 1,
9821 Depth: Depth + 1, VectorIndex, StartingIndex,
9822 ByteMask);
9823 case ISD::AND: {
9824 // Constants are canonicalized to the RHS of AND, so only operand 1 needs
9825 // to be checked.
9826 auto *MaskOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
9827 if (!MaskOp)
9828 return std::nullopt;
9829
9830 uint8_t MaskByte =
9831 MaskOp->getAPIntValue().extractBitsAsZExtValue(numBits: 8, bitPosition: Index * 8);
9832
9833 if (MaskByte == 0x00)
9834 return SDByteProvider::getConstantZero();
9835
9836 auto Result = calculateByteProvider(Op: Op->getOperand(Num: 0), Index, Depth: Depth + 1,
9837 VectorIndex, StartingIndex, ByteMask);
9838 if (!Result)
9839 return std::nullopt;
9840
9841 // Only record the mask if this byte is actually provided (not zero).
9842 // A ConstantZero result may be discarded by the OR handler in favor of
9843 // the other operand, so writing the mask here would corrupt ByteMask.
9844 if (MaskByte != 0xFF && !ByteMask.empty() && !Result->isConstantZero())
9845 ByteMask[StartingIndex] &= MaskByte;
9846
9847 return Result;
9848 }
9849 case ISD::EXTRACT_VECTOR_ELT: {
9850 auto OffsetOp = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1));
9851 if (!OffsetOp)
9852 return std::nullopt;
9853
9854 VectorIndex = OffsetOp->getZExtValue();
9855
9856 SDValue NarrowOp = Op->getOperand(Num: 0);
9857 unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
9858 if (NarrowBitWidth % 8 != 0)
9859 return std::nullopt;
9860 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
9861 // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
9862 // type, leaving the high bits undefined.
9863 if (Index >= NarrowByteWidth)
9864 return std::nullopt;
9865
9866 // Check to see if the position of the element in the vector corresponds
9867 // with the byte we are trying to provide for. In the case of a vector of
9868 // i8, this simply means the VectorIndex == StartingIndex. For non i8 cases,
9869 // the element will provide a range of bytes. For example, if we have a
9870 // vector of i16s, each element provides two bytes (V[1] provides byte 2 and
9871 // 3).
9872 if (*VectorIndex * NarrowByteWidth > StartingIndex)
9873 return std::nullopt;
9874 if ((*VectorIndex + 1) * NarrowByteWidth <= StartingIndex)
9875 return std::nullopt;
9876
9877 return calculateByteProvider(Op: Op->getOperand(Num: 0), Index, Depth: Depth + 1,
9878 VectorIndex, StartingIndex, ByteMask);
9879 }
9880 case ISD::LOAD: {
9881 auto L = cast<LoadSDNode>(Val: Op.getNode());
9882 if (!L->isSimple() || L->isIndexed())
9883 return std::nullopt;
9884
9885 unsigned NarrowBitWidth = L->getMemoryVT().getScalarSizeInBits();
9886 if (NarrowBitWidth % 8 != 0)
9887 return std::nullopt;
9888 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
9889
9890 // If the width of the load does not reach byte we are trying to provide for
9891 // and it is not a ZEXTLOAD, then the load does not provide for the byte in
9892 // question
9893 if (Index >= NarrowByteWidth)
9894 return L->getExtensionType() == ISD::ZEXTLOAD
9895 ? std::optional<SDByteProvider>(
9896 SDByteProvider::getConstantZero())
9897 : std::nullopt;
9898
9899 unsigned BPVectorIndex = VectorIndex.value_or(u: 0U);
9900 return SDByteProvider::getSrc(Val: L, ByteOffset: Index, VectorOffset: BPVectorIndex);
9901 }
9902 }
9903
9904 return std::nullopt;
9905}
9906
9907static unsigned littleEndianByteAt(unsigned BW, unsigned i) {
9908 return i;
9909}
9910
9911static unsigned bigEndianByteAt(unsigned BW, unsigned i) {
9912 return BW - i - 1;
9913}
9914
9915// Check if the bytes offsets we are looking at match with either big or
9916// little endian value loaded. Return true for big endian, false for little
9917// endian, and std::nullopt if match failed.
9918static std::optional<bool> isBigEndian(ArrayRef<int64_t> ByteOffsets,
9919 int64_t FirstOffset) {
9920 // The endian can be decided only when it is 2 bytes at least.
9921 unsigned Width = ByteOffsets.size();
9922 if (Width < 2)
9923 return std::nullopt;
9924
9925 bool BigEndian = true, LittleEndian = true;
9926 for (unsigned i = 0; i < Width; i++) {
9927 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
9928 LittleEndian &= CurrentByteOffset == littleEndianByteAt(BW: Width, i);
9929 BigEndian &= CurrentByteOffset == bigEndianByteAt(BW: Width, i);
9930 if (!BigEndian && !LittleEndian)
9931 return std::nullopt;
9932 }
9933
9934 assert((BigEndian != LittleEndian) && "It should be either big endian or"
9935 "little endian");
9936 return BigEndian;
9937}
9938
9939// Look through one layer of truncate or extend.
9940static SDValue stripTruncAndExt(SDValue Value) {
9941 switch (Value.getOpcode()) {
9942 case ISD::TRUNCATE:
9943 case ISD::ZERO_EXTEND:
9944 case ISD::SIGN_EXTEND:
9945 case ISD::ANY_EXTEND:
9946 return Value.getOperand(i: 0);
9947 }
9948 return SDValue();
9949}
9950
9951/// Match a pattern where a wide type scalar value is stored by several narrow
9952/// stores. Fold it into a single store or a BSWAP and a store if the targets
9953/// supports it.
9954///
9955/// Assuming little endian target:
9956/// i8 *p = ...
9957/// i32 val = ...
9958/// p[0] = (val >> 0) & 0xFF;
9959/// p[1] = (val >> 8) & 0xFF;
9960/// p[2] = (val >> 16) & 0xFF;
9961/// p[3] = (val >> 24) & 0xFF;
9962/// =>
9963/// *((i32)p) = val;
9964///
9965/// i8 *p = ...
9966/// i32 val = ...
9967/// p[0] = (val >> 24) & 0xFF;
9968/// p[1] = (val >> 16) & 0xFF;
9969/// p[2] = (val >> 8) & 0xFF;
9970/// p[3] = (val >> 0) & 0xFF;
9971/// =>
9972/// *((i32)p) = BSWAP(val);
9973SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
9974 // The matching looks for "store (trunc x)" patterns that appear early but are
9975 // likely to be replaced by truncating store nodes during combining.
9976 // TODO: If there is evidence that running this later would help, this
9977 // limitation could be removed. Legality checks may need to be added
9978 // for the created store and optional bswap/rotate.
9979 if (LegalOperations || OptLevel == CodeGenOptLevel::None)
9980 return SDValue();
9981
9982 // We only handle merging simple stores of 1-4 bytes.
9983 // TODO: Allow unordered atomics when wider type is legal (see D66309)
9984 EVT MemVT = N->getMemoryVT();
9985 if (!(MemVT == MVT::i8 || MemVT == MVT::i16 || MemVT == MVT::i32) ||
9986 !N->isSimple() || N->isIndexed())
9987 return SDValue();
9988
9989 // Collect all of the stores in the chain, upto the maximum store width (i64).
9990 SDValue Chain = N->getChain();
9991 SmallVector<StoreSDNode *, 8> Stores = {N};
9992 unsigned NarrowNumBits = MemVT.getScalarSizeInBits();
9993 unsigned MaxWideNumBits = 64;
9994 unsigned MaxStores = MaxWideNumBits / NarrowNumBits;
9995 while (auto *Store = dyn_cast<StoreSDNode>(Val&: Chain)) {
9996 // All stores must be the same size to ensure that we are writing all of the
9997 // bytes in the wide value.
9998 // This store should have exactly one use as a chain operand for another
9999 // store in the merging set. If there are other chain uses, then the
10000 // transform may not be safe because order of loads/stores outside of this
10001 // set may not be preserved.
10002 // TODO: We could allow multiple sizes by tracking each stored byte.
10003 if (Store->getMemoryVT() != MemVT || !Store->isSimple() ||
10004 Store->isIndexed() || !Store->hasOneUse())
10005 return SDValue();
10006 Stores.push_back(Elt: Store);
10007 Chain = Store->getChain();
10008 if (MaxStores < Stores.size())
10009 return SDValue();
10010 }
10011 // There is no reason to continue if we do not have at least a pair of stores.
10012 if (Stores.size() < 2)
10013 return SDValue();
10014
10015 // Handle simple types only.
10016 LLVMContext &Context = *DAG.getContext();
10017 unsigned NumStores = Stores.size();
10018 unsigned WideNumBits = NumStores * NarrowNumBits;
10019 if (WideNumBits != 16 && WideNumBits != 32 && WideNumBits != 64)
10020 return SDValue();
10021
10022 // Check if all bytes of the source value that we are looking at are stored
10023 // to the same base address. Collect offsets from Base address into OffsetMap.
10024 SDValue SourceValue;
10025 SmallVector<int64_t, 8> OffsetMap(NumStores, INT64_MAX);
10026 int64_t FirstOffset = INT64_MAX;
10027 StoreSDNode *FirstStore = nullptr;
10028 std::optional<BaseIndexOffset> Base;
10029 for (auto *Store : Stores) {
10030 // All the stores store different parts of the CombinedValue. A truncate is
10031 // required to get the partial value.
10032 SDValue Trunc = Store->getValue();
10033 if (Trunc.getOpcode() != ISD::TRUNCATE)
10034 return SDValue();
10035 // Other than the first/last part, a shift operation is required to get the
10036 // offset.
10037 int64_t Offset = 0;
10038 SDValue WideVal = Trunc.getOperand(i: 0);
10039 if ((WideVal.getOpcode() == ISD::SRL || WideVal.getOpcode() == ISD::SRA) &&
10040 isa<ConstantSDNode>(Val: WideVal.getOperand(i: 1))) {
10041 // The shift amount must be a constant multiple of the narrow type.
10042 // It is translated to the offset address in the wide source value "y".
10043 //
10044 // x = srl y, ShiftAmtC
10045 // i8 z = trunc x
10046 // store z, ...
10047 uint64_t ShiftAmtC = WideVal.getConstantOperandVal(i: 1);
10048 if (ShiftAmtC % NarrowNumBits != 0)
10049 return SDValue();
10050
10051 // Make sure we aren't reading bits that are shifted in.
10052 if (ShiftAmtC > WideVal.getScalarValueSizeInBits() - NarrowNumBits)
10053 return SDValue();
10054
10055 Offset = ShiftAmtC / NarrowNumBits;
10056 WideVal = WideVal.getOperand(i: 0);
10057 }
10058
10059 // Stores must share the same source value with different offsets.
10060 if (!SourceValue)
10061 SourceValue = WideVal;
10062 else if (SourceValue != WideVal) {
10063 // Truncate and extends can be stripped to see if the values are related.
10064 if (stripTruncAndExt(Value: SourceValue) != WideVal &&
10065 stripTruncAndExt(Value: WideVal) != SourceValue)
10066 return SDValue();
10067
10068 if (WideVal.getScalarValueSizeInBits() >
10069 SourceValue.getScalarValueSizeInBits())
10070 SourceValue = WideVal;
10071
10072 // Give up if the source value type is smaller than the store size.
10073 if (SourceValue.getScalarValueSizeInBits() < WideNumBits)
10074 return SDValue();
10075 }
10076
10077 // Stores must share the same base address.
10078 BaseIndexOffset Ptr = BaseIndexOffset::match(N: Store, DAG);
10079 int64_t ByteOffsetFromBase = 0;
10080 if (!Base)
10081 Base = Ptr;
10082 else if (!Base->equalBaseIndex(Other: Ptr, DAG, Off&: ByteOffsetFromBase))
10083 return SDValue();
10084
10085 // Remember the first store.
10086 if (ByteOffsetFromBase < FirstOffset) {
10087 FirstStore = Store;
10088 FirstOffset = ByteOffsetFromBase;
10089 }
10090 // Map the offset in the store and the offset in the combined value, and
10091 // early return if it has been set before.
10092 if (Offset < 0 || Offset >= NumStores || OffsetMap[Offset] != INT64_MAX)
10093 return SDValue();
10094 OffsetMap[Offset] = ByteOffsetFromBase;
10095 }
10096
10097 EVT WideVT = EVT::getIntegerVT(Context, BitWidth: WideNumBits);
10098
10099 assert(FirstOffset != INT64_MAX && "First byte offset must be set");
10100 assert(FirstStore && "First store must be set");
10101
10102 // Check that a store of the wide type is both allowed and fast on the target
10103 const DataLayout &Layout = DAG.getDataLayout();
10104 unsigned Fast = 0;
10105 bool Allowed = TLI.allowsMemoryAccess(Context, DL: Layout, VT: WideVT,
10106 MMO: *FirstStore->getMemOperand(), Fast: &Fast);
10107 if (!Allowed || !Fast)
10108 return SDValue();
10109
10110 // Check if the pieces of the value are going to the expected places in memory
10111 // to merge the stores.
10112 auto checkOffsets = [&](bool MatchLittleEndian) {
10113 if (MatchLittleEndian) {
10114 for (unsigned i = 0; i != NumStores; ++i)
10115 if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
10116 return false;
10117 } else { // MatchBigEndian by reversing loop counter.
10118 for (unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --j)
10119 if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
10120 return false;
10121 }
10122 return true;
10123 };
10124
10125 // Check if the offsets line up for the native data layout of this target.
10126 bool NeedBswap = false;
10127 bool NeedRotate = false;
10128 if (!checkOffsets(Layout.isLittleEndian())) {
10129 // Special-case: check if byte offsets line up for the opposite endian.
10130 if (NarrowNumBits == 8 && checkOffsets(Layout.isBigEndian()))
10131 NeedBswap = true;
10132 else if (NumStores == 2 && checkOffsets(Layout.isBigEndian()))
10133 NeedRotate = true;
10134 else
10135 return SDValue();
10136 }
10137
10138 SDLoc DL(N);
10139 if (WideVT != SourceValue.getValueType()) {
10140 assert(SourceValue.getValueType().getScalarSizeInBits() > WideNumBits &&
10141 "Unexpected store value to merge");
10142 SourceValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: WideVT, Operand: SourceValue);
10143 }
10144
10145 // Before legalize we can introduce illegal bswaps/rotates which will be later
10146 // converted to an explicit bswap sequence. This way we end up with a single
10147 // store and byte shuffling instead of several stores and byte shuffling.
10148 if (NeedBswap) {
10149 SourceValue = DAG.getNode(Opcode: ISD::BSWAP, DL, VT: WideVT, Operand: SourceValue);
10150 } else if (NeedRotate) {
10151 assert(WideNumBits % 2 == 0 && "Unexpected type for rotate");
10152 SDValue RotAmt = DAG.getConstant(Val: WideNumBits / 2, DL, VT: WideVT);
10153 SourceValue = DAG.getNode(Opcode: ISD::ROTR, DL, VT: WideVT, N1: SourceValue, N2: RotAmt);
10154 }
10155
10156 SDValue NewStore =
10157 DAG.getStore(Chain, dl: DL, Val: SourceValue, Ptr: FirstStore->getBasePtr(),
10158 PtrInfo: FirstStore->getPointerInfo(), Alignment: FirstStore->getAlign());
10159
10160 // Rely on other DAG combine rules to remove the other individual stores.
10161 DAG.ReplaceAllUsesWith(From: N, To: NewStore.getNode());
10162 return NewStore;
10163}
10164
10165/// Match a pattern where a wide type scalar value is loaded by several narrow
10166/// loads and combined by shifts and ors. Fold it into a single load or a load
10167/// and a BSWAP if the targets supports it.
10168///
10169/// Assuming little endian target:
10170/// i8 *a = ...
10171/// i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
10172/// =>
10173/// i32 val = *((i32)a)
10174///
10175/// i8 *a = ...
10176/// i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
10177/// =>
10178/// i32 val = BSWAP(*((i32)a))
10179///
10180/// TODO: This rule matches complex patterns with OR node roots and doesn't
10181/// interact well with the worklist mechanism. When a part of the pattern is
10182/// updated (e.g. one of the loads) its direct users are put into the worklist,
10183/// but the root node of the pattern which triggers the load combine is not
10184/// necessarily a direct user of the changed node. For example, once the address
10185/// of t28 load is reassociated load combine won't be triggered:
10186/// t25: i32 = add t4, Constant:i32<2>
10187/// t26: i64 = sign_extend t25
10188/// t27: i64 = add t2, t26
10189/// t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
10190/// t29: i32 = zero_extend t28
10191/// t32: i32 = shl t29, Constant:i8<8>
10192/// t33: i32 = or t23, t32
10193/// As a possible fix visitLoad can check if the load can be a part of a load
10194/// combine pattern and add corresponding OR roots to the worklist.
10195SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
10196 assert(N->getOpcode() == ISD::OR &&
10197 "Can only match load combining against OR nodes");
10198
10199 // Handles simple types only
10200 EVT VT = N->getValueType(ResNo: 0);
10201 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
10202 return SDValue();
10203 unsigned ByteWidth = VT.getSizeInBits() / 8;
10204
10205 bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
10206 auto MemoryByteOffset = [&](SDByteProvider P) {
10207 assert(P.hasSrc() && "Must be a memory byte provider");
10208 auto *Load = cast<LoadSDNode>(Val: P.Src.value());
10209
10210 unsigned LoadBitWidth = Load->getMemoryVT().getScalarSizeInBits();
10211
10212 assert(LoadBitWidth % 8 == 0 &&
10213 "can only analyze providers for individual bytes not bit");
10214 unsigned LoadByteWidth = LoadBitWidth / 8;
10215 return IsBigEndianTarget ? bigEndianByteAt(BW: LoadByteWidth, i: P.DestOffset)
10216 : littleEndianByteAt(BW: LoadByteWidth, i: P.DestOffset);
10217 };
10218
10219 std::optional<BaseIndexOffset> Base;
10220 SDValue Chain;
10221
10222 SmallPtrSet<LoadSDNode *, 8> Loads;
10223 std::optional<SDByteProvider> FirstByteProvider;
10224 int64_t FirstOffset = INT64_MAX;
10225
10226 // Check if all the bytes of the OR we are looking at are loaded from the same
10227 // base address. Collect bytes offsets from Base address in ByteOffsets.
10228 SmallVector<int64_t, 8> ByteOffsets(ByteWidth);
10229 SmallVector<uint8_t, 8> ByteMasks(ByteWidth, 0xFF);
10230 unsigned ZeroExtendedBytes = 0;
10231 for (int i = ByteWidth - 1; i >= 0; --i) {
10232 auto P =
10233 calculateByteProvider(Op: SDValue(N, 0), Index: i, Depth: 0, /*VectorIndex*/ std::nullopt,
10234 /*StartingIndex*/ i, ByteMask: ByteMasks);
10235 if (!P)
10236 return SDValue();
10237
10238 if (P->isConstantZero()) {
10239 // It's OK for the N most significant bytes to be 0, we can just
10240 // zero-extend the load.
10241 if (++ZeroExtendedBytes != (ByteWidth - static_cast<unsigned>(i)))
10242 return SDValue();
10243 continue;
10244 }
10245 assert(P->hasSrc() && "provenance should either be memory or zero");
10246 auto *L = cast<LoadSDNode>(Val: P->Src.value());
10247
10248 // All loads must share the same chain
10249 SDValue LChain = L->getChain();
10250 if (!Chain)
10251 Chain = LChain;
10252 else if (Chain != LChain)
10253 return SDValue();
10254
10255 // Loads must share the same base address
10256 BaseIndexOffset Ptr = BaseIndexOffset::match(N: L, DAG);
10257 int64_t ByteOffsetFromBase = 0;
10258
10259 // For vector loads, the expected load combine pattern will have an
10260 // ExtractElement for each index in the vector. While each of these
10261 // ExtractElements will be accessing the same base address as determined
10262 // by the load instruction, the actual bytes they interact with will differ
10263 // due to different ExtractElement indices. To accurately determine the
10264 // byte position of an ExtractElement, we offset the base load ptr with
10265 // the index multiplied by the byte size of each element in the vector.
10266 if (L->getMemoryVT().isVector()) {
10267 unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits();
10268 if (LoadWidthInBit % 8 != 0)
10269 return SDValue();
10270 unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
10271 Ptr.addToOffset(VectorOff: ByteOffsetFromVector);
10272 }
10273
10274 if (!Base)
10275 Base = Ptr;
10276
10277 else if (!Base->equalBaseIndex(Other: Ptr, DAG, Off&: ByteOffsetFromBase))
10278 return SDValue();
10279
10280 // Calculate the offset of the current byte from the base address
10281 ByteOffsetFromBase += MemoryByteOffset(*P);
10282 ByteOffsets[i] = ByteOffsetFromBase;
10283
10284 // Remember the first byte load
10285 if (ByteOffsetFromBase < FirstOffset) {
10286 FirstByteProvider = P;
10287 FirstOffset = ByteOffsetFromBase;
10288 }
10289
10290 Loads.insert(Ptr: L);
10291 }
10292
10293 assert(!Loads.empty() && "All the bytes of the value must be loaded from "
10294 "memory, so there must be at least one load which produces the value");
10295 assert(Base && "Base address of the accessed memory location must be set");
10296 assert(FirstOffset != INT64_MAX && "First byte offset must be set");
10297
10298 bool NeedsZext = ZeroExtendedBytes > 0;
10299
10300 EVT MemVT =
10301 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: (ByteWidth - ZeroExtendedBytes) * 8);
10302
10303 if (!MemVT.isSimple())
10304 return SDValue();
10305
10306 // Check if the bytes of the OR we are looking at match with either big or
10307 // little endian value load
10308 std::optional<bool> IsBigEndian = isBigEndian(
10309 ByteOffsets: ArrayRef(ByteOffsets).drop_back(N: ZeroExtendedBytes), FirstOffset);
10310 if (!IsBigEndian)
10311 return SDValue();
10312
10313 assert(FirstByteProvider && "must be set");
10314
10315 // Ensure that the first byte is loaded from zero offset of the first load.
10316 // So the combined value can be loaded from the first load address.
10317 if (MemoryByteOffset(*FirstByteProvider) != 0)
10318 return SDValue();
10319 auto *FirstLoad = cast<LoadSDNode>(Val: FirstByteProvider->Src.value());
10320
10321 // Before legalization we allow introducing loads that are wider than legal,
10322 // which will later be split into legally sized loads. This enables us to
10323 // combine, for example, i8 loads forming an i64 into an i64 load, which get
10324 // then gets split up into couple of i32 loads on 32 bit targets.
10325 if (LegalOperations &&
10326 !TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: FirstLoad->getAlign(),
10327 AddrSpace: FirstLoad->getAddressSpace(),
10328 ExtType: NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, Atomic: false))
10329 return SDValue();
10330
10331 // The node we are looking at matches with the pattern, check if we can
10332 // replace it with a single (possibly zero-extended) load and bswap + shift if
10333 // needed.
10334
10335 // If the load needs byte swap check if the target supports it
10336 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
10337
10338 // Before legalize we can introduce illegal bswaps which will be later
10339 // converted to an explicit bswap sequence. This way we end up with a single
10340 // load and byte shuffling instead of several loads and byte shuffling.
10341 // We do not introduce illegal bswaps when zero-extending as this tends to
10342 // introduce too many arithmetic instructions.
10343 if (NeedsBswap && (LegalOperations || NeedsZext) &&
10344 !TLI.isOperationLegal(Op: ISD::BSWAP, VT))
10345 return SDValue();
10346
10347 // If we need to bswap and zero extend, we have to insert a shift. Check that
10348 // it is legal.
10349 if (NeedsBswap && NeedsZext && LegalOperations &&
10350 !TLI.isOperationLegal(Op: ISD::SHL, VT))
10351 return SDValue();
10352
10353 // Check that a load of the wide type is both allowed and fast on the target
10354 unsigned Fast = 0;
10355 bool Allowed =
10356 TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: MemVT,
10357 MMO: *FirstLoad->getMemOperand(), Fast: &Fast);
10358 if (!Allowed || !Fast)
10359 return SDValue();
10360
10361 SDValue NewLoad =
10362 DAG.getExtLoad(ExtType: NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, dl: SDLoc(N), VT,
10363 Chain, Ptr: FirstLoad->getBasePtr(),
10364 PtrInfo: FirstLoad->getPointerInfo(), MemVT, Alignment: FirstLoad->getAlign());
10365
10366 // Transfer chain users from old loads to the new load.
10367 for (LoadSDNode *L : Loads)
10368 DAG.makeEquivalentMemoryOrdering(OldLoad: L, NewMemOp: NewLoad);
10369
10370 // Apply combined mask if any bytes were partially masked by AND operations.
10371 bool HasPartialMask = false;
10372 uint64_t CombinedMask = 0;
10373 for (unsigned i = 0; i < ByteWidth; ++i) {
10374 CombinedMask |= (uint64_t)ByteMasks[i] << (i * 8);
10375 if (ByteMasks[i] != 0xFF)
10376 HasPartialMask = true;
10377 }
10378
10379 if (!NeedsBswap) {
10380 if (HasPartialMask)
10381 NewLoad = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT, N1: NewLoad,
10382 N2: DAG.getConstant(Val: CombinedMask, DL: SDLoc(N), VT));
10383 return NewLoad;
10384 }
10385
10386 SDValue ShiftedLoad =
10387 NeedsZext ? DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(N), VT, N1: NewLoad,
10388 N2: DAG.getShiftAmountConstant(Val: ZeroExtendedBytes * 8,
10389 VT, DL: SDLoc(N)))
10390 : NewLoad;
10391 SDValue Result = DAG.getNode(Opcode: ISD::BSWAP, DL: SDLoc(N), VT, Operand: ShiftedLoad);
10392
10393 // The mask is built in final-result byte order (ByteMasks[i] corresponds to
10394 // byte i of the result), so it is correct to apply after the bswap.
10395 if (HasPartialMask)
10396 Result = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT, N1: Result,
10397 N2: DAG.getConstant(Val: CombinedMask, DL: SDLoc(N), VT));
10398
10399 return Result;
10400}
10401
10402// If the target has andn, bsl, or a similar bit-select instruction,
10403// we want to unfold masked merge, with canonical pattern of:
10404// | A | |B|
10405// ((x ^ y) & m) ^ y
10406// | D |
10407// Into:
10408// (x & m) | (y & ~m)
10409// If y is a constant, m is not a 'not', and the 'andn' does not work with
10410// immediates, we unfold into a different pattern:
10411// ~(~x & m) & (m | y)
10412// If x is a constant, m is a 'not', and the 'andn' does not work with
10413// immediates, we unfold into a different pattern:
10414// (x | ~m) & ~(~m & ~y)
10415// NOTE: we don't unfold the pattern if 'xor' is actually a 'not', because at
10416// the very least that breaks andnpd / andnps patterns, and because those
10417// patterns are simplified in IR and shouldn't be created in the DAG
10418SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
10419 assert(N->getOpcode() == ISD::XOR);
10420
10421 // Don't touch 'not' (i.e. where y = -1).
10422 if (isAllOnesOrAllOnesSplat(V: N->getOperand(Num: 1)))
10423 return SDValue();
10424
10425 EVT VT = N->getValueType(ResNo: 0);
10426
10427 // There are 3 commutable operators in the pattern,
10428 // so we have to deal with 8 possible variants of the basic pattern.
10429 SDValue X, Y, M;
10430 auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
10431 if (And.getOpcode() != ISD::AND || !And.hasOneUse())
10432 return false;
10433 SDValue Xor = And.getOperand(i: XorIdx);
10434 if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
10435 return false;
10436 SDValue Xor0 = Xor.getOperand(i: 0);
10437 SDValue Xor1 = Xor.getOperand(i: 1);
10438 // Don't touch 'not' (i.e. where y = -1).
10439 if (isAllOnesOrAllOnesSplat(V: Xor1))
10440 return false;
10441 if (Other == Xor0)
10442 std::swap(a&: Xor0, b&: Xor1);
10443 if (Other != Xor1)
10444 return false;
10445 X = Xor0;
10446 Y = Xor1;
10447 M = And.getOperand(i: XorIdx ? 0 : 1);
10448 return true;
10449 };
10450
10451 SDValue N0 = N->getOperand(Num: 0);
10452 SDValue N1 = N->getOperand(Num: 1);
10453 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
10454 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
10455 return SDValue();
10456
10457 // Don't do anything if the mask is constant. This should not be reachable.
10458 // InstCombine should have already unfolded this pattern, and DAGCombiner
10459 // probably shouldn't produce it, too.
10460 if (isa<ConstantSDNode>(Val: M.getNode()))
10461 return SDValue();
10462
10463 // We can transform if the target has AndNot
10464 if (!TLI.hasAndNot(X: M))
10465 return SDValue();
10466
10467 SDLoc DL(N);
10468
10469 // If Y is a constant, check that 'andn' works with immediates. Unless M is
10470 // a bitwise not that would already allow ANDN to be used.
10471 if (!TLI.hasAndNot(X: Y) && !isBitwiseNot(V: M)) {
10472 assert(TLI.hasAndNot(X) && "Only mask is a variable? Unreachable.");
10473 // If not, we need to do a bit more work to make sure andn is still used.
10474 SDValue NotX = DAG.getNOT(DL, Val: X, VT);
10475 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NotX, N2: M);
10476 SDValue NotLHS = DAG.getNOT(DL, Val: LHS, VT);
10477 SDValue RHS = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: M, N2: Y);
10478 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NotLHS, N2: RHS);
10479 }
10480
10481 // If X is a constant and M is a bitwise not, check that 'andn' works with
10482 // immediates.
10483 if (!TLI.hasAndNot(X) && isBitwiseNot(V: M)) {
10484 assert(TLI.hasAndNot(Y) && "Only mask is a variable? Unreachable.");
10485 // If not, we need to do a bit more work to make sure andn is still used.
10486 SDValue NotM = M.getOperand(i: 0);
10487 SDValue LHS = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: X, N2: NotM);
10488 SDValue NotY = DAG.getNOT(DL, Val: Y, VT);
10489 SDValue RHS = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NotM, N2: NotY);
10490 SDValue NotRHS = DAG.getNOT(DL, Val: RHS, VT);
10491 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LHS, N2: NotRHS);
10492 }
10493
10494 SDValue LHS = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: M);
10495 SDValue NotM = DAG.getNOT(DL, Val: M, VT);
10496 SDValue RHS = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Y, N2: NotM);
10497
10498 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LHS, N2: RHS);
10499}
10500
10501SDValue DAGCombiner::visitXOR(SDNode *N) {
10502 SDValue N0 = N->getOperand(Num: 0);
10503 SDValue N1 = N->getOperand(Num: 1);
10504 EVT VT = N0.getValueType();
10505 SDLoc DL(N);
10506
10507 // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
10508 if (N0.isUndef() && N1.isUndef())
10509 return DAG.getConstant(Val: 0, DL, VT);
10510
10511 // fold (xor x, undef) -> undef
10512 if (N0.isUndef())
10513 return N0;
10514 if (N1.isUndef())
10515 return N1;
10516
10517 // fold (xor c1, c2) -> c1^c2
10518 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::XOR, DL, VT, Ops: {N0, N1}))
10519 return C;
10520
10521 // canonicalize constant to RHS
10522 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
10523 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
10524 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1, N2: N0);
10525
10526 // fold vector ops
10527 if (VT.isVector()) {
10528 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
10529 return FoldedVOp;
10530
10531 // fold (xor x, 0) -> x, vector edition
10532 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
10533 return N0;
10534 }
10535
10536 // fold (xor x, 0) -> x
10537 if (isNullConstant(V: N1))
10538 return N0;
10539
10540 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
10541 return NewSel;
10542
10543 // reassociate xor
10544 if (SDValue RXOR = reassociateOps(Opc: ISD::XOR, DL, N0, N1, Flags: N->getFlags()))
10545 return RXOR;
10546
10547 // Fold xor(vecreduce(x), vecreduce(y)) -> vecreduce(xor(x, y))
10548 if (SDValue SD =
10549 reassociateReduction(RedOpc: ISD::VECREDUCE_XOR, Opc: ISD::XOR, DL, VT, N0, N1))
10550 return SD;
10551
10552 // fold (a^b) -> (a|b) iff a and b share no bits.
10553 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::OR, VT)) &&
10554 DAG.haveNoCommonBitsSet(A: N0, B: N1))
10555 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: N0, N2: N1, Flags: SDNodeFlags::Disjoint);
10556
10557 // look for 'add-like' folds:
10558 // XOR(N0,MIN_SIGNED_VALUE) == ADD(N0,MIN_SIGNED_VALUE)
10559 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::ADD, VT)) &&
10560 isMinSignedConstant(V: N1))
10561 if (SDValue Combined = visitADDLike(N))
10562 return Combined;
10563
10564 // fold not (setcc x, y, cc) -> setcc x y !cc
10565 // Avoid breaking: and (not(setcc x, y, cc), z) -> andn for vec
10566 unsigned N0Opcode = N0.getOpcode();
10567 SDValue LHS, RHS, CC;
10568 if (TLI.isConstTrueVal(N: N1) &&
10569 isSetCCEquivalent(N: N0, LHS, RHS, CC, /*MatchStrict*/ true) &&
10570 !(VT.isVector() && TLI.hasAndNot(X: SDValue(N, 0)) && N->hasOneUse() &&
10571 N->use_begin()->getUser()->getOpcode() == ISD::AND)) {
10572 ISD::CondCode NotCC = ISD::getSetCCInverse(Operation: cast<CondCodeSDNode>(Val&: CC)->get(),
10573 Type: LHS.getValueType());
10574 if (!LegalOperations ||
10575 TLI.isCondCodeLegal(CC: NotCC, VT: LHS.getSimpleValueType())) {
10576 // Propagate fast-math-flags.
10577 SDNodeFlags Flags = N0->getFlags();
10578 switch (N0Opcode) {
10579 default:
10580 llvm_unreachable("Unhandled SetCC Equivalent!");
10581 case ISD::SETCC:
10582 return DAG.getSetCC(DL: SDLoc(N0), VT, LHS, RHS, Cond: NotCC, Chain: SDValue(),
10583 /*IsSignaling=*/false, Flags);
10584 case ISD::SELECT_CC:
10585 return DAG.getSelectCC(DL: SDLoc(N0), LHS, RHS, True: N0.getOperand(i: 2),
10586 False: N0.getOperand(i: 3), Cond: NotCC, Flags);
10587 case ISD::STRICT_FSETCC:
10588 case ISD::STRICT_FSETCCS: {
10589 if (N0.hasOneUse()) {
10590 // FIXME Can we handle multiple uses? Could we token factor the chain
10591 // results from the new/old setcc?
10592 SDValue SetCC =
10593 DAG.getSetCC(DL: SDLoc(N0), VT, LHS, RHS, Cond: NotCC, Chain: N0.getOperand(i: 0),
10594 IsSignaling: N0Opcode == ISD::STRICT_FSETCCS, Flags);
10595 CombineTo(N, Res: SetCC);
10596 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: SetCC.getValue(R: 1));
10597 recursivelyDeleteUnusedNodes(N: N0.getNode());
10598 return SDValue(N, 0); // Return N so it doesn't get rechecked!
10599 }
10600 break;
10601 }
10602 }
10603 }
10604 }
10605
10606 // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
10607 if (isOneConstant(V: N1) && N0Opcode == ISD::ZERO_EXTEND && N0.hasOneUse() &&
10608 isSetCCEquivalent(N: N0.getOperand(i: 0), LHS, RHS, CC)){
10609 SDValue V = N0.getOperand(i: 0);
10610 SDLoc DL0(N0);
10611 V = DAG.getNode(Opcode: ISD::XOR, DL: DL0, VT: V.getValueType(), N1: V,
10612 N2: DAG.getConstant(Val: 1, DL: DL0, VT: V.getValueType()));
10613 AddToWorklist(N: V.getNode());
10614 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: V);
10615 }
10616
10617 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
10618 // fold (not (and x, y)) -> (or (not x), (not y)) iff x or y are setcc
10619 if (isOneConstant(V: N1) && VT == MVT::i1 && N0.hasOneUse() &&
10620 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) {
10621 SDValue N00 = N0.getOperand(i: 0), N01 = N0.getOperand(i: 1);
10622 if (isOneUseSetCC(N: N01) || isOneUseSetCC(N: N00)) {
10623 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND;
10624 N00 = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N00), VT, N1: N00, N2: N1); // N00 = ~N00
10625 N01 = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N01), VT, N1: N01, N2: N1); // N01 = ~N01
10626 AddToWorklist(N: N00.getNode()); AddToWorklist(N: N01.getNode());
10627 return DAG.getNode(Opcode: NewOpcode, DL, VT, N1: N00, N2: N01);
10628 }
10629 }
10630 // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
10631 // fold (not (and x, y)) -> (or (not x), (not y)) iff x or y are constants
10632 if (isAllOnesConstant(V: N1) && N0.hasOneUse() &&
10633 (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) {
10634 SDValue N00 = N0.getOperand(i: 0), N01 = N0.getOperand(i: 1);
10635 if (isa<ConstantSDNode>(Val: N01) || isa<ConstantSDNode>(Val: N00)) {
10636 unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND;
10637 N00 = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N00), VT, N1: N00, N2: N1); // N00 = ~N00
10638 N01 = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N01), VT, N1: N01, N2: N1); // N01 = ~N01
10639 AddToWorklist(N: N00.getNode()); AddToWorklist(N: N01.getNode());
10640 return DAG.getNode(Opcode: NewOpcode, DL, VT, N1: N00, N2: N01);
10641 }
10642 }
10643
10644 // fold (not (sub Y, X)) -> (add X, ~Y) if Y is a constant
10645 if (N0.getOpcode() == ISD::SUB && isAllOnesConstant(V: N1)) {
10646 SDValue Y = N0.getOperand(i: 0);
10647 SDValue X = N0.getOperand(i: 1);
10648
10649 if (auto *YConst = dyn_cast<ConstantSDNode>(Val&: Y)) {
10650 APInt NotYValue = ~YConst->getAPIntValue();
10651 SDValue NotY = DAG.getConstant(Val: NotYValue, DL, VT);
10652 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: X, N2: NotY, Flags: N->getFlags());
10653 }
10654 }
10655
10656 // fold (not (add X, -1)) -> (neg X)
10657 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && isAllOnesConstant(V: N1) &&
10658 isAllOnesOrAllOnesSplat(V: N0.getOperand(i: 1))) {
10659 return DAG.getNegative(Val: N0.getOperand(i: 0), DL, VT);
10660 }
10661
10662 // fold (xor (and x, y), y) -> (and (not x), y)
10663 if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(Num: 1) == N1) {
10664 SDValue X = N0.getOperand(i: 0);
10665 SDValue NotX = DAG.getNOT(DL: SDLoc(X), Val: X, VT);
10666 AddToWorklist(N: NotX.getNode());
10667 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NotX, N2: N1);
10668 }
10669
10670 // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
10671 if (!LegalOperations || hasOperation(Opcode: ISD::ABS, VT)) {
10672 SDValue A = N0Opcode == ISD::ADD ? N0 : N1;
10673 SDValue S = N0Opcode == ISD::SRA ? N0 : N1;
10674 if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
10675 SDValue A0 = A.getOperand(i: 0), A1 = A.getOperand(i: 1);
10676 SDValue S0 = S.getOperand(i: 0);
10677 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
10678 if (ConstantSDNode *C = isConstOrConstSplat(N: S.getOperand(i: 1)))
10679 if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1))
10680 return DAG.getNode(Opcode: ISD::ABS, DL, VT, Operand: S0);
10681 }
10682 }
10683
10684 // fold (xor x, x) -> 0
10685 if (N0 == N1)
10686 return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
10687
10688 // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
10689 // Here is a concrete example of this equivalence:
10690 // i16 x == 14
10691 // i16 shl == 1 << 14 == 16384 == 0b0100000000000000
10692 // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
10693 //
10694 // =>
10695 //
10696 // i16 ~1 == 0b1111111111111110
10697 // i16 rol(~1, 14) == 0b1011111111111111
10698 //
10699 // Some additional tips to help conceptualize this transform:
10700 // - Try to see the operation as placing a single zero in a value of all ones.
10701 // - There exists no value for x which would allow the result to contain zero.
10702 // - Values of x larger than the bitwidth are undefined and do not require a
10703 // consistent result.
10704 // - Pushing the zero left requires shifting one bits in from the right.
10705 // A rotate left of ~1 is a nice way of achieving the desired result.
10706 if (TLI.isOperationLegalOrCustom(Op: ISD::ROTL, VT) && N0Opcode == ISD::SHL &&
10707 isAllOnesConstant(V: N1) && isOneConstant(V: N0.getOperand(i: 0))) {
10708 return DAG.getNode(Opcode: ISD::ROTL, DL, VT, N1: DAG.getSignedConstant(Val: ~1, DL, VT),
10709 N2: N0.getOperand(i: 1));
10710 }
10711
10712 // Simplify: xor (op x...), (op y...) -> (op (xor x, y))
10713 if (N0Opcode == N1.getOpcode())
10714 if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
10715 return V;
10716
10717 if (SDValue R = foldLogicOfShifts(N, LogicOp: N0, ShiftOp: N1, DAG))
10718 return R;
10719 if (SDValue R = foldLogicOfShifts(N, LogicOp: N1, ShiftOp: N0, DAG))
10720 return R;
10721 if (SDValue R = foldLogicTreeOfShifts(N, LeftHand: N0, RightHand: N1, DAG))
10722 return R;
10723
10724 // Unfold ((x ^ y) & m) ^ y into (x & m) | (y & ~m) if profitable
10725 if (SDValue MM = unfoldMaskedMerge(N))
10726 return MM;
10727
10728 // Simplify the expression using non-local knowledge.
10729 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
10730 return SDValue(N, 0);
10731
10732 if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
10733 return Combined;
10734
10735 // fold (xor (smin(x, C), C)) -> select (x < C), xor(x, C), 0
10736 // fold (xor (smax(x, C), C)) -> select (x > C), xor(x, C), 0
10737 // fold (xor (umin(x, C), C)) -> select (x < C), xor(x, C), 0
10738 // fold (xor (umax(x, C), C)) -> select (x > C), xor(x, C), 0
10739 SDValue Op0;
10740 if (sd_match(N: N0, P: m_OneUse(P: m_AnyOf(preds: m_SMin(L: m_Value(N&: Op0), R: m_Specific(N: N1)),
10741 preds: m_SMax(L: m_Value(N&: Op0), R: m_Specific(N: N1)),
10742 preds: m_UMin(L: m_Value(N&: Op0), R: m_Specific(N: N1)),
10743 preds: m_UMax(L: m_Value(N&: Op0), R: m_Specific(N: N1)))))) {
10744
10745 if (isa<ConstantSDNode>(Val: N1) ||
10746 ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode())) {
10747 // For vectors, only optimize when the constant is zero or all-ones to
10748 // avoid generating more instructions
10749 if (VT.isVector()) {
10750 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
10751 if (!N1C || (!N1C->isZero() && !N1C->isAllOnes()))
10752 return SDValue();
10753 }
10754
10755 // Avoid the fold if the minmax operation is legal and select is expensive
10756 if (TLI.isOperationLegal(Op: N0.getOpcode(), VT) &&
10757 TLI.isPredictableSelectExpensive())
10758 return SDValue();
10759
10760 EVT CCVT = getSetCCResultType(VT);
10761 ISD::CondCode CC;
10762 switch (N0.getOpcode()) {
10763 case ISD::SMIN:
10764 CC = ISD::SETLT;
10765 break;
10766 case ISD::SMAX:
10767 CC = ISD::SETGT;
10768 break;
10769 case ISD::UMIN:
10770 CC = ISD::SETULT;
10771 break;
10772 case ISD::UMAX:
10773 CC = ISD::SETUGT;
10774 break;
10775 }
10776 SDValue FN1 = DAG.getFreeze(V: N1);
10777 SDValue Cmp = DAG.getSetCC(DL, VT: CCVT, LHS: Op0, RHS: FN1, Cond: CC);
10778 SDValue XorXC = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Op0, N2: FN1);
10779 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
10780 return DAG.getSelect(DL, VT, Cond: Cmp, LHS: XorXC, RHS: Zero);
10781 }
10782 }
10783
10784 return SDValue();
10785}
10786
10787/// If we have a shift-by-constant of a bitwise logic op that itself has a
10788/// shift-by-constant operand with identical opcode, we may be able to convert
10789/// that into 2 independent shifts followed by the logic op. This is a
10790/// throughput improvement.
10791static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG) {
10792 // Match a one-use bitwise logic op.
10793 SDValue LogicOp = Shift->getOperand(Num: 0);
10794 if (!LogicOp.hasOneUse())
10795 return SDValue();
10796
10797 unsigned LogicOpcode = LogicOp.getOpcode();
10798 if (LogicOpcode != ISD::AND && LogicOpcode != ISD::OR &&
10799 LogicOpcode != ISD::XOR)
10800 return SDValue();
10801
10802 // Find a matching one-use shift by constant.
10803 unsigned ShiftOpcode = Shift->getOpcode();
10804 SDValue C1 = Shift->getOperand(Num: 1);
10805 ConstantSDNode *C1Node = isConstOrConstSplat(N: C1);
10806 assert(C1Node && "Expected a shift with constant operand");
10807 const APInt &C1Val = C1Node->getAPIntValue();
10808 auto matchFirstShift = [&](SDValue V, SDValue &ShiftOp,
10809 const APInt *&ShiftAmtVal) {
10810 if (V.getOpcode() != ShiftOpcode || !V.hasOneUse())
10811 return false;
10812
10813 ConstantSDNode *ShiftCNode = isConstOrConstSplat(N: V.getOperand(i: 1));
10814 if (!ShiftCNode)
10815 return false;
10816
10817 // Capture the shifted operand and shift amount value.
10818 ShiftOp = V.getOperand(i: 0);
10819 ShiftAmtVal = &ShiftCNode->getAPIntValue();
10820
10821 // Shift amount types do not have to match their operand type, so check that
10822 // the constants are the same width.
10823 if (ShiftAmtVal->getBitWidth() != C1Val.getBitWidth())
10824 return false;
10825
10826 // The fold is not valid if the sum of the shift values doesn't fit in the
10827 // given shift amount type.
10828 bool Overflow = false;
10829 APInt NewShiftAmt = C1Val.uadd_ov(RHS: *ShiftAmtVal, Overflow);
10830 if (Overflow)
10831 return false;
10832
10833 // The fold is not valid if the sum of the shift values exceeds bitwidth.
10834 if (NewShiftAmt.uge(RHS: V.getScalarValueSizeInBits()))
10835 return false;
10836
10837 return true;
10838 };
10839
10840 // Logic ops are commutative, so check each operand for a match.
10841 SDValue X, Y;
10842 const APInt *C0Val;
10843 if (matchFirstShift(LogicOp.getOperand(i: 0), X, C0Val))
10844 Y = LogicOp.getOperand(i: 1);
10845 else if (matchFirstShift(LogicOp.getOperand(i: 1), X, C0Val))
10846 Y = LogicOp.getOperand(i: 0);
10847 else
10848 return SDValue();
10849
10850 // shift (logic (shift X, C0), Y), C1 -> logic (shift X, C0+C1), (shift Y, C1)
10851 SDLoc DL(Shift);
10852 EVT VT = Shift->getValueType(ResNo: 0);
10853 EVT ShiftAmtVT = Shift->getOperand(Num: 1).getValueType();
10854 SDValue ShiftSumC = DAG.getConstant(Val: *C0Val + C1Val, DL, VT: ShiftAmtVT);
10855 SDValue NewShift1 = DAG.getNode(Opcode: ShiftOpcode, DL, VT, N1: X, N2: ShiftSumC);
10856 SDValue NewShift2 = DAG.getNode(Opcode: ShiftOpcode, DL, VT, N1: Y, N2: C1);
10857 return DAG.getNode(Opcode: LogicOpcode, DL, VT, N1: NewShift1, N2: NewShift2,
10858 Flags: LogicOp->getFlags());
10859}
10860
10861/// Handle transforms common to the three shifts, when the shift amount is a
10862/// constant.
10863/// We are looking for: (shift being one of shl/sra/srl)
10864/// shift (binop X, C0), C1
10865/// And want to transform into:
10866/// binop (shift X, C1), (shift C0, C1)
10867SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
10868 assert(isConstOrConstSplat(N->getOperand(1)) && "Expected constant operand");
10869
10870 // Do not turn a 'not' into a regular xor.
10871 if (isBitwiseNot(V: N->getOperand(Num: 0)))
10872 return SDValue();
10873
10874 // The inner binop must be one-use, since we want to replace it.
10875 SDValue LHS = N->getOperand(Num: 0);
10876 if (!LHS.hasOneUse() || !TLI.isDesirableToCommuteWithShift(N, Level))
10877 return SDValue();
10878
10879 // Fold shift(bitop(shift(x,c1),y), c2) -> bitop(shift(x,c1+c2),shift(y,c2)).
10880 if (SDValue R = combineShiftOfShiftedLogic(Shift: N, DAG))
10881 return R;
10882
10883 // We want to pull some binops through shifts, so that we have (and (shift))
10884 // instead of (shift (and)), likewise for add, or, xor, etc. This sort of
10885 // thing happens with address calculations, so it's important to canonicalize
10886 // it.
10887 switch (LHS.getOpcode()) {
10888 default:
10889 return SDValue();
10890 case ISD::OR:
10891 case ISD::XOR:
10892 case ISD::AND:
10893 break;
10894 case ISD::ADD:
10895 if (N->getOpcode() != ISD::SHL)
10896 return SDValue(); // only shl(add) not sr[al](add).
10897 break;
10898 }
10899
10900 // FIXME: disable this unless the input to the binop is a shift by a constant
10901 // or is copy/select. Enable this in other cases when figure out it's exactly
10902 // profitable.
10903 SDValue BinOpLHSVal = LHS.getOperand(i: 0);
10904 bool IsShiftByConstant = (BinOpLHSVal.getOpcode() == ISD::SHL ||
10905 BinOpLHSVal.getOpcode() == ISD::SRA ||
10906 BinOpLHSVal.getOpcode() == ISD::SRL) &&
10907 isa<ConstantSDNode>(Val: BinOpLHSVal.getOperand(i: 1));
10908 bool IsCopyOrSelect = BinOpLHSVal.getOpcode() == ISD::CopyFromReg ||
10909 BinOpLHSVal.getOpcode() == ISD::SELECT;
10910
10911 if (!IsShiftByConstant && !IsCopyOrSelect)
10912 return SDValue();
10913
10914 if (IsCopyOrSelect && N->hasOneUse())
10915 return SDValue();
10916
10917 // Attempt to fold the constants, shifting the binop RHS by the shift amount.
10918 SDLoc DL(N);
10919 EVT VT = N->getValueType(ResNo: 0);
10920 if (SDValue NewRHS = DAG.FoldConstantArithmetic(
10921 Opcode: N->getOpcode(), DL, VT, Ops: {LHS.getOperand(i: 1), N->getOperand(Num: 1)})) {
10922 SDValue NewShift = DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: LHS.getOperand(i: 0),
10923 N2: N->getOperand(Num: 1));
10924 return DAG.getNode(Opcode: LHS.getOpcode(), DL, VT, N1: NewShift, N2: NewRHS);
10925 }
10926
10927 return SDValue();
10928}
10929
10930SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
10931 assert(N->getOpcode() == ISD::TRUNCATE);
10932 assert(N->getOperand(0).getOpcode() == ISD::AND);
10933
10934 // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
10935 EVT TruncVT = N->getValueType(ResNo: 0);
10936 if (N->hasOneUse() && N->getOperand(Num: 0).hasOneUse() &&
10937 TLI.isTypeDesirableForOp(ISD::AND, VT: TruncVT)) {
10938 SDValue N01 = N->getOperand(Num: 0).getOperand(i: 1);
10939 if (isConstantOrConstantVector(N: N01, /* NoOpaques */ true)) {
10940 SDLoc DL(N);
10941 SDValue N00 = N->getOperand(Num: 0).getOperand(i: 0);
10942 SDValue Trunc00 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: TruncVT, Operand: N00);
10943 SDValue Trunc01 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: TruncVT, Operand: N01);
10944 AddToWorklist(N: Trunc00.getNode());
10945 AddToWorklist(N: Trunc01.getNode());
10946 return DAG.getNode(Opcode: ISD::AND, DL, VT: TruncVT, N1: Trunc00, N2: Trunc01);
10947 }
10948 }
10949
10950 return SDValue();
10951}
10952
10953SDValue DAGCombiner::visitRotate(SDNode *N) {
10954 SDLoc dl(N);
10955 SDValue N0 = N->getOperand(Num: 0);
10956 SDValue N1 = N->getOperand(Num: 1);
10957 EVT VT = N->getValueType(ResNo: 0);
10958 unsigned Bitsize = VT.getScalarSizeInBits();
10959
10960 // fold (rot x, 0) -> x
10961 if (isNullOrNullSplat(V: N1))
10962 return N0;
10963
10964 // fold (rot x, c) -> x iff (c % BitSize) == 0
10965 if (isPowerOf2_32(Value: Bitsize) && Bitsize > 1) {
10966 APInt ModuloMask(N1.getScalarValueSizeInBits(), Bitsize - 1);
10967 if (DAG.MaskedValueIsZero(Op: N1, Mask: ModuloMask))
10968 return N0;
10969 }
10970
10971 // fold (rot x, c) -> (rot x, c % BitSize)
10972 bool OutOfRange = false;
10973 auto MatchOutOfRange = [Bitsize, &OutOfRange](ConstantSDNode *C) {
10974 OutOfRange |= C->getAPIntValue().uge(RHS: Bitsize);
10975 return true;
10976 };
10977 if (ISD::matchUnaryPredicate(Op: N1, Match: MatchOutOfRange) && OutOfRange) {
10978 EVT AmtVT = N1.getValueType();
10979 SDValue Bits = DAG.getConstant(Val: Bitsize, DL: dl, VT: AmtVT);
10980 if (SDValue Amt =
10981 DAG.FoldConstantArithmetic(Opcode: ISD::UREM, DL: dl, VT: AmtVT, Ops: {N1, Bits}))
10982 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT, N1: N0, N2: Amt);
10983 }
10984
10985 // rot i16 X, 8 --> bswap X
10986 auto *RotAmtC = isConstOrConstSplat(N: N1);
10987 if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
10988 VT.getScalarSizeInBits() == 16 && hasOperation(Opcode: ISD::BSWAP, VT))
10989 return DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT, Operand: N0);
10990
10991 // Simplify the operands using demanded-bits information.
10992 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
10993 return SDValue(N, 0);
10994
10995 // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
10996 if (N1.getOpcode() == ISD::TRUNCATE &&
10997 N1.getOperand(i: 0).getOpcode() == ISD::AND) {
10998 if (SDValue NewOp1 = distributeTruncateThroughAnd(N: N1.getNode()))
10999 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT, N1: N0, N2: NewOp1);
11000 }
11001
11002 unsigned NextOp = N0.getOpcode();
11003
11004 // fold (rot* (rot* x, c2), c1)
11005 // -> (rot* x, ((c1 % bitsize) +- (c2 % bitsize) + bitsize) % bitsize)
11006 if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
11007 bool C1 = DAG.isConstantIntBuildVectorOrConstantInt(N: N1);
11008 bool C2 = DAG.isConstantIntBuildVectorOrConstantInt(N: N0.getOperand(i: 1));
11009 if (C1 && C2 && N1.getValueType() == N0.getOperand(i: 1).getValueType()) {
11010 EVT ShiftVT = N1.getValueType();
11011 bool SameSide = (N->getOpcode() == NextOp);
11012 unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
11013 SDValue BitsizeC = DAG.getConstant(Val: Bitsize, DL: dl, VT: ShiftVT);
11014 SDValue Norm1 = DAG.FoldConstantArithmetic(Opcode: ISD::UREM, DL: dl, VT: ShiftVT,
11015 Ops: {N1, BitsizeC});
11016 SDValue Norm2 = DAG.FoldConstantArithmetic(Opcode: ISD::UREM, DL: dl, VT: ShiftVT,
11017 Ops: {N0.getOperand(i: 1), BitsizeC});
11018 if (Norm1 && Norm2)
11019 if (SDValue CombinedShift = DAG.FoldConstantArithmetic(
11020 Opcode: CombineOp, DL: dl, VT: ShiftVT, Ops: {Norm1, Norm2})) {
11021 CombinedShift = DAG.FoldConstantArithmetic(Opcode: ISD::ADD, DL: dl, VT: ShiftVT,
11022 Ops: {CombinedShift, BitsizeC});
11023 SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
11024 Opcode: ISD::UREM, DL: dl, VT: ShiftVT, Ops: {CombinedShift, BitsizeC});
11025 return DAG.getNode(Opcode: N->getOpcode(), DL: dl, VT, N1: N0->getOperand(Num: 0),
11026 N2: CombinedShiftNorm);
11027 }
11028 }
11029 }
11030 return SDValue();
11031}
11032
11033SDValue DAGCombiner::visitSHL(SDNode *N) {
11034 SDValue N0 = N->getOperand(Num: 0);
11035 SDValue N1 = N->getOperand(Num: 1);
11036 if (SDValue V = DAG.simplifyShift(X: N0, Y: N1))
11037 return V;
11038
11039 SDLoc DL(N);
11040 EVT VT = N0.getValueType();
11041 EVT ShiftVT = N1.getValueType();
11042 unsigned OpSizeInBits = VT.getScalarSizeInBits();
11043
11044 // fold (shl c1, c2) -> c1<<c2
11045 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::SHL, DL, VT, Ops: {N0, N1}))
11046 return C;
11047
11048 // fold vector ops
11049 if (VT.isVector()) {
11050 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
11051 return FoldedVOp;
11052
11053 BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(Val&: N1);
11054 // If setcc produces all-one true value then:
11055 // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
11056 if (N1CV && N1CV->isConstant()) {
11057 if (N0.getOpcode() == ISD::AND) {
11058 SDValue N00 = N0->getOperand(Num: 0);
11059 SDValue N01 = N0->getOperand(Num: 1);
11060 BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(Val&: N01);
11061
11062 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
11063 TLI.getBooleanContents(Type: N00.getOperand(i: 0).getValueType()) ==
11064 TargetLowering::ZeroOrNegativeOneBooleanContent) {
11065 if (SDValue C =
11066 DAG.FoldConstantArithmetic(Opcode: ISD::SHL, DL, VT, Ops: {N01, N1}))
11067 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N00, N2: C);
11068 }
11069 }
11070 }
11071 }
11072
11073 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
11074 return NewSel;
11075
11076 // if (shl x, c) is known to be zero, return 0
11077 if (DAG.MaskedValueIsZero(Op: SDValue(N, 0), Mask: APInt::getAllOnes(numBits: OpSizeInBits)))
11078 return DAG.getConstant(Val: 0, DL, VT);
11079
11080 // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
11081 if (N1.getOpcode() == ISD::TRUNCATE &&
11082 N1.getOperand(i: 0).getOpcode() == ISD::AND) {
11083 if (SDValue NewOp1 = distributeTruncateThroughAnd(N: N1.getNode()))
11084 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0, N2: NewOp1);
11085 }
11086
11087 // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
11088 if (N0.getOpcode() == ISD::SHL) {
11089 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
11090 ConstantSDNode *RHS) {
11091 APInt c1 = LHS->getAPIntValue();
11092 APInt c2 = RHS->getAPIntValue();
11093 zeroExtendToMatch(LHS&: c1, RHS&: c2, Offset: 1 /* Overflow Bit */);
11094 return (c1 + c2).uge(RHS: OpSizeInBits);
11095 };
11096 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: MatchOutOfRange))
11097 return DAG.getConstant(Val: 0, DL, VT);
11098
11099 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
11100 ConstantSDNode *RHS) {
11101 APInt c1 = LHS->getAPIntValue();
11102 APInt c2 = RHS->getAPIntValue();
11103 zeroExtendToMatch(LHS&: c1, RHS&: c2, Offset: 1 /* Overflow Bit */);
11104 return (c1 + c2).ult(RHS: OpSizeInBits);
11105 };
11106 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: MatchInRange)) {
11107 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ShiftVT, N1, N2: N0.getOperand(i: 1));
11108 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0.getOperand(i: 0), N2: Sum);
11109 }
11110 }
11111
11112 // fold (shl (ext (shl x, c1)), c2) -> (shl (ext x), (add c1, c2))
11113 // For this to be valid, the second form must not preserve any of the bits
11114 // that are shifted out by the inner shift in the first form. This means
11115 // the outer shift size must be >= the number of bits added by the ext.
11116 // As a corollary, we don't care what kind of ext it is.
11117 if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
11118 N0.getOpcode() == ISD::ANY_EXTEND ||
11119 N0.getOpcode() == ISD::SIGN_EXTEND) &&
11120 N0.getOperand(i: 0).getOpcode() == ISD::SHL) {
11121 SDValue N0Op0 = N0.getOperand(i: 0);
11122 SDValue InnerShiftAmt = N0Op0.getOperand(i: 1);
11123 EVT InnerVT = N0Op0.getValueType();
11124 uint64_t InnerBitwidth = InnerVT.getScalarSizeInBits();
11125
11126 auto MatchOutOfRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS,
11127 ConstantSDNode *RHS) {
11128 APInt c1 = LHS->getAPIntValue();
11129 APInt c2 = RHS->getAPIntValue();
11130 zeroExtendToMatch(LHS&: c1, RHS&: c2, Offset: 1 /* Overflow Bit */);
11131 return c2.uge(RHS: OpSizeInBits - InnerBitwidth) &&
11132 (c1 + c2).uge(RHS: OpSizeInBits);
11133 };
11134 if (ISD::matchBinaryPredicate(LHS: InnerShiftAmt, RHS: N1, Match: MatchOutOfRange,
11135 /*AllowUndefs*/ false,
11136 /*AllowTypeMismatch*/ true))
11137 return DAG.getConstant(Val: 0, DL, VT);
11138
11139 auto MatchInRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS,
11140 ConstantSDNode *RHS) {
11141 APInt c1 = LHS->getAPIntValue();
11142 APInt c2 = RHS->getAPIntValue();
11143 zeroExtendToMatch(LHS&: c1, RHS&: c2, Offset: 1 /* Overflow Bit */);
11144 return c2.uge(RHS: OpSizeInBits - InnerBitwidth) &&
11145 (c1 + c2).ult(RHS: OpSizeInBits);
11146 };
11147 if (ISD::matchBinaryPredicate(LHS: InnerShiftAmt, RHS: N1, Match: MatchInRange,
11148 /*AllowUndefs*/ false,
11149 /*AllowTypeMismatch*/ true)) {
11150 SDValue Ext = DAG.getNode(Opcode: N0.getOpcode(), DL, VT, Operand: N0Op0.getOperand(i: 0));
11151 SDValue Sum = DAG.getZExtOrTrunc(Op: InnerShiftAmt, DL, VT: ShiftVT);
11152 Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ShiftVT, N1: Sum, N2: N1);
11153 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Ext, N2: Sum);
11154 }
11155 }
11156
11157 // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
11158 // Only fold this if the inner zext has no other uses to avoid increasing
11159 // the total number of instructions.
11160 if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
11161 N0.getOperand(i: 0).getOpcode() == ISD::SRL) {
11162 SDValue N0Op0 = N0.getOperand(i: 0);
11163 SDValue InnerShiftAmt = N0Op0.getOperand(i: 1);
11164
11165 auto MatchEqual = [VT](ConstantSDNode *LHS, ConstantSDNode *RHS) {
11166 APInt c1 = LHS->getAPIntValue();
11167 APInt c2 = RHS->getAPIntValue();
11168 zeroExtendToMatch(LHS&: c1, RHS&: c2);
11169 return c1.ult(RHS: VT.getScalarSizeInBits()) && (c1 == c2);
11170 };
11171 if (ISD::matchBinaryPredicate(LHS: InnerShiftAmt, RHS: N1, Match: MatchEqual,
11172 /*AllowUndefs*/ false,
11173 /*AllowTypeMismatch*/ true)) {
11174 EVT InnerShiftAmtVT = N0Op0.getOperand(i: 1).getValueType();
11175 SDValue NewSHL = DAG.getZExtOrTrunc(Op: N1, DL, VT: InnerShiftAmtVT);
11176 NewSHL = DAG.getNode(Opcode: ISD::SHL, DL, VT: N0Op0.getValueType(), N1: N0Op0, N2: NewSHL);
11177 AddToWorklist(N: NewSHL.getNode());
11178 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(N0), VT, Operand: NewSHL);
11179 }
11180 }
11181
11182 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) {
11183 auto MatchShiftAmount = [OpSizeInBits](ConstantSDNode *LHS,
11184 ConstantSDNode *RHS) {
11185 const APInt &LHSC = LHS->getAPIntValue();
11186 const APInt &RHSC = RHS->getAPIntValue();
11187 return LHSC.ult(RHS: OpSizeInBits) && RHSC.ult(RHS: OpSizeInBits) &&
11188 LHSC.getZExtValue() <= RHSC.getZExtValue();
11189 };
11190
11191 // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2
11192 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 >= C2
11193 if (N0->getFlags().hasExact()) {
11194 if (ISD::matchBinaryPredicate(LHS: N0.getOperand(i: 1), RHS: N1, Match: MatchShiftAmount,
11195 /*AllowUndefs*/ false,
11196 /*AllowTypeMismatch*/ true)) {
11197 SDValue N01 = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 1), DL, VT: ShiftVT);
11198 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShiftVT, N1, N2: N01);
11199 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0.getOperand(i: 0), N2: Diff);
11200 }
11201 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: MatchShiftAmount,
11202 /*AllowUndefs*/ false,
11203 /*AllowTypeMismatch*/ true)) {
11204 SDValue N01 = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 1), DL, VT: ShiftVT);
11205 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShiftVT, N1: N01, N2: N1);
11206 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, N1: N0.getOperand(i: 0), N2: Diff);
11207 }
11208 }
11209
11210 // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
11211 // (and (srl x, (sub c1, c2), MASK)
11212 // Only fold this if the inner shift has no other uses -- if it does,
11213 // folding this will increase the total number of instructions.
11214 if (N0.getOpcode() == ISD::SRL &&
11215 (N0.getOperand(i: 1) == N1 || N0.hasOneUse()) &&
11216 TLI.shouldFoldConstantShiftPairToMask(N)) {
11217 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: MatchShiftAmount,
11218 /*AllowUndefs*/ false,
11219 /*AllowTypeMismatch*/ true)) {
11220 SDValue N01 = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 1), DL, VT: ShiftVT);
11221 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShiftVT, N1: N01, N2: N1);
11222 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11223 Mask = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Mask, N2: N01);
11224 Mask = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mask, N2: Diff);
11225 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0.getOperand(i: 0), N2: Diff);
11226 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shift, N2: Mask);
11227 }
11228 if (ISD::matchBinaryPredicate(LHS: N0.getOperand(i: 1), RHS: N1, Match: MatchShiftAmount,
11229 /*AllowUndefs*/ false,
11230 /*AllowTypeMismatch*/ true)) {
11231 SDValue N01 = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 1), DL, VT: ShiftVT);
11232 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShiftVT, N1, N2: N01);
11233 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11234 Mask = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Mask, N2: N1);
11235 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0.getOperand(i: 0), N2: Diff);
11236 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shift, N2: Mask);
11237 }
11238 }
11239 }
11240
11241 // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
11242 if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(i: 1) &&
11243 isConstantOrConstantVector(N: N1, /* No Opaques */ NoOpaques: true)) {
11244 SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
11245 SDValue HiBitsMask = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: AllBits, N2: N1);
11246 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: N0.getOperand(i: 0), N2: HiBitsMask);
11247 }
11248
11249 // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
11250 // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
11251 // Variant of version done on multiply, except mul by a power of 2 is turned
11252 // into a shift.
11253 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
11254 TLI.isDesirableToCommuteWithShift(N, Level)) {
11255 SDValue N01 = N0.getOperand(i: 1);
11256 if (SDValue Shl1 =
11257 DAG.FoldConstantArithmetic(Opcode: ISD::SHL, DL: SDLoc(N1), VT, Ops: {N01, N1})) {
11258 SDValue Shl0 = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(N0), VT, N1: N0.getOperand(i: 0), N2: N1);
11259 AddToWorklist(N: Shl0.getNode());
11260 SDNodeFlags Flags;
11261 // Preserve the disjoint flag for Or.
11262 if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint())
11263 Flags |= SDNodeFlags::Disjoint;
11264 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, N1: Shl0, N2: Shl1, Flags);
11265 }
11266 }
11267
11268 // fold (shl (sext (add_nsw x, c1)), c2) -> (add (shl (sext x), c2), c1 << c2)
11269 // TODO: Add zext/add_nuw variant with suitable test coverage
11270 // TODO: Should we limit this with isLegalAddImmediate?
11271 if (N0.getOpcode() == ISD::SIGN_EXTEND &&
11272 N0.getOperand(i: 0).getOpcode() == ISD::ADD &&
11273 N0.getOperand(i: 0)->getFlags().hasNoSignedWrap() &&
11274 TLI.isDesirableToCommuteWithShift(N, Level)) {
11275 SDValue Add = N0.getOperand(i: 0);
11276 SDLoc DL(N0);
11277 if (SDValue ExtC = DAG.FoldConstantArithmetic(Opcode: N0.getOpcode(), DL, VT,
11278 Ops: {Add.getOperand(i: 1)})) {
11279 if (SDValue ShlC =
11280 DAG.FoldConstantArithmetic(Opcode: ISD::SHL, DL, VT, Ops: {ExtC, N1})) {
11281 SDValue ExtX = DAG.getNode(Opcode: N0.getOpcode(), DL, VT, Operand: Add.getOperand(i: 0));
11282 SDValue ShlX = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ExtX, N2: N1);
11283 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: ShlX, N2: ShlC);
11284 }
11285 }
11286 }
11287
11288 // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
11289 if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) {
11290 SDValue N01 = N0.getOperand(i: 1);
11291 if (SDValue Shl =
11292 DAG.FoldConstantArithmetic(Opcode: ISD::SHL, DL: SDLoc(N1), VT, Ops: {N01, N1}))
11293 return DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N0.getOperand(i: 0), N2: Shl);
11294 }
11295
11296 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
11297 if (N1C && !N1C->isOpaque())
11298 if (SDValue NewSHL = visitShiftByConstant(N))
11299 return NewSHL;
11300
11301 // fold (shl X, cttz(Y)) -> (mul (Y & -Y), X) if cttz is unsupported on the
11302 // target.
11303 if (((N1.getOpcode() == ISD::CTTZ &&
11304 VT.getScalarSizeInBits() <= ShiftVT.getScalarSizeInBits()) ||
11305 N1.getOpcode() == ISD::CTTZ_ZERO_POISON) &&
11306 N1.hasOneUse() && !TLI.isOperationLegalOrCustom(Op: ISD::CTTZ, VT: ShiftVT) &&
11307 TLI.isOperationLegalOrCustom(Op: ISD::MUL, VT)) {
11308 SDValue Y = N1.getOperand(i: 0);
11309 SDLoc DL(N);
11310 SDValue NegY = DAG.getNegative(Val: Y, DL, VT: ShiftVT);
11311 SDValue And =
11312 DAG.getZExtOrTrunc(Op: DAG.getNode(Opcode: ISD::AND, DL, VT: ShiftVT, N1: Y, N2: NegY), DL, VT);
11313 return DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: And, N2: N0);
11314 }
11315
11316 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
11317 return SDValue(N, 0);
11318
11319 // Fold (shl (vscale * C0), C1) to (vscale * (C0 << C1)).
11320 if (N0.getOpcode() == ISD::VSCALE && N1C) {
11321 const APInt &C0 = N0.getConstantOperandAPInt(i: 0);
11322 const APInt &C1 = N1C->getAPIntValue();
11323 return DAG.getVScale(DL, VT, MulImm: C0 << C1);
11324 }
11325
11326 SDValue X;
11327 APInt VS0;
11328
11329 // fold (shl (X * vscale(VS0)), C1) -> (X * vscale(VS0 << C1))
11330 if (N1C && sd_match(N: N0, P: m_Mul(L: m_Value(N&: X), R: m_VScale(Op: m_ConstInt(V&: VS0))))) {
11331 SDNodeFlags Flags;
11332 Flags.setNoUnsignedWrap(N->getFlags().hasNoUnsignedWrap() &&
11333 N0->getFlags().hasNoUnsignedWrap());
11334
11335 SDValue VScale = DAG.getVScale(DL, VT, MulImm: VS0 << N1C->getAPIntValue());
11336 return DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: X, N2: VScale, Flags);
11337 }
11338
11339 // Fold (shl step_vector(C0), C1) to (step_vector(C0 << C1)).
11340 APInt ShlVal;
11341 if (N0.getOpcode() == ISD::STEP_VECTOR &&
11342 ISD::isConstantSplatVector(N: N1.getNode(), SplatValue&: ShlVal)) {
11343 const APInt &C0 = N0.getConstantOperandAPInt(i: 0);
11344 if (ShlVal.ult(RHS: C0.getBitWidth())) {
11345 APInt NewStep = C0 << ShlVal;
11346 return DAG.getStepVector(DL, ResVT: VT, StepVal: NewStep);
11347 }
11348 }
11349
11350 return SDValue();
11351}
11352
11353// Transform a right shift of a multiply into a multiply-high.
11354// Examples:
11355// (srl (mul (zext i32:$a to i64), (zext i32:$a to i64)), 32) -> (mulhu $a, $b)
11356// (sra (mul (sext i32:$a to i64), (sext i32:$a to i64)), 32) -> (mulhs $a, $b)
11357static SDValue combineShiftToMULH(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
11358 const TargetLowering &TLI) {
11359 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
11360 "SRL or SRA node is required here!");
11361
11362 // Check the shift amount. Proceed with the transformation if the shift
11363 // amount is constant.
11364 ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N: N->getOperand(Num: 1));
11365 if (!ShiftAmtSrc)
11366 return SDValue();
11367
11368 // The operation feeding into the shift must be a multiply.
11369 SDValue ShiftOperand = N->getOperand(Num: 0);
11370 if (ShiftOperand.getOpcode() != ISD::MUL)
11371 return SDValue();
11372
11373 // Both operands must be equivalent extend nodes.
11374 SDValue LeftOp = ShiftOperand.getOperand(i: 0);
11375 SDValue RightOp = ShiftOperand.getOperand(i: 1);
11376
11377 if (LeftOp.getOpcode() != ISD::SIGN_EXTEND &&
11378 LeftOp.getOpcode() != ISD::ZERO_EXTEND)
11379 std::swap(a&: LeftOp, b&: RightOp);
11380
11381 bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
11382 bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
11383
11384 if (!IsSignExt && !IsZeroExt)
11385 return SDValue();
11386
11387 EVT NarrowVT = LeftOp.getOperand(i: 0).getValueType();
11388 unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
11389
11390 // return true if U may use the lower bits of its operands
11391 auto UserOfLowerBits = [NarrowVTSize](SDNode *U) {
11392 if (U->getOpcode() != ISD::SRL && U->getOpcode() != ISD::SRA) {
11393 return true;
11394 }
11395 ConstantSDNode *UShiftAmtSrc = isConstOrConstSplat(N: U->getOperand(Num: 1));
11396 if (!UShiftAmtSrc) {
11397 return true;
11398 }
11399 unsigned UShiftAmt = UShiftAmtSrc->getZExtValue();
11400 return UShiftAmt < NarrowVTSize;
11401 };
11402
11403 // If the lower part of the MUL is also used and MUL_LOHI is supported
11404 // do not introduce the MULH in favor of MUL_LOHI
11405 unsigned MulLoHiOp = IsSignExt ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
11406 if (!ShiftOperand.hasOneUse() &&
11407 TLI.isOperationLegalOrCustom(Op: MulLoHiOp, VT: NarrowVT) &&
11408 llvm::any_of(Range: ShiftOperand->users(), P: UserOfLowerBits)) {
11409 return SDValue();
11410 }
11411
11412 SDValue MulhRightOp;
11413 if (LeftOp.getOpcode() != RightOp.getOpcode()) {
11414 if (IsZeroExt && ShiftOperand.hasOneUse() &&
11415 DAG.computeKnownBits(Op: RightOp).countMaxActiveBits() <= NarrowVTSize) {
11416 MulhRightOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: NarrowVT, Operand: RightOp);
11417 } else if (IsSignExt && ShiftOperand.hasOneUse() &&
11418 DAG.ComputeMaxSignificantBits(Op: RightOp) <= NarrowVTSize) {
11419 MulhRightOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: NarrowVT, Operand: RightOp);
11420 } else {
11421 return SDValue();
11422 }
11423 } else {
11424 // Check that the two extend nodes are the same type.
11425 if (NarrowVT != RightOp.getOperand(i: 0).getValueType())
11426 return SDValue();
11427 MulhRightOp = RightOp.getOperand(i: 0);
11428 }
11429
11430 EVT WideVT = LeftOp.getValueType();
11431 // Proceed with the transformation if the wide types match.
11432 assert((WideVT == RightOp.getValueType()) &&
11433 "Cannot have a multiply node with two different operand types.");
11434
11435 // Proceed with the transformation if the wide type is twice as large
11436 // as the narrow type.
11437 if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
11438 return SDValue();
11439
11440 // Check the shift amount with the narrow type size.
11441 // Proceed with the transformation if the shift amount is the width
11442 // of the narrow type.
11443 unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
11444 if (ShiftAmt != NarrowVTSize)
11445 return SDValue();
11446
11447 // If the operation feeding into the MUL is a sign extend (sext),
11448 // we use mulhs. Othewise, zero extends (zext) use mulhu.
11449 unsigned MulhOpcode = IsSignExt ? ISD::MULHS : ISD::MULHU;
11450
11451 // Combine to mulh if mulh is legal/custom for the narrow type on the target
11452 // or if it is a vector type then we could transform to an acceptable type and
11453 // rely on legalization to split/combine the result.
11454 EVT TransformVT = NarrowVT;
11455 if (NarrowVT.isVector()) {
11456 TransformVT = TLI.getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT: NarrowVT);
11457 if (TransformVT.getScalarType() != NarrowVT.getScalarType())
11458 return SDValue();
11459 }
11460 if (!TLI.isOperationLegalOrCustom(Op: MulhOpcode, VT: TransformVT))
11461 return SDValue();
11462
11463 SDValue Result =
11464 DAG.getNode(Opcode: MulhOpcode, DL, VT: NarrowVT, N1: LeftOp.getOperand(i: 0), N2: MulhRightOp);
11465 bool IsSigned = N->getOpcode() == ISD::SRA;
11466 return DAG.getExtOrTrunc(IsSigned, Op: Result, DL, VT: WideVT);
11467}
11468
11469// fold (bswap (logic_op(bswap(x),y))) -> logic_op(x,bswap(y))
11470// This helper function accept SDNode with opcode ISD::BSWAP and ISD::BITREVERSE
11471static SDValue foldBitOrderCrossLogicOp(SDNode *N, SelectionDAG &DAG) {
11472 unsigned Opcode = N->getOpcode();
11473 if (Opcode != ISD::BSWAP && Opcode != ISD::BITREVERSE)
11474 return SDValue();
11475
11476 SDValue N0 = N->getOperand(Num: 0);
11477 EVT VT = N->getValueType(ResNo: 0);
11478 SDLoc DL(N);
11479 SDValue X, Y;
11480
11481 // If both operands are bswap/bitreverse, ignore the multiuse
11482 if (sd_match(N: N0, P: m_OneUse(P: m_BitwiseLogic(L: m_UnaryOp(Opc: Opcode, Op: m_Value(N&: X)),
11483 R: m_UnaryOp(Opc: Opcode, Op: m_Value(N&: Y))))))
11484 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, N1: X, N2: Y);
11485
11486 // Otherwise need to ensure logic_op and bswap/bitreverse(x) have one use.
11487 if (sd_match(N: N0, P: m_OneUse(P: m_BitwiseLogic(
11488 L: m_OneUse(P: m_UnaryOp(Opc: Opcode, Op: m_Value(N&: X))), R: m_Value(N&: Y))))) {
11489 SDValue NewBitReorder = DAG.getNode(Opcode, DL, VT, Operand: Y);
11490 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, N1: X, N2: NewBitReorder);
11491 }
11492
11493 return SDValue();
11494}
11495
11496SDValue DAGCombiner::visitSRA(SDNode *N) {
11497 SDValue N0 = N->getOperand(Num: 0);
11498 SDValue N1 = N->getOperand(Num: 1);
11499 if (SDValue V = DAG.simplifyShift(X: N0, Y: N1))
11500 return V;
11501
11502 SDLoc DL(N);
11503 EVT VT = N0.getValueType();
11504 unsigned OpSizeInBits = VT.getScalarSizeInBits();
11505
11506 // fold (sra c1, c2) -> (sra c1, c2)
11507 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::SRA, DL, VT, Ops: {N0, N1}))
11508 return C;
11509
11510 // Arithmetic shifting an all-sign-bit value is a no-op.
11511 // fold (sra 0, x) -> 0
11512 // fold (sra -1, x) -> -1
11513 if (DAG.ComputeNumSignBits(Op: N0) == OpSizeInBits)
11514 return N0;
11515
11516 // fold vector ops
11517 if (VT.isVector())
11518 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
11519 return FoldedVOp;
11520
11521 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
11522 return NewSel;
11523
11524 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
11525
11526 // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
11527 // clamp (add c1, c2) to max shift.
11528 if (N0.getOpcode() == ISD::SRA) {
11529 EVT ShiftVT = N1.getValueType();
11530 EVT ShiftSVT = ShiftVT.getScalarType();
11531 SmallVector<SDValue, 16> ShiftValues;
11532
11533 auto SumOfShifts = [&](ConstantSDNode *LHS, ConstantSDNode *RHS) {
11534 APInt c1 = LHS->getAPIntValue();
11535 APInt c2 = RHS->getAPIntValue();
11536 zeroExtendToMatch(LHS&: c1, RHS&: c2, Offset: 1 /* Overflow Bit */);
11537 APInt Sum = c1 + c2;
11538 unsigned ShiftSum =
11539 Sum.uge(RHS: OpSizeInBits) ? (OpSizeInBits - 1) : Sum.getZExtValue();
11540 ShiftValues.push_back(Elt: DAG.getConstant(Val: ShiftSum, DL, VT: ShiftSVT));
11541 return true;
11542 };
11543 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: SumOfShifts)) {
11544 SDValue ShiftValue;
11545 if (N1.getOpcode() == ISD::BUILD_VECTOR)
11546 ShiftValue = DAG.getBuildVector(VT: ShiftVT, DL, Ops: ShiftValues);
11547 else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
11548 assert(ShiftValues.size() == 1 &&
11549 "Expected matchBinaryPredicate to return one element for "
11550 "SPLAT_VECTORs");
11551 ShiftValue = DAG.getSplatVector(VT: ShiftVT, DL, Op: ShiftValues[0]);
11552 } else
11553 ShiftValue = ShiftValues[0];
11554 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N0.getOperand(i: 0), N2: ShiftValue);
11555 }
11556 }
11557
11558 // fold (sra (xor (sra x, c1), -1), c2) -> (xor (sra x, c3), -1)
11559 // This allows merging two arithmetic shifts even when there's a NOT in
11560 // between.
11561 SDValue X;
11562 APInt C1;
11563 if (N1C && sd_match(N: N0, P: m_OneUse(P: m_Not(
11564 V: m_OneUse(P: m_Sra(L: m_Value(N&: X), R: m_ConstInt(V&: C1))))))) {
11565 APInt C2 = N1C->getAPIntValue();
11566 zeroExtendToMatch(LHS&: C1, RHS&: C2, Offset: 1 /* Overflow Bit */);
11567 APInt Sum = C1 + C2;
11568 unsigned ShiftSum = Sum.getLimitedValue(Limit: OpSizeInBits - 1);
11569 SDValue NewShift = DAG.getNode(
11570 Opcode: ISD::SRA, DL, VT, N1: X, N2: DAG.getShiftAmountConstant(Val: ShiftSum, VT, DL));
11571 return DAG.getNOT(DL, Val: NewShift, VT);
11572 }
11573
11574 // fold (sra (shl X, m), (sub result_size, n))
11575 // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
11576 // result_size - n != m.
11577 // If truncate is free for the target sext(shl) is likely to result in better
11578 // code.
11579 if (N0.getOpcode() == ISD::SHL && N1C) {
11580 // Get the two constants of the shifts, CN0 = m, CN = n.
11581 const ConstantSDNode *N01C = isConstOrConstSplat(N: N0.getOperand(i: 1));
11582 if (N01C) {
11583 LLVMContext &Ctx = *DAG.getContext();
11584 // Determine what the truncate's result bitsize and type would be.
11585 EVT TruncVT = VT.changeElementType(
11586 Context&: Ctx, EltVT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OpSizeInBits - N1C->getZExtValue()));
11587
11588 // Determine the residual right-shift amount.
11589 int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
11590
11591 // If the shift is not a no-op (in which case this should be just a sign
11592 // extend already), the truncated to type is legal, sign_extend is legal
11593 // on that type, and the truncate to that type is both legal and free,
11594 // perform the transform.
11595 if ((ShiftAmt > 0) &&
11596 TLI.isOperationLegalOrCustom(Op: ISD::SIGN_EXTEND, VT: TruncVT) &&
11597 TLI.isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT) &&
11598 TLI.isTruncateFree(FromVT: VT, ToVT: TruncVT)) {
11599 SDValue Amt = DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL);
11600 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT,
11601 N1: N0.getOperand(i: 0), N2: Amt);
11602 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: TruncVT,
11603 Operand: Shift);
11604 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL,
11605 VT: N->getValueType(ResNo: 0), Operand: Trunc);
11606 }
11607 }
11608 }
11609
11610 // We convert trunc/ext to opposing shifts in IR, but casts may be cheaper.
11611 // sra (add (shl X, N1C), AddC), N1C -->
11612 // sext (add (trunc X to (width - N1C)), AddC')
11613 // sra (sub AddC, (shl X, N1C)), N1C -->
11614 // sext (sub AddC1',(trunc X to (width - N1C)))
11615 if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB) && N1C &&
11616 N0.hasOneUse()) {
11617 bool IsAdd = N0.getOpcode() == ISD::ADD;
11618 SDValue Shl = N0.getOperand(i: IsAdd ? 0 : 1);
11619 if (Shl.getOpcode() == ISD::SHL && Shl.getOperand(i: 1) == N1 &&
11620 Shl.hasOneUse()) {
11621 // TODO: AddC does not need to be a splat.
11622 if (ConstantSDNode *AddC =
11623 isConstOrConstSplat(N: N0.getOperand(i: IsAdd ? 1 : 0))) {
11624 // Determine what the truncate's type would be and ask the target if
11625 // that is a free operation.
11626 LLVMContext &Ctx = *DAG.getContext();
11627 unsigned ShiftAmt = N1C->getZExtValue();
11628 EVT TruncVT = VT.changeElementType(
11629 Context&: Ctx, EltVT: EVT::getIntegerVT(Context&: Ctx, BitWidth: OpSizeInBits - ShiftAmt));
11630
11631 // TODO: The simple type check probably belongs in the default hook
11632 // implementation and/or target-specific overrides (because
11633 // non-simple types likely require masking when legalized), but
11634 // that restriction may conflict with other transforms.
11635 if (TruncVT.isSimple() && isTypeLegal(VT: TruncVT) &&
11636 TLI.isTruncateFree(FromVT: VT, ToVT: TruncVT)) {
11637 SDValue Trunc = DAG.getZExtOrTrunc(Op: Shl.getOperand(i: 0), DL, VT: TruncVT);
11638 SDValue ShiftC =
11639 DAG.getConstant(Val: AddC->getAPIntValue().lshr(shiftAmt: ShiftAmt).trunc(
11640 width: TruncVT.getScalarSizeInBits()),
11641 DL, VT: TruncVT);
11642 SDValue Add;
11643 if (IsAdd)
11644 Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: TruncVT, N1: Trunc, N2: ShiftC);
11645 else
11646 Add = DAG.getNode(Opcode: ISD::SUB, DL, VT: TruncVT, N1: ShiftC, N2: Trunc);
11647 return DAG.getSExtOrTrunc(Op: Add, DL, VT);
11648 }
11649 }
11650 }
11651 }
11652
11653 // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
11654 if (N1.getOpcode() == ISD::TRUNCATE &&
11655 N1.getOperand(i: 0).getOpcode() == ISD::AND) {
11656 if (SDValue NewOp1 = distributeTruncateThroughAnd(N: N1.getNode()))
11657 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N0, N2: NewOp1);
11658 }
11659
11660 // fold (sra (trunc (sra x, c1)), c2) -> (trunc (sra x, c1 + c2))
11661 // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
11662 // if c1 is equal to the number of bits the trunc removes
11663 // TODO - support non-uniform vector shift amounts.
11664 if (N0.getOpcode() == ISD::TRUNCATE &&
11665 (N0.getOperand(i: 0).getOpcode() == ISD::SRL ||
11666 N0.getOperand(i: 0).getOpcode() == ISD::SRA) &&
11667 N0.getOperand(i: 0).hasOneUse() &&
11668 N0.getOperand(i: 0).getOperand(i: 1).hasOneUse() && N1C) {
11669 SDValue N0Op0 = N0.getOperand(i: 0);
11670 if (ConstantSDNode *LargeShift = isConstOrConstSplat(N: N0Op0.getOperand(i: 1))) {
11671 EVT LargeVT = N0Op0.getValueType();
11672 unsigned TruncBits = LargeVT.getScalarSizeInBits() - OpSizeInBits;
11673 if (LargeShift->getAPIntValue() == TruncBits) {
11674 EVT LargeShiftVT = getShiftAmountTy(LHSTy: LargeVT);
11675 SDValue Amt = DAG.getZExtOrTrunc(Op: N1, DL, VT: LargeShiftVT);
11676 Amt = DAG.getNode(Opcode: ISD::ADD, DL, VT: LargeShiftVT, N1: Amt,
11677 N2: DAG.getConstant(Val: TruncBits, DL, VT: LargeShiftVT));
11678 SDValue SRA =
11679 DAG.getNode(Opcode: ISD::SRA, DL, VT: LargeVT, N1: N0Op0.getOperand(i: 0), N2: Amt);
11680 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: SRA);
11681 }
11682 }
11683 }
11684
11685 // fold (sra (add nsw X, C), D) -> (add nsw (sra X, D), C s>> D)
11686 // when C has D trailing zeros (so C s>> D is exact).
11687 if (N1C && N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
11688 N0->getFlags().hasNoSignedWrap()) {
11689 if (ConstantSDNode *AddC = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
11690 const APInt &ShAmt = N1C->getAPIntValue();
11691 const APInt &AddVal = AddC->getAPIntValue();
11692 if (ShAmt.ult(RHS: AddVal.countr_zero())) {
11693 SDNodeFlags ShiftFlags = N->getFlags();
11694 SDValue NewSra =
11695 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N0.getOperand(i: 0), N2: N1, Flags: ShiftFlags);
11696 SDValue NewC = DAG.getConstant(Val: AddVal.ashr(ShiftAmt: ShAmt), DL, VT);
11697 SDNodeFlags AddFlags = N0->getFlags();
11698 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NewSra, N2: NewC, Flags: AddFlags);
11699 }
11700 }
11701 }
11702
11703 // Simplify, based on bits shifted out of the LHS.
11704 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
11705 return SDValue(N, 0);
11706
11707 // If the sign bit is known to be zero, switch this to a SRL.
11708 if (DAG.SignBitIsZero(Op: N0))
11709 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0, N2: N1);
11710
11711 if (N1C && !N1C->isOpaque())
11712 if (SDValue NewSRA = visitShiftByConstant(N))
11713 return NewSRA;
11714
11715 // Try to transform this shift into a multiply-high if
11716 // it matches the appropriate pattern detected in combineShiftToMULH.
11717 if (SDValue MULH = combineShiftToMULH(N, DL, DAG, TLI))
11718 return MULH;
11719
11720 // Attempt to convert a sra of a load into a narrower sign-extending load.
11721 if (SDValue NarrowLoad = reduceLoadWidth(N))
11722 return NarrowLoad;
11723
11724 if (SDValue AVG = foldShiftToAvg(N, DL))
11725 return AVG;
11726
11727 return SDValue();
11728}
11729
11730SDValue DAGCombiner::visitSRL(SDNode *N) {
11731 SDValue N0 = N->getOperand(Num: 0);
11732 SDValue N1 = N->getOperand(Num: 1);
11733 if (SDValue V = DAG.simplifyShift(X: N0, Y: N1))
11734 return V;
11735
11736 SDLoc DL(N);
11737 EVT VT = N0.getValueType();
11738 EVT ShiftVT = N1.getValueType();
11739 unsigned OpSizeInBits = VT.getScalarSizeInBits();
11740
11741 // fold (srl c1, c2) -> c1 >>u c2
11742 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::SRL, DL, VT, Ops: {N0, N1}))
11743 return C;
11744
11745 // fold vector ops
11746 if (VT.isVector())
11747 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
11748 return FoldedVOp;
11749
11750 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
11751 return NewSel;
11752
11753 // if (srl x, c) is known to be zero, return 0
11754 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
11755 if (N1C &&
11756 DAG.MaskedValueIsZero(Op: SDValue(N, 0), Mask: APInt::getAllOnes(numBits: OpSizeInBits)))
11757 return DAG.getConstant(Val: 0, DL, VT);
11758
11759 // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
11760 if (N0.getOpcode() == ISD::SRL) {
11761 auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
11762 ConstantSDNode *RHS) {
11763 APInt c1 = LHS->getAPIntValue();
11764 APInt c2 = RHS->getAPIntValue();
11765 zeroExtendToMatch(LHS&: c1, RHS&: c2, Offset: 1 /* Overflow Bit */);
11766 return (c1 + c2).uge(RHS: OpSizeInBits);
11767 };
11768 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: MatchOutOfRange))
11769 return DAG.getConstant(Val: 0, DL, VT);
11770
11771 auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
11772 ConstantSDNode *RHS) {
11773 APInt c1 = LHS->getAPIntValue();
11774 APInt c2 = RHS->getAPIntValue();
11775 zeroExtendToMatch(LHS&: c1, RHS&: c2, Offset: 1 /* Overflow Bit */);
11776 return (c1 + c2).ult(RHS: OpSizeInBits);
11777 };
11778 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: MatchInRange)) {
11779 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL, VT: ShiftVT, N1, N2: N0.getOperand(i: 1));
11780 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0.getOperand(i: 0), N2: Sum);
11781 }
11782 }
11783
11784 if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
11785 N0.getOperand(i: 0).getOpcode() == ISD::SRL) {
11786 SDValue InnerShift = N0.getOperand(i: 0);
11787 // TODO - support non-uniform vector shift amounts.
11788 if (auto *N001C = isConstOrConstSplat(N: InnerShift.getOperand(i: 1))) {
11789 uint64_t c1 = N001C->getZExtValue();
11790 uint64_t c2 = N1C->getZExtValue();
11791 EVT InnerShiftVT = InnerShift.getValueType();
11792 EVT ShiftAmtVT = InnerShift.getOperand(i: 1).getValueType();
11793 uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
11794 // srl (trunc (srl x, c1)), c2 --> 0 or (trunc (srl x, (add c1, c2)))
11795 // This is only valid if the OpSizeInBits + c1 = size of inner shift.
11796 if (c1 + OpSizeInBits == InnerShiftSize) {
11797 if (c1 + c2 >= InnerShiftSize)
11798 return DAG.getConstant(Val: 0, DL, VT);
11799 SDValue NewShiftAmt = DAG.getConstant(Val: c1 + c2, DL, VT: ShiftAmtVT);
11800 SDValue NewShift = DAG.getNode(Opcode: ISD::SRL, DL, VT: InnerShiftVT,
11801 N1: InnerShift.getOperand(i: 0), N2: NewShiftAmt);
11802 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: NewShift);
11803 }
11804 // In the more general case, we can clear the high bits after the shift:
11805 // srl (trunc (srl x, c1)), c2 --> trunc (and (srl x, (c1+c2)), Mask)
11806 if (N0.hasOneUse() && InnerShift.hasOneUse() &&
11807 c1 + c2 < InnerShiftSize) {
11808 SDValue NewShiftAmt = DAG.getConstant(Val: c1 + c2, DL, VT: ShiftAmtVT);
11809 SDValue NewShift = DAG.getNode(Opcode: ISD::SRL, DL, VT: InnerShiftVT,
11810 N1: InnerShift.getOperand(i: 0), N2: NewShiftAmt);
11811 SDValue Mask = DAG.getConstant(Val: APInt::getLowBitsSet(numBits: InnerShiftSize,
11812 loBitsSet: OpSizeInBits - c2),
11813 DL, VT: InnerShiftVT);
11814 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: InnerShiftVT, N1: NewShift, N2: Mask);
11815 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: And);
11816 }
11817 }
11818 }
11819
11820 if (N0.getOpcode() == ISD::SHL) {
11821 // fold (srl (shl nuw x, c), c) -> x
11822 if (N0.getOperand(i: 1) == N1 && N0->getFlags().hasNoUnsignedWrap())
11823 return N0.getOperand(i: 0);
11824
11825 // fold (srl (shl x, c1), c2) -> (and (shl x, (sub c1, c2), MASK) or
11826 // (and (srl x, (sub c2, c1), MASK)
11827 if ((N0.getOperand(i: 1) == N1 || N0->hasOneUse()) &&
11828 TLI.shouldFoldConstantShiftPairToMask(N)) {
11829 auto MatchShiftAmount = [OpSizeInBits](ConstantSDNode *LHS,
11830 ConstantSDNode *RHS) {
11831 const APInt &LHSC = LHS->getAPIntValue();
11832 const APInt &RHSC = RHS->getAPIntValue();
11833 return LHSC.ult(RHS: OpSizeInBits) && RHSC.ult(RHS: OpSizeInBits) &&
11834 LHSC.getZExtValue() <= RHSC.getZExtValue();
11835 };
11836 if (ISD::matchBinaryPredicate(LHS: N1, RHS: N0.getOperand(i: 1), Match: MatchShiftAmount,
11837 /*AllowUndefs*/ false,
11838 /*AllowTypeMismatch*/ true)) {
11839 SDValue N01 = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 1), DL, VT: ShiftVT);
11840 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShiftVT, N1: N01, N2: N1);
11841 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11842 Mask = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mask, N2: N01);
11843 Mask = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Mask, N2: Diff);
11844 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0.getOperand(i: 0), N2: Diff);
11845 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shift, N2: Mask);
11846 }
11847 if (ISD::matchBinaryPredicate(LHS: N0.getOperand(i: 1), RHS: N1, Match: MatchShiftAmount,
11848 /*AllowUndefs*/ false,
11849 /*AllowTypeMismatch*/ true)) {
11850 SDValue N01 = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 1), DL, VT: ShiftVT);
11851 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: ShiftVT, N1, N2: N01);
11852 SDValue Mask = DAG.getAllOnesConstant(DL, VT);
11853 Mask = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Mask, N2: N1);
11854 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0.getOperand(i: 0), N2: Diff);
11855 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shift, N2: Mask);
11856 }
11857 }
11858 }
11859
11860 // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
11861 // TODO - support non-uniform vector shift amounts.
11862 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
11863 // Shifting in all undef bits?
11864 EVT SmallVT = N0.getOperand(i: 0).getValueType();
11865 unsigned BitSize = SmallVT.getScalarSizeInBits();
11866 if (N1C->getAPIntValue().uge(RHS: BitSize))
11867 return DAG.getUNDEF(VT);
11868
11869 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, VT: SmallVT)) {
11870 uint64_t ShiftAmt = N1C->getZExtValue();
11871 SDLoc DL0(N0);
11872 SDValue SmallShift =
11873 DAG.getNode(Opcode: ISD::SRL, DL: DL0, VT: SmallVT, N1: N0.getOperand(i: 0),
11874 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT: SmallVT, DL: DL0));
11875 AddToWorklist(N: SmallShift.getNode());
11876 APInt Mask = APInt::getLowBitsSet(numBits: OpSizeInBits, loBitsSet: OpSizeInBits - ShiftAmt);
11877 return DAG.getNode(Opcode: ISD::AND, DL, VT,
11878 N1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: SmallShift),
11879 N2: DAG.getConstant(Val: Mask, DL, VT));
11880 }
11881 }
11882
11883 // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign
11884 // bit, which is unmodified by sra.
11885 if (N1C && N1C->getAPIntValue() == (OpSizeInBits - 1)) {
11886 if (N0.getOpcode() == ISD::SRA)
11887 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0.getOperand(i: 0), N2: N1);
11888 }
11889
11890 // fold (srl (ctlz x), "5") -> x iff x has one bit set (the low bit), and x has a power
11891 // of two bitwidth. The "5" represents (log2 (bitwidth x)).
11892 if (N1C && N0.getOpcode() == ISD::CTLZ &&
11893 isPowerOf2_32(Value: OpSizeInBits) &&
11894 N1C->getAPIntValue() == Log2_32(Value: OpSizeInBits)) {
11895 KnownBits Known = DAG.computeKnownBits(Op: N0.getOperand(i: 0));
11896
11897 // If any of the input bits are KnownOne, then the input couldn't be all
11898 // zeros, thus the result of the srl will always be zero.
11899 if (Known.One.getBoolValue()) return DAG.getConstant(Val: 0, DL: SDLoc(N0), VT);
11900
11901 // If all of the bits input the to ctlz node are known to be zero, then
11902 // the result of the ctlz is "32" and the result of the shift is one.
11903 APInt UnknownBits = ~Known.Zero;
11904 if (UnknownBits == 0) return DAG.getConstant(Val: 1, DL: SDLoc(N0), VT);
11905
11906 // Otherwise, check to see if there is exactly one bit input to the ctlz.
11907 if (UnknownBits.isPowerOf2()) {
11908 // Okay, we know that only that the single bit specified by UnknownBits
11909 // could be set on input to the CTLZ node. If this bit is set, the SRL
11910 // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
11911 // to an SRL/XOR pair, which is likely to simplify more.
11912 unsigned ShAmt = UnknownBits.countr_zero();
11913 SDValue Op = N0.getOperand(i: 0);
11914
11915 if (ShAmt) {
11916 SDLoc DL(N0);
11917 Op = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Op,
11918 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT, DL));
11919 AddToWorklist(N: Op.getNode());
11920 }
11921 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Op, N2: DAG.getConstant(Val: 1, DL, VT));
11922 }
11923 }
11924
11925 // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
11926 if (N1.getOpcode() == ISD::TRUNCATE &&
11927 N1.getOperand(i: 0).getOpcode() == ISD::AND) {
11928 if (SDValue NewOp1 = distributeTruncateThroughAnd(N: N1.getNode()))
11929 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0, N2: NewOp1);
11930 }
11931
11932 // fold (srl (logic_op x, (shl (zext y), c1)), c1)
11933 // -> (logic_op (srl x, c1), (zext y))
11934 // c1 <= leadingzeros(zext(y))
11935 // TODO: Replace c1 with valuetracking?
11936 SDValue X, ZExtY;
11937 if (sd_match(
11938 N: N0,
11939 P: m_OneUse(P: m_BitwiseLogic(
11940 L: m_Value(N&: X),
11941 R: m_OneUse(P: m_Shl(L: m_Value(N&: ZExtY, P: m_SpecificOpc(Opcode: ISD::ZERO_EXTEND)),
11942 R: m_Specific(N: N1))))))) {
11943 unsigned NumLeadingZeros = ZExtY.getScalarValueSizeInBits() -
11944 ZExtY.getOperand(i: 0).getScalarValueSizeInBits();
11945 if (N1C && N1C->getZExtValue() <= NumLeadingZeros)
11946 return DAG.getNode(Opcode: N0.getOpcode(), DL: SDLoc(N0), VT,
11947 N1: DAG.getNode(Opcode: ISD::SRL, DL: SDLoc(N0), VT, N1: X, N2: N1), N2: ZExtY);
11948 }
11949
11950 // fold (srl (bitcast (build_vector e1, ..., eN)), (N-1) * eltsize)
11951 // -> (zext eN)
11952 if (N1C && VT.isScalarInteger() && DAG.getDataLayout().isLittleEndian()) {
11953 SDValue BV = peekThroughBitcasts(V: N0);
11954 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
11955 EVT BVVT = BV.getValueType();
11956 unsigned EltSizeInBits = BVVT.getScalarSizeInBits();
11957 unsigned NumElts = BVVT.getVectorNumElements();
11958 if (N1C->getZExtValue() == (NumElts - 1) * EltSizeInBits) {
11959 SDValue LastElt = BV.getOperand(i: NumElts - 1);
11960 assert(LastElt.getScalarValueSizeInBits() >= EltSizeInBits &&
11961 "Expected BUILD_VECTOR operand as wide as element type");
11962 EVT IntEltVT = LastElt.getValueType().changeTypeToInteger();
11963 if (!LegalTypes || TLI.isTypeLegal(VT: IntEltVT)) {
11964 LastElt = DAG.getBitcast(VT: IntEltVT, V: LastElt);
11965 SDValue Ext = DAG.getZExtOrTrunc(Op: LastElt, DL, VT);
11966 APInt Mask = APInt::getLowBitsSet(numBits: VT.getSizeInBits(), loBitsSet: EltSizeInBits);
11967 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Ext,
11968 N2: DAG.getConstant(Val: Mask, DL, VT));
11969 }
11970 }
11971 }
11972 }
11973
11974 // fold (srl (add nuw X, C), D) -> (add nuw (srl X, D), C u>> D)
11975 // when C has D trailing zeros (so C >> D is exact).
11976 if (N1C && N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
11977 N0->getFlags().hasNoUnsignedWrap()) {
11978 if (ConstantSDNode *AddC = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
11979 const APInt &ShAmt = N1C->getAPIntValue();
11980 const APInt &AddVal = AddC->getAPIntValue();
11981 if (ShAmt.ult(RHS: AddVal.countr_zero())) {
11982 SDNodeFlags ShiftFlags = N->getFlags();
11983 SDValue NewSrl =
11984 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: N0.getOperand(i: 0), N2: N1, Flags: ShiftFlags);
11985 SDValue NewC = DAG.getConstant(Val: AddVal.lshr(ShiftAmt: ShAmt), DL, VT);
11986 SDNodeFlags AddFlags = N0->getFlags();
11987 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NewSrl, N2: NewC, Flags: AddFlags);
11988 }
11989 }
11990 }
11991
11992 // fold operands of srl based on knowledge that the low bits are not
11993 // demanded.
11994 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
11995 return SDValue(N, 0);
11996
11997 if (N1C && !N1C->isOpaque())
11998 if (SDValue NewSRL = visitShiftByConstant(N))
11999 return NewSRL;
12000
12001 // Attempt to convert a srl of a load into a narrower zero-extending load.
12002 if (SDValue NarrowLoad = reduceLoadWidth(N))
12003 return NarrowLoad;
12004
12005 // Here is a common situation. We want to optimize:
12006 //
12007 // %a = ...
12008 // %b = and i32 %a, 2
12009 // %c = srl i32 %b, 1
12010 // brcond i32 %c ...
12011 //
12012 // into
12013 //
12014 // %a = ...
12015 // %b = and %a, 2
12016 // %c = setcc eq %b, 0
12017 // brcond %c ...
12018 //
12019 // However when after the source operand of SRL is optimized into AND, the SRL
12020 // itself may not be optimized further. Look for it and add the BRCOND into
12021 // the worklist.
12022 //
12023 // The also tends to happen for binary operations when SimplifyDemandedBits
12024 // is involved.
12025 //
12026 // FIXME: This is unecessary if we process the DAG in topological order,
12027 // which we plan to do. This workaround can be removed once the DAG is
12028 // processed in topological order.
12029 if (N->hasOneUse()) {
12030 SDNode *User = *N->user_begin();
12031
12032 // Look pass the truncate.
12033 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse())
12034 User = *User->user_begin();
12035
12036 if (User->getOpcode() == ISD::BRCOND || User->getOpcode() == ISD::AND ||
12037 User->getOpcode() == ISD::OR || User->getOpcode() == ISD::XOR)
12038 AddToWorklist(N: User);
12039 }
12040
12041 // Try to transform this shift into a multiply-high if
12042 // it matches the appropriate pattern detected in combineShiftToMULH.
12043 if (SDValue MULH = combineShiftToMULH(N, DL, DAG, TLI))
12044 return MULH;
12045
12046 if (SDValue AVG = foldShiftToAvg(N, DL))
12047 return AVG;
12048
12049 SDValue Y;
12050 if (VT.getScalarSizeInBits() % 2 == 0 && N1C) {
12051 // Fold clmul(zext(x), zext(y)) >> (BW - 1 | BW) -> clmul(r|h)(x, y).
12052 unsigned HalfBW = VT.getScalarSizeInBits() / 2;
12053 if (sd_match(N: N0, P: m_Clmul(L: m_ZExt(Op: m_Value(N&: X)), R: m_ZExt(Op: m_Value(N&: Y)))) &&
12054 X.getScalarValueSizeInBits() == HalfBW &&
12055 Y.getScalarValueSizeInBits() == HalfBW) {
12056 if (N1C->getZExtValue() == HalfBW - 1 &&
12057 (!LegalOperations ||
12058 TLI.isOperationLegalOrCustom(Op: ISD::CLMULR, VT: X.getValueType())))
12059 return DAG.getNode(
12060 Opcode: ISD::ZERO_EXTEND, DL, VT,
12061 Operand: DAG.getNode(Opcode: ISD::CLMULR, DL, VT: X.getValueType(), N1: X, N2: Y));
12062 if (N1C->getZExtValue() == HalfBW &&
12063 (!LegalOperations ||
12064 TLI.isOperationLegalOrCustom(Op: ISD::CLMULH, VT: X.getValueType())))
12065 return DAG.getNode(
12066 Opcode: ISD::ZERO_EXTEND, DL, VT,
12067 Operand: DAG.getNode(Opcode: ISD::CLMULH, DL, VT: X.getValueType(), N1: X, N2: Y));
12068 }
12069 }
12070
12071 // Fold bitreverse(clmul(bitreverse(x), bitreverse(y))) >> 1 ->
12072 // clmulh(x, y).
12073 if (N1C && N1C->getZExtValue() == 1 &&
12074 sd_match(N: N0, P: m_BitReverse(Op: m_Clmul(L: m_BitReverse(Op: m_Value(N&: X)),
12075 R: m_BitReverse(Op: m_Value(N&: Y))))))
12076 return DAG.getNode(Opcode: ISD::CLMULH, DL, VT, N1: X, N2: Y);
12077
12078 return SDValue();
12079}
12080
12081SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
12082 EVT VT = N->getValueType(ResNo: 0);
12083 SDValue N0 = N->getOperand(Num: 0);
12084 SDValue N1 = N->getOperand(Num: 1);
12085 SDValue N2 = N->getOperand(Num: 2);
12086 bool IsFSHL = N->getOpcode() == ISD::FSHL;
12087 unsigned BitWidth = VT.getScalarSizeInBits();
12088 SDLoc DL(N);
12089
12090 // fold (fshl/fshr C0, C1, C2) -> C3
12091 if (SDValue C =
12092 DAG.FoldConstantArithmetic(Opcode: N->getOpcode(), DL, VT, Ops: {N0, N1, N2}))
12093 return C;
12094
12095 // fold (fshl N0, N1, 0) -> N0
12096 // fold (fshr N0, N1, 0) -> N1
12097 if (isPowerOf2_32(Value: BitWidth))
12098 if (DAG.MaskedValueIsZero(
12099 Op: N2, Mask: APInt(N2.getScalarValueSizeInBits(), BitWidth - 1)))
12100 return IsFSHL ? N0 : N1;
12101
12102 auto IsUndefOrZero = [](SDValue V) {
12103 return V.isUndef() || isNullOrNullSplat(V, /*AllowUndefs*/ true);
12104 };
12105
12106 // TODO - support non-uniform vector shift amounts.
12107 if (ConstantSDNode *Cst = isConstOrConstSplat(N: N2)) {
12108 EVT ShAmtTy = N2.getValueType();
12109
12110 // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth)
12111 if (Cst->getAPIntValue().uge(RHS: BitWidth)) {
12112 uint64_t RotAmt = Cst->getAPIntValue().urem(RHS: BitWidth);
12113 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N0, N2: N1,
12114 N3: DAG.getConstant(Val: RotAmt, DL, VT: ShAmtTy));
12115 }
12116
12117 unsigned ShAmt = Cst->getZExtValue();
12118 if (ShAmt == 0)
12119 return IsFSHL ? N0 : N1;
12120
12121 // fold fshl(undef_or_zero, N1, C) -> lshr(N1, BW-C)
12122 // fold fshr(undef_or_zero, N1, C) -> lshr(N1, C)
12123 // fold fshl(N0, undef_or_zero, C) -> shl(N0, C)
12124 // fold fshr(N0, undef_or_zero, C) -> shl(N0, BW-C)
12125 if (IsUndefOrZero(N0))
12126 return DAG.getNode(
12127 Opcode: ISD::SRL, DL, VT, N1,
12128 N2: DAG.getConstant(Val: IsFSHL ? BitWidth - ShAmt : ShAmt, DL, VT: ShAmtTy));
12129 if (IsUndefOrZero(N1))
12130 return DAG.getNode(
12131 Opcode: ISD::SHL, DL, VT, N1: N0,
12132 N2: DAG.getConstant(Val: IsFSHL ? ShAmt : BitWidth - ShAmt, DL, VT: ShAmtTy));
12133
12134 // fold fshl(N0, N1, c) -> x and fshr(N0, N1, c) -> x
12135 // where N0 is any node that contributes "x >> C0" to the result:
12136 // lshr(x, C0) | fshr(_, x, C0) | fshl(_, x, C1)
12137 // and N1 is any node that contributes "x << C1" to the result:
12138 // shl(x, C1) | fshl(x, _, C1) | fshr(x, _, C0)
12139 // with C0 = IsFSHL ? amnt : BW-amnt, C1 = BW - C0
12140
12141 // ShAmt == 0 was handled above; uge(BitWidth) was reduced via modulo above.
12142 assert(ShAmt >= 1 && ShAmt < BitWidth &&
12143 "ShAmt must be in [1, BW-1] for the identity fold to be valid");
12144 SDValue Val;
12145 unsigned C0Expected = IsFSHL ? ShAmt : BitWidth - ShAmt;
12146 unsigned C1Expected = IsFSHL ? BitWidth - ShAmt : ShAmt;
12147
12148 if ((sd_match(N: N0, P: m_Srl(L: m_Value(N&: Val), R: m_SpecificInt(V: C0Expected))) ||
12149 sd_match(N: N0, P: m_Node(Opcode: ISD::FSHR, preds: m_Value(), preds: m_Value(N&: Val),
12150 preds: m_SpecificInt(V: C0Expected))) ||
12151 sd_match(N: N0, P: m_Node(Opcode: ISD::FSHL, preds: m_Value(), preds: m_Value(N&: Val),
12152 preds: m_SpecificInt(V: C1Expected)))) &&
12153 (sd_match(N: N1, P: m_Shl(L: m_Specific(N: Val), R: m_SpecificInt(V: C1Expected))) ||
12154 sd_match(N: N1, P: m_Node(Opcode: ISD::FSHL, preds: m_Specific(N: Val), preds: m_Value(),
12155 preds: m_SpecificInt(V: C1Expected))) ||
12156 sd_match(N: N1, P: m_Node(Opcode: ISD::FSHR, preds: m_Specific(N: Val), preds: m_Value(),
12157 preds: m_SpecificInt(V: C0Expected)))))
12158 return Val;
12159
12160 // fold (fshl ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive.
12161 // fold (fshr ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive.
12162 // TODO - bigendian support once we have test coverage.
12163 // TODO - can we merge this with CombineConseutiveLoads/MatchLoadCombine?
12164 // TODO - permit LHS EXTLOAD if extensions are shifted out.
12165 if ((BitWidth % 8) == 0 && (ShAmt % 8) == 0 && !VT.isVector() &&
12166 !DAG.getDataLayout().isBigEndian()) {
12167 auto *LHS = dyn_cast<LoadSDNode>(Val&: N0);
12168 auto *RHS = dyn_cast<LoadSDNode>(Val&: N1);
12169 if (LHS && RHS && LHS->isSimple() && RHS->isSimple() &&
12170 LHS->getAddressSpace() == RHS->getAddressSpace() &&
12171 (LHS->hasNUsesOfValue(NUses: 1, Value: 0) || RHS->hasNUsesOfValue(NUses: 1, Value: 0)) &&
12172 ISD::isNON_EXTLoad(N: RHS) && ISD::isNON_EXTLoad(N: LHS)) {
12173 if (DAG.areNonVolatileConsecutiveLoads(LD: LHS, Base: RHS, Bytes: BitWidth / 8, Dist: 1)) {
12174 SDLoc DL(RHS);
12175 uint64_t PtrOff =
12176 IsFSHL ? (((BitWidth - ShAmt) % BitWidth) / 8) : (ShAmt / 8);
12177 Align NewAlign = commonAlignment(A: RHS->getAlign(), Offset: PtrOff);
12178 unsigned Fast = 0;
12179 if (TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT,
12180 AddrSpace: RHS->getAddressSpace(), Alignment: NewAlign,
12181 Flags: RHS->getMemOperand()->getFlags(), Fast: &Fast) &&
12182 Fast) {
12183 SDValue NewPtr = DAG.getMemBasePlusOffset(
12184 Base: RHS->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: PtrOff), DL);
12185 AddToWorklist(N: NewPtr.getNode());
12186 SDValue Load = DAG.getLoad(
12187 VT, dl: DL, Chain: RHS->getChain(), Ptr: NewPtr,
12188 PtrInfo: RHS->getPointerInfo().getWithOffset(O: PtrOff), Alignment: NewAlign,
12189 MMOFlags: RHS->getMemOperand()->getFlags(), Metadata: RHS->getAAInfo());
12190 DAG.makeEquivalentMemoryOrdering(OldLoad: LHS, NewMemOp: Load.getValue(R: 1));
12191 DAG.makeEquivalentMemoryOrdering(OldLoad: RHS, NewMemOp: Load.getValue(R: 1));
12192 return Load;
12193 }
12194 }
12195 }
12196 }
12197 }
12198
12199 // fold fshr(undef_or_zero, N1, N2) -> lshr(N1, N2)
12200 // fold fshl(N0, undef_or_zero, N2) -> shl(N0, N2)
12201 // iff We know the shift amount is in range.
12202 // TODO: when is it worth doing SUB(BW, N2) as well?
12203 if (isPowerOf2_32(Value: BitWidth)) {
12204 APInt ModuloBits(N2.getScalarValueSizeInBits(), BitWidth - 1);
12205 if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(Op: N2, Mask: ~ModuloBits))
12206 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1, N2);
12207 if (IsUndefOrZero(N1) && IsFSHL && DAG.MaskedValueIsZero(Op: N2, Mask: ~ModuloBits))
12208 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0, N2);
12209 }
12210
12211 // fold (fshl N0, N0, N2) -> (rotl N0, N2)
12212 // fold (fshr N0, N0, N2) -> (rotr N0, N2)
12213 // TODO: Investigate flipping this rotate if only one is legal.
12214 // If funnel shift is legal as well we might be better off avoiding
12215 // non-constant (BW - N2).
12216 unsigned RotOpc = IsFSHL ? ISD::ROTL : ISD::ROTR;
12217 if (N0 == N1 && hasOperation(Opcode: RotOpc, VT))
12218 return DAG.getNode(Opcode: RotOpc, DL, VT, N1: N0, N2);
12219
12220 // Simplify, based on bits shifted out of N0/N1.
12221 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
12222 return SDValue(N, 0);
12223
12224 return SDValue();
12225}
12226
12227SDValue DAGCombiner::visitSHLSAT(SDNode *N) {
12228 SDValue N0 = N->getOperand(Num: 0);
12229 SDValue N1 = N->getOperand(Num: 1);
12230 if (SDValue V = DAG.simplifyShift(X: N0, Y: N1))
12231 return V;
12232
12233 SDLoc DL(N);
12234 EVT VT = N0.getValueType();
12235
12236 // fold (*shlsat c1, c2) -> c1<<c2
12237 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: N->getOpcode(), DL, VT, Ops: {N0, N1}))
12238 return C;
12239
12240 ConstantSDNode *N1C = isConstOrConstSplat(N: N1);
12241
12242 if (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::SHL, VT)) {
12243 // fold (sshlsat x, c) -> (shl x, c)
12244 if (N->getOpcode() == ISD::SSHLSAT && N1C &&
12245 N1C->getAPIntValue().ult(RHS: DAG.ComputeNumSignBits(Op: N0)))
12246 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0, N2: N1);
12247
12248 // fold (ushlsat x, c) -> (shl x, c)
12249 if (N->getOpcode() == ISD::USHLSAT && N1C &&
12250 N1C->getAPIntValue().ule(
12251 RHS: DAG.computeKnownBits(Op: N0).countMinLeadingZeros()))
12252 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0, N2: N1);
12253 }
12254
12255 return SDValue();
12256}
12257
12258// Given a ABS node, detect the following patterns:
12259// (ABS (SUB (EXTEND a), (EXTEND b))).
12260// (TRUNC (ABS (SUB (EXTEND a), (EXTEND b)))).
12261// Generates UABD/SABD instruction.
12262SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
12263 EVT SrcVT = N->getValueType(ResNo: 0);
12264
12265 if (N->getOpcode() == ISD::TRUNCATE)
12266 N = N->getOperand(Num: 0).getNode();
12267
12268 EVT VT = N->getValueType(ResNo: 0);
12269 SDValue Op0, Op1;
12270
12271 if (!sd_match(N, P: m_Abs(Op: m_AnyOf(preds: m_Sub(L: m_Value(N&: Op0), R: m_Value(N&: Op1)),
12272 preds: m_Add(L: m_Value(N&: Op0), R: m_Value(N&: Op1))))))
12273 return SDValue();
12274
12275 SDValue AbsOp0 = N->getOperand(Num: 0);
12276 bool IsAdd = AbsOp0.getOpcode() == ISD::ADD;
12277 // Make sure (abs B) is positive.
12278 if (IsAdd) {
12279 // Elements of Op1 must be constant and != VT.minSignedValue() (or undef)
12280 auto IsNotMinSignedInt = [VT](ConstantSDNode *C) {
12281 if (C == nullptr)
12282 return true;
12283 return !C->getAPIntValue()
12284 .trunc(width: VT.getScalarSizeInBits())
12285 .isMinSignedValue();
12286 };
12287
12288 if (!ISD::matchUnaryPredicate(Op: Op1, Match: IsNotMinSignedInt, /*AllowUndefs=*/true,
12289 /*AllowTruncation=*/true))
12290 return SDValue();
12291 }
12292
12293 unsigned Opc0 = Op0.getOpcode();
12294
12295 // Check if the operands of the sub are (zero|sign)-extended, otherwise
12296 // fallback to ValueTracking.
12297 if (Opc0 != Op1.getOpcode() ||
12298 (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND &&
12299 Opc0 != ISD::SIGN_EXTEND_INREG)) {
12300
12301 auto CreateZextedAbd = [&](unsigned AbdOpc) {
12302 if (IsAdd)
12303 Op1 = DAG.getNegative(Val: Op1, DL: SDLoc(Op1), VT);
12304 SDValue ABD = DAG.getNode(Opcode: AbdOpc, DL, VT, N1: Op0, N2: Op1);
12305 return DAG.getZExtOrTrunc(Op: ABD, DL, VT: SrcVT);
12306 };
12307
12308 // fold (abs (sub nsw x, y)) -> abds(x, y)
12309 // fold (abs (add nsw x, -y)) -> abds(x, y)
12310 bool AbsOpWillNSW =
12311 AbsOp0->getFlags().hasNoSignedWrap() ||
12312 (IsAdd ? DAG.willNotOverflowAdd(/*IsSigned=*/true, N0: Op0, N1: Op1)
12313 : DAG.willNotOverflowSub(/*IsSigned=*/true, N0: Op0, N1: Op1));
12314
12315 // Don't fold this for unsupported types as we lose the NSW handling.
12316 if (hasOperation(Opcode: ISD::ABDS, VT) && TLI.preferABDSToABSWithNSW(VT) &&
12317 AbsOpWillNSW)
12318 return CreateZextedAbd(ISD::ABDS);
12319
12320 // fold (abs (sub x, y)) -> abdu(x, y)
12321 bool AbsOpWillNUW =
12322 !IsAdd && DAG.SignBitIsZero(Op: Op0) && DAG.SignBitIsZero(Op: Op1);
12323
12324 if (hasOperation(Opcode: ISD::ABDU, VT) && AbsOpWillNUW)
12325 return CreateZextedAbd(ISD::ABDU);
12326
12327 return SDValue();
12328 }
12329
12330 // The IsAdd case explicitly checks for const/bv-of-const. This implies either
12331 // (Opc0 != Op1.getOpcode() || Opc0 is not in {zext/sext/sign_ext_inreg}. This
12332 // implies it was alrady handled by the above if statement.
12333 assert(!IsAdd && "Unexpected abs(add(x,y)) pattern");
12334
12335 EVT VT0, VT1;
12336 if (Opc0 == ISD::SIGN_EXTEND_INREG) {
12337 VT0 = cast<VTSDNode>(Val: Op0.getOperand(i: 1))->getVT();
12338 VT1 = cast<VTSDNode>(Val: Op1.getOperand(i: 1))->getVT();
12339 } else {
12340 VT0 = Op0.getOperand(i: 0).getValueType();
12341 VT1 = Op1.getOperand(i: 0).getValueType();
12342 }
12343 unsigned ABDOpcode = (Opc0 == ISD::ZERO_EXTEND) ? ISD::ABDU : ISD::ABDS;
12344
12345 // fold abs(sext(x) - sext(y)) -> zext(abds(x, y))
12346 // fold abs(zext(x) - zext(y)) -> zext(abdu(x, y))
12347 EVT MaxVT = VT0.bitsGT(VT: VT1) ? VT0 : VT1;
12348 if ((VT0 == MaxVT || Op0->hasOneUse()) &&
12349 (VT1 == MaxVT || Op1->hasOneUse()) &&
12350 (!LegalTypes || hasOperation(Opcode: ABDOpcode, VT: MaxVT))) {
12351 SDValue ABD = DAG.getNode(Opcode: ABDOpcode, DL, VT: MaxVT,
12352 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MaxVT, Operand: Op0),
12353 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MaxVT, Operand: Op1));
12354 ABD = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: ABD);
12355 return DAG.getZExtOrTrunc(Op: ABD, DL, VT: SrcVT);
12356 }
12357
12358 // fold abs(sext(x) - sext(y)) -> abds(sext(x), sext(y))
12359 // fold abs(zext(x) - zext(y)) -> abdu(zext(x), zext(y))
12360 if (!LegalOperations || hasOperation(Opcode: ABDOpcode, VT)) {
12361 SDValue ABD = DAG.getNode(Opcode: ABDOpcode, DL, VT, N1: Op0, N2: Op1);
12362 return DAG.getZExtOrTrunc(Op: ABD, DL, VT: SrcVT);
12363 }
12364
12365 return SDValue();
12366}
12367
12368SDValue DAGCombiner::visitABS(SDNode *N) {
12369 SDValue N0 = N->getOperand(Num: 0);
12370 EVT VT = N->getValueType(ResNo: 0);
12371 SDLoc DL(N);
12372
12373 // fold (abs c1) -> c2
12374 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::ABS, DL, VT, Ops: {N0}))
12375 return C;
12376 // fold (abs (abs x)) -> (abs x)
12377 // fold (abs (abs_min_poison x)) -> (abs_min_poison x)
12378 if (ISD::isAbsOpcode(Opcode: N0.getOpcode()))
12379 return N0;
12380 // fold (abs x) -> x iff not-negative
12381 if (DAG.SignBitIsZero(Op: N0))
12382 return N0;
12383
12384 if (SDValue ABD = foldABSToABD(N, DL))
12385 return ABD;
12386
12387 // fold (abs (sign_extend_inreg x)) -> (zero_extend (abs (truncate x)))
12388 // iff zero_extend/truncate are free.
12389 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
12390 EVT ExtVT = cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT();
12391 if (TLI.isTruncateFree(FromVT: VT, ToVT: ExtVT) && TLI.isZExtFree(FromTy: ExtVT, ToTy: VT) &&
12392 TLI.isTypeDesirableForOp(ISD::ABS, VT: ExtVT) &&
12393 hasOperation(Opcode: ISD::ABS, VT: ExtVT)) {
12394 return DAG.getNode(
12395 Opcode: ISD::ZERO_EXTEND, DL, VT,
12396 Operand: DAG.getNode(Opcode: ISD::ABS, DL, VT: ExtVT,
12397 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ExtVT, Operand: N0.getOperand(i: 0))));
12398 }
12399 }
12400
12401 return SDValue();
12402}
12403
12404SDValue DAGCombiner::visitABS_MIN_POISON(SDNode *N) {
12405 SDValue N0 = N->getOperand(Num: 0);
12406 EVT VT = N->getValueType(ResNo: 0);
12407 SDLoc DL(N);
12408
12409 // fold (abs_min_poison c1) -> c2 (or poison if c1 == INT_MIN)
12410 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::ABS_MIN_POISON, DL, VT, Ops: {N0}))
12411 return C;
12412 // fold (abs_min_poison (abs_min_poison x)) -> (abs_min_poison x)
12413 // fold (abs_min_poison (abs x)) -> (abs x)
12414 // fold (abs_min_poison (freeze (abs x))) -> (freeze (abs x))
12415 // fold (abs_min_poison (freeze (abs_min_poison x))) ->
12416 // (freeze (abs_min_poison x))
12417 //
12418 // Freeze case is valid because: for x != INT_MIN both sides equal abs(x);
12419 // for x == INT_MIN both forms produce a non-deterministic but well-defined
12420 // value since freeze already consumed the poison.
12421 if (ISD::isAbsOpcode(Opcode: peekThroughFreeze(V: N0).getOpcode()))
12422 return N0;
12423 // fold (abs_min_poison x) -> x iff not-negative
12424 if (DAG.SignBitIsZero(Op: N0))
12425 return N0;
12426
12427 if (SDValue ABD = foldABSToABD(N, DL))
12428 return ABD;
12429
12430 // fold (abs_min_poison (sign_extend_inreg x)) ->
12431 // (zero_extend (abs (truncate x)))
12432 // iff zero_extend/truncate are free. The sign_extend_inreg keeps the value
12433 // in the narrow type's range, so the wide abs_min_poison is never actually
12434 // poison.
12435 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
12436 EVT ExtVT = cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT();
12437 if (TLI.isTruncateFree(FromVT: VT, ToVT: ExtVT) && TLI.isZExtFree(FromTy: ExtVT, ToTy: VT) &&
12438 TLI.isTypeDesirableForOp(ISD::ABS, VT: ExtVT) &&
12439 hasOperation(Opcode: ISD::ABS, VT: ExtVT)) {
12440 return DAG.getNode(
12441 Opcode: ISD::ZERO_EXTEND, DL, VT,
12442 Operand: DAG.getNode(Opcode: ISD::ABS, DL, VT: ExtVT,
12443 Operand: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ExtVT, Operand: N0.getOperand(i: 0))));
12444 }
12445 }
12446
12447 return SDValue();
12448}
12449
12450SDValue DAGCombiner::visitCLMUL(SDNode *N) {
12451 unsigned Opcode = N->getOpcode();
12452 SDValue N0 = N->getOperand(Num: 0);
12453 SDValue N1 = N->getOperand(Num: 1);
12454 EVT VT = N->getValueType(ResNo: 0);
12455 SDLoc DL(N);
12456
12457 // fold (clmul c1, c2)
12458 if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, Ops: {N0, N1}))
12459 return C;
12460
12461 // canonicalize constant to RHS
12462 if (DAG.isConstantIntBuildVectorOrConstantInt(N: N0) &&
12463 !DAG.isConstantIntBuildVectorOrConstantInt(N: N1))
12464 return DAG.getNode(Opcode, DL, VT, N1, N2: N0);
12465
12466 // fold (clmul x, 0) -> 0
12467 if (isNullConstant(V: N1) || ISD::isConstantSplatVectorAllZeros(N: N1.getNode()))
12468 return DAG.getConstant(Val: 0, DL, VT);
12469
12470 // fold (clmul x, c_pow2) -> (shl x, log2(c_pow2))
12471 // This also handles (clmul x, 1) -> x since (shl x, 0) simplifies to x.
12472 if (Opcode == ISD::CLMUL) {
12473 if (ConstantSDNode *C = isConstOrConstSplat(N: N1)) {
12474 APInt CV = C->getAPIntValue().trunc(width: VT.getScalarSizeInBits());
12475 if (CV.isPowerOf2() &&
12476 (!LegalOperations || TLI.isOperationLegal(Op: ISD::SHL, VT)))
12477 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0,
12478 N2: DAG.getShiftAmountConstant(Val: CV.logBase2(), VT, DL));
12479 }
12480 }
12481
12482 return SDValue();
12483}
12484
12485SDValue DAGCombiner::visitPEXT(SDNode *N) {
12486 EVT VT = N->getValueType(ResNo: 0);
12487 SDValue N0 = N->getOperand(Num: 0);
12488 SDValue N1 = N->getOperand(Num: 1);
12489 SDLoc DL(N);
12490
12491 // pext(x, 0) -> 0
12492 if (isNullOrNullSplat(V: N1))
12493 return DAG.getConstant(Val: 0, DL, VT);
12494 // pext(x, -1) -> x (all bits selected, packed into low positions = x)
12495 if (isAllOnesOrAllOnesSplat(V: N1))
12496 return N0;
12497 // fold pext(c1, c2) -> c3
12498 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::PEXT, DL, VT, Ops: {N0, N1}))
12499 return C;
12500 return SDValue();
12501}
12502
12503SDValue DAGCombiner::visitPDEP(SDNode *N) {
12504 EVT VT = N->getValueType(ResNo: 0);
12505 SDValue N0 = N->getOperand(Num: 0);
12506 SDValue N1 = N->getOperand(Num: 1);
12507 SDLoc DL(N);
12508
12509 // pdep(x, 0) -> 0
12510 if (isNullOrNullSplat(V: N1))
12511 return DAG.getConstant(Val: 0, DL, VT);
12512
12513 // pdep(x, -1) -> x (all positions selected, bits deposited at identity)
12514 if (isAllOnesOrAllOnesSplat(V: N1))
12515 return N0;
12516
12517 // fold pdep(c1, c2) -> c3
12518 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::PDEP, DL, VT, Ops: {N0, N1}))
12519 return C;
12520
12521 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
12522 return SDValue(N, 0);
12523
12524 return SDValue();
12525}
12526
12527SDValue DAGCombiner::visitBSWAP(SDNode *N) {
12528 SDValue N0 = N->getOperand(Num: 0);
12529 EVT VT = N->getValueType(ResNo: 0);
12530 SDLoc DL(N);
12531
12532 // fold (bswap c1) -> c2
12533 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::BSWAP, DL, VT, Ops: {N0}))
12534 return C;
12535 // fold (bswap (bswap x)) -> x
12536 if (N0.getOpcode() == ISD::BSWAP)
12537 return N0.getOperand(i: 0);
12538
12539 // Canonicalize bswap(bitreverse(x)) -> bitreverse(bswap(x)). If bitreverse
12540 // isn't supported, it will be expanded to bswap followed by a manual reversal
12541 // of bits in each byte. By placing bswaps before bitreverse, we can remove
12542 // the two bswaps if the bitreverse gets expanded.
12543 if (N0.getOpcode() == ISD::BITREVERSE && N0.hasOneUse()) {
12544 SDValue BSwap = DAG.getNode(Opcode: ISD::BSWAP, DL, VT, Operand: N0.getOperand(i: 0));
12545 return DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT, Operand: BSwap);
12546 }
12547
12548 unsigned BW = VT.getScalarSizeInBits();
12549 // fold (bswap shl(x,c)) -> (zext(bswap(trunc(shl(x,sub(c,bw/2))))))
12550 // iff x >= bw/2 (i.e. lower half is known zero)
12551 if (BW >= 32 && N0.getOpcode() == ISD::SHL && N0.hasOneUse()) {
12552 auto *ShAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
12553 EVT HalfVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: BW / 2);
12554 if (ShAmt && ShAmt->getAPIntValue().ult(RHS: BW) &&
12555 ShAmt->getZExtValue() >= (BW / 2) && (ShAmt->getZExtValue() % 8) == 0 &&
12556 TLI.isTypeLegal(VT: HalfVT) && TLI.isTruncateFree(FromVT: VT, ToVT: HalfVT) &&
12557 (!LegalOperations || hasOperation(Opcode: ISD::BSWAP, VT: HalfVT))) {
12558 SDValue Res = N0.getOperand(i: 0);
12559 if (uint64_t NewShAmt = (ShAmt->getZExtValue() - (BW / 2)))
12560 Res = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Res,
12561 N2: DAG.getShiftAmountConstant(Val: NewShAmt, VT, DL));
12562 Res = DAG.getZExtOrTrunc(Op: Res, DL, VT: HalfVT);
12563 Res = DAG.getNode(Opcode: ISD::BSWAP, DL, VT: HalfVT, Operand: Res);
12564 return DAG.getZExtOrTrunc(Op: Res, DL, VT);
12565 }
12566 }
12567
12568 // Try to canonicalize bswap-of-logical-shift-by-8-bit-multiple as
12569 // inverse-shift-of-bswap:
12570 // bswap (X u<< C) --> (bswap X) u>> C
12571 // bswap (X u>> C) --> (bswap X) u<< C
12572 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
12573 N0.hasOneUse()) {
12574 auto *ShAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
12575 if (ShAmt && ShAmt->getAPIntValue().ult(RHS: BW) &&
12576 ShAmt->getZExtValue() % 8 == 0) {
12577 SDValue NewSwap = DAG.getNode(Opcode: ISD::BSWAP, DL, VT, Operand: N0.getOperand(i: 0));
12578 unsigned InverseShift = N0.getOpcode() == ISD::SHL ? ISD::SRL : ISD::SHL;
12579 return DAG.getNode(Opcode: InverseShift, DL, VT, N1: NewSwap, N2: N0.getOperand(i: 1));
12580 }
12581 }
12582
12583 if (SDValue V = foldBitOrderCrossLogicOp(N, DAG))
12584 return V;
12585
12586 // Folds that depend on computeKnownBits of the operand.
12587 KnownBits Known = DAG.computeKnownBits(Op: N0);
12588 // bswap(0) = 0. Catch cases that computeKnownBits can prove are zero but
12589 // that structural combines haven't simplified to a constant yet
12590 // (e.g. and of disjoint byte masks).
12591 if (Known.isZero())
12592 return DAG.getConstant(Val: 0, DL, VT);
12593 // If only one byte of the operand may be nonzero, bswap becomes a shift
12594 // to the mirror byte.
12595 unsigned TZ = alignDown(Value: Known.countMinTrailingZeros(), Align: 8);
12596 unsigned LZ = alignDown(Value: Known.countMinLeadingZeros(), Align: 8);
12597 if (BW - (LZ + TZ) == 8) {
12598 unsigned Opc = LZ > TZ ? ISD::SHL : ISD::SRL;
12599 // Skip if the target would re-expand the produced shift post-legalize.
12600 // Targets that custom-lower byte-multiple shifts via bswap (e.g. MSP430
12601 // for shl i16) would loop with this combine.
12602 if (!LegalOperations || hasOperation(Opcode: Opc, VT)) {
12603 unsigned Amt = AbsoluteDifference(X: LZ, Y: TZ);
12604 SDNodeFlags Flags =
12605 Opc == ISD::SHL ? SDNodeFlags::NoUnsignedWrap : SDNodeFlags::Exact;
12606 return DAG.getNode(Opcode: Opc, DL, VT, N1: N0,
12607 N2: DAG.getShiftAmountConstant(Val: Amt, VT, DL), Flags);
12608 }
12609 }
12610
12611 return SDValue();
12612}
12613
12614SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
12615 SDValue N0 = N->getOperand(Num: 0);
12616 EVT VT = N->getValueType(ResNo: 0);
12617 SDLoc DL(N);
12618
12619 // fold (bitreverse c1) -> c2
12620 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::BITREVERSE, DL, VT, Ops: {N0}))
12621 return C;
12622
12623 // fold (bitreverse (bitreverse x)) -> x
12624 if (N0.getOpcode() == ISD::BITREVERSE)
12625 return N0.getOperand(i: 0);
12626
12627 SDValue X, Y;
12628
12629 // fold (bitreverse (lshr (bitreverse x), y)) -> (shl x, y)
12630 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::SHL, VT)) &&
12631 sd_match(N: N0, P: m_Srl(L: m_BitReverse(Op: m_Value(N&: X)), R: m_Value(N&: Y))))
12632 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: Y);
12633
12634 // fold (bitreverse (shl (bitreverse x), y)) -> (lshr x, y)
12635 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::SRL, VT)) &&
12636 sd_match(N: N0, P: m_Shl(L: m_BitReverse(Op: m_Value(N&: X)), R: m_Value(N&: Y))))
12637 return DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: X, N2: Y);
12638
12639 // fold bitreverse(clmul(bitreverse(x), bitreverse(y))) -> clmulr(x, y)
12640 if ((!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::CLMULR, VT)) &&
12641 sd_match(N: N0, P: m_Clmul(L: m_BitReverse(Op: m_Value(N&: X)), R: m_BitReverse(Op: m_Value(N&: Y)))))
12642 return DAG.getNode(Opcode: ISD::CLMULR, DL, VT, N1: X, N2: Y);
12643
12644 return SDValue();
12645}
12646
12647// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
12648// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
12649SDValue DAGCombiner::foldCTLZToCTLS(SDValue Src, const SDLoc &DL) {
12650 EVT VT = Src.getValueType();
12651
12652 auto LK = TLI.getTypeConversion(Context&: *DAG.getContext(), VT);
12653 if ((LK.first != TargetLoweringBase::TypeLegal &&
12654 LK.first != TargetLoweringBase::TypePromoteInteger) ||
12655 !TLI.isOperationLegalOrCustom(Op: ISD::CTLS, VT: LK.second))
12656 return SDValue();
12657
12658 unsigned BitWidth = VT.getScalarSizeInBits();
12659
12660 bool NeedAdd = true;
12661
12662 SDValue X;
12663 if (sd_match(N: Src,
12664 P: m_OneUse(P: m_Or(L: m_OneUse(P: m_Shl(L: m_Value(N&: X), R: m_One())), R: m_One())))) {
12665 NeedAdd = false;
12666 Src = X;
12667 }
12668
12669 if (!sd_match(N: Src,
12670 P: m_OneUse(P: m_Xor(L: m_Value(N&: X),
12671 R: m_OneUse(P: m_Sra(L: m_Deferred(V&: X),
12672 R: m_SpecificInt(V: BitWidth - 1)))))))
12673 return SDValue();
12674
12675 SDValue Res = DAG.getNode(Opcode: ISD::CTLS, DL, VT, Operand: X);
12676 if (!NeedAdd)
12677 return Res;
12678
12679 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Res, N2: DAG.getConstant(Val: 1, DL, VT));
12680}
12681
12682SDValue DAGCombiner::visitCTLZ(SDNode *N) {
12683 SDValue N0 = N->getOperand(Num: 0);
12684 EVT VT = N->getValueType(ResNo: 0);
12685 SDLoc DL(N);
12686
12687 // fold (ctlz c1) -> c2
12688 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::CTLZ, DL, VT, Ops: {N0}))
12689 return C;
12690
12691 // If the value is known never to be zero, switch to the poison version.
12692 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::CTLZ_ZERO_POISON, VT))
12693 if (DAG.isKnownNeverZero(Op: N0))
12694 return DAG.getNode(Opcode: ISD::CTLZ_ZERO_POISON, DL, VT, Operand: N0);
12695
12696 if (SDValue V = foldCTLZToCTLS(Src: N0, DL))
12697 return V;
12698
12699 return SDValue();
12700}
12701
12702SDValue DAGCombiner::visitCTLZ_ZERO_POISON(SDNode *N) {
12703 SDValue N0 = N->getOperand(Num: 0);
12704 EVT VT = N->getValueType(ResNo: 0);
12705 SDLoc DL(N);
12706
12707 // fold (ctlz_zero_poison c1) -> c2
12708 if (SDValue C =
12709 DAG.FoldConstantArithmetic(Opcode: ISD::CTLZ_ZERO_POISON, DL, VT, Ops: {N0}))
12710 return C;
12711
12712 if (SDValue V = foldCTLZToCTLS(Src: N0, DL))
12713 return V;
12714
12715 return SDValue();
12716}
12717
12718SDValue DAGCombiner::visitCTTZ(SDNode *N) {
12719 SDValue N0 = N->getOperand(Num: 0);
12720 EVT VT = N->getValueType(ResNo: 0);
12721 SDLoc DL(N);
12722
12723 // fold (cttz c1) -> c2
12724 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::CTTZ, DL, VT, Ops: {N0}))
12725 return C;
12726
12727 // If the value is known never to be zero, switch to the poison version.
12728 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::CTTZ_ZERO_POISON, VT))
12729 if (DAG.isKnownNeverZero(Op: N0))
12730 return DAG.getNode(Opcode: ISD::CTTZ_ZERO_POISON, DL, VT, Operand: N0);
12731
12732 return SDValue();
12733}
12734
12735SDValue DAGCombiner::visitCTTZ_ZERO_POISON(SDNode *N) {
12736 SDValue N0 = N->getOperand(Num: 0);
12737 EVT VT = N->getValueType(ResNo: 0);
12738 SDLoc DL(N);
12739
12740 // fold (cttz_zero_poison c1) -> c2
12741 if (SDValue C =
12742 DAG.FoldConstantArithmetic(Opcode: ISD::CTTZ_ZERO_POISON, DL, VT, Ops: {N0}))
12743 return C;
12744 return SDValue();
12745}
12746
12747SDValue DAGCombiner::visitPARITY(SDNode *N) {
12748 SDValue N0 = N->getOperand(Num: 0);
12749 EVT VT = N->getValueType(ResNo: 0);
12750
12751 // fold (parity c1) -> c2
12752 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::PARITY, DL: SDLoc(N), VT, Ops: {N0}))
12753 return C;
12754
12755 return SDValue();
12756}
12757
12758SDValue DAGCombiner::visitCTPOP(SDNode *N) {
12759 SDValue N0 = N->getOperand(Num: 0);
12760 EVT VT = N->getValueType(ResNo: 0);
12761 unsigned NumBits = VT.getScalarSizeInBits();
12762 SDLoc DL(N);
12763
12764 // fold (ctpop c1) -> c2
12765 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::CTPOP, DL, VT, Ops: {N0}))
12766 return C;
12767
12768 // If the source is being shifted, but doesn't affect any active bits,
12769 // then we can call CTPOP on the shift source directly.
12770 if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SHL) {
12771 if (ConstantSDNode *AmtC = isConstOrConstSplat(N: N0.getOperand(i: 1))) {
12772 const APInt &Amt = AmtC->getAPIntValue();
12773 if (Amt.ult(RHS: NumBits)) {
12774 KnownBits KnownSrc = DAG.computeKnownBits(Op: N0.getOperand(i: 0));
12775 if ((N0.getOpcode() == ISD::SRL &&
12776 Amt.ule(RHS: KnownSrc.countMinTrailingZeros())) ||
12777 (N0.getOpcode() == ISD::SHL &&
12778 Amt.ule(RHS: KnownSrc.countMinLeadingZeros()))) {
12779 return DAG.getNode(Opcode: ISD::CTPOP, DL, VT, Operand: N0.getOperand(i: 0));
12780 }
12781 }
12782 }
12783 }
12784
12785 // If the upper bits are known to be zero, then see if its profitable to
12786 // only count the lower bits.
12787 if (VT.isScalarInteger() && NumBits > 8 && (NumBits & 1) == 0) {
12788 EVT HalfVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumBits / 2);
12789 if (hasOperation(Opcode: ISD::CTPOP, VT: HalfVT) &&
12790 TLI.isTypeDesirableForOp(ISD::CTPOP, VT: HalfVT) &&
12791 TLI.isTruncateFree(Val: N0, VT2: HalfVT) && TLI.isZExtFree(FromTy: HalfVT, ToTy: VT)) {
12792 APInt UpperBits = APInt::getHighBitsSet(numBits: NumBits, hiBitsSet: NumBits / 2);
12793 if (DAG.MaskedValueIsZero(Op: N0, Mask: UpperBits)) {
12794 SDValue PopCnt = DAG.getNode(Opcode: ISD::CTPOP, DL, VT: HalfVT,
12795 Operand: DAG.getZExtOrTrunc(Op: N0, DL, VT: HalfVT));
12796 return DAG.getZExtOrTrunc(Op: PopCnt, DL, VT);
12797 }
12798 }
12799 }
12800
12801 return SDValue();
12802}
12803
12804static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS,
12805 SDValue RHS,
12806 const SDNodeFlags SelectFlags,
12807 const SDNodeFlags CmpFlags,
12808 const TargetLowering &TLI) {
12809 EVT VT = LHS.getValueType();
12810 if (!VT.isFloatingPoint())
12811 return false;
12812
12813 return SelectFlags.hasNoSignedZeros() &&
12814 TLI.isProfitableToCombineMinNumMaxNum(VT) &&
12815 (SelectFlags.hasNoNaNs() || CmpFlags.hasNoNaNs() ||
12816 (DAG.isKnownNeverNaN(Op: RHS) && DAG.isKnownNeverNaN(Op: LHS)));
12817}
12818
12819static SDValue combineMinNumMaxNumImpl(const SDLoc &DL, EVT VT, SDValue LHS,
12820 SDValue RHS, SDValue True, SDValue False,
12821 ISD::CondCode CC,
12822 const TargetLowering &TLI,
12823 SelectionDAG &DAG) {
12824 EVT TransformVT = TLI.getLegalTypeToTransformTo(Context&: *DAG.getContext(), VT);
12825
12826 // We have checked nnan and nsz as pre-conditions for the transform.
12827 SDNodeFlags Flags = SDNodeFlags::NoNaNs | SDNodeFlags::NoSignedZeros;
12828
12829 switch (CC) {
12830 case ISD::SETOLT:
12831 case ISD::SETOLE:
12832 case ISD::SETLT:
12833 case ISD::SETLE:
12834 case ISD::SETULT:
12835 case ISD::SETULE: {
12836 // Since it's known never nan to get here already, either fminnum or
12837 // fminnum_ieee are OK. Try the ieee version first, since it's fminnum is
12838 // expanded in terms of it.
12839 unsigned IEEEOpcode = (LHS == True) ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
12840 if (TLI.isOperationLegalOrCustom(Op: IEEEOpcode, VT))
12841 return DAG.getNode(Opcode: IEEEOpcode, DL, VT, N1: LHS, N2: RHS, Flags);
12842
12843 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
12844 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: TransformVT))
12845 return DAG.getNode(Opcode, DL, VT, N1: LHS, N2: RHS, Flags);
12846 return SDValue();
12847 }
12848 case ISD::SETOGT:
12849 case ISD::SETOGE:
12850 case ISD::SETGT:
12851 case ISD::SETGE:
12852 case ISD::SETUGT:
12853 case ISD::SETUGE: {
12854 unsigned IEEEOpcode = (LHS == True) ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
12855 if (TLI.isOperationLegalOrCustom(Op: IEEEOpcode, VT))
12856 return DAG.getNode(Opcode: IEEEOpcode, DL, VT, N1: LHS, N2: RHS, Flags);
12857
12858 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
12859 if (TLI.isOperationLegalOrCustom(Op: Opcode, VT: TransformVT))
12860 return DAG.getNode(Opcode, DL, VT, N1: LHS, N2: RHS, Flags);
12861 return SDValue();
12862 }
12863 default:
12864 return SDValue();
12865 }
12866}
12867
12868// Convert (sr[al] (add n[su]w x, y)) -> (avgfloor[su] x, y)
12869SDValue DAGCombiner::foldShiftToAvg(SDNode *N, const SDLoc &DL) {
12870 const unsigned Opcode = N->getOpcode();
12871 if (Opcode != ISD::SRA && Opcode != ISD::SRL)
12872 return SDValue();
12873
12874 EVT VT = N->getValueType(ResNo: 0);
12875 bool IsUnsigned = Opcode == ISD::SRL;
12876
12877 // Captured values.
12878 SDValue A, B;
12879
12880 // Match floor average as it is common to both floor/ceil avgs, ensure the add
12881 // doesn't wrap.
12882 SDNodeFlags Flags =
12883 IsUnsigned ? SDNodeFlags::NoUnsignedWrap : SDNodeFlags::NoSignedWrap;
12884 if (sd_match(N, P: m_BinOp(Opc: Opcode,
12885 L: m_c_BinOp(Opc: ISD::ADD, L: m_Value(N&: A), R: m_Value(N&: B), Flgs: Flags),
12886 R: m_One()))) {
12887 // Decide whether signed or unsigned.
12888 unsigned FloorISD = IsUnsigned ? ISD::AVGFLOORU : ISD::AVGFLOORS;
12889 if (hasOperation(Opcode: FloorISD, VT))
12890 return DAG.getNode(Opcode: FloorISD, DL, VT, Ops: {A, B});
12891 }
12892
12893 return SDValue();
12894}
12895
12896SDValue DAGCombiner::foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT) {
12897 unsigned Opc = N->getOpcode();
12898 SDValue X, Y, Z;
12899 if (sd_match(
12900 N, P: m_BitwiseLogic(L: m_Value(N&: X), R: m_Add(L: m_Not(V: m_Value(N&: Y)), R: m_Value(N&: Z)))))
12901 return DAG.getNode(Opcode: Opc, DL, VT, N1: X,
12902 N2: DAG.getNOT(DL, Val: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Y, N2: Z), VT));
12903
12904 if (sd_match(N, P: m_BitwiseLogic(L: m_Value(N&: X), R: m_Sub(L: m_OneUse(P: m_Not(V: m_Value(N&: Y))),
12905 R: m_Value(N&: Z)))))
12906 return DAG.getNode(Opcode: Opc, DL, VT, N1: X,
12907 N2: DAG.getNOT(DL, Val: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Y, N2: Z), VT));
12908
12909 return SDValue();
12910}
12911
12912/// Generate Min/Max node
12913SDValue DAGCombiner::combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
12914 SDValue RHS, SDValue True,
12915 SDValue False, ISD::CondCode CC) {
12916 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
12917 return combineMinNumMaxNumImpl(DL, VT, LHS, RHS, True, False, CC, TLI, DAG);
12918
12919 // If we can't directly match this, try to see if we can pull an fneg out of
12920 // the select.
12921 SDValue NegTrue = TLI.getCheaperOrNeutralNegatedExpression(
12922 Op: True, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize);
12923 if (!NegTrue)
12924 return SDValue();
12925
12926 HandleSDNode NegTrueHandle(NegTrue);
12927
12928 // Try to unfold an fneg from the select if we are comparing the negated
12929 // constant.
12930 //
12931 // select (setcc x, K) (fneg x), -K -> fneg(minnum(x, K))
12932 //
12933 // TODO: Handle fabs
12934 if (LHS == NegTrue) {
12935 // If we can't directly match this, try to see if we can pull an fneg out of
12936 // the select.
12937 SDValue NegRHS = TLI.getCheaperOrNeutralNegatedExpression(
12938 Op: RHS, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize);
12939 if (NegRHS) {
12940 HandleSDNode NegRHSHandle(NegRHS);
12941 if (NegRHS == False) {
12942 SDValue Combined = combineMinNumMaxNumImpl(DL, VT, LHS, RHS, True: NegTrue,
12943 False, CC, TLI, DAG);
12944 if (Combined)
12945 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Combined);
12946 }
12947 }
12948 }
12949
12950 return SDValue();
12951}
12952
12953/// If a (v)select has a condition value that is a sign-bit test, try to smear
12954/// the condition operand sign-bit across the value width and use it as a mask.
12955static SDValue foldSelectOfConstantsUsingSra(SDNode *N, const SDLoc &DL,
12956 SelectionDAG &DAG) {
12957 SDValue Cond = N->getOperand(Num: 0);
12958 SDValue C1 = N->getOperand(Num: 1);
12959 SDValue C2 = N->getOperand(Num: 2);
12960 if (!isConstantOrConstantVector(N: C1) || !isConstantOrConstantVector(N: C2))
12961 return SDValue();
12962
12963 EVT VT = N->getValueType(ResNo: 0);
12964 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse() ||
12965 VT != Cond.getOperand(i: 0).getValueType())
12966 return SDValue();
12967
12968 // The inverted-condition + commuted-select variants of these patterns are
12969 // canonicalized to these forms in IR.
12970 SDValue X = Cond.getOperand(i: 0);
12971 SDValue CondC = Cond.getOperand(i: 1);
12972 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
12973 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(V: CondC) &&
12974 isAllOnesOrAllOnesSplat(V: C2)) {
12975 // i32 X > -1 ? C1 : -1 --> (X >>s 31) | C1
12976 SDValue ShAmtC = DAG.getConstant(Val: X.getScalarValueSizeInBits() - 1, DL, VT);
12977 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: X, N2: ShAmtC);
12978 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Sra, N2: C1);
12979 }
12980 if (CC == ISD::SETLT && isNullOrNullSplat(V: CondC) && isNullOrNullSplat(V: C2)) {
12981 // i8 X < 0 ? C1 : 0 --> (X >>s 7) & C1
12982 SDValue ShAmtC = DAG.getConstant(Val: X.getScalarValueSizeInBits() - 1, DL, VT);
12983 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: X, N2: ShAmtC);
12984 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Sra, N2: C1);
12985 }
12986 return SDValue();
12987}
12988
12989static bool shouldConvertSelectOfConstantsToMath(const SDValue &Cond, EVT VT,
12990 const TargetLowering &TLI) {
12991 if (!TLI.convertSelectOfConstantsToMath(VT))
12992 return false;
12993
12994 if (Cond.getOpcode() != ISD::SETCC || !Cond->hasOneUse())
12995 return true;
12996 if (!TLI.isOperationLegalOrCustom(Op: ISD::SELECT_CC, VT))
12997 return true;
12998
12999 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
13000 if (CC == ISD::SETLT && isNullOrNullSplat(V: Cond.getOperand(i: 1)))
13001 return true;
13002 if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(V: Cond.getOperand(i: 1)))
13003 return true;
13004
13005 return false;
13006}
13007
13008SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
13009 SDValue Cond = N->getOperand(Num: 0);
13010 SDValue N1 = N->getOperand(Num: 1);
13011 SDValue N2 = N->getOperand(Num: 2);
13012 EVT VT = N->getValueType(ResNo: 0);
13013 EVT CondVT = Cond.getValueType();
13014 SDLoc DL(N);
13015
13016 if (!VT.isInteger())
13017 return SDValue();
13018
13019 auto *C1 = dyn_cast<ConstantSDNode>(Val&: N1);
13020 auto *C2 = dyn_cast<ConstantSDNode>(Val&: N2);
13021 if (!C1 || !C2)
13022 return SDValue();
13023
13024 if (CondVT != MVT::i1 || LegalOperations) {
13025 // We can't do this reliably if integer based booleans have different contents
13026 // to floating point based booleans. This is because we can't tell whether we
13027 // have an integer-based boolean or a floating-point-based boolean unless we
13028 // can find the SETCC that produced it and inspect its operands. This is
13029 // fairly easy if C is the SETCC node, but it can potentially be
13030 // undiscoverable (or not reasonably discoverable). For example, it could be
13031 // in another basic block or it could require searching a complicated
13032 // expression.
13033 if (CondVT.isInteger() &&
13034 TLI.getBooleanContents(/*isVec*/false, /*isFloat*/true) ==
13035 TargetLowering::ZeroOrOneBooleanContent &&
13036 TLI.getBooleanContents(/*isVec*/false, /*isFloat*/false) ==
13037 TargetLowering::ZeroOrOneBooleanContent) {
13038 // fold (select Cond, 0, 1) -> (xor Cond, 1)
13039 if (C1->isZero() && C2->isOne()) {
13040 SDValue NotCond = DAG.getNode(Opcode: ISD::XOR, DL, VT: CondVT, N1: Cond,
13041 N2: DAG.getConstant(Val: 1, DL, VT: CondVT));
13042 if (VT.bitsEq(VT: CondVT))
13043 return NotCond;
13044 return DAG.getZExtOrTrunc(Op: NotCond, DL, VT);
13045 }
13046
13047 // fold (select Cond, 1, 0) -> Cond
13048 if (C1->isOne() && C2->isZero() && CondVT == VT)
13049 return Cond;
13050 }
13051
13052 return SDValue();
13053 }
13054
13055 // Only do this before legalization to avoid conflicting with target-specific
13056 // transforms in the other direction (create a select from a zext/sext). There
13057 // is also a target-independent combine here in DAGCombiner in the other
13058 // direction for (select Cond, -1, 0) when the condition is not i1.
13059 assert(CondVT == MVT::i1 && !LegalOperations);
13060
13061 // select Cond, 1, 0 --> zext (Cond)
13062 if (C1->isOne() && C2->isZero())
13063 return DAG.getZExtOrTrunc(Op: Cond, DL, VT);
13064
13065 // select Cond, -1, 0 --> sext (Cond)
13066 if (C1->isAllOnes() && C2->isZero())
13067 return DAG.getSExtOrTrunc(Op: Cond, DL, VT);
13068
13069 // select Cond, 0, 1 --> zext (!Cond)
13070 if (C1->isZero() && C2->isOne()) {
13071 SDValue NotCond = DAG.getNOT(DL, Val: Cond, VT: MVT::i1);
13072 NotCond = DAG.getZExtOrTrunc(Op: NotCond, DL, VT);
13073 return NotCond;
13074 }
13075
13076 // select Cond, 0, -1 --> sext (!Cond)
13077 if (C1->isZero() && C2->isAllOnes()) {
13078 SDValue NotCond = DAG.getNOT(DL, Val: Cond, VT: MVT::i1);
13079 NotCond = DAG.getSExtOrTrunc(Op: NotCond, DL, VT);
13080 return NotCond;
13081 }
13082
13083 // Use a target hook because some targets may prefer to transform in the
13084 // other direction.
13085 if (!shouldConvertSelectOfConstantsToMath(Cond, VT, TLI))
13086 return SDValue();
13087
13088 // For any constants that differ by 1, we can transform the select into
13089 // an extend and add.
13090 const APInt &C1Val = C1->getAPIntValue();
13091 const APInt &C2Val = C2->getAPIntValue();
13092
13093 // select Cond, C1, C1-1 --> add (zext Cond), C1-1
13094 if (C1Val - 1 == C2Val) {
13095 Cond = DAG.getZExtOrTrunc(Op: Cond, DL, VT);
13096 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Cond, N2);
13097 }
13098
13099 // select Cond, C1, C1+1 --> add (sext Cond), C1+1
13100 if (C1Val + 1 == C2Val) {
13101 Cond = DAG.getSExtOrTrunc(Op: Cond, DL, VT);
13102 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Cond, N2);
13103 }
13104
13105 // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
13106 if (C1Val.isPowerOf2() && C2Val.isZero()) {
13107 Cond = DAG.getZExtOrTrunc(Op: Cond, DL, VT);
13108 SDValue ShAmtC =
13109 DAG.getShiftAmountConstant(Val: C1Val.exactLogBase2(), VT, DL);
13110 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Cond, N2: ShAmtC);
13111 }
13112
13113 // select Cond, -1, C --> or (sext Cond), C
13114 if (C1->isAllOnes()) {
13115 Cond = DAG.getSExtOrTrunc(Op: Cond, DL, VT);
13116 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Cond, N2);
13117 }
13118
13119 // select Cond, C, -1 --> or (sext (not Cond)), C
13120 if (C2->isAllOnes()) {
13121 SDValue NotCond = DAG.getNOT(DL, Val: Cond, VT: MVT::i1);
13122 NotCond = DAG.getSExtOrTrunc(Op: NotCond, DL, VT);
13123 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: NotCond, N2: N1);
13124 }
13125
13126 if (SDValue V = foldSelectOfConstantsUsingSra(N, DL, DAG))
13127 return V;
13128
13129 return SDValue();
13130}
13131
13132static SDValue foldBoolSelectToLogic(SDNode *N, const SDLoc &DL,
13133 SelectionDAG &DAG) {
13134 assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT) &&
13135 "Expected a (v)select");
13136 SDValue Cond = N->getOperand(Num: 0);
13137 SDValue T = N->getOperand(Num: 1), F = N->getOperand(Num: 2);
13138 EVT VT = N->getValueType(ResNo: 0);
13139
13140 if (VT != Cond.getValueType() || VT.getScalarSizeInBits() != 1)
13141 return SDValue();
13142
13143 // select Cond, Cond, F --> or Cond, freeze(F)
13144 // select Cond, 1, F --> or Cond, freeze(F)
13145 if (Cond == T || isOneOrOneSplat(V: T, /* AllowUndefs */ true))
13146 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Cond, N2: DAG.getFreeze(V: F));
13147
13148 // select Cond, T, Cond --> and Cond, freeze(T)
13149 // select Cond, T, 0 --> and Cond, freeze(T)
13150 if (Cond == F || isNullOrNullSplat(V: F, /* AllowUndefs */ true))
13151 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Cond, N2: DAG.getFreeze(V: T));
13152
13153 // select Cond, T, 1 --> or (not Cond), freeze(T)
13154 if (isOneOrOneSplat(V: F, /* AllowUndefs */ true)) {
13155 SDValue NotCond =
13156 DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cond, N2: DAG.getAllOnesConstant(DL, VT));
13157 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: NotCond, N2: DAG.getFreeze(V: T));
13158 }
13159
13160 // select Cond, 0, F --> and (not Cond), freeze(F)
13161 if (isNullOrNullSplat(V: T, /* AllowUndefs */ true)) {
13162 SDValue NotCond =
13163 DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Cond, N2: DAG.getAllOnesConstant(DL, VT));
13164 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NotCond, N2: DAG.getFreeze(V: F));
13165 }
13166
13167 return SDValue();
13168}
13169
13170static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG) {
13171 SDValue N0 = N->getOperand(Num: 0);
13172 SDValue N1 = N->getOperand(Num: 1);
13173 SDValue N2 = N->getOperand(Num: 2);
13174 EVT VT = N->getValueType(ResNo: 0);
13175 unsigned EltSizeInBits = VT.getScalarSizeInBits();
13176
13177 SDValue Cond0, Cond1;
13178 ISD::CondCode CC;
13179 if (!sd_match(N: N0, P: m_OneUse(P: m_SetCC(LHS: m_Value(N&: Cond0), RHS: m_Value(N&: Cond1),
13180 CC: m_CondCode(CC)))) ||
13181 VT != Cond0.getValueType())
13182 return SDValue();
13183
13184 // Match a signbit check of Cond0 as "Cond0 s<0". Swap select operands if the
13185 // compare is inverted from that pattern ("Cond0 s> -1").
13186 if (CC == ISD::SETLT && isNullOrNullSplat(V: Cond1))
13187 ; // This is the pattern we are looking for.
13188 else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(V: Cond1))
13189 std::swap(a&: N1, b&: N2);
13190 else
13191 return SDValue();
13192
13193 // (Cond0 s< 0) ? N1 : 0 --> (Cond0 s>> BW-1) & freeze(N1)
13194 if (isNullOrNullSplat(V: N2)) {
13195 SDLoc DL(N);
13196 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: EltSizeInBits - 1, VT, DL);
13197 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Cond0, N2: ShiftAmt);
13198 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Sra, N2: DAG.getFreeze(V: N1));
13199 }
13200
13201 // (Cond0 s< 0) ? -1 : N2 --> (Cond0 s>> BW-1) | freeze(N2)
13202 if (isAllOnesOrAllOnesSplat(V: N1)) {
13203 SDLoc DL(N);
13204 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: EltSizeInBits - 1, VT, DL);
13205 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Cond0, N2: ShiftAmt);
13206 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Sra, N2: DAG.getFreeze(V: N2));
13207 }
13208
13209 // If we have to invert the sign bit mask, only do that transform if the
13210 // target has a bitwise 'and not' instruction (the invert is free).
13211 // (Cond0 s< -0) ? 0 : N2 --> ~(Cond0 s>> BW-1) & freeze(N2)
13212 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13213 if (isNullOrNullSplat(V: N1) && TLI.hasAndNot(X: N1)) {
13214 SDLoc DL(N);
13215 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: EltSizeInBits - 1, VT, DL);
13216 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Cond0, N2: ShiftAmt);
13217 SDValue Not = DAG.getNOT(DL, Val: Sra, VT);
13218 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Not, N2: DAG.getFreeze(V: N2));
13219 }
13220
13221 // TODO: There's another pattern in this family, but it may require
13222 // implementing hasOrNot() to check for profitability:
13223 // (Cond0 s> -1) ? -1 : N2 --> ~(Cond0 s>> BW-1) | freeze(N2)
13224
13225 return SDValue();
13226}
13227
13228// Match SELECTs with absolute difference patterns.
13229// (select (setcc a, b, set?gt), (sub a, b), (sub b, a)) --> (abd? a, b)
13230// (select (setcc a, b, set?ge), (sub a, b), (sub b, a)) --> (abd? a, b)
13231// (select (setcc a, b, set?lt), (sub b, a), (sub a, b)) --> (abd? a, b)
13232// (select (setcc a, b, set?le), (sub b, a), (sub a, b)) --> (abd? a, b)
13233SDValue DAGCombiner::foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
13234 SDValue False, ISD::CondCode CC,
13235 const SDLoc &DL) {
13236 bool IsSigned = isSignedIntSetCC(Code: CC);
13237 unsigned ABDOpc = IsSigned ? ISD::ABDS : ISD::ABDU;
13238 EVT VT = LHS.getValueType();
13239
13240 if (LegalOperations && !hasOperation(Opcode: ABDOpc, VT))
13241 return SDValue();
13242
13243 // (setcc 0, b set???) --> (setcc b, 0, set???)
13244 if (isZeroOrZeroSplat(N: LHS)) {
13245 std::swap(a&: LHS, b&: RHS);
13246 CC = ISD::getSetCCSwappedOperands(Operation: CC);
13247 }
13248
13249 // (setcc (add nsw A, Const), 0, sets??) --> (setcc A, -Const, sets??)
13250 SDValue A, B;
13251 if (ISD::isSignedIntSetCC(Code: CC) && LHS->getFlags().hasNoSignedWrap() &&
13252 isZeroOrZeroSplat(N: RHS) && sd_match(N: LHS, P: m_Add(L: m_Value(N&: A), R: m_Value(N&: B))) &&
13253 DAG.isConstantIntBuildVectorOrConstantInt(N: B)) {
13254 RHS = DAG.getNegative(Val: B, DL: LHS, VT: B.getValueType());
13255 LHS = A;
13256 }
13257
13258 bool IsTypeLegalOrPromote =
13259 TLI.isTypeLegal(VT) || TLI.getTypeAction(Context&: *DAG.getContext(), VT) ==
13260 TargetLowering::TypePromoteInteger;
13261
13262 switch (CC) {
13263 case ISD::SETGT:
13264 case ISD::SETGE:
13265 case ISD::SETUGT:
13266 case ISD::SETUGE:
13267 if (sd_match(N: True, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: LHS), R: m_Specific(N: RHS)),
13268 preds: m_Add(L: m_Specific(N: LHS), R: m_SpecificNeg(V: RHS)))) &&
13269 sd_match(N: False, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: RHS), R: m_Specific(N: LHS)),
13270 preds: m_Add(L: m_Specific(N: RHS), R: m_SpecificNeg(V: LHS)))))
13271 return DAG.getNode(Opcode: ABDOpc, DL, VT, N1: LHS, N2: RHS);
13272 if (sd_match(N: True, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: RHS), R: m_Specific(N: LHS)),
13273 preds: m_Add(L: m_Specific(N: RHS), R: m_SpecificNeg(V: LHS)))) &&
13274 sd_match(N: False, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: LHS), R: m_Specific(N: RHS)),
13275 preds: m_Add(L: m_Specific(N: LHS), R: m_SpecificNeg(V: RHS)))) &&
13276 IsTypeLegalOrPromote)
13277 return DAG.getNegative(Val: DAG.getNode(Opcode: ABDOpc, DL, VT, N1: LHS, N2: RHS), DL, VT);
13278 break;
13279 case ISD::SETLT:
13280 case ISD::SETLE:
13281 case ISD::SETULT:
13282 case ISD::SETULE:
13283 if (sd_match(N: True, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: RHS), R: m_Specific(N: LHS)),
13284 preds: m_Add(L: m_Specific(N: RHS), R: m_SpecificNeg(V: LHS)))) &&
13285 sd_match(N: False, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: LHS), R: m_Specific(N: RHS)),
13286 preds: m_Add(L: m_Specific(N: LHS), R: m_SpecificNeg(V: RHS)))))
13287 return DAG.getNode(Opcode: ABDOpc, DL, VT, N1: LHS, N2: RHS);
13288 if (sd_match(N: True, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: LHS), R: m_Specific(N: RHS)),
13289 preds: m_Add(L: m_Specific(N: LHS), R: m_SpecificNeg(V: RHS)))) &&
13290 sd_match(N: False, P: m_AnyOf(preds: m_Sub(L: m_Specific(N: RHS), R: m_Specific(N: LHS)),
13291 preds: m_Add(L: m_Specific(N: RHS), R: m_SpecificNeg(V: LHS)))) &&
13292 IsTypeLegalOrPromote)
13293 return DAG.getNegative(Val: DAG.getNode(Opcode: ABDOpc, DL, VT, N1: LHS, N2: RHS), DL, VT);
13294 break;
13295 default:
13296 break;
13297 }
13298
13299 return SDValue();
13300}
13301
13302// ([v]select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
13303// ([v]select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
13304SDValue DAGCombiner::foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
13305 SDValue False, ISD::CondCode CC,
13306 const SDLoc &DL) {
13307 APInt C;
13308 EVT VT = True.getValueType();
13309 if (sd_match(N: RHS, P: m_ConstInt(V&: C)) && hasUMin(VT)) {
13310 if (CC == ISD::SETUGT && LHS == False &&
13311 sd_match(N: True, P: m_Add(L: m_Specific(N: False), R: m_SpecificInt(V: ~C)))) {
13312 SDValue AddC = DAG.getConstant(Val: ~C, DL, VT);
13313 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: False, N2: AddC);
13314 return DAG.getNode(Opcode: ISD::UMIN, DL, VT, N1: Add, N2: False);
13315 }
13316 if (CC == ISD::SETULT && LHS == True &&
13317 sd_match(N: False, P: m_Add(L: m_Specific(N: True), R: m_SpecificInt(V: -C)))) {
13318 SDValue AddC = DAG.getConstant(Val: -C, DL, VT);
13319 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: True, N2: AddC);
13320 return DAG.getNode(Opcode: ISD::UMIN, DL, VT, N1: True, N2: Add);
13321 }
13322 }
13323 return SDValue();
13324}
13325
13326// Combine x olt y ? x : y to pseudo_fmin and x ogt y ? x : y to pseudo_fmax.
13327// Op0/Op1 are the setcc operands, LHS/RHS are the select operands, Flags are
13328// from the select.
13329// The return value is the opcode and its operands.
13330static std::tuple<unsigned, SDValue, SDValue> combineSelectCCToPseudoMinMax(
13331 SelectionDAG &DAG, const SDLoc &DL, ISD::CondCode CC, SDValue Op0,
13332 SDValue Op1, SDValue LHS, SDValue RHS, SDNodeFlags Flags, bool IsStrict) {
13333 std::tuple<unsigned, SDValue, SDValue> Invalid(0, {}, {});
13334 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13335 EVT VT = LHS.getValueType();
13336 if (!VT.isFloatingPoint())
13337 return Invalid;
13338
13339 // Check for x CC y ? x : y.
13340 if (!DAG.isEqualTo(A: LHS, B: Op0) || !DAG.isEqualTo(A: RHS, B: Op1)) {
13341 if (!DAG.isEqualTo(A: LHS, B: Op1) || !DAG.isEqualTo(A: RHS, B: Op0))
13342 return Invalid;
13343
13344 // Convert x CC y ? y : x to x inv(CC) y ? x : y.
13345 CC = ISD::getSetCCInverse(Operation: CC, Type: VT);
13346 std::swap(a&: LHS, b&: RHS);
13347 }
13348
13349 // Convert x CC y ? x : y to y swap(inv(CC)) x ? y : x
13350 // to convert an unordered into an ordered comparison.
13351 if (ISD::getUnorderedFlavor(Cond: CC) == 1) {
13352 CC = ISD::getSetCCSwappedOperands(Operation: ISD::getSetCCInverse(Operation: CC, Type: VT));
13353 std::swap(a&: LHS, b&: RHS);
13354 }
13355
13356 unsigned Opcode = 0;
13357 switch (CC) {
13358 default:
13359 break;
13360 case ISD::SETOLE:
13361 // Converting this to a min would handle comparisons between positive
13362 // and negative zero incorrectly.
13363 if (!Flags.hasNoSignedZeros() && !DAG.isKnownNeverLogicalZero(Op: LHS) &&
13364 !DAG.isKnownNeverLogicalZero(Op: RHS))
13365 break;
13366 Opcode = ISD::PSEUDO_FMIN;
13367 break;
13368 case ISD::SETLE:
13369 // Convert setle to setlt via inv+swap.
13370 std::swap(a&: LHS, b&: RHS);
13371 [[fallthrough]];
13372 case ISD::SETOLT:
13373 case ISD::SETLT:
13374 Opcode = ISD::PSEUDO_FMIN;
13375 break;
13376
13377 case ISD::SETOGE:
13378 // Converting this to a max would handle comparisons between positive
13379 // and negative zero incorrectly.
13380 if (!Flags.hasNoSignedZeros() && !DAG.isKnownNeverLogicalZero(Op: LHS) &&
13381 !DAG.isKnownNeverLogicalZero(Op: RHS))
13382 break;
13383 Opcode = ISD::PSEUDO_FMAX;
13384 break;
13385 case ISD::SETGE:
13386 // Convert setge to setgt via inv+swap.
13387 std::swap(a&: LHS, b&: RHS);
13388 [[fallthrough]];
13389 case ISD::SETOGT:
13390 case ISD::SETGT:
13391 Opcode = ISD::PSEUDO_FMAX;
13392 break;
13393 }
13394
13395 if (!Opcode)
13396 return Invalid;
13397
13398 if (IsStrict)
13399 Opcode = Opcode == ISD::PSEUDO_FMIN ? ISD::STRICT_PSEUDO_FMIN
13400 : ISD::STRICT_PSEUDO_FMAX;
13401 if (!TLI.isOperationLegalOrCustom(Op: Opcode, VT))
13402 return Invalid;
13403
13404 return {Opcode, LHS, RHS};
13405}
13406
13407static SDValue combineSelectToPseudoMinMax(SelectionDAG &DAG, SDNode *N) {
13408 SDLoc DL(N);
13409 SDValue Cond = N->getOperand(Num: 0);
13410 SDValue LHS = N->getOperand(Num: 1);
13411 SDValue RHS = N->getOperand(Num: 2);
13412 EVT VT = LHS.getValueType();
13413 if ((Cond.getOpcode() != ISD::SETCC &&
13414 Cond.getOpcode() != ISD::STRICT_FSETCCS))
13415 return SDValue();
13416
13417 bool IsStrict = Cond->isStrictFPOpcode();
13418 ISD::CondCode CC =
13419 cast<CondCodeSDNode>(Val: Cond.getOperand(i: IsStrict ? 3 : 2))->get();
13420 SDValue Op0 = Cond.getOperand(i: IsStrict ? 1 : 0);
13421 SDValue Op1 = Cond.getOperand(i: IsStrict ? 2 : 1);
13422 auto [Opcode, NewLHS, NewRHS] = combineSelectCCToPseudoMinMax(
13423 DAG, DL, CC, Op0, Op1, LHS, RHS, Flags: N->getFlags(), IsStrict);
13424 if (!Opcode)
13425 return SDValue();
13426
13427 // Propagate fast-math-flags.
13428 SelectionDAG::FlagInserter FlagsInserter(DAG, N->getFlags());
13429 if (IsStrict) {
13430 SDValue Ret = DAG.getNode(Opcode, DL, ResultTys: {VT, MVT::Other},
13431 Ops: {Cond.getOperand(i: 0), NewLHS, NewRHS});
13432 DAG.ReplaceAllUsesOfValueWith(From: Cond.getValue(R: 1), To: Ret.getValue(R: 1));
13433 return Ret;
13434 }
13435 return DAG.getNode(Opcode, DL, VT, N1: NewLHS, N2: NewRHS);
13436}
13437
13438/// Fold:
13439/// select_cc (select C, TV, FV), CmpC, TrueV, FalseV, seteq
13440/// -> select C, TrueV, FalseV
13441/// select_cc (select C, TV, FV), CmpC, TrueV, FalseV, setne
13442/// -> select C, FalseV, TrueV
13443/// and the same with CmpC on the LHS of the comparison. TV and FV must be
13444/// distinct integer constants. Also used for select (setcc ...).
13445static SDValue foldSelectOfSelectCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
13446 SDValue TrueV, SDValue FalseV,
13447 const SDLoc &DL, EVT VT, SelectionDAG &DAG,
13448 SDNodeFlags Flags) {
13449 if (CC != ISD::SETEQ && CC != ISD::SETNE)
13450 return SDValue();
13451
13452 SDValue InnerSel;
13453 SDValue CmpC;
13454 if (LHS.getOpcode() == ISD::SELECT) {
13455 InnerSel = LHS;
13456 CmpC = RHS;
13457 } else if (RHS.getOpcode() == ISD::SELECT) {
13458 InnerSel = RHS;
13459 CmpC = LHS;
13460 } else
13461 return SDValue();
13462
13463 SDValue Cond = InnerSel.getOperand(i: 0);
13464 SDValue InnerTV = InnerSel.getOperand(i: 1);
13465 SDValue InnerFV = InnerSel.getOperand(i: 2);
13466
13467 auto *CTV = dyn_cast<ConstantSDNode>(Val&: InnerTV);
13468 auto *CFV = dyn_cast<ConstantSDNode>(Val&: InnerFV);
13469 auto *CCnst = dyn_cast<ConstantSDNode>(Val&: CmpC);
13470 if (!CTV || !CFV || !CCnst)
13471 return SDValue();
13472
13473 // If one of the constants is opaque, the SDNodes may differ while the values
13474 // are the same. Check APInt to avoid miscompiles.
13475 if (CTV->getAPIntValue() == CFV->getAPIntValue())
13476 return SDValue();
13477
13478 const APInt &CmpVal = CCnst->getAPIntValue();
13479 bool MatchesTV = CmpVal == CTV->getAPIntValue();
13480 bool MatchesFV = CmpVal == CFV->getAPIntValue();
13481 if (!MatchesTV && !MatchesFV)
13482 return SDValue();
13483
13484 SDValue SelTrueV = TrueV;
13485 SDValue SelFalseV = FalseV;
13486 if (CC == ISD::SETEQ) {
13487 if (MatchesFV)
13488 std::swap(a&: SelTrueV, b&: SelFalseV);
13489 } else {
13490 if (MatchesTV)
13491 std::swap(a&: SelTrueV, b&: SelFalseV);
13492 }
13493
13494 return DAG.getSelect(DL, VT, Cond, LHS: SelTrueV, RHS: SelFalseV, Flags);
13495}
13496
13497static SDValue foldSelectCCOfSelect(SDNode *N, SelectionDAG &DAG) {
13498 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
13499 return foldSelectOfSelectCmp(LHS: N->getOperand(Num: 0), RHS: N->getOperand(Num: 1), CC,
13500 TrueV: N->getOperand(Num: 2), FalseV: N->getOperand(Num: 3), DL: SDLoc(N),
13501 VT: N->getValueType(ResNo: 0), DAG, Flags: N->getFlags());
13502}
13503
13504SDValue DAGCombiner::visitSELECT(SDNode *N) {
13505 SDValue N0 = N->getOperand(Num: 0);
13506 SDValue N1 = N->getOperand(Num: 1);
13507 SDValue N2 = N->getOperand(Num: 2);
13508 EVT VT = N->getValueType(ResNo: 0);
13509 EVT VT0 = N0.getValueType();
13510 SDLoc DL(N);
13511 SDNodeFlags Flags = N->getFlags();
13512
13513 if (SDValue V = DAG.simplifySelect(Cond: N0, TVal: N1, FVal: N2))
13514 return V;
13515
13516 if (SDValue V = foldBoolSelectToLogic(N, DL, DAG))
13517 return V;
13518
13519 // select (not Cond), N1, N2 -> select Cond, N2, N1
13520 if (SDValue F = extractBooleanFlip(V: N0, DAG, TLI, Force: false))
13521 return DAG.getSelect(DL, VT, Cond: F, LHS: N2, RHS: N1, Flags);
13522
13523 if (SDValue V = foldSelectOfConstants(N))
13524 return V;
13525
13526 // select (setcc (select C, TV, FV), CmpC, cc), TrueV, FalseV
13527 // -> select C, TrueV, FalseV (or swapped FalseV/TrueV)
13528 if (N0.getOpcode() == ISD::SETCC) {
13529 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
13530 if (SDValue R = foldSelectOfSelectCmp(LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1),
13531 CC, TrueV: N1, FalseV: N2, DL, VT, DAG, Flags))
13532 return R;
13533 }
13534
13535 // If we can fold this based on the true/false value, do so.
13536 if (SimplifySelectOps(SELECT: N, LHS: N1, RHS: N2))
13537 return SDValue(N, 0); // Don't revisit N.
13538
13539 if (VT0 == MVT::i1) {
13540 // The code in this block deals with the following 2 equivalences:
13541 // select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
13542 // select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
13543 // The target can specify its preferred form with the
13544 // shouldNormalizeToSelectSequence() callback. However we always transform
13545 // to the right anyway if we find the inner select exists in the DAG anyway
13546 // and we always transform to the left side if we know that we can further
13547 // optimize the combination of the conditions.
13548 bool normalizeToSequence =
13549 TLI.shouldNormalizeToSelectSequence(Context&: *DAG.getContext(), VT, CCVT: VT0);
13550 // select (and Cond0, Cond1), X, Y
13551 // -> select Cond0, (select Cond1, X, Y), Y
13552 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
13553 SDValue Cond0 = N0->getOperand(Num: 0);
13554 SDValue Cond1 = N0->getOperand(Num: 1);
13555 SDValue InnerSelect =
13556 DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(), N1: Cond1, N2: N1, N3: N2, Flags);
13557 if (normalizeToSequence || !InnerSelect.use_empty())
13558 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(), N1: Cond0,
13559 N2: InnerSelect, N3: N2, Flags);
13560 // Cleanup on failure.
13561 if (InnerSelect.use_empty())
13562 recursivelyDeleteUnusedNodes(N: InnerSelect.getNode());
13563 }
13564 // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
13565 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
13566 SDValue Cond0 = N0->getOperand(Num: 0);
13567 SDValue Cond1 = N0->getOperand(Num: 1);
13568 SDValue InnerSelect = DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(),
13569 N1: Cond1, N2: N1, N3: N2, Flags);
13570 if (normalizeToSequence || !InnerSelect.use_empty())
13571 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(), N1: Cond0, N2: N1,
13572 N3: InnerSelect, Flags);
13573 // Cleanup on failure.
13574 if (InnerSelect.use_empty())
13575 recursivelyDeleteUnusedNodes(N: InnerSelect.getNode());
13576 }
13577
13578 // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
13579 if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
13580 SDValue N1_0 = N1->getOperand(Num: 0);
13581 SDValue N1_1 = N1->getOperand(Num: 1);
13582 SDValue N1_2 = N1->getOperand(Num: 2);
13583 if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
13584 // Create the actual and node if we can generate good code for it.
13585 if (!normalizeToSequence) {
13586 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: N0.getValueType(), N1: N0, N2: N1_0);
13587 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(), N1: And, N2: N1_1,
13588 N3: N2, Flags);
13589 }
13590 // Otherwise see if we can optimize the "and" to a better pattern.
13591 if (SDValue Combined = visitANDLike(N0, N1: N1_0, N)) {
13592 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(), N1: Combined, N2: N1_1,
13593 N3: N2, Flags);
13594 }
13595 }
13596 }
13597 // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y
13598 if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
13599 SDValue N2_0 = N2->getOperand(Num: 0);
13600 SDValue N2_1 = N2->getOperand(Num: 1);
13601 SDValue N2_2 = N2->getOperand(Num: 2);
13602 if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
13603 // Create the actual or node if we can generate good code for it.
13604 if (!normalizeToSequence) {
13605 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT: N0.getValueType(), N1: N0, N2: N2_0);
13606 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(), N1: Or, N2: N1,
13607 N3: N2_2, Flags);
13608 }
13609 // Otherwise see if we can optimize to a better pattern.
13610 if (SDValue Combined = visitORLike(N0, N1: N2_0, DL))
13611 return DAG.getNode(Opcode: ISD::SELECT, DL, VT: N1.getValueType(), N1: Combined, N2: N1,
13612 N3: N2_2, Flags);
13613 }
13614 }
13615
13616 // select usubo(x, y).overflow, (sub y, x), (usubo x, y) -> abdu(x, y)
13617 if (N0.getOpcode() == ISD::USUBO && N0.getResNo() == 1 &&
13618 N2.getNode() == N0.getNode() && N2.getResNo() == 0 &&
13619 N1.getOpcode() == ISD::SUB && N2.getOperand(i: 0) == N1.getOperand(i: 1) &&
13620 N2.getOperand(i: 1) == N1.getOperand(i: 0) &&
13621 (!LegalOperations || TLI.isOperationLegal(Op: ISD::ABDU, VT)))
13622 return DAG.getNode(Opcode: ISD::ABDU, DL, VT, N1: N0.getOperand(i: 0), N2: N0.getOperand(i: 1));
13623
13624 // select usubo(x, y).overflow, (usubo x, y), (sub y, x) -> neg (abdu x, y)
13625 if (N0.getOpcode() == ISD::USUBO && N0.getResNo() == 1 &&
13626 N1.getNode() == N0.getNode() && N1.getResNo() == 0 &&
13627 N2.getOpcode() == ISD::SUB && N2.getOperand(i: 0) == N1.getOperand(i: 1) &&
13628 N2.getOperand(i: 1) == N1.getOperand(i: 0) &&
13629 (!LegalOperations || TLI.isOperationLegal(Op: ISD::ABDU, VT)))
13630 return DAG.getNegative(
13631 Val: DAG.getNode(Opcode: ISD::ABDU, DL, VT, N1: N0.getOperand(i: 0), N2: N0.getOperand(i: 1)),
13632 DL, VT);
13633 }
13634
13635 // Fold selects based on a setcc into other things, such as min/max/abs.
13636 if (N0.getOpcode() == ISD::SETCC) {
13637 SDValue Cond0 = N0.getOperand(i: 0), Cond1 = N0.getOperand(i: 1);
13638 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
13639
13640 // select (fcmp lt x, y), x, y -> fminnum x, y
13641 // select (fcmp gt x, y), x, y -> fmaxnum x, y
13642 //
13643 // This is OK if we don't care what happens if either operand is a NaN.
13644 if (N0.hasOneUse() &&
13645 isLegalToCombineMinNumMaxNum(DAG, LHS: N1, RHS: N2, SelectFlags: Flags, CmpFlags: N0->getFlags(), TLI))
13646 if (SDValue FMinMax =
13647 combineMinNumMaxNum(DL, VT, LHS: Cond0, RHS: Cond1, True: N1, False: N2, CC))
13648 return FMinMax;
13649
13650 // Use 'unsigned add with overflow' to optimize an unsigned saturating add.
13651 // This is conservatively limited to pre-legal-operations to give targets
13652 // a chance to reverse the transform if they want to do that. Also, it is
13653 // unlikely that the pattern would be formed late, so it's probably not
13654 // worth going through the other checks.
13655 if (!LegalOperations && TLI.isOperationLegalOrCustom(Op: ISD::UADDO, VT) &&
13656 CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(V: N1) &&
13657 N2.getOpcode() == ISD::ADD && Cond0 == N2.getOperand(i: 0)) {
13658 auto *C = dyn_cast<ConstantSDNode>(Val: N2.getOperand(i: 1));
13659 auto *NotC = dyn_cast<ConstantSDNode>(Val&: Cond1);
13660 if (C && NotC && C->getAPIntValue() == ~NotC->getAPIntValue()) {
13661 // select (setcc Cond0, ~C, ugt), -1, (add Cond0, C) -->
13662 // uaddo Cond0, C; select uaddo.1, -1, uaddo.0
13663 //
13664 // The IR equivalent of this transform would have this form:
13665 // %a = add %x, C
13666 // %c = icmp ugt %x, ~C
13667 // %r = select %c, -1, %a
13668 // =>
13669 // %u = call {iN,i1} llvm.uadd.with.overflow(%x, C)
13670 // %u0 = extractvalue %u, 0
13671 // %u1 = extractvalue %u, 1
13672 // %r = select %u1, -1, %u0
13673 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: VT0);
13674 SDValue UAO = DAG.getNode(Opcode: ISD::UADDO, DL, VTList: VTs, N1: Cond0, N2: N2.getOperand(i: 1));
13675 return DAG.getSelect(DL, VT, Cond: UAO.getValue(R: 1), LHS: N1, RHS: UAO.getValue(R: 0));
13676 }
13677 }
13678
13679 if (SDValue S = performNanGuardFpToSatCombine(N, DAG))
13680 return S;
13681
13682 if (TLI.isOperationLegal(Op: ISD::SELECT_CC, VT) ||
13683 (!LegalOperations &&
13684 TLI.isOperationLegalOrCustom(Op: ISD::SELECT_CC, VT))) {
13685 // Any flags available in a select/setcc fold will be on the setcc as they
13686 // migrated from fcmp
13687 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT, N1: Cond0, N2: Cond1, N3: N1, N4: N2,
13688 N5: N0.getOperand(i: 2), Flags: N0->getFlags());
13689 }
13690
13691 if (SDValue ABD = foldSelectToABD(LHS: Cond0, RHS: Cond1, True: N1, False: N2, CC, DL))
13692 return ABD;
13693
13694 if (SDValue NewSel = SimplifySelect(DL, N0, N1, N2))
13695 return NewSel;
13696
13697 // (select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
13698 // (select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
13699 if (SDValue UMin = foldSelectToUMin(LHS: Cond0, RHS: Cond1, True: N1, False: N2, CC, DL))
13700 return UMin;
13701 }
13702
13703 if (!VT.isVector())
13704 if (SDValue BinOp = foldSelectOfBinops(N))
13705 return BinOp;
13706
13707 if (SDValue R = combineSelectAsExtAnd(Cond: N0, T: N1, F: N2, DL, DAG))
13708 return R;
13709
13710 if (SDValue R = combineSelectToPseudoMinMax(DAG, N))
13711 return R;
13712
13713 return SDValue();
13714}
13715
13716// This function assumes all the vselect's arguments are CONCAT_VECTOR
13717// nodes and that the condition is a BV of ConstantSDNodes (or undefs).
13718static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
13719 SDLoc DL(N);
13720 SDValue Cond = N->getOperand(Num: 0);
13721 SDValue LHS = N->getOperand(Num: 1);
13722 SDValue RHS = N->getOperand(Num: 2);
13723 EVT VT = N->getValueType(ResNo: 0);
13724 int NumElems = VT.getVectorNumElements();
13725 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&
13726 RHS.getOpcode() == ISD::CONCAT_VECTORS &&
13727 Cond.getOpcode() == ISD::BUILD_VECTOR);
13728
13729 // CONCAT_VECTOR can take an arbitrary number of arguments. We only care about
13730 // binary ones here.
13731 if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
13732 return SDValue();
13733
13734 // We're sure we have an even number of elements due to the
13735 // concat_vectors we have as arguments to vselect.
13736 // Skip BV elements until we find one that's not an UNDEF
13737 // After we find an UNDEF element, keep looping until we get to half the
13738 // length of the BV and see if all the non-undef nodes are the same.
13739 ConstantSDNode *BottomHalf = nullptr;
13740 for (int i = 0; i < NumElems / 2; ++i) {
13741 if (Cond->getOperand(Num: i)->isUndef())
13742 continue;
13743
13744 if (BottomHalf == nullptr)
13745 BottomHalf = cast<ConstantSDNode>(Val: Cond.getOperand(i));
13746 else if (Cond->getOperand(Num: i).getNode() != BottomHalf)
13747 return SDValue();
13748 }
13749
13750 // Do the same for the second half of the BuildVector
13751 ConstantSDNode *TopHalf = nullptr;
13752 for (int i = NumElems / 2; i < NumElems; ++i) {
13753 if (Cond->getOperand(Num: i)->isUndef())
13754 continue;
13755
13756 if (TopHalf == nullptr)
13757 TopHalf = cast<ConstantSDNode>(Val: Cond.getOperand(i));
13758 else if (Cond->getOperand(Num: i).getNode() != TopHalf)
13759 return SDValue();
13760 }
13761
13762 assert(TopHalf && BottomHalf &&
13763 "One half of the selector was all UNDEFs and the other was all the "
13764 "same value. This should have been addressed before this function.");
13765 return DAG.getNode(
13766 Opcode: ISD::CONCAT_VECTORS, DL, VT,
13767 N1: BottomHalf->isZero() ? RHS->getOperand(Num: 0) : LHS->getOperand(Num: 0),
13768 N2: TopHalf->isZero() ? RHS->getOperand(Num: 1) : LHS->getOperand(Num: 1));
13769}
13770
13771bool refineUniformBase(SDValue &BasePtr, SDValue &Index, bool IndexIsScaled,
13772 SelectionDAG &DAG, const SDLoc &DL) {
13773
13774 // Only perform the transformation when existing operands can be reused.
13775 if (IndexIsScaled)
13776 return false;
13777
13778 if (!isNullConstant(V: BasePtr) && !Index.hasOneUse())
13779 return false;
13780
13781 EVT VT = BasePtr.getValueType();
13782
13783 if (SDValue SplatVal = DAG.getSplatValue(V: Index);
13784 SplatVal && !isNullConstant(V: SplatVal) &&
13785 SplatVal.getValueType() == VT) {
13786 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: BasePtr, N2: SplatVal);
13787 Index = DAG.getSplat(VT: Index.getValueType(), DL, Op: DAG.getConstant(Val: 0, DL, VT));
13788 return true;
13789 }
13790
13791 if (Index.getOpcode() != ISD::ADD)
13792 return false;
13793
13794 if (SDValue SplatVal = DAG.getSplatValue(V: Index.getOperand(i: 0));
13795 SplatVal && SplatVal.getValueType() == VT) {
13796 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: BasePtr, N2: SplatVal);
13797 Index = Index.getOperand(i: 1);
13798 return true;
13799 }
13800 if (SDValue SplatVal = DAG.getSplatValue(V: Index.getOperand(i: 1));
13801 SplatVal && SplatVal.getValueType() == VT) {
13802 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: BasePtr, N2: SplatVal);
13803 Index = Index.getOperand(i: 0);
13804 return true;
13805 }
13806 return false;
13807}
13808
13809// Fold sext/zext of index into index type.
13810bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT,
13811 SelectionDAG &DAG) {
13812 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
13813
13814 // It's always safe to look through zero extends.
13815 if (Index.getOpcode() == ISD::ZERO_EXTEND) {
13816 if (TLI.shouldRemoveExtendFromGSIndex(Extend: Index, DataVT)) {
13817 IndexType = ISD::UNSIGNED_SCALED;
13818 Index = Index.getOperand(i: 0);
13819 return true;
13820 }
13821 if (ISD::isIndexTypeSigned(IndexType)) {
13822 IndexType = ISD::UNSIGNED_SCALED;
13823 return true;
13824 }
13825 }
13826
13827 // It's only safe to look through sign extends when Index is signed.
13828 if (Index.getOpcode() == ISD::SIGN_EXTEND &&
13829 ISD::isIndexTypeSigned(IndexType) &&
13830 TLI.shouldRemoveExtendFromGSIndex(Extend: Index, DataVT)) {
13831 Index = Index.getOperand(i: 0);
13832 return true;
13833 }
13834
13835 return false;
13836}
13837
13838SDValue DAGCombiner::visitVPSCATTER(SDNode *N) {
13839 VPScatterSDNode *MSC = cast<VPScatterSDNode>(Val: N);
13840 SDValue Mask = MSC->getMask();
13841 SDValue Chain = MSC->getChain();
13842 SDValue Index = MSC->getIndex();
13843 SDValue Scale = MSC->getScale();
13844 SDValue StoreVal = MSC->getValue();
13845 SDValue BasePtr = MSC->getBasePtr();
13846 SDValue VL = MSC->getVectorLength();
13847 ISD::MemIndexType IndexType = MSC->getIndexType();
13848 SDLoc DL(N);
13849
13850 // Zap scatters with a zero mask.
13851 if (ISD::isConstantSplatVectorAllZeros(N: Mask.getNode()))
13852 return Chain;
13853
13854 if (refineUniformBase(BasePtr, Index, IndexIsScaled: MSC->isIndexScaled(), DAG, DL)) {
13855 SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL};
13856 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: MSC->getMemoryVT(),
13857 dl: DL, Ops, MMO: MSC->getMemOperand(), IndexType);
13858 }
13859
13860 if (refineIndexType(Index, IndexType, DataVT: StoreVal.getValueType(), DAG)) {
13861 SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL};
13862 return DAG.getScatterVP(VTs: DAG.getVTList(VT: MVT::Other), VT: MSC->getMemoryVT(),
13863 dl: DL, Ops, MMO: MSC->getMemOperand(), IndexType);
13864 }
13865
13866 return SDValue();
13867}
13868
13869SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
13870 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(Val: N);
13871 SDValue Mask = MSC->getMask();
13872 SDValue Chain = MSC->getChain();
13873 SDValue Index = MSC->getIndex();
13874 SDValue Scale = MSC->getScale();
13875 SDValue StoreVal = MSC->getValue();
13876 SDValue BasePtr = MSC->getBasePtr();
13877 ISD::MemIndexType IndexType = MSC->getIndexType();
13878 SDLoc DL(N);
13879
13880 // Zap scatters with a zero mask.
13881 if (ISD::isConstantSplatVectorAllZeros(N: Mask.getNode()))
13882 return Chain;
13883
13884 if (refineUniformBase(BasePtr, Index, IndexIsScaled: MSC->isIndexScaled(), DAG, DL)) {
13885 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
13886 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: MSC->getMemoryVT(),
13887 dl: DL, Ops, MMO: MSC->getMemOperand(), IndexType,
13888 IsTruncating: MSC->isTruncatingStore());
13889 }
13890
13891 if (refineIndexType(Index, IndexType, DataVT: StoreVal.getValueType(), DAG)) {
13892 SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
13893 return DAG.getMaskedScatter(VTs: DAG.getVTList(VT: MVT::Other), MemVT: MSC->getMemoryVT(),
13894 dl: DL, Ops, MMO: MSC->getMemOperand(), IndexType,
13895 IsTruncating: MSC->isTruncatingStore());
13896 }
13897
13898 return SDValue();
13899}
13900
13901SDValue DAGCombiner::visitMSTORE(SDNode *N) {
13902 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(Val: N);
13903 SDValue Mask = MST->getMask();
13904 SDValue Chain = MST->getChain();
13905 SDValue Value = MST->getValue();
13906 SDValue Ptr = MST->getBasePtr();
13907
13908 // Zap masked stores with a zero mask.
13909 if (ISD::isConstantSplatVectorAllZeros(N: Mask.getNode()))
13910 return Chain;
13911
13912 // Remove a masked store if base pointers and masks are equal.
13913 if (MaskedStoreSDNode *MST1 = dyn_cast<MaskedStoreSDNode>(Val&: Chain)) {
13914 if (MST->isUnindexed() && MST->isSimple() && MST1->isUnindexed() &&
13915 MST1->isSimple() && MST1->getBasePtr() == Ptr &&
13916 !MST->getBasePtr().isUndef() &&
13917 ((Mask == MST1->getMask() && MST->getMemoryVT().getStoreSize() ==
13918 MST1->getMemoryVT().getStoreSize()) ||
13919 ISD::isConstantSplatVectorAllOnes(N: Mask.getNode())) &&
13920 TypeSize::isKnownLE(LHS: MST1->getMemoryVT().getStoreSize(),
13921 RHS: MST->getMemoryVT().getStoreSize())) {
13922 CombineTo(N: MST1, Res: MST1->getChain());
13923 if (N->getOpcode() != ISD::DELETED_NODE)
13924 AddToWorklist(N);
13925 return SDValue(N, 0);
13926 }
13927 }
13928
13929 // If this is a masked load with an all ones mask, we can use a unmasked load.
13930 // FIXME: Can we do this for indexed, compressing, or truncating stores?
13931 if (ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()) && MST->isUnindexed() &&
13932 !MST->isCompressingStore() && !MST->isTruncatingStore())
13933 return DAG.getStore(Chain: MST->getChain(), dl: SDLoc(N), Val: MST->getValue(),
13934 Ptr: MST->getBasePtr(), PtrInfo: MST->getPointerInfo(),
13935 Alignment: MST->getBaseAlign(), MMOFlags: MST->getMemOperand()->getFlags(),
13936 Metadata: MST->getAAInfo());
13937
13938 // Try transforming N to an indexed store.
13939 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
13940 return SDValue(N, 0);
13941
13942 if (MST->isTruncatingStore() && MST->isUnindexed() &&
13943 Value.getValueType().isInteger() &&
13944 (!isa<ConstantSDNode>(Val: Value) ||
13945 !cast<ConstantSDNode>(Val&: Value)->isOpaque())) {
13946 APInt TruncDemandedBits =
13947 APInt::getLowBitsSet(numBits: Value.getScalarValueSizeInBits(),
13948 loBitsSet: MST->getMemoryVT().getScalarSizeInBits());
13949
13950 // See if we can simplify the operation with
13951 // SimplifyDemandedBits, which only works if the value has a single use.
13952 if (SimplifyDemandedBits(Op: Value, DemandedBits: TruncDemandedBits)) {
13953 // Re-visit the store if anything changed and the store hasn't been merged
13954 // with another node (N is deleted) SimplifyDemandedBits will add Value's
13955 // node back to the worklist if necessary, but we also need to re-visit
13956 // the Store node itself.
13957 if (N->getOpcode() != ISD::DELETED_NODE)
13958 AddToWorklist(N);
13959 return SDValue(N, 0);
13960 }
13961 }
13962
13963 // If this is a TRUNC followed by a masked store, fold this into a masked
13964 // truncating store. We can do this even if this is already a masked
13965 // truncstore.
13966 // TODO: Try combine to masked compress store if possiable.
13967 if ((Value.getOpcode() == ISD::TRUNCATE) && Value->hasOneUse() &&
13968 MST->isUnindexed() && !MST->isCompressingStore() &&
13969 TLI.canCombineTruncStore(ValVT: Value.getOperand(i: 0).getValueType(),
13970 MemVT: MST->getMemoryVT(), Alignment: MST->getAlign(),
13971 AddrSpace: MST->getAddressSpace(), LegalOnly: LegalOperations)) {
13972 auto Mask = TLI.promoteTargetBoolean(DAG, Bool: MST->getMask(),
13973 ValVT: Value.getOperand(i: 0).getValueType());
13974 return DAG.getMaskedStore(Chain, dl: SDLoc(N), Val: Value.getOperand(i: 0), Base: Ptr,
13975 Offset: MST->getOffset(), Mask, MemVT: MST->getMemoryVT(),
13976 MMO: MST->getMemOperand(), AM: MST->getAddressingMode(),
13977 /*IsTruncating=*/true);
13978 }
13979
13980 return SDValue();
13981}
13982
13983SDValue DAGCombiner::visitVP_STRIDED_STORE(SDNode *N) {
13984 auto *SST = cast<VPStridedStoreSDNode>(Val: N);
13985 EVT EltVT = SST->getValue().getValueType().getVectorElementType();
13986 // Combine strided stores with unit-stride to a regular VP store.
13987 if (auto *CStride = dyn_cast<ConstantSDNode>(Val: SST->getStride());
13988 CStride && CStride->getZExtValue() == EltVT.getStoreSize()) {
13989 return DAG.getStoreVP(Chain: SST->getChain(), dl: SDLoc(N), Val: SST->getValue(),
13990 Ptr: SST->getBasePtr(), Offset: SST->getOffset(), Mask: SST->getMask(),
13991 EVL: SST->getVectorLength(), MemVT: SST->getMemoryVT(),
13992 MMO: SST->getMemOperand(), AM: SST->getAddressingMode(),
13993 IsTruncating: SST->isTruncatingStore(), IsCompressing: SST->isCompressingStore());
13994 }
13995 return SDValue();
13996}
13997
13998SDValue DAGCombiner::visitVECTOR_COMPRESS(SDNode *N) {
13999 SDLoc DL(N);
14000 SDValue Vec = N->getOperand(Num: 0);
14001 SDValue Mask = N->getOperand(Num: 1);
14002 SDValue Passthru = N->getOperand(Num: 2);
14003 EVT VecVT = Vec.getValueType();
14004
14005 bool HasPassthru = !Passthru.isUndef();
14006
14007 APInt SplatVal;
14008 if (ISD::isConstantSplatVector(N: Mask.getNode(), SplatValue&: SplatVal))
14009 return TLI.isConstTrueVal(N: Mask) ? Vec : Passthru;
14010
14011 if (Vec.isUndef() || Mask.isUndef())
14012 return Passthru;
14013
14014 // No need for potentially expensive compress if the mask is constant.
14015 if (ISD::isBuildVectorOfConstantSDNodes(N: Mask.getNode())) {
14016 SmallVector<SDValue, 16> Ops;
14017 EVT ScalarVT = VecVT.getVectorElementType();
14018 unsigned NumSelected = 0;
14019 unsigned NumElmts = VecVT.getVectorNumElements();
14020 for (unsigned I = 0; I < NumElmts; ++I) {
14021 SDValue MaskI = Mask.getOperand(i: I);
14022 // We treat undef mask entries as "false".
14023 if (MaskI.isUndef())
14024 continue;
14025
14026 if (TLI.isConstTrueVal(N: MaskI)) {
14027 SDValue VecI = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ScalarVT, N1: Vec,
14028 N2: DAG.getVectorIdxConstant(Val: I, DL));
14029 Ops.push_back(Elt: VecI);
14030 NumSelected++;
14031 }
14032 }
14033 for (unsigned Rest = NumSelected; Rest < NumElmts; ++Rest) {
14034 SDValue Val =
14035 HasPassthru
14036 ? DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ScalarVT, N1: Passthru,
14037 N2: DAG.getVectorIdxConstant(Val: Rest, DL))
14038 : DAG.getUNDEF(VT: ScalarVT);
14039 Ops.push_back(Elt: Val);
14040 }
14041 return DAG.getBuildVector(VT: VecVT, DL, Ops);
14042 }
14043
14044 return SDValue();
14045}
14046
14047SDValue DAGCombiner::visitVPGATHER(SDNode *N) {
14048 VPGatherSDNode *MGT = cast<VPGatherSDNode>(Val: N);
14049 SDValue Mask = MGT->getMask();
14050 SDValue Chain = MGT->getChain();
14051 SDValue Index = MGT->getIndex();
14052 SDValue Scale = MGT->getScale();
14053 SDValue BasePtr = MGT->getBasePtr();
14054 SDValue VL = MGT->getVectorLength();
14055 ISD::MemIndexType IndexType = MGT->getIndexType();
14056 SDLoc DL(N);
14057
14058 if (refineUniformBase(BasePtr, Index, IndexIsScaled: MGT->isIndexScaled(), DAG, DL)) {
14059 SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL};
14060 return DAG.getGatherVP(
14061 VTs: DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other), VT: MGT->getMemoryVT(), dl: DL,
14062 Ops, MMO: MGT->getMemOperand(), IndexType);
14063 }
14064
14065 if (refineIndexType(Index, IndexType, DataVT: N->getValueType(ResNo: 0), DAG)) {
14066 SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL};
14067 return DAG.getGatherVP(
14068 VTs: DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other), VT: MGT->getMemoryVT(), dl: DL,
14069 Ops, MMO: MGT->getMemOperand(), IndexType);
14070 }
14071
14072 return SDValue();
14073}
14074
14075SDValue DAGCombiner::visitMGATHER(SDNode *N) {
14076 MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(Val: N);
14077 SDValue Mask = MGT->getMask();
14078 SDValue Chain = MGT->getChain();
14079 SDValue Index = MGT->getIndex();
14080 SDValue Scale = MGT->getScale();
14081 SDValue PassThru = MGT->getPassThru();
14082 SDValue BasePtr = MGT->getBasePtr();
14083 ISD::MemIndexType IndexType = MGT->getIndexType();
14084 SDLoc DL(N);
14085
14086 // Zap gathers with a zero mask.
14087 if (ISD::isConstantSplatVectorAllZeros(N: Mask.getNode()))
14088 return CombineTo(N, Res0: PassThru, Res1: MGT->getChain());
14089
14090 if (refineUniformBase(BasePtr, Index, IndexIsScaled: MGT->isIndexScaled(), DAG, DL)) {
14091 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
14092 return DAG.getMaskedGather(
14093 VTs: DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other), MemVT: MGT->getMemoryVT(), dl: DL,
14094 Ops, MMO: MGT->getMemOperand(), IndexType, ExtTy: MGT->getExtensionType());
14095 }
14096
14097 if (refineIndexType(Index, IndexType, DataVT: N->getValueType(ResNo: 0), DAG)) {
14098 SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
14099 return DAG.getMaskedGather(
14100 VTs: DAG.getVTList(VT1: N->getValueType(ResNo: 0), VT2: MVT::Other), MemVT: MGT->getMemoryVT(), dl: DL,
14101 Ops, MMO: MGT->getMemOperand(), IndexType, ExtTy: MGT->getExtensionType());
14102 }
14103
14104 return SDValue();
14105}
14106
14107SDValue DAGCombiner::visitMLOAD(SDNode *N) {
14108 MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(Val: N);
14109 SDValue Mask = MLD->getMask();
14110
14111 // Zap masked loads with a zero mask.
14112 if (ISD::isConstantSplatVectorAllZeros(N: Mask.getNode()))
14113 return CombineTo(N, Res0: MLD->getPassThru(), Res1: MLD->getChain());
14114
14115 // If this is a masked load with an all ones mask, we can use a unmasked load.
14116 // FIXME: Can we do this for indexed, expanding, or extending loads?
14117 if (ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()) && MLD->isUnindexed() &&
14118 !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) {
14119 SDValue NewLd =
14120 DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(N), Chain: MLD->getChain(),
14121 Ptr: MLD->getBasePtr(), PtrInfo: MLD->getPointerInfo(),
14122 Alignment: MLD->getBaseAlign(), MMOFlags: MLD->getMemOperand()->getFlags(),
14123 Metadata: MMOMetadata(MLD->getAAInfo(), MLD->getRanges()));
14124 return CombineTo(N, Res0: NewLd, Res1: NewLd.getValue(R: 1));
14125 }
14126
14127 // Try transforming N to an indexed load.
14128 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
14129 return SDValue(N, 0);
14130
14131 return SDValue();
14132}
14133
14134SDValue DAGCombiner::visitMHISTOGRAM(SDNode *N) {
14135 MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(Val: N);
14136 SDValue Chain = HG->getChain();
14137 SDValue Inc = HG->getInc();
14138 SDValue Mask = HG->getMask();
14139 SDValue BasePtr = HG->getBasePtr();
14140 SDValue Index = HG->getIndex();
14141 SDLoc DL(HG);
14142
14143 EVT MemVT = HG->getMemoryVT();
14144 EVT DataVT = Index.getValueType();
14145 MachineMemOperand *MMO = HG->getMemOperand();
14146 ISD::MemIndexType IndexType = HG->getIndexType();
14147
14148 if (ISD::isConstantSplatVectorAllZeros(N: Mask.getNode()))
14149 return Chain;
14150
14151 if (refineUniformBase(BasePtr, Index, IndexIsScaled: HG->isIndexScaled(), DAG, DL) ||
14152 refineIndexType(Index, IndexType, DataVT, DAG)) {
14153 SDValue Ops[] = {Chain, Inc, Mask, BasePtr, Index,
14154 HG->getScale(), HG->getIntID()};
14155 return DAG.getMaskedHistogram(VTs: DAG.getVTList(VT: MVT::Other), MemVT, dl: DL, Ops,
14156 MMO, IndexType);
14157 }
14158
14159 return SDValue();
14160}
14161
14162SDValue DAGCombiner::visitPARTIAL_REDUCE_MLA(SDNode *N) {
14163 if (SDValue Res = foldPartialReduceMLAMulOp(N))
14164 return Res;
14165 if (SDValue Res = foldPartialReduceAdd(N))
14166 return Res;
14167 return SDValue();
14168}
14169
14170// partial_reduce_*mla(acc, mul(*ext(a), *ext(b)), splat(1))
14171// -> partial_reduce_*mla(acc, a, b)
14172//
14173// partial_reduce_*mla(acc, mul(*ext(x), splat(C)), splat(1))
14174// -> partial_reduce_*mla(acc, x, splat(C))
14175//
14176// partial_reduce_*mla(acc, sel(p, mul(*ext(a), *ext(b)), splat(0)), splat(1))
14177// -> partial_reduce_*mla(acc, sel(p, a, splat(0)), b)
14178//
14179// partial_reduce_*mla(acc, sel(p, mul(*ext(a), splat(C)), splat(0)), splat(1))
14180// -> partial_reduce_*mla(acc, sel(p, a, splat(0)), splat(C))
14181//
14182// `sel` could either be VSELECT or VP_MERGE.
14183SDValue DAGCombiner::foldPartialReduceMLAMulOp(SDNode *N) {
14184 SDLoc DL(N);
14185 auto *Context = DAG.getContext();
14186 SDValue Tmp;
14187 SDValue Acc = N->getOperand(Num: 0);
14188 SDValue Op1 = N->getOperand(Num: 1);
14189 SDValue OrigOp1 = Op1;
14190 SDValue Op2 = N->getOperand(Num: 2);
14191 unsigned Opc = Op1->getOpcode();
14192
14193 // Handle predication by moving the VSELECT / VP_MERGE into the operand of the
14194 // MUL.
14195 SDValue Pred;
14196 if ((Opc == ISD::VSELECT || Opc == ISD::VP_MERGE) &&
14197 (isZeroOrZeroSplat(N: Op1->getOperand(Num: 2)) ||
14198 isZeroOrZeroSplatFP(N: Op1->getOperand(Num: 2)))) {
14199 Pred = Op1->getOperand(Num: 0);
14200 Op1 = Op1->getOperand(Num: 1);
14201 Opc = Op1->getOpcode();
14202 }
14203
14204 // Handle negation (sub-reduction).
14205 bool IsMLS = false;
14206 if (sd_match(N: Op1, P: m_Neg(V: m_Value(N&: Tmp)))) {
14207 Op1 = Tmp;
14208 Opc = Op1->getOpcode();
14209 IsMLS = true;
14210 }
14211
14212 if (Opc != ISD::MUL && Opc != ISD::FMUL && Opc != ISD::SHL)
14213 return SDValue();
14214
14215 SDValue LHS = Op1->getOperand(Num: 0);
14216 SDValue RHS = Op1->getOperand(Num: 1);
14217
14218 // After instcombine, negation for FP operations is on the RHS, so implement:
14219 // fmul(fpext(a), fneg(fpext(b)))
14220 //-> fmul(fpext(a), fpext(fneg(b)))
14221 if (sd_match(N: RHS, P: m_FNeg(Op: m_Value(N&: Tmp)))) {
14222 RHS = Tmp;
14223 IsMLS = true;
14224 }
14225
14226 // Try to treat (shl %a, %c) as (mul %a, (1 << %c)) for constant %c.
14227 if (Opc == ISD::SHL) {
14228 APInt C;
14229 if (!ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: C))
14230 return SDValue();
14231
14232 RHS =
14233 DAG.getSplatVector(VT: RHS.getValueType(), DL,
14234 Op: DAG.getConstant(Val: APInt(C.getBitWidth(), 1).shl(ShiftAmt: C), DL,
14235 VT: RHS.getValueType().getScalarType()));
14236 Opc = ISD::MUL;
14237 }
14238
14239 if (!(Opc == ISD::MUL && llvm::isOneOrOneSplat(V: Op2)) &&
14240 !(Opc == ISD::FMUL && llvm::isOneOrOneSplatFP(V: Op2)))
14241 return SDValue();
14242
14243 auto IsIntOrFPExtOpcode = [](unsigned int Opcode) {
14244 return (ISD::isExtOpcode(Opcode) || Opcode == ISD::FP_EXTEND);
14245 };
14246
14247 unsigned LHSOpcode = LHS->getOpcode();
14248 if (!IsIntOrFPExtOpcode(LHSOpcode))
14249 return SDValue();
14250
14251 SDValue LHSExtOp = LHS->getOperand(Num: 0);
14252 EVT LHSExtOpVT = LHSExtOp.getValueType();
14253
14254 // When Pred is non-zero, set Op = select(Pred, Op, splat(0)) and freeze
14255 // OtherOp to keep the same semantics when moving the selects into the MUL
14256 // operands.
14257 auto ApplyPredicate = [&](SDValue &Op, SDValue &OtherOp) {
14258 if (Pred) {
14259 EVT OpVT = Op.getValueType();
14260 SDValue Zero = OpVT.isFloatingPoint() ? DAG.getConstantFP(Val: 0.0, DL, VT: OpVT)
14261 : DAG.getConstant(Val: 0, DL, VT: OpVT);
14262 if (OrigOp1.getOpcode() == ISD::VP_MERGE)
14263 Op = DAG.getNode(Opcode: ISD::VP_MERGE, DL, VT: OpVT, N1: Pred, N2: Op, N3: Zero,
14264 N4: OrigOp1.getOperand(i: 3));
14265 else
14266 Op = DAG.getSelect(DL, VT: OpVT, Cond: Pred, LHS: Op, RHS: Zero);
14267 OtherOp = DAG.getFreeze(V: OtherOp);
14268 }
14269 };
14270
14271 // Generate an MLA or MLS.
14272 auto GetMLA = [&](unsigned Opc, SDValue Acc, SDValue LHS,
14273 SDValue RHS) -> SDValue {
14274 EVT AccVT = Acc.getValueType();
14275 return IsMLS ? DAG.getPartialReduceMLS(Opc, DL, Acc, LHS, RHS)
14276 : DAG.getNode(Opcode: Opc, DL, VT: AccVT, N1: Acc, N2: LHS, N3: RHS);
14277 };
14278
14279 // partial_reduce_*mla(acc, mul(ext(x), splat(C)), splat(1))
14280 // -> partial_reduce_*mla(acc, x, C)
14281 APInt C;
14282 if (ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: C)) {
14283 // TODO: Make use of partial_reduce_sumla here
14284 APInt CTrunc = C.trunc(width: LHSExtOpVT.getScalarSizeInBits());
14285 unsigned LHSBits = LHS.getValueType().getScalarSizeInBits();
14286 if ((LHSOpcode != ISD::ZERO_EXTEND || CTrunc.zext(width: LHSBits) != C) &&
14287 (LHSOpcode != ISD::SIGN_EXTEND || CTrunc.sext(width: LHSBits) != C))
14288 return SDValue();
14289
14290 unsigned NewOpcode = LHSOpcode == ISD::SIGN_EXTEND
14291 ? ISD::PARTIAL_REDUCE_SMLA
14292 : ISD::PARTIAL_REDUCE_UMLA;
14293
14294 // Only perform these combines if the target supports folding
14295 // the extends into the operation.
14296 if (!TLI.isPartialReduceMLALegalOrCustom(
14297 Opc: NewOpcode, AccVT: TLI.getTypeToTransformTo(Context&: *Context, VT: N->getValueType(ResNo: 0)),
14298 InputVT: TLI.getTypeToTransformTo(Context&: *Context, VT: LHSExtOpVT)))
14299 return SDValue();
14300
14301 SDValue C = DAG.getConstant(Val: CTrunc, DL, VT: LHSExtOpVT);
14302 ApplyPredicate(C, LHSExtOp);
14303 return GetMLA(NewOpcode, Acc, LHSExtOp, C);
14304 }
14305
14306 unsigned RHSOpcode = RHS->getOpcode();
14307 if (!IsIntOrFPExtOpcode(RHSOpcode))
14308 return SDValue();
14309
14310 SDValue RHSExtOp = RHS->getOperand(Num: 0);
14311 if (LHSExtOpVT != RHSExtOp.getValueType())
14312 return SDValue();
14313
14314 unsigned NewOpc;
14315 if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::SIGN_EXTEND)
14316 NewOpc = ISD::PARTIAL_REDUCE_SMLA;
14317 else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
14318 NewOpc = ISD::PARTIAL_REDUCE_UMLA;
14319 else if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
14320 NewOpc = ISD::PARTIAL_REDUCE_SUMLA;
14321 else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::SIGN_EXTEND) {
14322 NewOpc = ISD::PARTIAL_REDUCE_SUMLA;
14323 std::swap(a&: LHSExtOp, b&: RHSExtOp);
14324 } else if (LHSOpcode == ISD::FP_EXTEND && RHSOpcode == ISD::FP_EXTEND) {
14325 NewOpc = ISD::PARTIAL_REDUCE_FMLA;
14326 } else
14327 return SDValue();
14328 // For a 2-stage extend the signedness of both of the extends must match
14329 // If the mul has the same type, there is no outer extend, and thus we
14330 // can simply use the inner extends to pick the result node.
14331 // TODO: extend to handle nonneg zext as sext
14332 EVT AccElemVT = Acc.getValueType().getVectorElementType();
14333 if (Op1.getValueType().getVectorElementType() != AccElemVT &&
14334 NewOpc != N->getOpcode())
14335 return SDValue();
14336
14337 // Only perform these combines if the target supports folding
14338 // the extends into the operation.
14339 if (!TLI.isPartialReduceMLALegalOrCustom(
14340 Opc: NewOpc, AccVT: TLI.getTypeToTransformTo(Context&: *Context, VT: N->getValueType(ResNo: 0)),
14341 InputVT: TLI.getTypeToTransformTo(Context&: *Context, VT: LHSExtOpVT)))
14342 return SDValue();
14343
14344 ApplyPredicate(RHSExtOp, LHSExtOp);
14345 return GetMLA(NewOpc, Acc, LHSExtOp, RHSExtOp);
14346}
14347
14348// partial.reduce.*mla(acc, *ext(op), splat(1))
14349// -> partial.reduce.*mla(acc, op, splat(trunc(1)))
14350// partial.reduce.sumla(acc, sext(op), splat(1))
14351// -> partial.reduce.smla(acc, op, splat(trunc(1)))
14352//
14353// partial.reduce.*mla(acc, sel(p, *ext(op), splat(0)), splat(1))
14354// -> partial.reduce.*mla(acc, sel(p, op, splat(0)), splat(trunc(1)))
14355SDValue DAGCombiner::foldPartialReduceAdd(SDNode *N) {
14356 SDLoc DL(N);
14357 SDValue Tmp;
14358 SDValue Acc = N->getOperand(Num: 0);
14359 SDValue Op1 = N->getOperand(Num: 1);
14360 SDValue Op2 = N->getOperand(Num: 2);
14361
14362 if (!llvm::isOneOrOneSplat(V: Op2) && !llvm::isOneOrOneSplatFP(V: Op2))
14363 return SDValue();
14364
14365 SDValue Pred;
14366 unsigned Op1Opcode = Op1.getOpcode();
14367 if (Op1Opcode == ISD::VSELECT && (isZeroOrZeroSplat(N: Op1->getOperand(Num: 2)) ||
14368 isZeroOrZeroSplatFP(N: Op1->getOperand(Num: 2)))) {
14369 Pred = Op1->getOperand(Num: 0);
14370 Op1 = Op1->getOperand(Num: 1);
14371 Op1Opcode = Op1->getOpcode();
14372 }
14373
14374 // Handle negation (sub-reduction).
14375 bool IsMLS = false;
14376 if (sd_match(N: Op1, P: m_AnyOf(preds: m_Neg(V: m_Value(N&: Tmp)), preds: m_FNeg(Op: m_Value(N&: Tmp))))) {
14377 Op1 = Tmp;
14378 Op1Opcode = Op1.getOpcode();
14379 IsMLS = true;
14380 }
14381
14382 if (!ISD::isExtOpcode(Opcode: Op1Opcode) && Op1Opcode != ISD::FP_EXTEND)
14383 return SDValue();
14384
14385 bool Op1IsSigned =
14386 Op1Opcode == ISD::SIGN_EXTEND || Op1Opcode == ISD::FP_EXTEND;
14387 bool NodeIsSigned = N->getOpcode() != ISD::PARTIAL_REDUCE_UMLA;
14388 EVT AccElemVT = Acc.getValueType().getVectorElementType();
14389 if (Op1IsSigned != NodeIsSigned &&
14390 Op1.getValueType().getVectorElementType() != AccElemVT)
14391 return SDValue();
14392
14393 unsigned NewOpcode = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
14394 ? ISD::PARTIAL_REDUCE_FMLA
14395 : Op1IsSigned ? ISD::PARTIAL_REDUCE_SMLA
14396 : ISD::PARTIAL_REDUCE_UMLA;
14397
14398 SDValue UnextOp1 = Op1.getOperand(i: 0);
14399 EVT UnextOp1VT = UnextOp1.getValueType();
14400 auto *Context = DAG.getContext();
14401 EVT PromOp1VT = TLI.getTypeToTransformTo(Context&: *Context, VT: UnextOp1VT);
14402 if (!TLI.isPartialReduceMLALegalOrCustom(
14403 Opc: NewOpcode, AccVT: TLI.getTypeToTransformTo(Context&: *Context, VT: N->getValueType(ResNo: 0)),
14404 InputVT: PromOp1VT))
14405 return SDValue();
14406
14407 // The multiplier below is built at the operand type, where a splat of 1 in i1
14408 // sign extends to -1. Extend i1 masks to the promoted type first.
14409 if (Op1IsSigned && UnextOp1VT.getVectorElementType() == MVT::i1) {
14410 if (PromOp1VT == UnextOp1VT)
14411 return SDValue();
14412 UnextOp1VT = PromOp1VT;
14413 UnextOp1 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: UnextOp1VT, Operand: UnextOp1);
14414 }
14415
14416 SDValue Constant = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
14417 ? DAG.getConstantFP(Val: 1, DL, VT: UnextOp1VT)
14418 : DAG.getConstant(Val: 1, DL, VT: UnextOp1VT);
14419
14420 if (Pred) {
14421 SDValue Zero = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
14422 ? DAG.getConstantFP(Val: 0, DL, VT: UnextOp1VT)
14423 : DAG.getConstant(Val: 0, DL, VT: UnextOp1VT);
14424 Constant = DAG.getSelect(DL, VT: UnextOp1VT, Cond: Pred, LHS: Constant, RHS: Zero);
14425 }
14426 EVT AccVT = Acc.getValueType();
14427 return IsMLS ? DAG.getPartialReduceMLS(Opc: NewOpcode, DL, Acc, LHS: UnextOp1, RHS: Constant)
14428 : DAG.getNode(Opcode: NewOpcode, DL, VT: AccVT, N1: Acc, N2: UnextOp1, N3: Constant);
14429}
14430
14431SDValue DAGCombiner::visitLOOP_DEPENDENCE_MASK(SDNode *N) {
14432 SDLoc DL(N);
14433 EVT VT = N->getValueType(ResNo: 0);
14434 unsigned LaneOffset = N->getConstantOperandVal(Num: 3);
14435
14436 // The first lane is always active, so v1i1 => true.
14437 if (LaneOffset == 0 &&
14438 VT.getVectorElementCount() == ElementCount::getFixed(MinVal: 1))
14439 return DAG.getBoolConstant(V: true, DL, VT, OpVT: VT);
14440
14441 return SDValue();
14442}
14443
14444SDValue DAGCombiner::visitVP_STRIDED_LOAD(SDNode *N) {
14445 auto *SLD = cast<VPStridedLoadSDNode>(Val: N);
14446 EVT EltVT = SLD->getValueType(ResNo: 0).getVectorElementType();
14447 // Combine strided loads with unit-stride to a regular VP load.
14448 if (auto *CStride = dyn_cast<ConstantSDNode>(Val: SLD->getStride());
14449 CStride && CStride->getZExtValue() == EltVT.getStoreSize()) {
14450 SDValue NewLd = DAG.getLoadVP(
14451 AM: SLD->getAddressingMode(), ExtType: SLD->getExtensionType(), VT: SLD->getValueType(ResNo: 0),
14452 dl: SDLoc(N), Chain: SLD->getChain(), Ptr: SLD->getBasePtr(), Offset: SLD->getOffset(),
14453 Mask: SLD->getMask(), EVL: SLD->getVectorLength(), MemVT: SLD->getMemoryVT(),
14454 MMO: SLD->getMemOperand(), IsExpanding: SLD->isExpandingLoad());
14455 return CombineTo(N, Res0: NewLd, Res1: NewLd.getValue(R: 1));
14456 }
14457 return SDValue();
14458}
14459
14460/// A vector select of 2 constant vectors can be simplified to math/logic to
14461/// avoid a variable select instruction and possibly avoid constant loads.
14462SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
14463 SDValue Cond = N->getOperand(Num: 0);
14464 SDValue N1 = N->getOperand(Num: 1);
14465 SDValue N2 = N->getOperand(Num: 2);
14466 EVT VT = N->getValueType(ResNo: 0);
14467 if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
14468 !shouldConvertSelectOfConstantsToMath(Cond, VT, TLI) ||
14469 !ISD::isBuildVectorOfConstantSDNodes(N: N1.getNode()) ||
14470 !ISD::isBuildVectorOfConstantSDNodes(N: N2.getNode()))
14471 return SDValue();
14472
14473 // Check if we can use the condition value to increment/decrement a single
14474 // constant value. This simplifies a select to an add and removes a constant
14475 // load/materialization from the general case.
14476 bool AllAddOne = true;
14477 bool AllSubOne = true;
14478 unsigned Elts = VT.getVectorNumElements();
14479 for (unsigned i = 0; i != Elts; ++i) {
14480 SDValue N1Elt = N1.getOperand(i);
14481 SDValue N2Elt = N2.getOperand(i);
14482 if (N1Elt.isUndef())
14483 continue;
14484 // N2 should not contain undef values since it will be reused in the fold.
14485 if (N2Elt.isUndef() || N1Elt.getValueType() != N2Elt.getValueType()) {
14486 AllAddOne = false;
14487 AllSubOne = false;
14488 break;
14489 }
14490
14491 const APInt &C1 = N1Elt->getAsAPIntVal();
14492 const APInt &C2 = N2Elt->getAsAPIntVal();
14493 if (C1 != C2 + 1)
14494 AllAddOne = false;
14495 if (C1 != C2 - 1)
14496 AllSubOne = false;
14497 }
14498
14499 // Further simplifications for the extra-special cases where the constants are
14500 // all 0 or all -1 should be implemented as folds of these patterns.
14501 SDLoc DL(N);
14502 if (AllAddOne || AllSubOne) {
14503 // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C
14504 // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C
14505 auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
14506 SDValue ExtendedCond = DAG.getNode(Opcode: ExtendOpcode, DL, VT, Operand: Cond);
14507 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: ExtendedCond, N2);
14508 }
14509
14510 // select Cond, Pow2C, 0 --> (zext Cond) << log2(Pow2C)
14511 APInt Pow2C;
14512 if (ISD::isConstantSplatVector(N: N1.getNode(), SplatValue&: Pow2C) && Pow2C.isPowerOf2() &&
14513 isNullOrNullSplat(V: N2)) {
14514 SDValue ZextCond = DAG.getZExtOrTrunc(Op: Cond, DL, VT);
14515 SDValue ShAmtC = DAG.getConstant(Val: Pow2C.exactLogBase2(), DL, VT);
14516 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ZextCond, N2: ShAmtC);
14517 }
14518
14519 if (SDValue V = foldSelectOfConstantsUsingSra(N, DL, DAG))
14520 return V;
14521
14522 // The general case for select-of-constants:
14523 // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2
14524 // ...but that only makes sense if a vselect is slower than 2 logic ops, so
14525 // leave that to a machine-specific pass.
14526 return SDValue();
14527}
14528
14529static SDValue combineVSelectWithAllOnesOrZeros(SDValue Cond, SDValue TVal,
14530 SDValue FVal,
14531 const TargetLowering &TLI,
14532 SelectionDAG &DAG,
14533 const SDLoc &DL) {
14534 EVT VT = TVal.getValueType();
14535 if (!TLI.isTypeLegal(VT))
14536 return SDValue();
14537
14538 EVT CondVT = Cond.getValueType();
14539 assert(CondVT.isVector() && "Vector select expects a vector selector!");
14540
14541 bool IsTAllZero = ISD::isConstantSplatVectorAllZeros(N: TVal.getNode());
14542 bool IsTAllOne = ISD::isConstantSplatVectorAllOnes(N: TVal.getNode());
14543 bool IsFAllZero = ISD::isConstantSplatVectorAllZeros(N: FVal.getNode());
14544 bool IsFAllOne = ISD::isConstantSplatVectorAllOnes(N: FVal.getNode());
14545
14546 // no vselect(cond, 0/-1, X) or vselect(cond, X, 0/-1), return
14547 if (!IsTAllZero && !IsTAllOne && !IsFAllZero && !IsFAllOne)
14548 return SDValue();
14549
14550 // select Cond, 0, 0 → 0
14551 if (IsTAllZero && IsFAllZero) {
14552 return VT.isFloatingPoint() ? DAG.getConstantFP(Val: 0.0, DL, VT)
14553 : DAG.getConstant(Val: 0, DL, VT);
14554 }
14555
14556 // check select(setgt lhs, -1), 1, -1 --> or (sra lhs, bitwidth - 1), 1
14557 APInt TValAPInt;
14558 if (Cond.getOpcode() == ISD::SETCC &&
14559 Cond.getOperand(i: 2) == DAG.getCondCode(Cond: ISD::SETGT) &&
14560 Cond.getOperand(i: 0).getValueType() == VT && VT.isSimple() &&
14561 ISD::isConstantSplatVector(N: TVal.getNode(), SplatValue&: TValAPInt) &&
14562 TValAPInt.isOne() &&
14563 ISD::isConstantSplatVectorAllOnes(N: Cond.getOperand(i: 1).getNode()) &&
14564 ISD::isConstantSplatVectorAllOnes(N: FVal.getNode()) &&
14565 !TLI.shouldAvoidTransformToShift(VT, Amount: VT.getScalarSizeInBits() - 1)) {
14566 SDValue LHS = Cond.getOperand(i: 0);
14567 SDValue ShiftC =
14568 DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL);
14569 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS, N2: ShiftC);
14570 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Shift, N2: TVal);
14571 }
14572
14573 // To use the condition operand as a bitwise mask, it must have elements that
14574 // are the same size as the select elements. i.e, the condition operand must
14575 // have already been promoted from the IR select condition type <N x i1>.
14576 // Don't check if the types themselves are equal because that excludes
14577 // vector floating-point selects.
14578 if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
14579 return SDValue();
14580
14581 // Cond value must be 'sign splat' to be converted to a logical op.
14582 if (DAG.ComputeNumSignBits(Op: Cond) != CondVT.getScalarSizeInBits())
14583 return SDValue();
14584
14585 // Try inverting Cond and swapping T/F if it gives all-ones/all-zeros form
14586 if (!IsTAllOne && !IsFAllZero && Cond.hasOneUse() &&
14587 Cond.getOpcode() == ISD::SETCC &&
14588 TLI.getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT) ==
14589 CondVT) {
14590 if (IsTAllZero || IsFAllOne) {
14591 SDValue CC = Cond.getOperand(i: 2);
14592 ISD::CondCode InverseCC = ISD::getSetCCInverse(
14593 Operation: cast<CondCodeSDNode>(Val&: CC)->get(), Type: Cond.getOperand(i: 0).getValueType());
14594 Cond = DAG.getSetCC(DL, VT: CondVT, LHS: Cond.getOperand(i: 0), RHS: Cond.getOperand(i: 1),
14595 Cond: InverseCC);
14596 std::swap(a&: TVal, b&: FVal);
14597 std::swap(a&: IsTAllOne, b&: IsFAllOne);
14598 std::swap(a&: IsTAllZero, b&: IsFAllZero);
14599 }
14600 }
14601
14602 assert(DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits() &&
14603 "Select condition no longer all-sign bits");
14604
14605 // select Cond, -1, 0 → bitcast Cond
14606 if (IsTAllOne && IsFAllZero)
14607 return DAG.getBitcast(VT, V: Cond);
14608
14609 // select Cond, -1, x → or Cond, x
14610 if (IsTAllOne) {
14611 SDValue X = DAG.getBitcast(VT: CondVT, V: DAG.getFreeze(V: FVal));
14612 SDValue Or = DAG.getNode(Opcode: ISD::OR, DL, VT: CondVT, N1: Cond, N2: X);
14613 return DAG.getBitcast(VT, V: Or);
14614 }
14615
14616 // select Cond, x, 0 → and Cond, x
14617 if (IsFAllZero) {
14618 SDValue X = DAG.getBitcast(VT: CondVT, V: DAG.getFreeze(V: TVal));
14619 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: CondVT, N1: Cond, N2: X);
14620 return DAG.getBitcast(VT, V: And);
14621 }
14622
14623 // select Cond, 0, x -> and not(Cond), x
14624 if (IsTAllZero &&
14625 (isBitwiseNot(V: peekThroughBitcasts(V: Cond)) || TLI.hasAndNot(X: Cond))) {
14626 SDValue X = DAG.getBitcast(VT: CondVT, V: DAG.getFreeze(V: FVal));
14627 SDValue And =
14628 DAG.getNode(Opcode: ISD::AND, DL, VT: CondVT, N1: DAG.getNOT(DL, Val: Cond, VT: CondVT), N2: X);
14629 return DAG.getBitcast(VT, V: And);
14630 }
14631
14632 return SDValue();
14633}
14634
14635SDValue DAGCombiner::visitVSELECT(SDNode *N) {
14636 SDValue N0 = N->getOperand(Num: 0);
14637 SDValue N1 = N->getOperand(Num: 1);
14638 SDValue N2 = N->getOperand(Num: 2);
14639 EVT VT = N->getValueType(ResNo: 0);
14640 SDLoc DL(N);
14641
14642 if (SDValue V = DAG.simplifySelect(Cond: N0, TVal: N1, FVal: N2))
14643 return V;
14644
14645 if (SDValue V = foldBoolSelectToLogic(N, DL, DAG))
14646 return V;
14647
14648 // vselect (not Cond), N1, N2 -> vselect Cond, N2, N1
14649 if (!TLI.isTargetCanonicalSelect(N))
14650 if (SDValue F = extractBooleanFlip(V: N0, DAG, TLI, Force: false))
14651 return DAG.getSelect(DL, VT, Cond: F, LHS: N2, RHS: N1, Flags: N->getFlags());
14652
14653 // select (sext m), (add X, C), X --> (add X, (and C, (sext m))))
14654 if (N1.getOpcode() == ISD::ADD && N1.getOperand(i: 0) == N2 && N1->hasOneUse() &&
14655 DAG.isConstantIntBuildVectorOrConstantInt(N: N1.getOperand(i: 1)) &&
14656 N0.getScalarValueSizeInBits() == N1.getScalarValueSizeInBits() &&
14657 TLI.getBooleanContents(Type: N0.getValueType()) ==
14658 TargetLowering::ZeroOrNegativeOneBooleanContent) {
14659 return DAG.getNode(
14660 Opcode: ISD::ADD, DL, VT: N1.getValueType(), N1: N2,
14661 N2: DAG.getNode(Opcode: ISD::AND, DL, VT: N0.getValueType(), N1: N1.getOperand(i: 1), N2: N0));
14662 }
14663
14664 // Canonicalize integer abs.
14665 // vselect (setg[te] X, 0), X, -X ->
14666 // vselect (setgt X, -1), X, -X ->
14667 // vselect (setl[te] X, 0), -X, X ->
14668 // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
14669 if (N0.getOpcode() == ISD::SETCC) {
14670 SDValue LHS = N0.getOperand(i: 0), RHS = N0.getOperand(i: 1);
14671 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
14672 bool isAbs = false;
14673 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(N: RHS.getNode());
14674
14675 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
14676 (ISD::isBuildVectorAllOnes(N: RHS.getNode()) && CC == ISD::SETGT)) &&
14677 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(i: 1))
14678 isAbs = ISD::isBuildVectorAllZeros(N: N2.getOperand(i: 0).getNode());
14679 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
14680 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(i: 1))
14681 isAbs = ISD::isBuildVectorAllZeros(N: N1.getOperand(i: 0).getNode());
14682
14683 if (isAbs) {
14684 if (TLI.isOperationLegalOrCustom(Op: ISD::ABS, VT))
14685 return DAG.getNode(Opcode: ISD::ABS, DL, VT, Operand: LHS);
14686
14687 SDValue Shift = DAG.getNode(
14688 Opcode: ISD::SRA, DL, VT, N1: LHS,
14689 N2: DAG.getShiftAmountConstant(Val: VT.getScalarSizeInBits() - 1, VT, DL));
14690 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: Shift);
14691 AddToWorklist(N: Shift.getNode());
14692 AddToWorklist(N: Add.getNode());
14693 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Add, N2: Shift);
14694 }
14695
14696 // vselect x, y (fcmp lt x, y) -> fminnum x, y
14697 // vselect x, y (fcmp gt x, y) -> fmaxnum x, y
14698 //
14699 // This is OK if we don't care about what happens if either operand is a
14700 // NaN.
14701 //
14702 if (N0.hasOneUse() &&
14703 isLegalToCombineMinNumMaxNum(DAG, LHS, RHS, SelectFlags: N->getFlags(),
14704 CmpFlags: N0->getFlags(), TLI)) {
14705 if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, LHS, RHS, True: N1, False: N2, CC))
14706 return FMinMax;
14707 }
14708
14709 if (SDValue S = PerformMinMaxFpToSatCombine(N0: LHS, N1: RHS, N2: N1, N3: N2, CC, DAG))
14710 return S;
14711 if (SDValue S = PerformUMinFpToSatCombine(N0: LHS, N1: RHS, N2: N1, N3: N2, CC, DAG))
14712 return S;
14713 if (SDValue S = performNanGuardFpToSatCombine(N, DAG))
14714 return S;
14715
14716 // If this select has a condition (setcc) with narrower operands than the
14717 // select, try to widen the compare to match the select width.
14718 // TODO: This should be extended to handle any constant.
14719 // TODO: This could be extended to handle non-loading patterns, but that
14720 // requires thorough testing to avoid regressions.
14721 if (isNullOrNullSplat(V: RHS)) {
14722 EVT NarrowVT = LHS.getValueType();
14723 EVT WideVT = N1.getValueType().changeVectorElementTypeToInteger();
14724 EVT SetCCVT = getSetCCResultType(VT: LHS.getValueType());
14725 unsigned SetCCWidth = SetCCVT.getScalarSizeInBits();
14726 unsigned WideWidth = WideVT.getScalarSizeInBits();
14727 bool IsSigned = isSignedIntSetCC(Code: CC);
14728 auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
14729 if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && SetCCWidth != 1 &&
14730 SetCCWidth < WideWidth &&
14731 TLI.isOperationLegalOrCustom(Op: ISD::SETCC, VT: WideVT)) {
14732 LoadSDNode *Ld = cast<LoadSDNode>(Val&: LHS);
14733
14734 if (TLI.isLoadLegalOrCustom(ValVT: WideVT, MemVT: NarrowVT, Alignment: Ld->getAlign(),
14735 AddrSpace: Ld->getAddressSpace(), ExtType: LoadExtOpcode,
14736 Atomic: false)) {
14737 // Both compare operands can be widened for free. The LHS can use an
14738 // extended load, and the RHS is a constant:
14739 // vselect (ext (setcc load(X), C)), N1, N2 -->
14740 // vselect (setcc extload(X), C'), N1, N2
14741 auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
14742 SDValue WideLHS = DAG.getNode(Opcode: ExtOpcode, DL, VT: WideVT, Operand: LHS);
14743 SDValue WideRHS = DAG.getNode(Opcode: ExtOpcode, DL, VT: WideVT, Operand: RHS);
14744 EVT WideSetCCVT = getSetCCResultType(VT: WideVT);
14745 SDValue WideSetCC =
14746 DAG.getSetCC(DL, VT: WideSetCCVT, LHS: WideLHS, RHS: WideRHS, Cond: CC);
14747 return DAG.getSelect(DL, VT: N1.getValueType(), Cond: WideSetCC, LHS: N1, RHS: N2);
14748 }
14749 }
14750 }
14751
14752 if (SDValue ABD = foldSelectToABD(LHS, RHS, True: N1, False: N2, CC, DL))
14753 return ABD;
14754
14755 // Match VSELECTs into add with unsigned saturation.
14756 if (hasOperation(Opcode: ISD::UADDSAT, VT)) {
14757 // Check if one of the arms of the VSELECT is vector with all bits set.
14758 // If it's on the left side invert the predicate to simplify logic below.
14759 SDValue Other;
14760 ISD::CondCode SatCC = CC;
14761 if (ISD::isConstantSplatVectorAllOnes(N: N1.getNode())) {
14762 Other = N2;
14763 SatCC = ISD::getSetCCInverse(Operation: SatCC, Type: VT.getScalarType());
14764 } else if (ISD::isConstantSplatVectorAllOnes(N: N2.getNode())) {
14765 Other = N1;
14766 }
14767
14768 if (Other && Other.getOpcode() == ISD::ADD) {
14769 SDValue CondLHS = LHS, CondRHS = RHS;
14770 SDValue OpLHS = Other.getOperand(i: 0), OpRHS = Other.getOperand(i: 1);
14771
14772 // Canonicalize condition operands.
14773 if (SatCC == ISD::SETUGE) {
14774 std::swap(a&: CondLHS, b&: CondRHS);
14775 SatCC = ISD::SETULE;
14776 }
14777
14778 // We can test against either of the addition operands.
14779 // x <= x+y ? x+y : ~0 --> uaddsat x, y
14780 // x+y >= x ? x+y : ~0 --> uaddsat x, y
14781 if (SatCC == ISD::SETULE && Other == CondRHS &&
14782 (OpLHS == CondLHS || OpRHS == CondLHS))
14783 return DAG.getNode(Opcode: ISD::UADDSAT, DL, VT, N1: OpLHS, N2: OpRHS);
14784
14785 if (OpRHS.getOpcode() == CondRHS.getOpcode() &&
14786 (OpRHS.getOpcode() == ISD::BUILD_VECTOR ||
14787 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) &&
14788 CondLHS == OpLHS) {
14789 // If the RHS is a constant we have to reverse the const
14790 // canonicalization.
14791 // x >= ~C ? x+C : ~0 --> uaddsat x, C
14792 auto MatchUADDSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
14793 return Cond->getAPIntValue() == ~Op->getAPIntValue();
14794 };
14795 if (SatCC == ISD::SETULE &&
14796 ISD::matchBinaryPredicate(LHS: OpRHS, RHS: CondRHS, Match: MatchUADDSAT))
14797 return DAG.getNode(Opcode: ISD::UADDSAT, DL, VT, N1: OpLHS, N2: OpRHS);
14798 }
14799 }
14800 }
14801
14802 // Match VSELECTs into sub with unsigned saturation.
14803 if (hasOperation(Opcode: ISD::USUBSAT, VT)) {
14804 // Check if one of the arms of the VSELECT is a zero vector. If it's on
14805 // the left side invert the predicate to simplify logic below.
14806 SDValue Other;
14807 ISD::CondCode SatCC = CC;
14808 if (ISD::isConstantSplatVectorAllZeros(N: N1.getNode())) {
14809 Other = N2;
14810 SatCC = ISD::getSetCCInverse(Operation: SatCC, Type: VT.getScalarType());
14811 } else if (ISD::isConstantSplatVectorAllZeros(N: N2.getNode())) {
14812 Other = N1;
14813 }
14814
14815 // zext(x) >= y ? trunc(zext(x) - y) : 0
14816 // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit)))
14817 // zext(x) > y ? trunc(zext(x) - y) : 0
14818 // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit)))
14819 if (Other && Other.getOpcode() == ISD::TRUNCATE &&
14820 Other.getOperand(i: 0).getOpcode() == ISD::SUB &&
14821 (SatCC == ISD::SETUGE || SatCC == ISD::SETUGT)) {
14822 SDValue OpLHS = Other.getOperand(i: 0).getOperand(i: 0);
14823 SDValue OpRHS = Other.getOperand(i: 0).getOperand(i: 1);
14824 if (LHS == OpLHS && RHS == OpRHS && LHS.getOpcode() == ISD::ZERO_EXTEND)
14825 if (SDValue R = getTruncatedUSUBSAT(DstVT: VT, SrcVT: LHS.getValueType(), LHS, RHS,
14826 DAG, DL))
14827 return R;
14828 }
14829
14830 if (Other && Other.getNumOperands() == 2) {
14831 SDValue CondRHS = RHS;
14832 SDValue OpLHS = Other.getOperand(i: 0), OpRHS = Other.getOperand(i: 1);
14833
14834 if (OpLHS == LHS) {
14835 // Look for a general sub with unsigned saturation first.
14836 // x >= y ? x-y : 0 --> usubsat x, y
14837 // x > y ? x-y : 0 --> usubsat x, y
14838 if ((SatCC == ISD::SETUGE || SatCC == ISD::SETUGT) &&
14839 Other.getOpcode() == ISD::SUB && OpRHS == CondRHS)
14840 return DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: OpLHS, N2: OpRHS);
14841
14842 if (OpRHS.getOpcode() == ISD::BUILD_VECTOR ||
14843 OpRHS.getOpcode() == ISD::SPLAT_VECTOR) {
14844 if (CondRHS.getOpcode() == ISD::BUILD_VECTOR ||
14845 CondRHS.getOpcode() == ISD::SPLAT_VECTOR) {
14846 // If the RHS is a constant we have to reverse the const
14847 // canonicalization.
14848 // x > C-1 ? x+-C : 0 --> usubsat x, C
14849 auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
14850 return (!Op && !Cond) ||
14851 (Op && Cond &&
14852 Cond->getAPIntValue() == (-Op->getAPIntValue() - 1));
14853 };
14854 if (SatCC == ISD::SETUGT && Other.getOpcode() == ISD::ADD &&
14855 ISD::matchBinaryPredicate(LHS: OpRHS, RHS: CondRHS, Match: MatchUSUBSAT,
14856 /*AllowUndefs*/ true)) {
14857 OpRHS = DAG.getNegative(Val: OpRHS, DL, VT);
14858 return DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: OpLHS, N2: OpRHS);
14859 }
14860
14861 // Another special case: If C was a sign bit, the sub has been
14862 // canonicalized into a xor.
14863 // FIXME: Would it be better to use computeKnownBits to
14864 // determine whether it's safe to decanonicalize the xor?
14865 // x s< 0 ? x^C : 0 --> usubsat x, C
14866 APInt SplatValue;
14867 if (SatCC == ISD::SETLT && Other.getOpcode() == ISD::XOR &&
14868 ISD::isConstantSplatVector(N: OpRHS.getNode(), SplatValue) &&
14869 ISD::isConstantSplatVectorAllZeros(N: CondRHS.getNode()) &&
14870 SplatValue.isSignMask()) {
14871 // Note that we have to rebuild the RHS constant here to
14872 // ensure we don't rely on particular values of undef lanes.
14873 OpRHS = DAG.getConstant(Val: SplatValue, DL, VT);
14874 return DAG.getNode(Opcode: ISD::USUBSAT, DL, VT, N1: OpLHS, N2: OpRHS);
14875 }
14876 }
14877 }
14878 }
14879 }
14880 }
14881
14882 // (vselect (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
14883 // (vselect (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
14884 if (SDValue UMin = foldSelectToUMin(LHS, RHS, True: N1, False: N2, CC, DL))
14885 return UMin;
14886 }
14887
14888 if (SimplifySelectOps(SELECT: N, LHS: N1, RHS: N2))
14889 return SDValue(N, 0); // Don't revisit N.
14890
14891 // Fold (vselect all_ones, N1, N2) -> N1
14892 if (ISD::isConstantSplatVectorAllOnes(N: N0.getNode()))
14893 return N1;
14894 // Fold (vselect all_zeros, N1, N2) -> N2
14895 if (ISD::isConstantSplatVectorAllZeros(N: N0.getNode()))
14896 return N2;
14897
14898 // The ConvertSelectToConcatVector function is assuming both the above
14899 // checks for (vselect (build_vector all{ones,zeros) ...) have been made
14900 // and addressed.
14901 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
14902 N2.getOpcode() == ISD::CONCAT_VECTORS &&
14903 ISD::isBuildVectorOfConstantSDNodes(N: N0.getNode())) {
14904 if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
14905 return CV;
14906 }
14907
14908 if (SDValue V = foldVSelectOfConstants(N))
14909 return V;
14910
14911 if (hasOperation(Opcode: ISD::SRA, VT))
14912 if (SDValue V = foldVSelectToSignBitSplatMask(N, DAG))
14913 return V;
14914
14915 if (SimplifyDemandedVectorElts(Op: SDValue(N, 0)))
14916 return SDValue(N, 0);
14917
14918 if (SDValue V = combineVSelectWithAllOnesOrZeros(Cond: N0, TVal: N1, FVal: N2, TLI, DAG, DL))
14919 return V;
14920
14921 if (SDValue R = combineSelectToPseudoMinMax(DAG, N))
14922 return R;
14923
14924 return SDValue();
14925}
14926
14927SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
14928 SDValue N0 = N->getOperand(Num: 0);
14929 SDValue N1 = N->getOperand(Num: 1);
14930 SDValue N2 = N->getOperand(Num: 2);
14931 SDValue N3 = N->getOperand(Num: 3);
14932 SDValue N4 = N->getOperand(Num: 4);
14933 ISD::CondCode CC = cast<CondCodeSDNode>(Val&: N4)->get();
14934 SDLoc DL(N);
14935
14936 // fold select_cc lhs, rhs, x, x, cc -> x
14937 if (N2 == N3)
14938 return N2;
14939
14940 if (SDValue R = foldSelectCCOfSelect(N, DAG))
14941 return R;
14942
14943 // select_cc bool, 0, x, y, seteq -> select bool, y, x
14944 if (CC == ISD::SETEQ && !LegalTypes && N0.getValueType() == MVT::i1 &&
14945 isNullConstant(V: N1))
14946 return DAG.getSelect(DL, VT: N2.getValueType(), Cond: N0, LHS: N3, RHS: N2);
14947
14948 // Determine if the condition we're dealing with is constant
14949 if (SDValue SCC = SimplifySetCC(VT: getSetCCResultType(VT: N0.getValueType()), N0, N1,
14950 Cond: CC, DL, foldBooleans: false)) {
14951 AddToWorklist(N: SCC.getNode());
14952
14953 // cond always true -> true val
14954 // cond always false -> false val
14955 if (auto *SCCC = dyn_cast<ConstantSDNode>(Val: SCC.getNode()))
14956 return SCCC->isZero() ? N3 : N2;
14957
14958 // When the condition is UNDEF, just return the first operand. This is
14959 // coherent the DAG creation, no setcc node is created in this case
14960 if (SCC->isUndef())
14961 return N2;
14962
14963 // Fold to a simpler select_cc
14964 if (SCC.getOpcode() == ISD::SETCC) {
14965 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: N2.getValueType(),
14966 N1: SCC.getOperand(i: 0), N2: SCC.getOperand(i: 1), N3: N2, N4: N3,
14967 N5: SCC.getOperand(i: 2), Flags: SCC->getFlags());
14968 }
14969 }
14970
14971 // If we can fold this based on the true/false value, do so.
14972 if (SimplifySelectOps(SELECT: N, LHS: N2, RHS: N3))
14973 return SDValue(N, 0); // Don't revisit N.
14974
14975 auto [Opcode, NewLHS, NewRHS] = combineSelectCCToPseudoMinMax(
14976 DAG, DL, CC, Op0: N0, Op1: N1, LHS: N2, RHS: N3, Flags: N->getFlags(), /*IsStrict=*/false);
14977 if (Opcode)
14978 return DAG.getNode(Opcode, DL, VT: N->getValueType(ResNo: 0), N1: NewLHS, N2: NewRHS,
14979 Flags: N->getFlags());
14980
14981 // fold select_cc into other things, such as min/max/abs
14982 return SimplifySelectCC(DL, N0, N1, N2, N3, CC);
14983}
14984
14985SDValue DAGCombiner::visitSETCC(SDNode *N) {
14986 // setcc is very commonly used as an argument to brcond or cond_loop. This
14987 // pattern also lend itself to numerous combines and, as a result, it is
14988 // desired we keep the argument to a brcond as a setcc as much as possible.
14989 bool PreferSetCC =
14990 N->hasOneUse() && (N->user_begin()->getOpcode() == ISD::BRCOND ||
14991 N->user_begin()->getOpcode() == ISD::COND_LOOP);
14992
14993 ISD::CondCode Cond = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
14994 EVT VT = N->getValueType(ResNo: 0);
14995 SDValue N0 = N->getOperand(Num: 0), N1 = N->getOperand(Num: 1);
14996 SDLoc DL(N);
14997
14998 if (SDValue Combined = SimplifySetCC(VT, N0, N1, Cond, DL, foldBooleans: !PreferSetCC)) {
14999 // If we prefer to have a setcc, and we don't, we'll try our best to
15000 // recreate one using rebuildSetCC.
15001 if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
15002 SDValue NewSetCC = rebuildSetCC(N: Combined);
15003
15004 // We don't have anything interesting to combine to.
15005 if (NewSetCC.getNode() == N)
15006 return SDValue();
15007
15008 if (NewSetCC)
15009 return NewSetCC;
15010 }
15011 return Combined;
15012 }
15013
15014 // Optimize
15015 // 1) (icmp eq/ne (and X, C0), (shift X, C1))
15016 // or
15017 // 2) (icmp eq/ne X, (rotate X, C1))
15018 // If C0 is a mask or shifted mask and the shift amt (C1) isolates the
15019 // remaining bits (i.e something like `(x64 & UINT32_MAX) == (x64 >> 32)`)
15020 // Then:
15021 // If C1 divides the bit width, then the rotate and shift+and versions are
15022 // equivalent, so we can interchange them depending on target preference.
15023 // Otherwise, if we have the shift+and version we can interchange srl/shl
15024 // which inturn affects the constant C0. We can use this to get better
15025 // constants again determined by target preference.
15026 if (Cond == ISD::SETNE || Cond == ISD::SETEQ) {
15027 auto IsAndWithShift = [](SDValue A, SDValue B) {
15028 return A.getOpcode() == ISD::AND &&
15029 (B.getOpcode() == ISD::SRL || B.getOpcode() == ISD::SHL) &&
15030 A.getOperand(i: 0) == B.getOperand(i: 0);
15031 };
15032 auto IsRotateWithOp = [](SDValue A, SDValue B) {
15033 return (B.getOpcode() == ISD::ROTL || B.getOpcode() == ISD::ROTR) &&
15034 B.getOperand(i: 0) == A;
15035 };
15036 SDValue AndOrOp = SDValue(), ShiftOrRotate = SDValue();
15037 bool IsRotate = false;
15038
15039 // Find either shift+and or rotate pattern.
15040 if (IsAndWithShift(N0, N1)) {
15041 AndOrOp = N0;
15042 ShiftOrRotate = N1;
15043 } else if (IsAndWithShift(N1, N0)) {
15044 AndOrOp = N1;
15045 ShiftOrRotate = N0;
15046 } else if (IsRotateWithOp(N0, N1)) {
15047 IsRotate = true;
15048 AndOrOp = N0;
15049 ShiftOrRotate = N1;
15050 } else if (IsRotateWithOp(N1, N0)) {
15051 IsRotate = true;
15052 AndOrOp = N1;
15053 ShiftOrRotate = N0;
15054 }
15055
15056 if (AndOrOp && ShiftOrRotate && ShiftOrRotate.hasOneUse() &&
15057 (IsRotate || AndOrOp.hasOneUse())) {
15058 EVT OpVT = N0.getValueType();
15059 // Get constant shift/rotate amount and possibly mask (if its shift+and
15060 // variant).
15061 auto GetAPIntValue = [](SDValue Op) -> std::optional<APInt> {
15062 ConstantSDNode *CNode = isConstOrConstSplat(N: Op, /*AllowUndefs*/ false,
15063 /*AllowTrunc*/ AllowTruncation: false);
15064 if (CNode == nullptr)
15065 return std::nullopt;
15066 return CNode->getAPIntValue();
15067 };
15068 std::optional<APInt> AndCMask =
15069 IsRotate ? std::nullopt : GetAPIntValue(AndOrOp.getOperand(i: 1));
15070 std::optional<APInt> ShiftCAmt =
15071 GetAPIntValue(ShiftOrRotate.getOperand(i: 1));
15072 unsigned NumBits = OpVT.getScalarSizeInBits();
15073
15074 // We found constants.
15075 if (ShiftCAmt && (IsRotate || AndCMask) && ShiftCAmt->ult(RHS: NumBits)) {
15076 unsigned ShiftOpc = ShiftOrRotate.getOpcode();
15077 // Check that the constants meet the constraints.
15078 bool CanTransform = IsRotate;
15079 if (!CanTransform) {
15080 // Check that mask and shift compliment eachother
15081 CanTransform = *ShiftCAmt == (~*AndCMask).popcount();
15082 // Check that we are comparing all bits
15083 CanTransform &= (*ShiftCAmt + AndCMask->popcount()) == NumBits;
15084 // Check that the and mask is correct for the shift
15085 CanTransform &=
15086 ShiftOpc == ISD::SHL ? (~*AndCMask).isMask() : AndCMask->isMask();
15087 }
15088
15089 // The rotate and shift+and forms are only equivalent if the shift
15090 // amount divides the bit width.
15091 bool MayTransformRotate =
15092 !ShiftCAmt->isZero() && NumBits % ShiftCAmt->getZExtValue() == 0;
15093 // See if target prefers another shift/rotate opcode.
15094 unsigned NewShiftOpc = TLI.preferedOpcodeForCmpEqPiecesOfOperand(
15095 VT: OpVT, ShiftOpc, MayTransformRotate, ShiftOrRotateAmt: *ShiftCAmt, AndMask: AndCMask);
15096 // Transform is valid and we have a new preference.
15097 if (CanTransform && NewShiftOpc != ShiftOpc) {
15098 SDValue NewShiftOrRotate =
15099 DAG.getNode(Opcode: NewShiftOpc, DL, VT: OpVT, N1: ShiftOrRotate.getOperand(i: 0),
15100 N2: ShiftOrRotate.getOperand(i: 1));
15101 SDValue NewAndOrOp = SDValue();
15102
15103 if (NewShiftOpc == ISD::SHL || NewShiftOpc == ISD::SRL) {
15104 APInt NewMask =
15105 NewShiftOpc == ISD::SHL
15106 ? APInt::getHighBitsSet(numBits: NumBits,
15107 hiBitsSet: NumBits - ShiftCAmt->getZExtValue())
15108 : APInt::getLowBitsSet(numBits: NumBits,
15109 loBitsSet: NumBits - ShiftCAmt->getZExtValue());
15110 NewAndOrOp =
15111 DAG.getNode(Opcode: ISD::AND, DL, VT: OpVT, N1: ShiftOrRotate.getOperand(i: 0),
15112 N2: DAG.getConstant(Val: NewMask, DL, VT: OpVT));
15113 } else {
15114 NewAndOrOp = ShiftOrRotate.getOperand(i: 0);
15115 }
15116
15117 return DAG.getSetCC(DL, VT, LHS: NewAndOrOp, RHS: NewShiftOrRotate, Cond);
15118 }
15119 }
15120 }
15121 }
15122 return SDValue();
15123}
15124
15125SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
15126 SDValue LHS = N->getOperand(Num: 0);
15127 SDValue RHS = N->getOperand(Num: 1);
15128 SDValue Carry = N->getOperand(Num: 2);
15129 SDValue Cond = N->getOperand(Num: 3);
15130
15131 // If Carry is false, fold to a regular SETCC.
15132 if (isNullConstant(V: Carry))
15133 return DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N), VTList: N->getVTList(), N1: LHS, N2: RHS, N3: Cond);
15134
15135 return SDValue();
15136}
15137
15138/// Check if N satisfies:
15139/// N is used once.
15140/// N is a Load.
15141/// The load is compatible with ExtOpcode. It means
15142/// If load has explicit zero/sign extension, ExpOpcode must have the same
15143/// extension.
15144/// Otherwise returns true.
15145static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode) {
15146 if (!N.hasOneUse())
15147 return false;
15148
15149 if (!isa<LoadSDNode>(Val: N))
15150 return false;
15151
15152 LoadSDNode *Load = cast<LoadSDNode>(Val&: N);
15153 ISD::LoadExtType LoadExt = Load->getExtensionType();
15154 if (LoadExt == ISD::NON_EXTLOAD || LoadExt == ISD::EXTLOAD)
15155 return true;
15156
15157 // Now LoadExt is either SEXTLOAD or ZEXTLOAD, ExtOpcode must have the same
15158 // extension.
15159 if ((LoadExt == ISD::SEXTLOAD && ExtOpcode != ISD::SIGN_EXTEND) ||
15160 (LoadExt == ISD::ZEXTLOAD && ExtOpcode != ISD::ZERO_EXTEND))
15161 return false;
15162
15163 return true;
15164}
15165
15166/// Fold
15167/// (sext (select c, load x, load y)) -> (select c, sextload x, sextload y)
15168/// (zext (select c, load x, load y)) -> (select c, zextload x, zextload y)
15169/// (aext (select c, load x, load y)) -> (select c, extload x, extload y)
15170/// This function is called by the DAGCombiner when visiting sext/zext/aext
15171/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
15172static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI,
15173 SelectionDAG &DAG, const SDLoc &DL,
15174 CombineLevel Level) {
15175 unsigned Opcode = N->getOpcode();
15176 SDValue N0 = N->getOperand(Num: 0);
15177 EVT VT = N->getValueType(ResNo: 0);
15178 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
15179 Opcode == ISD::ANY_EXTEND) &&
15180 "Expected EXTEND dag node in input!");
15181
15182 SDValue Cond, Op1, Op2;
15183 if (!sd_match(N: N0, P: m_OneUse(P: m_SelectLike(Cond: m_Value(N&: Cond), T: m_Value(N&: Op1),
15184 F: m_Value(N&: Op2)))))
15185 return SDValue();
15186
15187 if (!isCompatibleLoad(N: Op1, ExtOpcode: Opcode) || !isCompatibleLoad(N: Op2, ExtOpcode: Opcode))
15188 return SDValue();
15189
15190 auto ExtLoadOpcode = ISD::EXTLOAD;
15191 if (Opcode == ISD::SIGN_EXTEND)
15192 ExtLoadOpcode = ISD::SEXTLOAD;
15193 else if (Opcode == ISD::ZERO_EXTEND)
15194 ExtLoadOpcode = ISD::ZEXTLOAD;
15195
15196 // Illegal VSELECT may ISel fail if happen after legalization (DAG
15197 // Combine2), so we should conservatively check the OperationAction.
15198 LoadSDNode *Load1 = cast<LoadSDNode>(Val&: Op1);
15199 LoadSDNode *Load2 = cast<LoadSDNode>(Val&: Op2);
15200 if (!TLI.isLoadLegal(ValVT: VT, MemVT: Load1->getMemoryVT(), Alignment: Load1->getAlign(),
15201 AddrSpace: Load1->getAddressSpace(), ExtType: ExtLoadOpcode, Atomic: false) ||
15202 !TLI.isLoadLegal(ValVT: VT, MemVT: Load2->getMemoryVT(), Alignment: Load2->getAlign(),
15203 AddrSpace: Load2->getAddressSpace(), ExtType: ExtLoadOpcode, Atomic: false) ||
15204 (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes &&
15205 TLI.getOperationAction(Op: ISD::VSELECT, VT) != TargetLowering::Legal))
15206 return SDValue();
15207
15208 SDValue Ext1 = DAG.getNode(Opcode, DL, VT, Operand: Op1);
15209 SDValue Ext2 = DAG.getNode(Opcode, DL, VT, Operand: Op2);
15210 return DAG.getSelect(DL, VT, Cond, LHS: Ext1, RHS: Ext2);
15211}
15212
15213/// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
15214/// a build_vector of constants.
15215/// This function is called by the DAGCombiner when visiting sext/zext/aext
15216/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
15217/// Vector extends are not folded if operations are legal; this is to
15218/// avoid introducing illegal build_vector dag nodes.
15219static SDValue tryToFoldExtendOfConstant(SDNode *N, const SDLoc &DL,
15220 const TargetLowering &TLI,
15221 SelectionDAG &DAG, bool LegalTypes) {
15222 unsigned Opcode = N->getOpcode();
15223 SDValue N0 = N->getOperand(Num: 0);
15224 EVT VT = N->getValueType(ResNo: 0);
15225
15226 assert((ISD::isExtOpcode(Opcode) || ISD::isExtVecInRegOpcode(Opcode)) &&
15227 "Expected EXTEND dag node in input!");
15228
15229 // fold (sext c1) -> c1
15230 // fold (zext c1) -> c1
15231 // fold (aext c1) -> c1
15232 if (isa<ConstantSDNode>(Val: N0))
15233 return DAG.getNode(Opcode, DL, VT, Operand: N0);
15234
15235 // fold (sext (select cond, c1, c2)) -> (select cond, sext c1, sext c2)
15236 // fold (zext (select cond, c1, c2)) -> (select cond, zext c1, zext c2)
15237 // fold (aext (select cond, c1, c2)) -> (select cond, sext c1, sext c2)
15238 if (N0->getOpcode() == ISD::SELECT) {
15239 SDValue Op1 = N0->getOperand(Num: 1);
15240 SDValue Op2 = N0->getOperand(Num: 2);
15241 if (isa<ConstantSDNode>(Val: Op1) && isa<ConstantSDNode>(Val: Op2) &&
15242 (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(FromTy: N0.getValueType(), ToTy: VT))) {
15243 // For any_extend, choose sign extension of the constants to allow a
15244 // possible further transform to sign_extend_inreg.i.e.
15245 //
15246 // t1: i8 = select t0, Constant:i8<-1>, Constant:i8<0>
15247 // t2: i64 = any_extend t1
15248 // -->
15249 // t3: i64 = select t0, Constant:i64<-1>, Constant:i64<0>
15250 // -->
15251 // t4: i64 = sign_extend_inreg t3
15252 unsigned FoldOpc = Opcode;
15253 if (FoldOpc == ISD::ANY_EXTEND)
15254 FoldOpc = ISD::SIGN_EXTEND;
15255 return DAG.getSelect(DL, VT, Cond: N0->getOperand(Num: 0),
15256 LHS: DAG.getNode(Opcode: FoldOpc, DL, VT, Operand: Op1),
15257 RHS: DAG.getNode(Opcode: FoldOpc, DL, VT, Operand: Op2));
15258 }
15259 }
15260
15261 // fold (sext (build_vector AllConstants) -> (build_vector AllConstants)
15262 // fold (zext (build_vector AllConstants) -> (build_vector AllConstants)
15263 // fold (aext (build_vector AllConstants) -> (build_vector AllConstants)
15264 EVT SVT = VT.getScalarType();
15265 if (!(VT.isVector() && (!LegalTypes || TLI.isTypeLegal(VT: SVT)) &&
15266 ISD::isBuildVectorOfConstantSDNodes(N: N0.getNode())))
15267 return SDValue();
15268
15269 // We can fold this node into a build_vector.
15270 unsigned VTBits = SVT.getSizeInBits();
15271 unsigned EVTBits = N0->getValueType(ResNo: 0).getScalarSizeInBits();
15272 SmallVector<SDValue, 8> Elts;
15273 unsigned NumElts = VT.getVectorNumElements();
15274
15275 for (unsigned i = 0; i != NumElts; ++i) {
15276 SDValue Op = N0.getOperand(i);
15277 if (Op.isUndef()) {
15278 if (Opcode == ISD::ANY_EXTEND || Opcode == ISD::ANY_EXTEND_VECTOR_INREG)
15279 Elts.push_back(Elt: DAG.getUNDEF(VT: SVT));
15280 else
15281 Elts.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: SVT));
15282 continue;
15283 }
15284
15285 SDLoc DL(Op);
15286 // Get the constant value and if needed trunc it to the size of the type.
15287 // Nodes like build_vector might have constants wider than the scalar type.
15288 APInt C = Op->getAsAPIntVal().zextOrTrunc(width: EVTBits);
15289 if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
15290 Elts.push_back(Elt: DAG.getConstant(Val: C.sext(width: VTBits), DL, VT: SVT));
15291 else
15292 Elts.push_back(Elt: DAG.getConstant(Val: C.zext(width: VTBits), DL, VT: SVT));
15293 }
15294
15295 return DAG.getBuildVector(VT, DL, Ops: Elts);
15296}
15297
15298// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
15299// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
15300// transformation. Returns true if extension are possible and the above
15301// mentioned transformation is profitable.
15302static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0,
15303 unsigned ExtOpc,
15304 SmallVectorImpl<SDNode *> &ExtendNodes,
15305 const TargetLowering &TLI) {
15306 bool HasCopyToRegUses = false;
15307 bool isTruncFree = TLI.isTruncateFree(FromVT: VT, ToVT: N0.getValueType());
15308 for (SDUse &Use : N0->uses()) {
15309 SDNode *User = Use.getUser();
15310 if (User == N)
15311 continue;
15312 if (Use.getResNo() != N0.getResNo())
15313 continue;
15314 // FIXME: Only extend SETCC N, N and SETCC N, c for now.
15315 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
15316 ISD::CondCode CC = cast<CondCodeSDNode>(Val: User->getOperand(Num: 2))->get();
15317 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(Code: CC))
15318 // Sign bits will be lost after a zext.
15319 return false;
15320 bool Add = false;
15321 for (unsigned i = 0; i != 2; ++i) {
15322 SDValue UseOp = User->getOperand(Num: i);
15323 if (UseOp == N0)
15324 continue;
15325 if (!isa<ConstantSDNode>(Val: UseOp))
15326 return false;
15327 Add = true;
15328 }
15329 if (Add)
15330 ExtendNodes.push_back(Elt: User);
15331 continue;
15332 }
15333 // If truncates aren't free and there are users we can't
15334 // extend, it isn't worthwhile.
15335 if (!isTruncFree)
15336 return false;
15337 // Remember if this value is live-out.
15338 if (User->getOpcode() == ISD::CopyToReg)
15339 HasCopyToRegUses = true;
15340 }
15341
15342 if (HasCopyToRegUses) {
15343 bool BothLiveOut = false;
15344 for (SDUse &Use : N->uses()) {
15345 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
15346 BothLiveOut = true;
15347 break;
15348 }
15349 }
15350 if (BothLiveOut)
15351 // Both unextended and extended values are live out. There had better be
15352 // a good reason for the transformation.
15353 return !ExtendNodes.empty();
15354 }
15355 return true;
15356}
15357
15358void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
15359 SDValue OrigLoad, SDValue ExtLoad,
15360 ISD::NodeType ExtType) {
15361 // Extend SetCC uses if necessary.
15362 SDLoc DL(ExtLoad);
15363 for (SDNode *SetCC : SetCCs) {
15364 SmallVector<SDValue, 4> Ops;
15365
15366 for (unsigned j = 0; j != 2; ++j) {
15367 SDValue SOp = SetCC->getOperand(Num: j);
15368 if (SOp == OrigLoad)
15369 Ops.push_back(Elt: ExtLoad);
15370 else
15371 Ops.push_back(Elt: DAG.getNode(Opcode: ExtType, DL, VT: ExtLoad->getValueType(ResNo: 0), Operand: SOp));
15372 }
15373
15374 Ops.push_back(Elt: SetCC->getOperand(Num: 2));
15375 CombineTo(N: SetCC, Res: DAG.getNode(Opcode: ISD::SETCC, DL, VT: SetCC->getValueType(ResNo: 0), Ops));
15376 }
15377}
15378
15379// FIXME: Bring more similar combines here, common to sext/zext (maybe aext?).
15380SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
15381 SDValue N0 = N->getOperand(Num: 0);
15382 EVT DstVT = N->getValueType(ResNo: 0);
15383 EVT SrcVT = N0.getValueType();
15384
15385 assert((N->getOpcode() == ISD::SIGN_EXTEND ||
15386 N->getOpcode() == ISD::ZERO_EXTEND) &&
15387 "Unexpected node type (not an extend)!");
15388
15389 // fold (sext (load x)) to multiple smaller sextloads; same for zext.
15390 // For example, on a target with legal v4i32, but illegal v8i32, turn:
15391 // (v8i32 (sext (v8i16 (load x))))
15392 // into:
15393 // (v8i32 (concat_vectors (v4i32 (sextload x)),
15394 // (v4i32 (sextload (x + 16)))))
15395 // Where uses of the original load, i.e.:
15396 // (v8i16 (load x))
15397 // are replaced with:
15398 // (v8i16 (truncate
15399 // (v8i32 (concat_vectors (v4i32 (sextload x)),
15400 // (v4i32 (sextload (x + 16)))))))
15401 //
15402 // This combine is only applicable to illegal, but splittable, vectors.
15403 // All legal types, and illegal non-vector types, are handled elsewhere.
15404 // This combine is controlled by TargetLowering::isVectorLoadExtDesirable.
15405 //
15406 if (N0->getOpcode() != ISD::LOAD)
15407 return SDValue();
15408
15409 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
15410
15411 if (!ISD::isNON_EXTLoad(N: LN0) || !ISD::isUNINDEXEDLoad(N: LN0) ||
15412 !N0.hasOneUse() || !LN0->isSimple() ||
15413 !DstVT.isVector() || !DstVT.isPow2VectorType() ||
15414 !TLI.isVectorLoadExtDesirable(ExtVal: SDValue(N, 0)))
15415 return SDValue();
15416
15417 SmallVector<SDNode *, 4> SetCCs;
15418 if (!ExtendUsesToFormExtLoad(VT: DstVT, N, N0, ExtOpc: N->getOpcode(), ExtendNodes&: SetCCs, TLI))
15419 return SDValue();
15420
15421 ISD::LoadExtType ExtType =
15422 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
15423
15424 // Try to split the vector types to get down to legal types.
15425 EVT SplitSrcVT = SrcVT;
15426 EVT SplitDstVT = DstVT;
15427 while (!TLI.isLoadLegalOrCustom(ValVT: SplitDstVT, MemVT: SplitSrcVT, Alignment: LN0->getAlign(),
15428 AddrSpace: LN0->getAddressSpace(), ExtType, Atomic: false) &&
15429 SplitSrcVT.getVectorNumElements() > 1) {
15430 SplitDstVT = DAG.GetSplitDestVTs(VT: SplitDstVT).first;
15431 SplitSrcVT = DAG.GetSplitDestVTs(VT: SplitSrcVT).first;
15432 }
15433
15434 if (!TLI.isLoadLegalOrCustom(ValVT: SplitDstVT, MemVT: SplitSrcVT, Alignment: LN0->getAlign(),
15435 AddrSpace: LN0->getAddressSpace(), ExtType, Atomic: false))
15436 return SDValue();
15437
15438 assert(!DstVT.isScalableVector() && "Unexpected scalable vector type");
15439
15440 SDLoc DL(N);
15441 const unsigned NumSplits =
15442 DstVT.getVectorNumElements() / SplitDstVT.getVectorNumElements();
15443 const unsigned Stride = SplitSrcVT.getStoreSize();
15444 SmallVector<SDValue, 4> Loads;
15445 SmallVector<SDValue, 4> Chains;
15446
15447 SDValue BasePtr = LN0->getBasePtr();
15448 for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
15449 const unsigned Offset = Idx * Stride;
15450
15451 SDValue SplitLoad =
15452 DAG.getExtLoad(ExtType, dl: SDLoc(LN0), VT: SplitDstVT, Chain: LN0->getChain(),
15453 Ptr: BasePtr, PtrInfo: LN0->getPointerInfo().getWithOffset(O: Offset),
15454 MemVT: SplitSrcVT, Alignment: LN0->getBaseAlign(),
15455 MMOFlags: LN0->getMemOperand()->getFlags(), Metadata: LN0->getAAInfo());
15456
15457 BasePtr = DAG.getMemBasePlusOffset(Base: BasePtr, Offset: TypeSize::getFixed(ExactSize: Stride), DL);
15458
15459 Loads.push_back(Elt: SplitLoad.getValue(R: 0));
15460 Chains.push_back(Elt: SplitLoad.getValue(R: 1));
15461 }
15462
15463 SDValue NewChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
15464 SDValue NewValue = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: DstVT, Ops: Loads);
15465
15466 // Simplify TF.
15467 AddToWorklist(N: NewChain.getNode());
15468
15469 CombineTo(N, Res: NewValue);
15470
15471 // Replace uses of the original load (before extension)
15472 // with a truncate of the concatenated sextloaded vectors.
15473 SDValue Trunc =
15474 DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N0), VT: N0.getValueType(), Operand: NewValue);
15475 ExtendSetCCUses(SetCCs, OrigLoad: N0, ExtLoad: NewValue, ExtType: (ISD::NodeType)N->getOpcode());
15476 CombineTo(N: N0.getNode(), Res0: Trunc, Res1: NewChain);
15477 return SDValue(N, 0); // Return N so it doesn't get rechecked!
15478}
15479
15480// fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
15481// (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
15482SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
15483 assert(N->getOpcode() == ISD::ZERO_EXTEND);
15484 EVT VT = N->getValueType(ResNo: 0);
15485 EVT OrigVT = N->getOperand(Num: 0).getValueType();
15486 if (TLI.isZExtFree(FromTy: OrigVT, ToTy: VT))
15487 return SDValue();
15488
15489 // and/or/xor
15490 SDValue N0 = N->getOperand(Num: 0);
15491 if (!ISD::isBitwiseLogicOp(Opcode: N0.getOpcode()) ||
15492 N0.getOperand(i: 1).getOpcode() != ISD::Constant ||
15493 (LegalOperations && !TLI.isOperationLegal(Op: N0.getOpcode(), VT)))
15494 return SDValue();
15495
15496 // shl/shr
15497 SDValue N1 = N0->getOperand(Num: 0);
15498 if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) ||
15499 N1.getOperand(i: 1).getOpcode() != ISD::Constant ||
15500 (LegalOperations && !TLI.isOperationLegal(Op: N1.getOpcode(), VT)))
15501 return SDValue();
15502
15503 // load
15504 if (!isa<LoadSDNode>(Val: N1.getOperand(i: 0)))
15505 return SDValue();
15506 LoadSDNode *Load = cast<LoadSDNode>(Val: N1.getOperand(i: 0));
15507 EVT MemVT = Load->getMemoryVT();
15508 if (!TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: Load->getAlign(), AddrSpace: Load->getAddressSpace(),
15509 ExtType: ISD::ZEXTLOAD, Atomic: false) ||
15510 Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed())
15511 return SDValue();
15512
15513
15514 // If the shift op is SHL, the logic op must be AND, otherwise the result
15515 // will be wrong.
15516 if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND)
15517 return SDValue();
15518
15519 if (!N0.hasOneUse() || !N1.hasOneUse())
15520 return SDValue();
15521
15522 SmallVector<SDNode*, 4> SetCCs;
15523 if (!ExtendUsesToFormExtLoad(VT, N: N1.getNode(), N0: N1.getOperand(i: 0),
15524 ExtOpc: ISD::ZERO_EXTEND, ExtendNodes&: SetCCs, TLI))
15525 return SDValue();
15526
15527 // Actually do the transformation.
15528 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(Load), VT,
15529 Chain: Load->getChain(), Ptr: Load->getBasePtr(),
15530 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
15531
15532 SDLoc DL1(N1);
15533 SDValue Shift = DAG.getNode(Opcode: N1.getOpcode(), DL: DL1, VT, N1: ExtLoad,
15534 N2: N1.getOperand(i: 1));
15535
15536 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
15537 SDLoc DL0(N0);
15538 SDValue And = DAG.getNode(Opcode: N0.getOpcode(), DL: DL0, VT, N1: Shift,
15539 N2: DAG.getConstant(Val: Mask, DL: DL0, VT));
15540
15541 ExtendSetCCUses(SetCCs, OrigLoad: N1.getOperand(i: 0), ExtLoad, ExtType: ISD::ZERO_EXTEND);
15542 CombineTo(N, Res: And);
15543 if (SDValue(Load, 0).hasOneUse()) {
15544 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: ExtLoad.getValue(R: 1));
15545 } else {
15546 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Load),
15547 VT: Load->getValueType(ResNo: 0), Operand: ExtLoad);
15548 CombineTo(N: Load, Res0: Trunc, Res1: ExtLoad.getValue(R: 1));
15549 }
15550
15551 // N0 is dead at this point.
15552 recursivelyDeleteUnusedNodes(N: N0.getNode());
15553
15554 return SDValue(N,0); // Return N so it doesn't get rechecked!
15555}
15556
15557/// If we're narrowing or widening the result of a vector select and the final
15558/// size is the same size as a setcc (compare) feeding the select, then try to
15559/// apply the cast operation to the select's operands because matching vector
15560/// sizes for a select condition and other operands should be more efficient.
15561SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
15562 unsigned CastOpcode = Cast->getOpcode();
15563 assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
15564 CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
15565 CastOpcode == ISD::FP_ROUND) &&
15566 "Unexpected opcode for vector select narrowing/widening");
15567
15568 // We only do this transform before legal ops because the pattern may be
15569 // obfuscated by target-specific operations after legalization. Do not create
15570 // an illegal select op, however, because that may be difficult to lower.
15571 EVT VT = Cast->getValueType(ResNo: 0);
15572 if (LegalOperations || !TLI.isOperationLegalOrCustom(Op: ISD::VSELECT, VT))
15573 return SDValue();
15574
15575 SDValue VSel = Cast->getOperand(Num: 0);
15576 if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() ||
15577 VSel.getOperand(i: 0).getOpcode() != ISD::SETCC)
15578 return SDValue();
15579
15580 // Does the setcc have the same vector size as the casted select?
15581 SDValue SetCC = VSel.getOperand(i: 0);
15582 EVT SetCCVT = getSetCCResultType(VT: SetCC.getOperand(i: 0).getValueType());
15583 if (SetCCVT.getSizeInBits() != VT.getSizeInBits())
15584 return SDValue();
15585
15586 // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B)
15587 SDValue A = VSel.getOperand(i: 1);
15588 SDValue B = VSel.getOperand(i: 2);
15589 SDValue CastA, CastB;
15590 SDLoc DL(Cast);
15591 if (CastOpcode == ISD::FP_ROUND) {
15592 // FP_ROUND (fptrunc) has an extra flag operand to pass along.
15593 CastA = DAG.getNode(Opcode: CastOpcode, DL, VT, N1: A, N2: Cast->getOperand(Num: 1));
15594 CastB = DAG.getNode(Opcode: CastOpcode, DL, VT, N1: B, N2: Cast->getOperand(Num: 1));
15595 } else {
15596 CastA = DAG.getNode(Opcode: CastOpcode, DL, VT, Operand: A);
15597 CastB = DAG.getNode(Opcode: CastOpcode, DL, VT, Operand: B);
15598 }
15599 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SetCC, N2: CastA, N3: CastB);
15600}
15601
15602// fold ([s|z]ext ([s|z]extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
15603// fold ([s|z]ext ( extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
15604static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
15605 const TargetLowering &TLI, EVT VT,
15606 bool LegalOperations, SDNode *N,
15607 SDValue N0, ISD::LoadExtType ExtLoadType) {
15608 bool Frozen = N0.getOpcode() == ISD::FREEZE;
15609 auto *OldExtLoad = dyn_cast<LoadSDNode>(Val: Frozen ? N0.getOperand(i: 0) : N0);
15610 if (!OldExtLoad)
15611 return SDValue();
15612
15613 bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD)
15614 ? ISD::isSEXTLoad(N: OldExtLoad)
15615 : ISD::isZEXTLoad(N: OldExtLoad);
15616 if ((!isAExtLoad && !ISD::isEXTLoad(N: OldExtLoad)) ||
15617 !ISD::isUNINDEXEDLoad(N: OldExtLoad) || !OldExtLoad->hasNUsesOfValue(NUses: 1, Value: 0))
15618 return SDValue();
15619
15620 EVT MemVT = OldExtLoad->getMemoryVT();
15621 if ((LegalOperations || !OldExtLoad->isSimple() || VT.isVector()) &&
15622 !TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: OldExtLoad->getAlign(),
15623 AddrSpace: OldExtLoad->getAddressSpace(), ExtType: ExtLoadType, Atomic: false))
15624 return SDValue();
15625
15626 SDLoc DL(OldExtLoad);
15627 SDValue ExtLoad = DAG.getExtLoad(ExtType: ExtLoadType, dl: DL, VT, Chain: OldExtLoad->getChain(),
15628 Ptr: OldExtLoad->getBasePtr(), MemVT,
15629 MMO: OldExtLoad->getMemOperand());
15630 SDValue Res = ExtLoad;
15631 if (Frozen) {
15632 Res = DAG.getFreeze(V: ExtLoad);
15633 Res = DAG.getNode(
15634 Opcode: ExtLoadType == ISD::SEXTLOAD ? ISD::AssertSext : ISD::AssertZext, DL,
15635 VT: Res.getValueType(), N1: Res,
15636 N2: DAG.getValueType(OldExtLoad->getValueType(ResNo: 0).getScalarType()));
15637 }
15638 Combiner.CombineTo(N, Res);
15639 DAG.ReplaceAllUsesOfValueWith(From: SDValue(OldExtLoad, 1), To: ExtLoad.getValue(R: 1));
15640 if (N0->use_empty())
15641 Combiner.recursivelyDeleteUnusedNodes(N: N0.getNode());
15642 return SDValue(N, 0); // Return N so it doesn't get rechecked!
15643}
15644
15645// fold ([s|z]ext (load x)) -> ([s|z]ext (truncate ([s|z]extload x)))
15646// Only generate vector extloads when 1) they're legal, and 2) they are
15647// deemed desirable by the target. NonNegZExt can be set to true if a zero
15648// extend has the nonneg flag to allow use of sextload if profitable.
15649static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
15650 const TargetLowering &TLI, EVT VT,
15651 bool LegalOperations, SDNode *N, SDValue N0,
15652 ISD::LoadExtType ExtLoadType,
15653 ISD::NodeType ExtOpc,
15654 bool NonNegZExt = false) {
15655
15656 bool Frozen = N0.getOpcode() == ISD::FREEZE;
15657 SDValue Freeze = Frozen ? N0 : SDValue();
15658 auto *Load = dyn_cast<LoadSDNode>(Val: Frozen ? N0.getOperand(i: 0) : N0);
15659 // TODO: Support multiple uses of the load when frozen.
15660 if (!Load || !ISD::isNON_EXTLoad(N: Load) || !ISD::isUNINDEXEDLoad(N: Load) ||
15661 (Frozen && !Load->hasNUsesOfValue(NUses: 1, Value: 0)))
15662 return {};
15663
15664 // If this is zext nneg, see if it would make sense to treat it as a sext.
15665 if (NonNegZExt) {
15666 assert(ExtLoadType == ISD::ZEXTLOAD && ExtOpc == ISD::ZERO_EXTEND &&
15667 "Unexpected load type or opcode");
15668 for (SDNode *User : Load->users()) {
15669 if (User->getOpcode() == ISD::SETCC) {
15670 ISD::CondCode CC = cast<CondCodeSDNode>(Val: User->getOperand(Num: 2))->get();
15671 if (ISD::isSignedIntSetCC(Code: CC)) {
15672 ExtLoadType = ISD::SEXTLOAD;
15673 ExtOpc = ISD::SIGN_EXTEND;
15674 break;
15675 }
15676 }
15677 }
15678 }
15679
15680 // TODO: isFixedLengthVector() should be removed and any negative effects on
15681 // code generation being the result of that target's implementation of
15682 // isVectorLoadExtDesirable().
15683 if ((LegalOperations || VT.isFixedLengthVector() || !Load->isSimple()) &&
15684 !TLI.isLoadLegal(ValVT: VT, MemVT: Load->getValueType(ResNo: 0), Alignment: Load->getAlign(),
15685 AddrSpace: Load->getAddressSpace(), ExtType: ExtLoadType, Atomic: false))
15686 return {};
15687
15688 bool DoXform = true;
15689 SmallVector<SDNode *, 4> SetCCs;
15690 if (!N0->hasOneUse())
15691 DoXform = ExtendUsesToFormExtLoad(VT, N, N0: Frozen ? Freeze : SDValue(Load, 0),
15692 ExtOpc, ExtendNodes&: SetCCs, TLI);
15693 if (VT.isVector())
15694 DoXform &= TLI.isVectorLoadExtDesirable(ExtVal: SDValue(N, 0));
15695 if (!DoXform)
15696 return {};
15697
15698 SDLoc DL(Load);
15699
15700 auto SalvageDbgValue = [&](SDDbgValue *Dbg, SDValue Old, SDValue New,
15701 unsigned OldBits, unsigned NewBits,
15702 bool IsSigned) {
15703 SmallVector<SDDbgOperand> Locs = Dbg->copyLocationOps();
15704 bool Changed = false;
15705
15706 bool IsVariadic = Dbg->isVariadic();
15707 SmallVector<unsigned, 2> AffectedArgs;
15708
15709 for (unsigned I = 0, E = Locs.size(); I != E; ++I) {
15710 SDDbgOperand &Op = Locs[I];
15711 if (Op.getKind() != SDDbgOperand::SDNODE)
15712 continue;
15713
15714 if (Op.getSDNode() == Old.getNode() && Op.getResNo() == Old.getResNo()) {
15715 Op = SDDbgOperand::fromNode(Node: New.getNode(), ResNo: New.getResNo());
15716 Changed = true;
15717
15718 if (IsVariadic)
15719 AffectedArgs.push_back(Elt: I);
15720 }
15721 }
15722
15723 if (!Changed)
15724 return;
15725
15726 const DIExpression *OldExpr = Dbg->getExpression();
15727 const DIExpression *NewExpr = nullptr;
15728
15729 if (!IsVariadic) {
15730 // Do not introduce DW_OP_LLVM_arg into ordinary single-location
15731 // DBG_VALUEs.
15732 NewExpr = DIExpression::appendExt(Expr: OldExpr, FromSize: NewBits, ToSize: OldBits, Signed: IsSigned);
15733 } else {
15734 auto ExtOps = DIExpression::getExtOps(FromSize: NewBits, ToSize: OldBits, Signed: IsSigned);
15735
15736 NewExpr = DIExpression::convertToVariadicExpression(Expr: OldExpr);
15737
15738 for (unsigned ArgNo : AffectedArgs)
15739 NewExpr = DIExpression::appendOpsToArg(Expr: NewExpr, Ops: ExtOps, ArgNo,
15740 /*StackValue=*/false);
15741 }
15742
15743 SDDbgValue *NewDV = DAG.getDbgValueList(
15744 Var: Dbg->getVariable(), Expr: const_cast<DIExpression *>(NewExpr), Locs,
15745 Dependencies: Dbg->getAdditionalDependencies(), IsIndirect: Dbg->isIndirect(), DL: Dbg->getDebugLoc(),
15746 O: Dbg->getOrder(), IsVariadic: Dbg->isVariadic());
15747
15748 Dbg->setIsInvalidated();
15749 Dbg->setIsEmitted();
15750 DAG.AddDbgValue(DB: NewDV, /*isParameter=*/false);
15751 };
15752
15753 // Because we are replacing a load and a s|z ext with a load-s|z ext
15754 // instruction, the dbg_value attached to the load will be of a smaller bit
15755 // width, and we have to add a DW_OP_LLVM_convert expression to get the
15756 // correct size.
15757 auto SalvageToOldLoadSize = [&](SDValue Old, SDValue New, bool IsSigned) {
15758 SmallVector<SDDbgValue *, 4> DbgVals(
15759 DAG.GetDbgValues(SD: Old.getNode()).begin(),
15760 DAG.GetDbgValues(SD: Old.getNode()).end());
15761
15762 unsigned VarBitsOld = Old.getValueSizeInBits();
15763 unsigned VarBitsNew = New.getValueSizeInBits();
15764
15765 for (SDDbgValue *Dbg : DbgVals) {
15766 if (Dbg->isInvalidated())
15767 continue;
15768
15769 SalvageDbgValue(Dbg, Old, New, VarBitsOld, VarBitsNew, IsSigned);
15770 }
15771 };
15772
15773 SDValue ExtLoad =
15774 DAG.getExtLoad(ExtType: ExtLoadType, dl: DL, VT, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
15775 MemVT: Load->getValueType(ResNo: 0), MMO: Load->getMemOperand());
15776 SDValue Res = ExtLoad;
15777 if (Frozen) {
15778 Res = DAG.getFreeze(V: ExtLoad);
15779 Res = DAG.getNode(Opcode: ExtLoadType == ISD::SEXTLOAD ? ISD::AssertSext
15780 : ISD::AssertZext,
15781 DL, VT: Res.getValueType(), N1: Res,
15782 N2: DAG.getValueType(Load->getValueType(ResNo: 0).getScalarType()));
15783 }
15784 Combiner.ExtendSetCCUses(SetCCs, OrigLoad: N0, ExtLoad: Res, ExtType: ExtOpc);
15785 // If the load value is used only by N, replace it via CombineTo N.
15786 bool NoReplaceTrunc = N0.hasOneUse();
15787 if (N->getHasDebugValue()) {
15788 SDValue OldExtValue(N, 0);
15789 DAG.transferDbgValues(From: OldExtValue, To: ExtLoad);
15790 }
15791 if (NoReplaceTrunc) {
15792 bool IsSigned = N->getOpcode() == ISD::SIGN_EXTEND;
15793 if (Load->getHasDebugValue()) {
15794 SDValue OldLoadVal(Load, 0);
15795 SalvageToOldLoadSize(OldLoadVal, ExtLoad, IsSigned);
15796 }
15797 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: ExtLoad.getValue(R: 1));
15798 Combiner.CombineTo(N, Res);
15799 Combiner.recursivelyDeleteUnusedNodes(N: N0.getNode());
15800 } else {
15801 Combiner.CombineTo(N, Res);
15802 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Load->getValueType(ResNo: 0), Operand: Res);
15803 if (Frozen) {
15804 Combiner.CombineTo(N: Freeze.getNode(), Res: Trunc);
15805 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Load, 1), To: ExtLoad.getValue(R: 1));
15806 } else {
15807 Combiner.CombineTo(N: Load, Res0: Trunc, Res1: ExtLoad.getValue(R: 1));
15808 }
15809 }
15810 return SDValue(N, 0); // Return N so it doesn't get rechecked!
15811}
15812
15813static SDValue
15814tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT,
15815 bool LegalOperations, SDNode *N, SDValue N0,
15816 ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc) {
15817 if (!N0.hasOneUse())
15818 return SDValue();
15819
15820 MaskedLoadSDNode *Ld = dyn_cast<MaskedLoadSDNode>(Val&: N0);
15821 if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD)
15822 return SDValue();
15823
15824 if ((LegalOperations || !cast<MaskedLoadSDNode>(Val&: N0)->isSimple()) &&
15825 !TLI.isLoadLegalOrCustom(ValVT: VT, MemVT: Ld->getValueType(ResNo: 0), Alignment: Ld->getAlign(),
15826 AddrSpace: Ld->getAddressSpace(), ExtType: ExtLoadType, Atomic: false))
15827 return SDValue();
15828
15829 if (!TLI.isVectorLoadExtDesirable(ExtVal: SDValue(N, 0)))
15830 return SDValue();
15831
15832 SDLoc dl(Ld);
15833 SDValue PassThru = DAG.getNode(Opcode: ExtOpc, DL: dl, VT, Operand: Ld->getPassThru());
15834 SDValue NewLoad = DAG.getMaskedLoad(
15835 VT, dl, Chain: Ld->getChain(), Base: Ld->getBasePtr(), Offset: Ld->getOffset(), Mask: Ld->getMask(),
15836 Src0: PassThru, MemVT: Ld->getMemoryVT(), MMO: Ld->getMemOperand(), AM: Ld->getAddressingMode(),
15837 ExtLoadType, IsExpanding: Ld->isExpandingLoad());
15838 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Ld, 1), To: SDValue(NewLoad.getNode(), 1));
15839 return NewLoad;
15840}
15841
15842// fold ([s|z]ext (atomic_load)) -> ([s|z]ext (truncate ([s|z]ext atomic_load)))
15843static SDValue tryToFoldExtOfAtomicLoad(SelectionDAG &DAG,
15844 const TargetLowering &TLI, EVT VT,
15845 SDValue N0,
15846 ISD::LoadExtType ExtLoadType) {
15847 auto *ALoad = dyn_cast<AtomicSDNode>(Val&: N0);
15848 if (!ALoad || ALoad->getOpcode() != ISD::ATOMIC_LOAD)
15849 return {};
15850 EVT MemoryVT = ALoad->getMemoryVT();
15851 if (!TLI.isLoadLegal(ValVT: VT, MemVT: MemoryVT, Alignment: ALoad->getAlign(),
15852 AddrSpace: ALoad->getAddressSpace(), ExtType: ExtLoadType, Atomic: true))
15853 return {};
15854 // Can't fold into ALoad if it is already extending differently.
15855 ISD::LoadExtType ALoadExtTy = ALoad->getExtensionType();
15856 if ((ALoadExtTy == ISD::ZEXTLOAD && ExtLoadType == ISD::SEXTLOAD) ||
15857 (ALoadExtTy == ISD::SEXTLOAD && ExtLoadType == ISD::ZEXTLOAD))
15858 return {};
15859
15860 EVT OrigVT = ALoad->getValueType(ResNo: 0);
15861 assert(OrigVT.getSizeInBits() < VT.getSizeInBits() && "VT should be wider.");
15862 auto *NewALoad = cast<AtomicSDNode>(Val: DAG.getAtomicLoad(
15863 ExtType: ExtLoadType, dl: SDLoc(ALoad), MemVT: MemoryVT, VT, Chain: ALoad->getChain(),
15864 Ptr: ALoad->getBasePtr(), MMO: ALoad->getMemOperand()));
15865 DAG.ReplaceAllUsesOfValueWith(
15866 From: SDValue(ALoad, 0),
15867 To: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(ALoad), VT: OrigVT, Operand: SDValue(NewALoad, 0)));
15868 // Update the chain uses.
15869 DAG.ReplaceAllUsesOfValueWith(From: SDValue(ALoad, 1), To: SDValue(NewALoad, 1));
15870 return SDValue(NewALoad, 0);
15871}
15872
15873static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG,
15874 bool LegalOperations) {
15875 assert((N->getOpcode() == ISD::SIGN_EXTEND ||
15876 N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext");
15877
15878 SDValue SetCC = N->getOperand(Num: 0);
15879 if (LegalOperations || SetCC.getOpcode() != ISD::SETCC ||
15880 !SetCC.hasOneUse() || SetCC.getValueType() != MVT::i1)
15881 return SDValue();
15882
15883 SDValue X = SetCC.getOperand(i: 0);
15884 SDValue Ones = SetCC.getOperand(i: 1);
15885 ISD::CondCode CC = cast<CondCodeSDNode>(Val: SetCC.getOperand(i: 2))->get();
15886 EVT VT = N->getValueType(ResNo: 0);
15887 EVT XVT = X.getValueType();
15888 // setge X, C is canonicalized to setgt, so we do not need to match that
15889 // pattern. The setlt sibling is folded in SimplifySelectCC() because it does
15890 // not require the 'not' op.
15891 if (CC == ISD::SETGT && isAllOnesConstant(V: Ones) && VT == XVT) {
15892 // Invert and smear/shift the sign bit:
15893 // sext i1 (setgt iN X, -1) --> sra (not X), (N - 1)
15894 // zext i1 (setgt iN X, -1) --> srl (not X), (N - 1)
15895 SDLoc DL(N);
15896 unsigned ShCt = VT.getSizeInBits() - 1;
15897 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15898 if (!TLI.shouldAvoidTransformToShift(VT, Amount: ShCt)) {
15899 SDValue NotX = DAG.getNOT(DL, Val: X, VT);
15900 SDValue ShiftAmount = DAG.getConstant(Val: ShCt, DL, VT);
15901 auto ShiftOpcode =
15902 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL;
15903 return DAG.getNode(Opcode: ShiftOpcode, DL, VT, N1: NotX, N2: ShiftAmount);
15904 }
15905 }
15906 return SDValue();
15907}
15908
15909SDValue DAGCombiner::foldSextSetcc(SDNode *N) {
15910 SDValue N0 = N->getOperand(Num: 0);
15911 if (N0.getOpcode() != ISD::SETCC)
15912 return SDValue();
15913
15914 SDValue N00 = N0.getOperand(i: 0);
15915 SDValue N01 = N0.getOperand(i: 1);
15916 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
15917 EVT VT = N->getValueType(ResNo: 0);
15918 EVT N00VT = N00.getValueType();
15919 SDLoc DL(N);
15920
15921 // Propagate fast-math-flags.
15922 SDNodeFlags Flags = N0->getFlags();
15923
15924 // On some architectures (such as SSE/NEON/etc) the SETCC result type is
15925 // the same size as the compared operands. Try to optimize sext(setcc())
15926 // if this is the case.
15927 if (VT.isVector() && !LegalOperations &&
15928 TLI.getBooleanContents(Type: N00VT) ==
15929 TargetLowering::ZeroOrNegativeOneBooleanContent) {
15930 EVT SVT = getSetCCResultType(VT: N00VT);
15931
15932 // If we already have the desired type, don't change it.
15933 if (SVT != N0.getValueType()) {
15934 // We know that the # elements of the results is the same as the
15935 // # elements of the compare (and the # elements of the compare result
15936 // for that matter). Check to see that they are the same size. If so,
15937 // we know that the element size of the sext'd result matches the
15938 // element size of the compare operands.
15939 if (VT.getSizeInBits() == SVT.getSizeInBits())
15940 return DAG.getSetCC(DL, VT, LHS: N00, RHS: N01, Cond: CC, /*Chain=*/{},
15941 /*Signaling=*/IsSignaling: false, Flags);
15942
15943 // If the desired elements are smaller or larger than the source
15944 // elements, we can use a matching integer vector type and then
15945 // truncate/sign extend.
15946 EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
15947 if (SVT == MatchingVecType) {
15948 SDValue VsetCC = DAG.getSetCC(DL, VT: MatchingVecType, LHS: N00, RHS: N01, Cond: CC,
15949 /*Chain=*/{}, /*Signaling=*/IsSignaling: false, Flags);
15950 return DAG.getSExtOrTrunc(Op: VsetCC, DL, VT);
15951 }
15952 }
15953
15954 // Try to eliminate the sext of a setcc by zexting the compare operands.
15955 if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(Op: ISD::SETCC, VT) &&
15956 !TLI.isOperationLegalOrCustom(Op: ISD::SETCC, VT: SVT)) {
15957 bool IsSignedCmp = ISD::isSignedIntSetCC(Code: CC);
15958 unsigned LoadOpcode = IsSignedCmp ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
15959 unsigned ExtOpcode = IsSignedCmp ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
15960
15961 // We have an unsupported narrow vector compare op that would be legal
15962 // if extended to the destination type. See if the compare operands
15963 // can be freely extended to the destination type.
15964 auto IsFreeToExtend = [&](SDValue V) {
15965 if (isConstantOrConstantVector(N: V, /*NoOpaques*/ true))
15966 return true;
15967 // Match a simple, non-extended load that can be converted to a
15968 // legal {z/s}ext-load.
15969 // TODO: Allow widening of an existing {z/s}ext-load?
15970 if (!(ISD::isNON_EXTLoad(N: V.getNode()) &&
15971 ISD::isUNINDEXEDLoad(N: V.getNode())))
15972 return false;
15973
15974 LoadSDNode *Ld = cast<LoadSDNode>(Val: V.getNode());
15975
15976 if (!Ld->isSimple() ||
15977 !TLI.isLoadLegal(ValVT: VT, MemVT: V.getValueType(), Alignment: Ld->getAlign(),
15978 AddrSpace: Ld->getAddressSpace(), ExtType: LoadOpcode, Atomic: false))
15979 return false;
15980
15981 // Non-chain users of this value must either be the setcc in this
15982 // sequence or extends that can be folded into the new {z/s}ext-load.
15983 for (SDUse &Use : V->uses()) {
15984 // Skip uses of the chain and the setcc.
15985 SDNode *User = Use.getUser();
15986 if (Use.getResNo() != 0 || User == N0.getNode())
15987 continue;
15988 // Extra users must have exactly the same cast we are about to create.
15989 // TODO: This restriction could be eased if ExtendUsesToFormExtLoad()
15990 // is enhanced similarly.
15991 if (User->getOpcode() != ExtOpcode || User->getValueType(ResNo: 0) != VT)
15992 return false;
15993 }
15994 return true;
15995 };
15996
15997 if (IsFreeToExtend(N00) && IsFreeToExtend(N01)) {
15998 SDValue Ext0 = DAG.getNode(Opcode: ExtOpcode, DL, VT, Operand: N00);
15999 SDValue Ext1 = DAG.getNode(Opcode: ExtOpcode, DL, VT, Operand: N01);
16000 return DAG.getSetCC(DL, VT, LHS: Ext0, RHS: Ext1, Cond: CC, /*Chain=*/{},
16001 /*Signaling=*/IsSignaling: false, Flags);
16002 }
16003 }
16004 }
16005
16006 // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0)
16007 // Here, T can be 1 or -1, depending on the type of the setcc and
16008 // getBooleanContents().
16009 unsigned SetCCWidth = N0.getScalarValueSizeInBits();
16010
16011 // To determine the "true" side of the select, we need to know the high bit
16012 // of the value returned by the setcc if it evaluates to true.
16013 // If the type of the setcc is i1, then the true case of the select is just
16014 // sext(i1 1), that is, -1.
16015 // If the type of the setcc is larger (say, i8) then the value of the high
16016 // bit depends on getBooleanContents(), so ask TLI for a real "true" value
16017 // of the appropriate width.
16018 SDValue ExtTrueVal = (SetCCWidth == 1)
16019 ? DAG.getAllOnesConstant(DL, VT)
16020 : DAG.getBoolConstant(V: true, DL, VT, OpVT: N00VT);
16021 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
16022 if (SDValue SCC = SimplifySelectCC(DL, N0: N00, N1: N01, N2: ExtTrueVal, N3: Zero, CC, NotExtCompare: true))
16023 return SCC;
16024
16025 if (!VT.isVector() && !shouldConvertSelectOfConstantsToMath(Cond: N0, VT, TLI)) {
16026 EVT SetCCVT = getSetCCResultType(VT: N00VT);
16027 // Don't do this transform for i1 because there's a select transform
16028 // that would reverse it.
16029 // TODO: We should not do this transform at all without a target hook
16030 // because a sext is likely cheaper than a select?
16031 if (SetCCVT.getScalarSizeInBits() != 1 &&
16032 (!LegalOperations || TLI.isOperationLegal(Op: ISD::SETCC, VT: N00VT))) {
16033 SDValue SetCC = DAG.getSetCC(DL, VT: SetCCVT, LHS: N00, RHS: N01, Cond: CC, /*Chain=*/{},
16034 /*Signaling=*/IsSignaling: false, Flags);
16035 return DAG.getSelect(DL, VT, Cond: SetCC, LHS: ExtTrueVal, RHS: Zero, Flags);
16036 }
16037 }
16038
16039 return SDValue();
16040}
16041
16042SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
16043 SDValue N0 = N->getOperand(Num: 0);
16044 EVT VT = N->getValueType(ResNo: 0);
16045 SDLoc DL(N);
16046
16047 if (VT.isVector())
16048 if (SDValue FoldedVOp = SimplifyVCastOp(N, DL))
16049 return FoldedVOp;
16050
16051 // sext(undef) = 0 because the top bit will all be the same.
16052 if (N0.isUndef())
16053 return DAG.getConstant(Val: 0, DL, VT);
16054
16055 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
16056 return Res;
16057
16058 // fold (sext (sext x)) -> (sext x)
16059 // fold (sext (aext x)) -> (sext x)
16060 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
16061 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: N0.getOperand(i: 0));
16062
16063 // fold (sext (aext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
16064 // fold (sext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
16065 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
16066 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG)
16067 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL: SDLoc(N), VT,
16068 Operand: N0.getOperand(i: 0));
16069
16070 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
16071 SDValue N00 = N0.getOperand(i: 0);
16072 EVT ExtVT = cast<VTSDNode>(Val: N0->getOperand(Num: 1))->getVT();
16073 if (N00.getOpcode() == ISD::TRUNCATE || TLI.isTruncateFree(Val: N00, VT2: ExtVT)) {
16074 // fold (sext (sext_inreg x)) -> (sext (trunc x))
16075 if ((!LegalTypes || TLI.isTypeLegal(VT: ExtVT))) {
16076 SDValue T = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ExtVT, Operand: N00);
16077 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: T);
16078 }
16079
16080 // If the trunc wasn't legal, try to fold to (sext_inreg (anyext x))
16081 if (!LegalTypes || TLI.isTypeLegal(VT)) {
16082 SDValue ExtSrc = DAG.getAnyExtOrTrunc(Op: N00, DL, VT);
16083 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: ExtSrc,
16084 N2: N0->getOperand(Num: 1));
16085 }
16086 }
16087 }
16088
16089 if (N0.getOpcode() == ISD::TRUNCATE) {
16090 // fold (sext (truncate (load x))) -> (sext (smaller load x))
16091 // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
16092 if (SDValue NarrowLoad = reduceLoadWidth(N: N0.getNode())) {
16093 SDNode *oye = N0.getOperand(i: 0).getNode();
16094 if (NarrowLoad.getNode() != N0.getNode()) {
16095 CombineTo(N: N0.getNode(), Res: NarrowLoad);
16096 // CombineTo deleted the truncate, if needed, but not what's under it.
16097 AddToWorklist(N: oye);
16098 }
16099 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16100 }
16101
16102 // See if the value being truncated is already sign extended. If so, just
16103 // eliminate the trunc/sext pair.
16104 SDValue Op = N0.getOperand(i: 0);
16105 unsigned OpBits = Op.getScalarValueSizeInBits();
16106 unsigned MidBits = N0.getScalarValueSizeInBits();
16107 unsigned DestBits = VT.getScalarSizeInBits();
16108
16109 if (N0->getFlags().hasNoSignedWrap() ||
16110 DAG.ComputeNumSignBits(Op) > OpBits - MidBits) {
16111 if (OpBits == DestBits) {
16112 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
16113 // bits, it is already ready.
16114 return Op;
16115 }
16116
16117 if (OpBits < DestBits) {
16118 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
16119 // bits, just sext from i32.
16120 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op);
16121 }
16122
16123 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
16124 // bits, just truncate to i32.
16125 SDNodeFlags Flags;
16126 Flags.setNoSignedWrap(true);
16127 Flags.setNoUnsignedWrap(N0->getFlags().hasNoUnsignedWrap());
16128 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op, Flags);
16129 }
16130
16131 // fold (sext (truncate x)) -> (sextinreg x).
16132 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::SIGN_EXTEND_INREG,
16133 VT: N0.getValueType())) {
16134 if (OpBits < DestBits)
16135 Op = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N0), VT, Operand: Op);
16136 else if (OpBits > DestBits)
16137 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N0), VT, Operand: Op);
16138 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: Op,
16139 N2: DAG.getValueType(N0.getValueType()));
16140 }
16141 }
16142
16143 // Try to simplify (sext (load x)).
16144 if (SDValue foldedExt =
16145 tryToFoldExtOfLoad(DAG, Combiner&: *this, TLI, VT, LegalOperations, N, N0,
16146 ExtLoadType: ISD::SEXTLOAD, ExtOpc: ISD::SIGN_EXTEND))
16147 return foldedExt;
16148
16149 if (SDValue foldedExt =
16150 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0,
16151 ExtLoadType: ISD::SEXTLOAD, ExtOpc: ISD::SIGN_EXTEND))
16152 return foldedExt;
16153
16154 // fold (sext (load x)) to multiple smaller sextloads.
16155 // Only on illegal but splittable vectors.
16156 if (SDValue ExtLoad = CombineExtLoad(N))
16157 return ExtLoad;
16158
16159 // Try to simplify (sext (sextload x)).
16160 if (SDValue foldedExt = tryToFoldExtOfExtload(
16161 DAG, Combiner&: *this, TLI, VT, LegalOperations, N, N0, ExtLoadType: ISD::SEXTLOAD))
16162 return foldedExt;
16163
16164 // Try to simplify (sext (atomic_load x)).
16165 if (SDValue foldedExt =
16166 tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ExtLoadType: ISD::SEXTLOAD))
16167 return foldedExt;
16168
16169 // fold (sext (and/or/xor (load x), cst)) ->
16170 // (and/or/xor (sextload x), (sext cst))
16171 if (ISD::isBitwiseLogicOp(Opcode: N0.getOpcode()) &&
16172 isa<LoadSDNode>(Val: N0.getOperand(i: 0)) &&
16173 N0.getOperand(i: 1).getOpcode() == ISD::Constant &&
16174 (!LegalOperations && TLI.isOperationLegal(Op: N0.getOpcode(), VT))) {
16175 LoadSDNode *LN00 = cast<LoadSDNode>(Val: N0.getOperand(i: 0));
16176 EVT MemVT = LN00->getMemoryVT();
16177 if (TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: LN00->getAlign(), AddrSpace: LN00->getAddressSpace(),
16178 ExtType: ISD::SEXTLOAD, Atomic: false) &&
16179 LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
16180 SmallVector<SDNode*, 4> SetCCs;
16181 bool DoXform = ExtendUsesToFormExtLoad(VT, N: N0.getNode(), N0: N0.getOperand(i: 0),
16182 ExtOpc: ISD::SIGN_EXTEND, ExtendNodes&: SetCCs, TLI);
16183 if (DoXform) {
16184 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::SEXTLOAD, dl: SDLoc(LN00), VT,
16185 Chain: LN00->getChain(), Ptr: LN00->getBasePtr(),
16186 MemVT: LN00->getMemoryVT(),
16187 MMO: LN00->getMemOperand());
16188 APInt Mask = N0.getConstantOperandAPInt(i: 1).sext(width: VT.getSizeInBits());
16189 SDValue And = DAG.getNode(Opcode: N0.getOpcode(), DL, VT,
16190 N1: ExtLoad, N2: DAG.getConstant(Val: Mask, DL, VT));
16191 ExtendSetCCUses(SetCCs, OrigLoad: N0.getOperand(i: 0), ExtLoad, ExtType: ISD::SIGN_EXTEND);
16192 bool NoReplaceTruncAnd = !N0.hasOneUse();
16193 bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
16194 CombineTo(N, Res: And);
16195 // If N0 has multiple uses, change other uses as well.
16196 if (NoReplaceTruncAnd) {
16197 SDValue TruncAnd =
16198 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: And);
16199 CombineTo(N: N0.getNode(), Res: TruncAnd);
16200 }
16201 if (NoReplaceTrunc) {
16202 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN00, 1), To: ExtLoad.getValue(R: 1));
16203 } else {
16204 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(LN00),
16205 VT: LN00->getValueType(ResNo: 0), Operand: ExtLoad);
16206 CombineTo(N: LN00, Res0: Trunc, Res1: ExtLoad.getValue(R: 1));
16207 }
16208 return SDValue(N,0); // Return N so it doesn't get rechecked!
16209 }
16210 }
16211 }
16212
16213 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
16214 return V;
16215
16216 if (SDValue V = foldSextSetcc(N))
16217 return V;
16218
16219 // fold (sext x) -> (zext x) if the sign bit is known zero.
16220 if (!TLI.isSExtCheaperThanZExt(FromTy: N0.getValueType(), ToTy: VT) &&
16221 (!LegalOperations || TLI.isOperationLegal(Op: ISD::ZERO_EXTEND, VT)) &&
16222 DAG.SignBitIsZero(Op: N0))
16223 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: N0, Flags: SDNodeFlags::NonNeg);
16224
16225 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(Cast: N))
16226 return NewVSel;
16227
16228 // Eliminate this sign extend by doing a negation in the destination type:
16229 // sext i32 (0 - (zext i8 X to i32)) to i64 --> 0 - (zext i8 X to i64)
16230 if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
16231 isNullOrNullSplat(V: N0.getOperand(i: 0)) &&
16232 N0.getOperand(i: 1).getOpcode() == ISD::ZERO_EXTEND &&
16233 TLI.isOperationLegalOrCustom(Op: ISD::SUB, VT)) {
16234 SDValue Zext = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 1).getOperand(i: 0), DL, VT);
16235 return DAG.getNegative(Val: Zext, DL, VT);
16236 }
16237 // Eliminate this sign extend by doing a decrement in the destination type:
16238 // sext i32 ((zext i8 X to i32) + (-1)) to i64 --> (zext i8 X to i64) + (-1)
16239 if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
16240 isAllOnesOrAllOnesSplat(V: N0.getOperand(i: 1)) &&
16241 N0.getOperand(i: 0).getOpcode() == ISD::ZERO_EXTEND &&
16242 TLI.isOperationLegalOrCustom(Op: ISD::ADD, VT)) {
16243 SDValue Zext = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 0).getOperand(i: 0), DL, VT);
16244 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Zext, N2: DAG.getAllOnesConstant(DL, VT));
16245 }
16246
16247 // fold sext (not i1 X) -> add (zext i1 X), -1
16248 // TODO: This could be extended to handle bool vectors.
16249 if (N0.getValueType() == MVT::i1 && isBitwiseNot(V: N0) && N0.hasOneUse() &&
16250 (!LegalOperations || (TLI.isOperationLegal(Op: ISD::ZERO_EXTEND, VT) &&
16251 TLI.isOperationLegal(Op: ISD::ADD, VT)))) {
16252 // If we can eliminate the 'not', the sext form should be better
16253 if (SDValue NewXor = visitXOR(N: N0.getNode())) {
16254 // Returning N0 is a form of in-visit replacement that may have
16255 // invalidated N0.
16256 if (NewXor.getNode() == N0.getNode()) {
16257 // Return SDValue here as the xor should have already been replaced in
16258 // this sext.
16259 return SDValue();
16260 }
16261
16262 // Return a new sext with the new xor.
16263 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: NewXor);
16264 }
16265
16266 SDValue Zext = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: N0.getOperand(i: 0));
16267 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Zext, N2: DAG.getAllOnesConstant(DL, VT));
16268 }
16269
16270 if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
16271 return Res;
16272
16273 return SDValue();
16274}
16275
16276/// Given an extending node with a pop-count operand, if the target does not
16277/// support a pop-count in the narrow source type but does support it in the
16278/// destination type, widen the pop-count to the destination type.
16279static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG, const SDLoc &DL) {
16280 assert((Extend->getOpcode() == ISD::ZERO_EXTEND ||
16281 Extend->getOpcode() == ISD::ANY_EXTEND) &&
16282 "Expected extend op");
16283
16284 SDValue CtPop = Extend->getOperand(Num: 0);
16285 if (CtPop.getOpcode() != ISD::CTPOP || !CtPop.hasOneUse())
16286 return SDValue();
16287
16288 EVT VT = Extend->getValueType(ResNo: 0);
16289 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16290 if (TLI.isOperationLegalOrCustom(Op: ISD::CTPOP, VT: CtPop.getValueType()) ||
16291 !TLI.isOperationLegalOrCustom(Op: ISD::CTPOP, VT))
16292 return SDValue();
16293
16294 // zext (ctpop X) --> ctpop (zext X)
16295 SDValue NewZext = DAG.getZExtOrTrunc(Op: CtPop.getOperand(i: 0), DL, VT);
16296 return DAG.getNode(Opcode: ISD::CTPOP, DL, VT, Operand: NewZext);
16297}
16298
16299// If we have (zext (abs X)) where X is a type that will be promoted by type
16300// legalization, convert to (abs_min_poison (sext X)). But do not extend
16301// past a legal type.
16302static SDValue widenAbs(SDNode *Extend, SelectionDAG &DAG) {
16303 assert(Extend->getOpcode() == ISD::ZERO_EXTEND && "Expected zero extend.");
16304
16305 EVT VT = Extend->getValueType(ResNo: 0);
16306 if (VT.isVector())
16307 return SDValue();
16308
16309 SDValue Abs = Extend->getOperand(Num: 0);
16310 if (!ISD::isAbsOpcode(Opcode: Abs.getOpcode()) || !Abs.hasOneUse())
16311 return SDValue();
16312
16313 EVT AbsVT = Abs.getValueType();
16314 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16315 if (TLI.getTypeAction(Context&: *DAG.getContext(), VT: AbsVT) !=
16316 TargetLowering::TypePromoteInteger)
16317 return SDValue();
16318
16319 EVT LegalVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: AbsVT);
16320
16321 SDValue SExt =
16322 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(Abs), VT: LegalVT, Operand: Abs.getOperand(i: 0));
16323 SDValue NewAbs = DAG.getNode(Opcode: ISD::ABS_MIN_POISON, DL: SDLoc(Abs), VT: LegalVT, Operand: SExt);
16324 return DAG.getZExtOrTrunc(Op: NewAbs, DL: SDLoc(Extend), VT);
16325}
16326
16327SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
16328 SDValue N0 = N->getOperand(Num: 0);
16329 EVT VT = N->getValueType(ResNo: 0);
16330 SDLoc DL(N);
16331
16332 if (VT.isVector())
16333 if (SDValue FoldedVOp = SimplifyVCastOp(N, DL))
16334 return FoldedVOp;
16335
16336 // zext(undef) = 0
16337 if (N0.isUndef())
16338 return DAG.getConstant(Val: 0, DL, VT);
16339
16340 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
16341 return Res;
16342
16343 // fold (zext (zext x)) -> (zext x)
16344 // fold (zext (aext x)) -> (zext x)
16345 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
16346 SDNodeFlags Flags;
16347 if (N0.getOpcode() == ISD::ZERO_EXTEND)
16348 Flags.setNonNeg(N0->getFlags().hasNonNeg());
16349 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: N0.getOperand(i: 0), Flags);
16350 }
16351
16352 // fold (zext (aext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
16353 // fold (zext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
16354 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
16355 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)
16356 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Operand: N0.getOperand(i: 0));
16357
16358 // fold (zext (truncate x)) -> (zext x) or
16359 // (zext (truncate x)) -> (truncate x)
16360 // This is valid when the truncated bits of x are already zero.
16361 SDValue Op;
16362 KnownBits Known;
16363 if (isTruncateOf(DAG, N: N0, Op, Known)) {
16364 APInt TruncatedBits =
16365 (Op.getScalarValueSizeInBits() == N0.getScalarValueSizeInBits()) ?
16366 APInt(Op.getScalarValueSizeInBits(), 0) :
16367 APInt::getBitsSet(numBits: Op.getScalarValueSizeInBits(),
16368 loBit: N0.getScalarValueSizeInBits(),
16369 hiBit: std::min(a: Op.getScalarValueSizeInBits(),
16370 b: VT.getScalarSizeInBits()));
16371 if (TruncatedBits.isSubsetOf(RHS: Known.Zero)) {
16372 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT);
16373 DAG.salvageDebugInfo(N&: *N0.getNode());
16374
16375 return ZExtOrTrunc;
16376 }
16377 }
16378
16379 // fold (zext (truncate x)) -> (and x, mask)
16380 if (N0.getOpcode() == ISD::TRUNCATE) {
16381 // fold (zext (truncate (load x))) -> (zext (smaller load x))
16382 // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
16383 if (SDValue NarrowLoad = reduceLoadWidth(N: N0.getNode())) {
16384 SDNode *oye = N0.getOperand(i: 0).getNode();
16385 if (NarrowLoad.getNode() != N0.getNode()) {
16386 CombineTo(N: N0.getNode(), Res: NarrowLoad);
16387 // CombineTo deleted the truncate, if needed, but not what's under it.
16388 AddToWorklist(N: oye);
16389 }
16390 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16391 }
16392
16393 EVT SrcVT = N0.getOperand(i: 0).getValueType();
16394 EVT MinVT = N0.getValueType();
16395
16396 if (N->getFlags().hasNonNeg()) {
16397 SDValue Op = N0.getOperand(i: 0);
16398 unsigned OpBits = SrcVT.getScalarSizeInBits();
16399 unsigned MidBits = MinVT.getScalarSizeInBits();
16400 unsigned DestBits = VT.getScalarSizeInBits();
16401
16402 if (N0->getFlags().hasNoSignedWrap() ||
16403 DAG.ComputeNumSignBits(Op) > OpBits - MidBits) {
16404 if (OpBits == DestBits) {
16405 // Op is i32, Mid is i8, and Dest is i32. If Op has more than 24 sign
16406 // bits, it is already ready.
16407 return Op;
16408 }
16409
16410 if (OpBits < DestBits) {
16411 // Op is i32, Mid is i8, and Dest is i64. If Op has more than 24 sign
16412 // bits, just sext from i32.
16413 // FIXME: This can probably be ZERO_EXTEND nneg?
16414 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: Op);
16415 }
16416
16417 // Op is i64, Mid is i8, and Dest is i32. If Op has more than 56 sign
16418 // bits, just truncate to i32.
16419 SDNodeFlags Flags;
16420 Flags.setNoSignedWrap(true);
16421 Flags.setNoUnsignedWrap(true);
16422 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Op, Flags);
16423 }
16424 }
16425
16426 // Try to mask before the extension to avoid having to generate a larger mask,
16427 // possibly over several sub-vectors.
16428 if (SrcVT.bitsLT(VT) && VT.isVector()) {
16429 if (!LegalOperations || (TLI.isOperationLegal(Op: ISD::AND, VT: SrcVT) &&
16430 TLI.isOperationLegal(Op: ISD::ZERO_EXTEND, VT))) {
16431 SDValue Op = N0.getOperand(i: 0);
16432 Op = DAG.getZeroExtendInReg(Op, DL, VT: MinVT);
16433 AddToWorklist(N: Op.getNode());
16434 SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT);
16435 // Transfer the debug info; the new node is equivalent to N0.
16436 DAG.transferDbgValues(From: N0, To: ZExtOrTrunc);
16437 return ZExtOrTrunc;
16438 }
16439 }
16440
16441 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::AND, VT)) {
16442 SDValue Op = DAG.getAnyExtOrTrunc(Op: N0.getOperand(i: 0), DL, VT);
16443 AddToWorklist(N: Op.getNode());
16444 SDValue And = DAG.getZeroExtendInReg(Op, DL, VT: MinVT);
16445 // We may safely transfer the debug info describing the truncate node over
16446 // to the equivalent and operation.
16447 DAG.transferDbgValues(From: N0, To: And);
16448 return And;
16449 }
16450 }
16451
16452 // Fold (zext (and (trunc x), cst)) -> (and x, cst),
16453 // if either of the casts is not free.
16454 // Also handles (zext (and (bitcast (extract_subvector vNi1, 0)) cst))
16455 // by treating the bitcast+extract as equivalent to a truncate of the
16456 // wider bitcast, e.g. on AVX512DQ where v8i1 extract replaces truncate.
16457 if (N0.getOpcode() == ISD::AND &&
16458 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
16459 SDValue AndSrc = N0.getOperand(i: 0);
16460 SDValue X;
16461 if (AndSrc.getOpcode() == ISD::TRUNCATE) {
16462 X = AndSrc.getOperand(i: 0);
16463 } else if (AndSrc.getOpcode() == ISD::BITCAST &&
16464 AndSrc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
16465 AndSrc.getOperand(i: 0).getConstantOperandVal(i: 1) == 0) {
16466 // (bitcast (extract_subvector vNi1, 0) -> iK) is equivalent to
16467 // (truncate (bitcast vNi1 -> iN) -> iK); use the wider vNi1 as X.
16468 SDValue Src = AndSrc.getOperand(i: 0).getOperand(i: 0);
16469 EVT SrcVT = Src.getValueType();
16470 if (SrcVT.isFixedLengthVectorOf(EltVT: MVT::i1)) {
16471 EVT WideIntVT =
16472 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getSizeInBits());
16473 if (TLI.isTypeLegal(VT: WideIntVT))
16474 X = DAG.getBitcast(VT: WideIntVT, V: Src);
16475 }
16476 }
16477 if (X && (!TLI.isTruncateFree(Val: X, VT2: N0.getValueType()) ||
16478 !TLI.isZExtFree(FromTy: N0.getValueType(), ToTy: VT))) {
16479 X = DAG.getAnyExtOrTrunc(Op: X, DL: SDLoc(X), VT);
16480 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
16481 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: DAG.getConstant(Val: Mask, DL, VT));
16482 }
16483 }
16484
16485 // Try to simplify (zext (load x)).
16486 if (SDValue foldedExt = tryToFoldExtOfLoad(
16487 DAG, Combiner&: *this, TLI, VT, LegalOperations, N, N0, ExtLoadType: ISD::ZEXTLOAD,
16488 ExtOpc: ISD::ZERO_EXTEND, NonNegZExt: N->getFlags().hasNonNeg()))
16489 return foldedExt;
16490
16491 if (SDValue foldedExt =
16492 tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0,
16493 ExtLoadType: ISD::ZEXTLOAD, ExtOpc: ISD::ZERO_EXTEND))
16494 return foldedExt;
16495
16496 // fold (zext (load x)) to multiple smaller zextloads.
16497 // Only on illegal but splittable vectors.
16498 if (SDValue ExtLoad = CombineExtLoad(N))
16499 return ExtLoad;
16500
16501 // Try to simplify (zext (atomic_load x)).
16502 if (SDValue foldedExt =
16503 tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ExtLoadType: ISD::ZEXTLOAD))
16504 return foldedExt;
16505
16506 // fold (zext (and/or/xor (load x), cst)) ->
16507 // (and/or/xor (zextload x), (zext cst))
16508 // Unless (and (load x) cst) will match as a zextload already and has
16509 // additional users, or the zext is already free.
16510 if (ISD::isBitwiseLogicOp(Opcode: N0.getOpcode()) && !TLI.isZExtFree(Val: N0, VT2: VT) &&
16511 isa<LoadSDNode>(Val: N0.getOperand(i: 0)) &&
16512 N0.getOperand(i: 1).getOpcode() == ISD::Constant &&
16513 (!LegalOperations && TLI.isOperationLegal(Op: N0.getOpcode(), VT))) {
16514 LoadSDNode *LN00 = cast<LoadSDNode>(Val: N0.getOperand(i: 0));
16515 EVT MemVT = LN00->getMemoryVT();
16516 if (TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: LN00->getAlign(), AddrSpace: LN00->getAddressSpace(),
16517 ExtType: ISD::ZEXTLOAD, Atomic: false) &&
16518 LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) {
16519 bool DoXform = true;
16520 SmallVector<SDNode*, 4> SetCCs;
16521 if (!N0.hasOneUse()) {
16522 if (N0.getOpcode() == ISD::AND) {
16523 auto *AndC = cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
16524 EVT LoadResultTy = AndC->getValueType(ResNo: 0);
16525 EVT ExtVT;
16526 if (isAndLoadExtLoad(AndC, LoadN: LN00, LoadResultTy, ExtVT))
16527 DoXform = false;
16528 }
16529 }
16530 if (DoXform)
16531 DoXform = ExtendUsesToFormExtLoad(VT, N: N0.getNode(), N0: N0.getOperand(i: 0),
16532 ExtOpc: ISD::ZERO_EXTEND, ExtendNodes&: SetCCs, TLI);
16533 if (DoXform) {
16534 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl: SDLoc(LN00), VT,
16535 Chain: LN00->getChain(), Ptr: LN00->getBasePtr(),
16536 MemVT: LN00->getMemoryVT(),
16537 MMO: LN00->getMemOperand());
16538 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
16539 SDValue And = DAG.getNode(Opcode: N0.getOpcode(), DL, VT,
16540 N1: ExtLoad, N2: DAG.getConstant(Val: Mask, DL, VT));
16541 ExtendSetCCUses(SetCCs, OrigLoad: N0.getOperand(i: 0), ExtLoad, ExtType: ISD::ZERO_EXTEND);
16542 bool NoReplaceTruncAnd = !N0.hasOneUse();
16543 bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
16544 CombineTo(N, Res: And);
16545 // If N0 has multiple uses, change other uses as well.
16546 if (NoReplaceTruncAnd) {
16547 SDValue TruncAnd =
16548 DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N0.getValueType(), Operand: And);
16549 CombineTo(N: N0.getNode(), Res: TruncAnd);
16550 }
16551 if (NoReplaceTrunc) {
16552 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN00, 1), To: ExtLoad.getValue(R: 1));
16553 } else {
16554 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(LN00),
16555 VT: LN00->getValueType(ResNo: 0), Operand: ExtLoad);
16556 CombineTo(N: LN00, Res0: Trunc, Res1: ExtLoad.getValue(R: 1));
16557 }
16558 return SDValue(N,0); // Return N so it doesn't get rechecked!
16559 }
16560 }
16561 }
16562
16563 // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
16564 // (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
16565 if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
16566 return ZExtLoad;
16567
16568 // Try to simplify (zext (zextload x)).
16569 if (SDValue foldedExt = tryToFoldExtOfExtload(
16570 DAG, Combiner&: *this, TLI, VT, LegalOperations, N, N0, ExtLoadType: ISD::ZEXTLOAD))
16571 return foldedExt;
16572
16573 if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
16574 return V;
16575
16576 if (N0.getOpcode() == ISD::SETCC) {
16577 // Propagate fast-math-flags.
16578 SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags());
16579
16580 // Only do this before legalize for now.
16581 if (!LegalOperations && VT.isVector() &&
16582 N0.getValueType().getVectorElementType() == MVT::i1) {
16583 EVT N00VT = N0.getOperand(i: 0).getValueType();
16584 if (getSetCCResultType(VT: N00VT) == N0.getValueType())
16585 return SDValue();
16586
16587 // We know that the # elements of the results is the same as the #
16588 // elements of the compare (and the # elements of the compare result for
16589 // that matter). Check to see that they are the same size. If so, we know
16590 // that the element size of the sext'd result matches the element size of
16591 // the compare operands.
16592 if (VT.getSizeInBits() == N00VT.getSizeInBits()) {
16593 // zext(setcc) -> zext_in_reg(vsetcc) for vectors.
16594 SDValue VSetCC = DAG.getNode(Opcode: ISD::SETCC, DL, VT, N1: N0.getOperand(i: 0),
16595 N2: N0.getOperand(i: 1), N3: N0.getOperand(i: 2));
16596 return DAG.getZeroExtendInReg(Op: VSetCC, DL, VT: N0.getValueType());
16597 }
16598
16599 // If the desired elements are smaller or larger than the source
16600 // elements we can use a matching integer vector type and then
16601 // truncate/any extend followed by zext_in_reg.
16602 EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
16603 SDValue VsetCC =
16604 DAG.getNode(Opcode: ISD::SETCC, DL, VT: MatchingVectorType, N1: N0.getOperand(i: 0),
16605 N2: N0.getOperand(i: 1), N3: N0.getOperand(i: 2));
16606 return DAG.getZeroExtendInReg(Op: DAG.getAnyExtOrTrunc(Op: VsetCC, DL, VT), DL,
16607 VT: N0.getValueType());
16608 }
16609
16610 // zext(setcc x,y,cc) -> zext(select x, y, true, false, cc)
16611 EVT N0VT = N0.getValueType();
16612 EVT N00VT = N0.getOperand(i: 0).getValueType();
16613 if (SDValue SCC = SimplifySelectCC(
16614 DL, N0: N0.getOperand(i: 0), N1: N0.getOperand(i: 1),
16615 N2: DAG.getBoolConstant(V: true, DL, VT: N0VT, OpVT: N00VT),
16616 N3: DAG.getBoolConstant(V: false, DL, VT: N0VT, OpVT: N00VT),
16617 CC: cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get(), NotExtCompare: true))
16618 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: SCC);
16619 }
16620
16621 // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
16622 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
16623 !TLI.isZExtFree(Val: N0, VT2: VT)) {
16624 SDValue ShVal = N0.getOperand(i: 0);
16625 SDValue ShAmt = N0.getOperand(i: 1);
16626 if (auto *ShAmtC = dyn_cast<ConstantSDNode>(Val&: ShAmt)) {
16627 if (ShVal.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse()) {
16628 if (N0.getOpcode() == ISD::SHL) {
16629 // If the original shl may be shifting out bits, do not perform this
16630 // transformation.
16631 unsigned KnownZeroBits = ShVal.getValueSizeInBits() -
16632 ShVal.getOperand(i: 0).getValueSizeInBits();
16633 if (ShAmtC->getAPIntValue().ugt(RHS: KnownZeroBits)) {
16634 // If the shift is too large, then see if we can deduce that the
16635 // shift is safe anyway.
16636
16637 // Check if the bits being shifted out are known to be zero.
16638 KnownBits KnownShVal = DAG.computeKnownBits(Op: ShVal);
16639 if (ShAmtC->getAPIntValue().ugt(RHS: KnownShVal.countMinLeadingZeros()))
16640 return SDValue();
16641 }
16642 }
16643
16644 // Ensure that the shift amount is wide enough for the shifted value.
16645 if (Log2_32_Ceil(Value: VT.getSizeInBits()) > ShAmt.getValueSizeInBits())
16646 ShAmt = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i32, Operand: ShAmt);
16647
16648 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT,
16649 N1: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: ShVal), N2: ShAmt);
16650 }
16651 }
16652 }
16653
16654 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(Cast: N))
16655 return NewVSel;
16656
16657 if (SDValue NewCtPop = widenCtPop(Extend: N, DAG, DL))
16658 return NewCtPop;
16659
16660 if (SDValue V = widenAbs(Extend: N, DAG))
16661 return V;
16662
16663 if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
16664 return Res;
16665
16666 // CSE zext nneg with sext if the zext is not free.
16667 if (N->getFlags().hasNonNeg() && !TLI.isZExtFree(FromTy: N0.getValueType(), ToTy: VT)) {
16668 SDNode *CSENode = DAG.getNodeIfExists(Opcode: ISD::SIGN_EXTEND, VTList: N->getVTList(), Ops: N0);
16669 if (CSENode)
16670 return SDValue(CSENode, 0);
16671 }
16672
16673 return SDValue();
16674}
16675
16676SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
16677 SDValue N0 = N->getOperand(Num: 0);
16678 EVT VT = N->getValueType(ResNo: 0);
16679 SDLoc DL(N);
16680
16681 // aext(undef) = undef
16682 if (N0.isUndef())
16683 return DAG.getUNDEF(VT);
16684
16685 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
16686 return Res;
16687
16688 // fold (aext (aext x)) -> (aext x)
16689 // fold (aext (zext x)) -> (zext x)
16690 // fold (aext (sext x)) -> (sext x)
16691 if (N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::ZERO_EXTEND ||
16692 N0.getOpcode() == ISD::SIGN_EXTEND) {
16693 SDNodeFlags Flags;
16694 if (N0.getOpcode() == ISD::ZERO_EXTEND)
16695 Flags.setNonNeg(N0->getFlags().hasNonNeg());
16696 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, Operand: N0.getOperand(i: 0), Flags);
16697 }
16698
16699 // fold (aext (aext_extend_vector_inreg x)) -> (aext_extend_vector_inreg x)
16700 // fold (aext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
16701 // fold (aext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
16702 if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
16703 N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
16704 N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG)
16705 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, Operand: N0.getOperand(i: 0));
16706
16707 // fold (aext (truncate (load x))) -> (aext (smaller load x))
16708 // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
16709 if (N0.getOpcode() == ISD::TRUNCATE) {
16710 if (SDValue NarrowLoad = reduceLoadWidth(N: N0.getNode())) {
16711 SDNode *oye = N0.getOperand(i: 0).getNode();
16712 if (NarrowLoad.getNode() != N0.getNode()) {
16713 CombineTo(N: N0.getNode(), Res: NarrowLoad);
16714 // CombineTo deleted the truncate, if needed, but not what's under it.
16715 AddToWorklist(N: oye);
16716 }
16717 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16718 }
16719 }
16720
16721 // fold (aext (truncate x))
16722 if (N0.getOpcode() == ISD::TRUNCATE)
16723 return DAG.getAnyExtOrTrunc(Op: N0.getOperand(i: 0), DL, VT);
16724
16725 // Fold (aext (and (trunc x), cst)) -> (and x, cst)
16726 // if either of the casts is not free, and sign-extending the narrow type is
16727 // not cheaper than zero-extending it (which would indicate the target prefers
16728 // to keep operations at the narrower width).
16729 // Also handles (aext (and (bitcast (extract_subvector vNi1, 0)) cst))
16730 // which arises on AVX512DQ where v8i1 extract replaces truncate.
16731 if (N0.getOpcode() == ISD::AND &&
16732 N0.getOperand(i: 1).getOpcode() == ISD::Constant) {
16733 SDValue AndSrc = N0.getOperand(i: 0);
16734 SDValue X;
16735 if (AndSrc.getOpcode() == ISD::TRUNCATE) {
16736 X = AndSrc.getOperand(i: 0);
16737 } else if (AndSrc.getOpcode() == ISD::BITCAST &&
16738 AndSrc.getOperand(i: 0).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
16739 AndSrc.getOperand(i: 0).getConstantOperandVal(i: 1) == 0) {
16740 SDValue Src = AndSrc.getOperand(i: 0).getOperand(i: 0);
16741 EVT SrcVT = Src.getValueType();
16742 if (SrcVT.isFixedLengthVectorOf(EltVT: MVT::i1)) {
16743 EVT WideIntVT =
16744 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SrcVT.getSizeInBits());
16745 if (TLI.isTypeLegal(VT: WideIntVT))
16746 X = DAG.getBitcast(VT: WideIntVT, V: Src);
16747 }
16748 }
16749 if (X && (!TLI.isTruncateFree(Val: X, VT2: N0.getValueType()) ||
16750 (!TLI.isZExtFree(FromTy: N0.getValueType(), ToTy: VT) &&
16751 !TLI.isSExtCheaperThanZExt(FromTy: N0.getValueType(), ToTy: VT)))) {
16752 X = DAG.getAnyExtOrTrunc(Op: X, DL, VT);
16753 APInt Mask = N0.getConstantOperandAPInt(i: 1).zext(width: VT.getSizeInBits());
16754 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: X, N2: DAG.getConstant(Val: Mask, DL, VT));
16755 }
16756 }
16757
16758 // fold (aext (load x)) -> (aext (truncate (extload x)))
16759 // None of the supported targets knows how to perform load and any_ext
16760 // on vectors in one instruction, so attempt to fold to zext instead.
16761 if (VT.isVector()) {
16762 // Try to simplify (zext (load x)).
16763 if (SDValue foldedExt =
16764 tryToFoldExtOfLoad(DAG, Combiner&: *this, TLI, VT, LegalOperations, N, N0,
16765 ExtLoadType: ISD::ZEXTLOAD, ExtOpc: ISD::ZERO_EXTEND))
16766 return foldedExt;
16767 } else if (ISD::isNON_EXTLoad(N: N0.getNode()) &&
16768 ISD::isUNINDEXEDLoad(N: N0.getNode())) {
16769 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
16770 if (TLI.isLoadLegalOrCustom(ValVT: VT, MemVT: N0.getValueType(), Alignment: LN0->getAlign(),
16771 AddrSpace: LN0->getAddressSpace(), ExtType: ISD::EXTLOAD, Atomic: false)) {
16772 bool DoXform = true;
16773 SmallVector<SDNode *, 4> SetCCs;
16774 if (!N0.hasOneUse())
16775 DoXform =
16776 ExtendUsesToFormExtLoad(VT, N, N0, ExtOpc: ISD::ANY_EXTEND, ExtendNodes&: SetCCs, TLI);
16777 if (DoXform) {
16778 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT, Chain: LN0->getChain(),
16779 Ptr: LN0->getBasePtr(), MemVT: N0.getValueType(),
16780 MMO: LN0->getMemOperand());
16781 ExtendSetCCUses(SetCCs, OrigLoad: N0, ExtLoad, ExtType: ISD::ANY_EXTEND);
16782 // If the load value is used only by N, replace it via CombineTo N.
16783 bool NoReplaceTrunc = N0.hasOneUse();
16784 CombineTo(N, Res: ExtLoad);
16785 if (NoReplaceTrunc) {
16786 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN0, 1), To: ExtLoad.getValue(R: 1));
16787 recursivelyDeleteUnusedNodes(N: LN0);
16788 } else {
16789 SDValue Trunc =
16790 DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N0), VT: N0.getValueType(), Operand: ExtLoad);
16791 CombineTo(N: LN0, Res0: Trunc, Res1: ExtLoad.getValue(R: 1));
16792 }
16793 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16794 }
16795 }
16796 }
16797
16798 // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
16799 // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
16800 // fold (aext ( extload x)) -> (aext (truncate (extload x)))
16801 if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N: N0.getNode()) &&
16802 ISD::isUNINDEXEDLoad(N: N0.getNode()) && N0.hasOneUse()) {
16803 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
16804 ISD::LoadExtType ExtType = LN0->getExtensionType();
16805 EVT MemVT = LN0->getMemoryVT();
16806 if (!LegalOperations ||
16807 TLI.isLoadLegal(ValVT: VT, MemVT, Alignment: LN0->getAlign(), AddrSpace: LN0->getAddressSpace(),
16808 ExtType, Atomic: false)) {
16809 SDValue ExtLoad =
16810 DAG.getExtLoad(ExtType, dl: DL, VT, Chain: LN0->getChain(), Ptr: LN0->getBasePtr(),
16811 MemVT, MMO: LN0->getMemOperand());
16812 CombineTo(N, Res: ExtLoad);
16813 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LN0, 1), To: ExtLoad.getValue(R: 1));
16814 recursivelyDeleteUnusedNodes(N: LN0);
16815 return SDValue(N, 0); // Return N so it doesn't get rechecked!
16816 }
16817 }
16818
16819 if (N0.getOpcode() == ISD::SETCC) {
16820 // Propagate fast-math-flags.
16821 SDNodeFlags Flags = N0->getFlags();
16822 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
16823
16824 // For vectors:
16825 // aext(setcc) -> vsetcc
16826 // aext(setcc) -> truncate(vsetcc)
16827 // aext(setcc) -> aext(vsetcc)
16828 // Only do this before legalize for now.
16829 if (VT.isVector() && !LegalOperations) {
16830 EVT N00VT = N0.getOperand(i: 0).getValueType();
16831 if (getSetCCResultType(VT: N00VT) == N0.getValueType())
16832 return SDValue();
16833
16834 // We know that the # elements of the results is the same as the
16835 // # elements of the compare (and the # elements of the compare result
16836 // for that matter). Check to see that they are the same size. If so,
16837 // we know that the element size of the sext'd result matches the
16838 // element size of the compare operands.
16839 if (VT.getSizeInBits() == N00VT.getSizeInBits())
16840 return DAG.getSetCC(DL, VT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1),
16841 Cond: cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get(),
16842 /*Chain=*/{}, /*Signaling=*/IsSignaling: false, Flags);
16843
16844 // If the desired elements are smaller or larger than the source
16845 // elements we can use a matching integer vector type and then
16846 // truncate/any extend
16847 EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
16848 SDValue VsetCC = DAG.getSetCC(
16849 DL, VT: MatchingVectorType, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1),
16850 Cond: cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get(), /*Chain=*/{},
16851 /*Signaling=*/IsSignaling: false, Flags);
16852 return DAG.getAnyExtOrTrunc(Op: VsetCC, DL, VT);
16853 }
16854
16855 // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
16856 if (SDValue SCC = SimplifySelectCC(
16857 DL, N0: N0.getOperand(i: 0), N1: N0.getOperand(i: 1), N2: DAG.getConstant(Val: 1, DL, VT),
16858 N3: DAG.getConstant(Val: 0, DL, VT),
16859 CC: cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get(), NotExtCompare: true))
16860 return SCC;
16861 }
16862
16863 if (SDValue NewCtPop = widenCtPop(Extend: N, DAG, DL))
16864 return NewCtPop;
16865
16866 if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
16867 return Res;
16868
16869 return SDValue();
16870}
16871
16872SDValue DAGCombiner::visitAssertExt(SDNode *N) {
16873 unsigned Opcode = N->getOpcode();
16874 SDValue N0 = N->getOperand(Num: 0);
16875 SDValue N1 = N->getOperand(Num: 1);
16876 EVT AssertVT = cast<VTSDNode>(Val&: N1)->getVT();
16877
16878 // fold (assert?ext (assert?ext x, vt), vt) -> (assert?ext x, vt)
16879 if (N0.getOpcode() == Opcode &&
16880 AssertVT == cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT())
16881 return N0;
16882
16883 // fold (assert?ext c, vt) -> c
16884 if (isa<ConstantSDNode>(Val: N0))
16885 return N0;
16886
16887 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
16888 N0.getOperand(i: 0).getOpcode() == Opcode) {
16889 // We have an assert, truncate, assert sandwich. Make one stronger assert
16890 // by asserting on the smallest asserted type to the larger source type.
16891 // This eliminates the later assert:
16892 // assert (trunc (assert X, i8) to iN), i1 --> trunc (assert X, i1) to iN
16893 // assert (trunc (assert X, i1) to iN), i8 --> trunc (assert X, i1) to iN
16894 SDLoc DL(N);
16895 SDValue BigA = N0.getOperand(i: 0);
16896 EVT BigA_AssertVT = cast<VTSDNode>(Val: BigA.getOperand(i: 1))->getVT();
16897 EVT MinAssertVT = AssertVT.bitsLT(VT: BigA_AssertVT) ? AssertVT : BigA_AssertVT;
16898 SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
16899 SDValue NewAssert = DAG.getNode(Opcode, DL, VT: BigA.getValueType(),
16900 N1: BigA.getOperand(i: 0), N2: MinAssertVTVal);
16901 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewAssert);
16902 }
16903
16904 // If we have (AssertZext (truncate (AssertSext X, iX)), iY) and Y is smaller
16905 // than X. Just move the AssertZext in front of the truncate and drop the
16906 // AssertSExt.
16907 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
16908 N0.getOperand(i: 0).getOpcode() == ISD::AssertSext &&
16909 Opcode == ISD::AssertZext) {
16910 SDValue BigA = N0.getOperand(i: 0);
16911 EVT BigA_AssertVT = cast<VTSDNode>(Val: BigA.getOperand(i: 1))->getVT();
16912 if (AssertVT.bitsLT(VT: BigA_AssertVT)) {
16913 SDLoc DL(N);
16914 SDValue NewAssert = DAG.getNode(Opcode, DL, VT: BigA.getValueType(),
16915 N1: BigA.getOperand(i: 0), N2: N1);
16916 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewAssert);
16917 }
16918 }
16919
16920 if (Opcode == ISD::AssertZext && N0.getOpcode() == ISD::AND &&
16921 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
16922 const APInt &Mask = N0.getConstantOperandAPInt(i: 1);
16923
16924 // If we have (AssertZext (and (AssertSext X, iX), M), iY) and Y is smaller
16925 // than X, and the And doesn't change the lower iX bits, we can move the
16926 // AssertZext in front of the And and drop the AssertSext.
16927 if (N0.getOperand(i: 0).getOpcode() == ISD::AssertSext && N0.hasOneUse()) {
16928 SDValue BigA = N0.getOperand(i: 0);
16929 EVT BigA_AssertVT = cast<VTSDNode>(Val: BigA.getOperand(i: 1))->getVT();
16930 if (AssertVT.bitsLT(VT: BigA_AssertVT) &&
16931 Mask.countr_one() >= BigA_AssertVT.getScalarSizeInBits()) {
16932 SDLoc DL(N);
16933 SDValue NewAssert =
16934 DAG.getNode(Opcode, DL, VT: N->getValueType(ResNo: 0), N1: BigA.getOperand(i: 0), N2: N1);
16935 return DAG.getNode(Opcode: ISD::AND, DL, VT: N->getValueType(ResNo: 0), N1: NewAssert,
16936 N2: N0.getOperand(i: 1));
16937 }
16938 }
16939
16940 // Remove AssertZext entirely if the mask guarantees the assertion cannot
16941 // fail.
16942 // TODO: Use KB countMinLeadingZeros to handle non-constant masks?
16943 if (Mask.isIntN(N: AssertVT.getScalarSizeInBits()))
16944 return N0;
16945 }
16946
16947 return SDValue();
16948}
16949
16950SDValue DAGCombiner::visitAssertAlign(SDNode *N) {
16951 SDLoc DL(N);
16952
16953 Align AL = cast<AssertAlignSDNode>(Val: N)->getAlign();
16954 SDValue N0 = N->getOperand(Num: 0);
16955
16956 // Fold (assertalign (assertalign x, AL0), AL1) ->
16957 // (assertalign x, max(AL0, AL1))
16958 if (auto *AAN = dyn_cast<AssertAlignSDNode>(Val&: N0))
16959 return DAG.getAssertAlign(DL, V: N0.getOperand(i: 0),
16960 A: std::max(a: AL, b: AAN->getAlign()));
16961
16962 // In rare cases, there are trivial arithmetic ops in source operands. Sink
16963 // this assert down to source operands so that those arithmetic ops could be
16964 // exposed to the DAG combining.
16965 switch (N0.getOpcode()) {
16966 default:
16967 break;
16968 case ISD::ADD:
16969 case ISD::PTRADD:
16970 case ISD::SUB: {
16971 unsigned AlignShift = Log2(A: AL);
16972 SDValue LHS = N0.getOperand(i: 0);
16973 SDValue RHS = N0.getOperand(i: 1);
16974 unsigned LHSAlignShift = DAG.computeKnownBits(Op: LHS).countMinTrailingZeros();
16975 unsigned RHSAlignShift = DAG.computeKnownBits(Op: RHS).countMinTrailingZeros();
16976 if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
16977 if (LHSAlignShift < AlignShift)
16978 LHS = DAG.getAssertAlign(DL, V: LHS, A: AL);
16979 if (RHSAlignShift < AlignShift)
16980 RHS = DAG.getAssertAlign(DL, V: RHS, A: AL);
16981 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT: N0.getValueType(), N1: LHS, N2: RHS);
16982 }
16983 break;
16984 }
16985 }
16986
16987 return SDValue();
16988}
16989
16990SDValue DAGCombiner::visitIS_FPCLASS(SDNode *N) {
16991 SDValue Src = N->getOperand(Num: 0);
16992 FPClassTest Mask = static_cast<FPClassTest>(N->getConstantOperandVal(Num: 1));
16993 EVT VT = N->getValueType(ResNo: 0);
16994 SDLoc DL(N);
16995
16996 // is.fpclass(poison, mask) -> poison
16997 if (Src.getOpcode() == ISD::POISON)
16998 return DAG.getPOISON(VT);
16999
17000 KnownFPClass Known = DAG.computeKnownFPClass(Op: Src, InterestedClasses: Mask);
17001
17002 // All possible classes are within the mask: result is always true.
17003 if ((~Mask & Known.getKnownFPClasses()) == fcNone)
17004 return DAG.getBoolConstant(V: true, DL, VT, OpVT: Src.getValueType());
17005
17006 // Clear test bits we know must be false from the source value.
17007 // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
17008 // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other
17009 if ((Mask & Known.getKnownFPClasses()) != Mask) {
17010 return DAG.getNode(
17011 Opcode: ISD::IS_FPCLASS, DL, VT, N1: Src,
17012 N2: DAG.getTargetConstant(Val: Mask & Known.getKnownFPClasses(), DL, VT: MVT::i32),
17013 Flags: N->getFlags());
17014 }
17015
17016 return SDValue();
17017}
17018
17019/// If the result of a load is shifted/masked/truncated to an effectively
17020/// narrower type, try to transform the load to a narrower type and/or
17021/// use an extending load.
17022SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
17023 unsigned Opc = N->getOpcode();
17024
17025 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
17026 SDValue N0 = N->getOperand(Num: 0);
17027 EVT VT = N->getValueType(ResNo: 0);
17028 EVT ExtVT = VT;
17029
17030 // This transformation isn't valid for vector loads.
17031 if (VT.isVector())
17032 return SDValue();
17033
17034 // The ShAmt variable is used to indicate that we've consumed a right
17035 // shift. I.e. we want to narrow the width of the load by skipping to load the
17036 // ShAmt least significant bits.
17037 unsigned ShAmt = 0;
17038 // A special case is when the least significant bits from the load are masked
17039 // away, but using an AND rather than a right shift. HasShiftedOffset is used
17040 // to indicate that the narrowed load should be left-shifted ShAmt bits to get
17041 // the result.
17042 unsigned ShiftedOffset = 0;
17043 // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
17044 // extended to VT.
17045 if (Opc == ISD::SIGN_EXTEND_INREG) {
17046 ExtType = ISD::SEXTLOAD;
17047 ExtVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
17048 } else if (Opc == ISD::SRL || Opc == ISD::SRA) {
17049 // Another special-case: SRL/SRA is basically zero/sign-extending a narrower
17050 // value, or it may be shifting a higher subword, half or byte into the
17051 // lowest bits.
17052
17053 // Only handle shift with constant shift amount, and the shiftee must be a
17054 // load.
17055 auto *LN = dyn_cast<LoadSDNode>(Val&: N0);
17056 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17057 if (!N1C || !LN)
17058 return SDValue();
17059 // If the shift amount is larger than the memory type then we're not
17060 // accessing any of the loaded bytes.
17061 ShAmt = N1C->getZExtValue();
17062 uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits();
17063 if (MemoryWidth <= ShAmt)
17064 return SDValue();
17065 // Attempt to fold away the SRL by using ZEXTLOAD and SRA by using SEXTLOAD.
17066 ExtType = Opc == ISD::SRL ? ISD::ZEXTLOAD : ISD::SEXTLOAD;
17067 ExtVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemoryWidth - ShAmt);
17068 // If original load is a SEXTLOAD then we can't simply replace it by a
17069 // ZEXTLOAD (we could potentially replace it by a more narrow SEXTLOAD
17070 // followed by a ZEXT, but that is not handled at the moment). Similarly if
17071 // the original load is a ZEXTLOAD and we want to use a SEXTLOAD.
17072 if ((LN->getExtensionType() == ISD::SEXTLOAD ||
17073 LN->getExtensionType() == ISD::ZEXTLOAD) &&
17074 LN->getExtensionType() != ExtType)
17075 return SDValue();
17076 } else if (Opc == ISD::AND) {
17077 // An AND with a constant mask is the same as a truncate + zero-extend.
17078 auto AndC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17079 if (!AndC)
17080 return SDValue();
17081
17082 const APInt &Mask = AndC->getAPIntValue();
17083 unsigned ActiveBits = 0;
17084 if (Mask.isMask()) {
17085 ActiveBits = Mask.countr_one();
17086 } else if (Mask.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: ActiveBits)) {
17087 ShiftedOffset = ShAmt;
17088 } else {
17089 return SDValue();
17090 }
17091
17092 ExtType = ISD::ZEXTLOAD;
17093 ExtVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ActiveBits);
17094 }
17095
17096 // In case Opc==SRL we've already prepared ExtVT/ExtType/ShAmt based on doing
17097 // a right shift. Here we redo some of those checks, to possibly adjust the
17098 // ExtVT even further based on "a masking AND". We could also end up here for
17099 // other reasons (e.g. based on Opc==TRUNCATE) and that is why some checks
17100 // need to be done here as well.
17101 if (Opc == ISD::SRL || N0.getOpcode() == ISD::SRL) {
17102 SDValue SRL = Opc == ISD::SRL ? SDValue(N, 0) : N0;
17103 // Bail out when the SRL has more than one use. This is done for historical
17104 // (undocumented) reasons. Maybe intent was to guard the AND-masking below
17105 // check below? And maybe it could be non-profitable to do the transform in
17106 // case the SRL has multiple uses and we get here with Opc!=ISD::SRL?
17107 // FIXME: Can't we just skip this check for the Opc==ISD::SRL case.
17108 if (!SRL.hasOneUse())
17109 return SDValue();
17110
17111 // Only handle shift with constant shift amount, and the shiftee must be a
17112 // load.
17113 auto *LN = dyn_cast<LoadSDNode>(Val: SRL.getOperand(i: 0));
17114 auto *SRL1C = dyn_cast<ConstantSDNode>(Val: SRL.getOperand(i: 1));
17115 if (!SRL1C || !LN)
17116 return SDValue();
17117
17118 // If the shift amount is larger than the input type then we're not
17119 // accessing any of the loaded bytes. If the load was a zextload/extload
17120 // then the result of the shift+trunc is zero/undef (handled elsewhere).
17121 ShAmt = SRL1C->getZExtValue();
17122 uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits();
17123 if (ShAmt >= MemoryWidth)
17124 return SDValue();
17125
17126 // Because a SRL must be assumed to *need* to zero-extend the high bits
17127 // (as opposed to anyext the high bits), we can't combine the zextload
17128 // lowering of SRL and an sextload.
17129 if (LN->getExtensionType() == ISD::SEXTLOAD)
17130 return SDValue();
17131
17132 // Avoid reading outside the memory accessed by the original load (could
17133 // happened if we only adjust the load base pointer by ShAmt). Instead we
17134 // try to narrow the load even further. The typical scenario here is:
17135 // (i64 (truncate (i96 (srl (load x), 64)))) ->
17136 // (i64 (truncate (i96 (zextload (load i32 + offset) from i32))))
17137 if (ExtVT.getScalarSizeInBits() > MemoryWidth - ShAmt) {
17138 // Don't replace sextload by zextload.
17139 if (ExtType == ISD::SEXTLOAD)
17140 return SDValue();
17141 // Narrow the load.
17142 ExtType = ISD::ZEXTLOAD;
17143 ExtVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemoryWidth - ShAmt);
17144 }
17145
17146 // If the SRL is only used by a masking AND, we may be able to adjust
17147 // the ExtVT to make the AND redundant.
17148 SDNode *Mask = *(SRL->user_begin());
17149 if (SRL.hasOneUse() && Mask->getOpcode() == ISD::AND &&
17150 isa<ConstantSDNode>(Val: Mask->getOperand(Num: 1))) {
17151 unsigned Offset, ActiveBits;
17152 const APInt& ShiftMask = Mask->getConstantOperandAPInt(Num: 1);
17153 if (ShiftMask.isMask()) {
17154 EVT MaskedVT =
17155 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ShiftMask.countr_one());
17156 // If the mask is smaller, recompute the type.
17157 if ((ExtVT.getScalarSizeInBits() > MaskedVT.getScalarSizeInBits()) &&
17158 TLI.isLoadLegal(ValVT: SRL.getValueType(), MemVT: MaskedVT, Alignment: LN->getAlign(),
17159 AddrSpace: LN->getAddressSpace(), ExtType, Atomic: false))
17160 ExtVT = MaskedVT;
17161 } else if (ExtType == ISD::ZEXTLOAD &&
17162 ShiftMask.isShiftedMask(MaskIdx&: Offset, MaskLen&: ActiveBits) &&
17163 (Offset + ShAmt) < VT.getScalarSizeInBits()) {
17164 EVT MaskedVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ActiveBits);
17165 // If the mask is shifted we can use a narrower load and a shl to insert
17166 // the trailing zeros.
17167 if (((Offset + ActiveBits) <= ExtVT.getScalarSizeInBits()) &&
17168 TLI.isLoadLegal(ValVT: SRL.getValueType(), MemVT: MaskedVT, Alignment: LN->getAlign(),
17169 AddrSpace: LN->getAddressSpace(), ExtType, Atomic: false)) {
17170 ExtVT = MaskedVT;
17171 ShAmt = Offset + ShAmt;
17172 ShiftedOffset = Offset;
17173 }
17174 }
17175 }
17176
17177 N0 = SRL.getOperand(i: 0);
17178 }
17179
17180 // If the load is shifted left (and the result isn't shifted back right), we
17181 // can fold a truncate through the shift. The typical scenario is that N
17182 // points at a TRUNCATE here so the attempted fold is:
17183 // (truncate (shl (load x), c))) -> (shl (narrow load x), c)
17184 // ShLeftAmt will indicate how much a narrowed load should be shifted left.
17185 unsigned ShLeftAmt = 0;
17186 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
17187 ExtVT == VT && TLI.isNarrowingProfitable(N, SrcVT: N0.getValueType(), DestVT: VT)) {
17188 if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
17189 ShLeftAmt = N01->getZExtValue();
17190 N0 = N0.getOperand(i: 0);
17191 }
17192 }
17193
17194 // Look through a freeze if present between the operation and the load.
17195 // The freeze will be preserved on the narrowed result.
17196 SDValue FreezeNode;
17197 if (N0.getOpcode() == ISD::FREEZE) {
17198 FreezeNode = N0;
17199 N0 = N0.getOperand(i: 0);
17200 }
17201
17202 // If we haven't found a load, we can't narrow it.
17203 if (!isa<LoadSDNode>(Val: N0))
17204 return SDValue();
17205
17206 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
17207 // Reducing the width of a volatile load is illegal. For atomics, we may be
17208 // able to reduce the width provided we never widen again. (see D66309)
17209 if (!LN0->isSimple() ||
17210 !isLegalNarrowLdSt(LDST: LN0, ExtType, MemVT&: ExtVT, ShAmt))
17211 return SDValue();
17212
17213 // Bail early when looking through a multi-use freeze, since other users of
17214 // the freeze can depend on the full load value. But its still safe to change
17215 // the extension type from anyext to zext.
17216 if (FreezeNode && !FreezeNode.hasOneUse() &&
17217 (LN0->getMemoryVT().bitsGT(VT: ExtVT) || ExtType != ISD::ZEXTLOAD ||
17218 (LN0->getExtensionType() != ISD::EXTLOAD &&
17219 LN0->getExtensionType() != ISD::ZEXTLOAD)))
17220 return SDValue();
17221
17222 auto AdjustBigEndianShift = [&](unsigned ShAmt) {
17223 unsigned LVTStoreBits =
17224 LN0->getMemoryVT().getStoreSizeInBits().getFixedValue();
17225 unsigned EVTStoreBits = ExtVT.getStoreSizeInBits().getFixedValue();
17226 return LVTStoreBits - EVTStoreBits - ShAmt;
17227 };
17228
17229 // We need to adjust the pointer to the load by ShAmt bits in order to load
17230 // the correct bytes.
17231 unsigned PtrAdjustmentInBits =
17232 DAG.getDataLayout().isBigEndian() ? AdjustBigEndianShift(ShAmt) : ShAmt;
17233
17234 uint64_t PtrOff = PtrAdjustmentInBits / 8;
17235 SDLoc DL(LN0);
17236 // The original load itself didn't wrap, so an offset within it doesn't.
17237 SDValue NewPtr =
17238 DAG.getMemBasePlusOffset(Base: LN0->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: PtrOff),
17239 DL, Flags: SDNodeFlags::NoUnsignedWrap);
17240 AddToWorklist(N: NewPtr.getNode());
17241
17242 SDValue Load;
17243 if (ExtType == ISD::NON_EXTLOAD) {
17244 const MDNode *OldRanges = LN0->getRanges();
17245 const MDNode *NewRanges = nullptr;
17246 // If LSBs are loaded and the truncated ConstantRange for the OldRanges
17247 // metadata is not the full-set for the new width then create a NewRanges
17248 // metadata for the truncated load
17249 if (ShAmt == 0 && OldRanges) {
17250 ConstantRange CR = getConstantRangeFromMetadata(RangeMD: *OldRanges);
17251 unsigned BitSize = VT.getScalarSizeInBits();
17252
17253 // It is possible for an 8-bit extending load with 8-bit range
17254 // metadata to be narrowed to an 8-bit load. This guard is necessary to
17255 // ensure that truncation is strictly smaller.
17256 if (CR.getBitWidth() > BitSize) {
17257 ConstantRange TruncatedCR = CR.truncate(BitWidth: BitSize);
17258 if (!TruncatedCR.isFullSet()) {
17259 Metadata *Bounds[2] = {
17260 ConstantAsMetadata::get(
17261 C: ConstantInt::get(Context&: *DAG.getContext(), V: TruncatedCR.getLower())),
17262 ConstantAsMetadata::get(
17263 C: ConstantInt::get(Context&: *DAG.getContext(), V: TruncatedCR.getUpper()))};
17264 NewRanges = MDNode::get(Context&: *DAG.getContext(), MDs: Bounds);
17265 }
17266 } else if (CR.getBitWidth() == BitSize)
17267 NewRanges = OldRanges;
17268 }
17269 Load = DAG.getLoad(VT, dl: DL, Chain: LN0->getChain(), Ptr: NewPtr,
17270 PtrInfo: LN0->getPointerInfo().getWithOffset(O: PtrOff),
17271 Alignment: LN0->getBaseAlign(), MMOFlags: LN0->getMemOperand()->getFlags(),
17272 Metadata: MMOMetadata(LN0->getAAInfo(), NewRanges));
17273 } else
17274 Load = DAG.getExtLoad(ExtType, dl: DL, VT, Chain: LN0->getChain(), Ptr: NewPtr,
17275 PtrInfo: LN0->getPointerInfo().getWithOffset(O: PtrOff), MemVT: ExtVT,
17276 Alignment: LN0->getBaseAlign(), MMOFlags: LN0->getMemOperand()->getFlags(),
17277 Metadata: LN0->getAAInfo());
17278
17279 // Replace the old load's chain with the new load's chain.
17280 WorklistRemover DeadNodes(*this);
17281 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: Load.getValue(R: 1));
17282
17283 // Replace old load value for multi-use freeze so all users benefit.
17284 if (FreezeNode && !FreezeNode.hasOneUse())
17285 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 0), To: Load.getValue(R: 0));
17286
17287 // If we looked through a freeze, rewrap the narrowed result and add an
17288 // Assert node so downstream analyses can see the range.
17289 SDValue Result = Load;
17290 if (FreezeNode) {
17291 Result = DAG.getNode(Opcode: ISD::FREEZE, DL, VT, Operand: Result);
17292 if (ExtType == ISD::ZEXTLOAD)
17293 Result =
17294 DAG.getNode(Opcode: ISD::AssertZext, DL, VT, N1: Result, N2: DAG.getValueType(ExtVT));
17295 else if (ExtType == ISD::SEXTLOAD)
17296 Result =
17297 DAG.getNode(Opcode: ISD::AssertSext, DL, VT, N1: Result, N2: DAG.getValueType(ExtVT));
17298 }
17299
17300 // Shift the result left, if we've swallowed a left shift.
17301 if (ShLeftAmt != 0) {
17302 // If the shift amount is as large as the result size (but, presumably,
17303 // no larger than the source) then the useful bits of the result are
17304 // zero; we can't simply return the shortened shift, because the result
17305 // of that operation is undefined.
17306 if (ShLeftAmt >= VT.getScalarSizeInBits())
17307 Result = DAG.getConstant(Val: 0, DL, VT);
17308 else
17309 Result = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Result,
17310 N2: DAG.getShiftAmountConstant(Val: ShLeftAmt, VT, DL));
17311 }
17312
17313 if (ShiftedOffset != 0) {
17314 // We're using a shifted mask, so the load now has an offset. This means
17315 // that data has been loaded into the lower bytes than it would have been
17316 // before, so we need to shl the loaded data into the correct position in the
17317 // register.
17318 SDValue ShiftC = DAG.getConstant(Val: ShiftedOffset, DL, VT);
17319 Result = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Result, N2: ShiftC);
17320 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result);
17321 }
17322
17323 // Return the new loaded value.
17324 return Result;
17325}
17326
17327SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
17328 SDValue N0 = N->getOperand(Num: 0);
17329 SDValue N1 = N->getOperand(Num: 1);
17330 EVT VT = N->getValueType(ResNo: 0);
17331 EVT ExtVT = cast<VTSDNode>(Val&: N1)->getVT();
17332 unsigned VTBits = VT.getScalarSizeInBits();
17333 unsigned ExtVTBits = ExtVT.getScalarSizeInBits();
17334 SDLoc DL(N);
17335
17336 // sext_vector_inreg(undef) = 0 because the top bit will all be the same.
17337 if (N0.isUndef())
17338 return DAG.getConstant(Val: 0, DL, VT);
17339
17340 // fold (sext_in_reg c1) -> c1
17341 if (SDValue C =
17342 DAG.FoldConstantArithmetic(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, Ops: {N0, N1}))
17343 return C;
17344
17345 // If the input is already sign extended, just drop the extension.
17346 if (ExtVTBits >= DAG.ComputeMaxSignificantBits(Op: N0))
17347 return N0;
17348
17349 // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
17350 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
17351 ExtVT.bitsLT(VT: cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT()))
17352 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: N0.getOperand(i: 0), N2: N1);
17353
17354 // fold (sext_in_reg (sext x)) -> (sext x)
17355 // fold (sext_in_reg (aext x)) -> (sext x)
17356 // if x is small enough or if we know that x has more than 1 sign bit and the
17357 // sign_extend_inreg is extending from one of them.
17358 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
17359 SDValue N00 = N0.getOperand(i: 0);
17360 unsigned N00Bits = N00.getScalarValueSizeInBits();
17361 if ((N00Bits <= ExtVTBits ||
17362 DAG.ComputeMaxSignificantBits(Op: N00) <= ExtVTBits) &&
17363 (!LegalOperations || TLI.isOperationLegal(Op: ISD::SIGN_EXTEND, VT)))
17364 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: N00);
17365 }
17366
17367 // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
17368 // if x is small enough or if we know that x has more than 1 sign bit and the
17369 // sign_extend_inreg is extending from one of them.
17370 if (ISD::isExtVecInRegOpcode(Opcode: N0.getOpcode())) {
17371 SDValue N00 = N0.getOperand(i: 0);
17372 unsigned N00Bits = N00.getScalarValueSizeInBits();
17373 bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
17374 if ((N00Bits == ExtVTBits ||
17375 (!IsZext && (N00Bits < ExtVTBits ||
17376 DAG.ComputeMaxSignificantBits(Op: N00) <= ExtVTBits))) &&
17377 (!LegalOperations ||
17378 TLI.isOperationLegal(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT)))
17379 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, Operand: N00);
17380 }
17381
17382 // fold (sext_in_reg (zext x)) -> (sext x)
17383 // iff we are extending the source sign bit.
17384 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
17385 SDValue N00 = N0.getOperand(i: 0);
17386 if (N00.getScalarValueSizeInBits() == ExtVTBits &&
17387 (!LegalOperations || TLI.isOperationLegal(Op: ISD::SIGN_EXTEND, VT)))
17388 return DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT, Operand: N00);
17389 }
17390
17391 // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
17392 if (DAG.MaskedValueIsZero(Op: N0, Mask: APInt::getOneBitSet(numBits: VTBits, BitNo: ExtVTBits - 1)))
17393 return DAG.getZeroExtendInReg(Op: N0, DL, VT: ExtVT);
17394
17395 // fold operands of sext_in_reg based on knowledge that the top bits are not
17396 // demanded.
17397 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
17398 return SDValue(N, 0);
17399
17400 // fold (sext_in_reg (load x)) -> (smaller sextload x)
17401 // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
17402 if (SDValue NarrowLoad = reduceLoadWidth(N))
17403 return NarrowLoad;
17404
17405 // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
17406 // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
17407 // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
17408 if (N0.getOpcode() == ISD::SRL) {
17409 if (auto *ShAmt = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1)))
17410 if (ShAmt->getAPIntValue().ule(RHS: VTBits - ExtVTBits)) {
17411 // We can turn this into an SRA iff the input to the SRL is already sign
17412 // extended enough.
17413 unsigned InSignBits = DAG.ComputeNumSignBits(Op: N0.getOperand(i: 0));
17414 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
17415 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N0.getOperand(i: 0),
17416 N2: N0.getOperand(i: 1));
17417 }
17418 }
17419
17420 // fold (sext_inreg (extload x)) -> (sextload x)
17421 // If sextload is not supported by target, we can only do the combine when
17422 // load has one use. Doing otherwise can block folding the extload with other
17423 // extends that the target does support.
17424 if (ISD::isEXTLoad(N: N0.getNode()) && ISD::isUNINDEXEDLoad(N: N0.getNode())) {
17425 auto *LN0 = cast<LoadSDNode>(Val&: N0);
17426 if (ExtVT == LN0->getMemoryVT() &&
17427 ((!LegalOperations && LN0->isSimple() && N0.hasOneUse()) ||
17428 TLI.isLoadLegal(ValVT: VT, MemVT: ExtVT, Alignment: LN0->getAlign(), AddrSpace: LN0->getAddressSpace(),
17429 ExtType: ISD::SEXTLOAD, Atomic: false))) {
17430 SDValue ExtLoad =
17431 DAG.getExtLoad(ExtType: ISD::SEXTLOAD, dl: DL, VT, Chain: LN0->getChain(),
17432 Ptr: LN0->getBasePtr(), MemVT: ExtVT, MMO: LN0->getMemOperand());
17433 CombineTo(N, Res: ExtLoad);
17434 CombineTo(N: N0.getNode(), Res0: ExtLoad, Res1: ExtLoad.getValue(R: 1));
17435 AddToWorklist(N: ExtLoad.getNode());
17436 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17437 }
17438 }
17439
17440 // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
17441 if (ISD::isZEXTLoad(N: N0.getNode()) && ISD::isUNINDEXEDLoad(N: N0.getNode())) {
17442 auto *LN0 = cast<LoadSDNode>(Val&: N0);
17443
17444 if (N0.hasOneUse() && ExtVT == LN0->getMemoryVT() &&
17445 ((!LegalOperations && LN0->isSimple()) &&
17446 TLI.isLoadLegal(ValVT: VT, MemVT: ExtVT, Alignment: LN0->getAlign(), AddrSpace: LN0->getAddressSpace(),
17447 ExtType: ISD::SEXTLOAD, Atomic: false))) {
17448 SDValue ExtLoad =
17449 DAG.getExtLoad(ExtType: ISD::SEXTLOAD, dl: DL, VT, Chain: LN0->getChain(),
17450 Ptr: LN0->getBasePtr(), MemVT: ExtVT, MMO: LN0->getMemOperand());
17451 CombineTo(N, Res: ExtLoad);
17452 CombineTo(N: N0.getNode(), Res0: ExtLoad, Res1: ExtLoad.getValue(R: 1));
17453 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17454 }
17455 }
17456
17457 // fold (sext_inreg (masked_load x)) -> (sext_masked_load x)
17458 // ignore it if the masked load is already sign extended
17459 bool Frozen = N0.getOpcode() == ISD::FREEZE && N0.hasOneUse();
17460 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Val: Frozen ? N0.getOperand(i: 0) : N0)) {
17461 if (ExtVT == Ld->getMemoryVT() && Ld->hasNUsesOfValue(NUses: 1, Value: 0) &&
17462 Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD &&
17463 TLI.isLoadLegal(ValVT: VT, MemVT: ExtVT, Alignment: Ld->getAlign(), AddrSpace: Ld->getAddressSpace(),
17464 ExtType: ISD::SEXTLOAD, Atomic: false)) {
17465 SDValue ExtMaskedLoad = DAG.getMaskedLoad(
17466 VT, dl: DL, Chain: Ld->getChain(), Base: Ld->getBasePtr(), Offset: Ld->getOffset(),
17467 Mask: Ld->getMask(), Src0: Ld->getPassThru(), MemVT: ExtVT, MMO: Ld->getMemOperand(),
17468 AM: Ld->getAddressingMode(), ISD::SEXTLOAD, IsExpanding: Ld->isExpandingLoad());
17469 CombineTo(N, Res: Frozen ? N0 : ExtMaskedLoad);
17470 CombineTo(N: Ld, Res0: ExtMaskedLoad, Res1: ExtMaskedLoad.getValue(R: 1));
17471 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17472 }
17473 }
17474
17475 // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
17476 if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(Val&: N0)) {
17477 if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
17478 TLI.isVectorLoadExtDesirable(ExtVal: SDValue(N, 0))) {
17479 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
17480 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
17481
17482 SDValue ExtLoad = DAG.getMaskedGather(
17483 VTs: DAG.getVTList(VT1: VT, VT2: MVT::Other), MemVT: ExtVT, dl: DL, Ops, MMO: GN0->getMemOperand(),
17484 IndexType: GN0->getIndexType(), ExtTy: ISD::SEXTLOAD);
17485
17486 CombineTo(N, Res: ExtLoad);
17487 CombineTo(N: N0.getNode(), Res0: ExtLoad, Res1: ExtLoad.getValue(R: 1));
17488 AddToWorklist(N: ExtLoad.getNode());
17489 return SDValue(N, 0); // Return N so it doesn't get rechecked!
17490 }
17491 }
17492
17493 // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
17494 if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) {
17495 if (SDValue BSwap = MatchBSwapHWordLow(N: N0.getNode(), N0: N0.getOperand(i: 0),
17496 N1: N0.getOperand(i: 1), DemandHighBits: false))
17497 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: BSwap, N2: N1);
17498 }
17499
17500 // Fold (iM_signext_inreg
17501 // (extract_subvector (zext|anyext|sext iN_v to _) _)
17502 // from iN)
17503 // -> (extract_subvector (signext iN_v to iM))
17504 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() &&
17505 ISD::isExtOpcode(Opcode: N0.getOperand(i: 0).getOpcode())) {
17506 SDValue InnerExt = N0.getOperand(i: 0);
17507 EVT InnerExtVT = InnerExt->getValueType(ResNo: 0);
17508 SDValue Extendee = InnerExt->getOperand(Num: 0);
17509
17510 if (ExtVTBits == Extendee.getValueType().getScalarSizeInBits() &&
17511 (!LegalOperations ||
17512 TLI.isOperationLegal(Op: ISD::SIGN_EXTEND, VT: InnerExtVT))) {
17513 SDValue SignExtExtendee =
17514 DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: InnerExtVT, Operand: Extendee);
17515 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: SignExtExtendee,
17516 N2: N0.getOperand(i: 1));
17517 }
17518 }
17519
17520 return SDValue();
17521}
17522
17523static SDValue foldExtendVectorInregToExtendOfSubvector(
17524 SDNode *N, const SDLoc &DL, const TargetLowering &TLI, SelectionDAG &DAG,
17525 bool LegalOperations) {
17526 unsigned InregOpcode = N->getOpcode();
17527 unsigned Opcode = DAG.getOpcode_EXTEND(Opcode: InregOpcode);
17528
17529 SDValue Src = N->getOperand(Num: 0);
17530 EVT VT = N->getValueType(ResNo: 0);
17531 EVT SrcVT = VT.changeVectorElementType(
17532 Context&: *DAG.getContext(), EltVT: Src.getValueType().getVectorElementType());
17533
17534 assert(ISD::isExtVecInRegOpcode(InregOpcode) &&
17535 "Expected EXTEND_VECTOR_INREG dag node in input!");
17536
17537 // Profitability check: our operand must be an one-use CONCAT_VECTORS.
17538 // FIXME: one-use check may be overly restrictive
17539 if (!Src.hasOneUse() || Src.getOpcode() != ISD::CONCAT_VECTORS)
17540 return SDValue();
17541
17542 // Profitability check: we must be extending exactly one of it's operands.
17543 // FIXME: this is probably overly restrictive.
17544 Src = Src.getOperand(i: 0);
17545 if (Src.getValueType() != SrcVT)
17546 return SDValue();
17547
17548 if (LegalOperations && !TLI.isOperationLegal(Op: Opcode, VT))
17549 return SDValue();
17550
17551 return DAG.getNode(Opcode, DL, VT, Operand: Src);
17552}
17553
17554SDValue DAGCombiner::visitEXTEND_VECTOR_INREG(SDNode *N) {
17555 SDValue N0 = N->getOperand(Num: 0);
17556 EVT VT = N->getValueType(ResNo: 0);
17557 SDLoc DL(N);
17558
17559 if (N0.isUndef()) {
17560 // aext_vector_inreg(undef) = undef because the top bits are undefined.
17561 // {s/z}ext_vector_inreg(undef) = 0 because the top bits must be the same.
17562 return N->getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG
17563 ? DAG.getUNDEF(VT)
17564 : DAG.getConstant(Val: 0, DL, VT);
17565 }
17566
17567 if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
17568 return Res;
17569
17570 if (SimplifyDemandedVectorElts(Op: SDValue(N, 0)))
17571 return SDValue(N, 0);
17572
17573 if (SDValue R = foldExtendVectorInregToExtendOfSubvector(N, DL, TLI, DAG,
17574 LegalOperations))
17575 return R;
17576
17577 return SDValue();
17578}
17579
17580SDValue DAGCombiner::visitTRUNCATE_USAT_U(SDNode *N) {
17581 EVT VT = N->getValueType(ResNo: 0);
17582 SDValue N0 = N->getOperand(Num: 0);
17583
17584 SDValue FPVal;
17585 if (sd_match(N: N0, P: m_FPToUI(Op: m_Value(N&: FPVal))) &&
17586 DAG.getTargetLoweringInfo().shouldConvertFpToSat(
17587 Op: ISD::FP_TO_UINT_SAT, FPVT: FPVal.getValueType(), VT))
17588 return DAG.getNode(Opcode: ISD::FP_TO_UINT_SAT, DL: SDLoc(N0), VT, N1: FPVal,
17589 N2: DAG.getValueType(VT.getScalarType()));
17590
17591 return SDValue();
17592}
17593
17594/// Detect patterns of truncation with unsigned saturation:
17595///
17596/// (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
17597/// Return the source value x to be truncated or SDValue() if the pattern was
17598/// not matched.
17599///
17600static SDValue detectUSatUPattern(SDValue In, EVT VT) {
17601 unsigned NumDstBits = VT.getScalarSizeInBits();
17602 unsigned NumSrcBits = In.getScalarValueSizeInBits();
17603 // Saturation with truncation. We truncate from InVT to VT.
17604 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
17605
17606 SDValue Min;
17607 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
17608 if (sd_match(N: In, P: m_UMin(L: m_Value(N&: Min), R: m_SpecificInt(V: UnsignedMax))))
17609 return Min;
17610
17611 return SDValue();
17612}
17613
17614/// Detect patterns of truncation with signed saturation:
17615/// (truncate (smin (smax (x, signed_min_of_dest_type),
17616/// signed_max_of_dest_type)) to dest_type)
17617/// or:
17618/// (truncate (smax (smin (x, signed_max_of_dest_type),
17619/// signed_min_of_dest_type)) to dest_type).
17620///
17621/// Return the source value to be truncated or SDValue() if the pattern was not
17622/// matched.
17623static SDValue detectSSatSPattern(SDValue In, EVT VT) {
17624 unsigned NumDstBits = VT.getScalarSizeInBits();
17625 unsigned NumSrcBits = In.getScalarValueSizeInBits();
17626 // Saturation with truncation. We truncate from InVT to VT.
17627 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
17628
17629 SDValue Val;
17630 APInt SignedMax = APInt::getSignedMaxValue(numBits: NumDstBits).sext(width: NumSrcBits);
17631 APInt SignedMin = APInt::getSignedMinValue(numBits: NumDstBits).sext(width: NumSrcBits);
17632
17633 if (sd_match(N: In, P: m_SMin(L: m_SMax(L: m_Value(N&: Val), R: m_SpecificInt(V: SignedMin)),
17634 R: m_SpecificInt(V: SignedMax))))
17635 return Val;
17636
17637 if (sd_match(N: In, P: m_SMax(L: m_SMin(L: m_Value(N&: Val), R: m_SpecificInt(V: SignedMax)),
17638 R: m_SpecificInt(V: SignedMin))))
17639 return Val;
17640
17641 return SDValue();
17642}
17643
17644/// Detect patterns of truncation with unsigned saturation:
17645static SDValue detectSSatUPattern(SDValue In, EVT VT, SelectionDAG &DAG,
17646 const SDLoc &DL) {
17647 unsigned NumDstBits = VT.getScalarSizeInBits();
17648 unsigned NumSrcBits = In.getScalarValueSizeInBits();
17649 // Saturation with truncation. We truncate from InVT to VT.
17650 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
17651
17652 SDValue Val;
17653 APInt UnsignedMax = APInt::getMaxValue(numBits: NumDstBits).zext(width: NumSrcBits);
17654 // Min == 0, Max is unsigned max of destination type.
17655 if (sd_match(N: In, P: m_SMax(L: m_SMin(L: m_Value(N&: Val), R: m_SpecificInt(V: UnsignedMax)),
17656 R: m_Zero())))
17657 return Val;
17658
17659 if (sd_match(N: In, P: m_SMin(L: m_SMax(L: m_Value(N&: Val), R: m_Zero()),
17660 R: m_SpecificInt(V: UnsignedMax))))
17661 return Val;
17662
17663 if (sd_match(N: In, P: m_UMin(L: m_SMax(L: m_Value(N&: Val), R: m_Zero()),
17664 R: m_SpecificInt(V: UnsignedMax))))
17665 return Val;
17666
17667 return SDValue();
17668}
17669
17670static SDValue foldToSaturated(SDNode *N, EVT &VT, SDValue &Src, EVT &SrcVT,
17671 SDLoc &DL, const TargetLowering &TLI,
17672 SelectionDAG &DAG) {
17673 auto AllowedTruncateSat = [&](unsigned Opc, EVT SrcVT, EVT VT) -> bool {
17674 return (TLI.isOperationLegalOrCustom(Op: Opc, VT: SrcVT) &&
17675 TLI.isTypeDesirableForOp(Opc, VT));
17676 };
17677
17678 if (Src.getOpcode() == ISD::SMIN || Src.getOpcode() == ISD::SMAX) {
17679 if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_S, SrcVT, VT))
17680 if (SDValue SSatVal = detectSSatSPattern(In: Src, VT))
17681 return DAG.getNode(Opcode: ISD::TRUNCATE_SSAT_S, DL, VT, Operand: SSatVal);
17682 if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_U, SrcVT, VT))
17683 if (SDValue SSatVal = detectSSatUPattern(In: Src, VT, DAG, DL))
17684 return DAG.getNode(Opcode: ISD::TRUNCATE_SSAT_U, DL, VT, Operand: SSatVal);
17685 } else if (Src.getOpcode() == ISD::UMIN) {
17686 if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_U, SrcVT, VT))
17687 if (SDValue SSatVal = detectSSatUPattern(In: Src, VT, DAG, DL))
17688 return DAG.getNode(Opcode: ISD::TRUNCATE_SSAT_U, DL, VT, Operand: SSatVal);
17689 if (AllowedTruncateSat(ISD::TRUNCATE_USAT_U, SrcVT, VT))
17690 if (SDValue USatVal = detectUSatUPattern(In: Src, VT))
17691 return DAG.getNode(Opcode: ISD::TRUNCATE_USAT_U, DL, VT, Operand: USatVal);
17692 }
17693
17694 return SDValue();
17695}
17696
17697SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
17698 SDValue N0 = N->getOperand(Num: 0);
17699 EVT VT = N->getValueType(ResNo: 0);
17700 EVT SrcVT = N0.getValueType();
17701 bool isLE = DAG.getDataLayout().isLittleEndian();
17702 SDLoc DL(N);
17703
17704 // trunc(undef) = undef
17705 if (N0.isUndef())
17706 return DAG.getUNDEF(VT);
17707
17708 // fold (truncate (truncate x)) -> (truncate x)
17709 if (N0.getOpcode() == ISD::TRUNCATE)
17710 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0.getOperand(i: 0));
17711
17712 // fold saturated truncate
17713 if (SDValue SaturatedTR = foldToSaturated(N, VT, Src&: N0, SrcVT, DL, TLI, DAG))
17714 return SaturatedTR;
17715
17716 // fold (truncate c1) -> c1
17717 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::TRUNCATE, DL, VT, Ops: {N0}))
17718 return C;
17719
17720 // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
17721 if (N0.getOpcode() == ISD::ZERO_EXTEND ||
17722 N0.getOpcode() == ISD::SIGN_EXTEND ||
17723 N0.getOpcode() == ISD::ANY_EXTEND) {
17724 // if the source is smaller than the dest, we still need an extend.
17725 if (N0.getOperand(i: 0).getValueType().bitsLT(VT)) {
17726 SDNodeFlags Flags;
17727 if (N0.getOpcode() == ISD::ZERO_EXTEND)
17728 Flags.setNonNeg(N0->getFlags().hasNonNeg());
17729 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, Operand: N0.getOperand(i: 0), Flags);
17730 }
17731 // if the source is larger than the dest, than we just need the truncate.
17732 if (N0.getOperand(i: 0).getValueType().bitsGT(VT))
17733 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0.getOperand(i: 0));
17734 // if the source and dest are the same type, we can drop both the extend
17735 // and the truncate.
17736 return N0.getOperand(i: 0);
17737 }
17738
17739 // Try to narrow a truncate-of-sext_in_reg to the destination type:
17740 // trunc (sign_ext_inreg X, iM) to iN --> sign_ext_inreg (trunc X to iN), iM
17741 if (!LegalTypes && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
17742 N0.hasOneUse()) {
17743 SDValue X = N0.getOperand(i: 0);
17744 SDValue ExtVal = N0.getOperand(i: 1);
17745 EVT ExtVT = cast<VTSDNode>(Val&: ExtVal)->getVT();
17746 if (ExtVT.bitsLT(VT) && TLI.preferSextInRegOfTruncate(TruncVT: VT, VT: SrcVT, ExtVT)) {
17747 SDValue TrX = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: X);
17748 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT, N1: TrX, N2: ExtVal);
17749 }
17750 }
17751
17752 // If this is anyext(trunc), don't fold it, allow ourselves to be folded.
17753 if (N->hasOneUse() && (N->user_begin()->getOpcode() == ISD::ANY_EXTEND))
17754 return SDValue();
17755
17756 // Fold extract-and-trunc into a narrow extract. For example:
17757 // i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
17758 // i32 y = TRUNCATE(i64 x)
17759 // -- becomes --
17760 // v16i8 b = BITCAST (v2i64 val)
17761 // i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
17762 //
17763 // Note: We only run this optimization after type legalization (which often
17764 // creates this pattern) and before operation legalization after which
17765 // we need to be more careful about the vector instructions that we generate.
17766 if (LegalTypes && !LegalOperations && VT.isScalarInteger() && VT != MVT::i1 &&
17767 N0->hasOneUse()) {
17768 EVT TrTy = N->getValueType(ResNo: 0);
17769 SDValue Src = N0;
17770
17771 // Check for cases where we shift down an upper element before truncation.
17772 int EltOffset = 0;
17773 if (Src.getOpcode() == ISD::SRL && Src.getOperand(i: 0)->hasOneUse()) {
17774 if (auto ShAmt = DAG.getValidShiftAmount(V: Src)) {
17775 if ((*ShAmt % TrTy.getSizeInBits()) == 0) {
17776 Src = Src.getOperand(i: 0);
17777 EltOffset = *ShAmt / TrTy.getSizeInBits();
17778 }
17779 }
17780 }
17781
17782 if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17783 EVT VecTy = Src.getOperand(i: 0).getValueType();
17784 EVT ExTy = Src.getValueType();
17785
17786 auto EltCnt = VecTy.getVectorElementCount();
17787 unsigned SizeRatio = ExTy.getSizeInBits() / TrTy.getSizeInBits();
17788 auto NewEltCnt = EltCnt * SizeRatio;
17789
17790 EVT NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: TrTy, EC: NewEltCnt);
17791 assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
17792
17793 SDValue EltNo = Src->getOperand(Num: 1);
17794 if (isa<ConstantSDNode>(Val: EltNo) && isTypeLegal(VT: NVT)) {
17795 int Elt = EltNo->getAsZExtVal();
17796 int Index = isLE ? (Elt * SizeRatio + EltOffset)
17797 : (Elt * SizeRatio + (SizeRatio - 1) - EltOffset);
17798 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: TrTy,
17799 N1: DAG.getBitcast(VT: NVT, V: Src.getOperand(i: 0)),
17800 N2: DAG.getVectorIdxConstant(Val: Index, DL));
17801 }
17802 }
17803 }
17804
17805 // trunc (select c, a, b) -> select c, (trunc a), (trunc b)
17806 if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse() &&
17807 TLI.isTruncateFree(FromVT: SrcVT, ToVT: VT)) {
17808 if (!LegalOperations ||
17809 (TLI.isOperationLegal(Op: ISD::SELECT, VT: SrcVT) &&
17810 TLI.isNarrowingProfitable(N: N0.getNode(), SrcVT, DestVT: VT))) {
17811 SDLoc SL(N0);
17812 SDValue Cond = N0.getOperand(i: 0);
17813 SDValue TruncOp0 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: N0.getOperand(i: 1));
17814 SDValue TruncOp1 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SL, VT, Operand: N0.getOperand(i: 2));
17815 return DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: Cond, N2: TruncOp0, N3: TruncOp1);
17816 }
17817 }
17818
17819 // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits()
17820 if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
17821 (!LegalOperations || TLI.isOperationLegal(Op: ISD::SHL, VT)) &&
17822 TLI.isTypeDesirableForOp(ISD::SHL, VT)) {
17823 SDValue Amt = N0.getOperand(i: 1);
17824 KnownBits Known = DAG.computeKnownBits(Op: Amt);
17825 unsigned Size = VT.getScalarSizeInBits();
17826 if (Known.countMaxActiveBits() <= Log2_32(Value: Size)) {
17827 EVT AmtVT = TLI.getShiftAmountTy(LHSTy: VT, DL: DAG.getDataLayout());
17828 SDValue Trunc = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0.getOperand(i: 0));
17829 if (AmtVT != Amt.getValueType()) {
17830 Amt = DAG.getZExtOrTrunc(Op: Amt, DL, VT: AmtVT);
17831 AddToWorklist(N: Amt.getNode());
17832 }
17833 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Trunc, N2: Amt);
17834 }
17835 }
17836
17837 if (SDValue V = foldSubToUSubSat(DstVT: VT, N: N0.getNode(), DL))
17838 return V;
17839
17840 if (SDValue ABD = foldABSToABD(N, DL))
17841 return ABD;
17842
17843 // Attempt to pre-truncate BUILD_VECTOR sources.
17844 if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations &&
17845 N0.hasOneUse() &&
17846 // Avoid creating illegal types if running after type legalizer.
17847 (!LegalTypes || TLI.isTypeLegal(VT: VT.getScalarType()))) {
17848 if (TLI.isTruncateFree(FromVT: SrcVT.getScalarType(), ToVT: VT.getScalarType()))
17849 return DAG.UnrollVectorOp(N);
17850
17851 // trunc(build_vector(ext(x), ext(x)) -> build_vector(x,x)
17852 if (SDValue SplatVal = DAG.getSplatValue(V: N0)) {
17853 if (ISD::isExtOpcode(Opcode: SplatVal.getOpcode()) &&
17854 SrcVT.getScalarType() == SplatVal.getValueType())
17855 return DAG.UnrollVectorOp(N);
17856 }
17857 }
17858
17859 // trunc (splat_vector x) -> splat_vector (trunc x)
17860 if (N0.getOpcode() == ISD::SPLAT_VECTOR &&
17861 (!LegalTypes || TLI.isTypeLegal(VT: VT.getScalarType())) &&
17862 (!LegalOperations || TLI.isOperationLegal(Op: ISD::SPLAT_VECTOR, VT))) {
17863 EVT SVT = VT.getScalarType();
17864 return DAG.getSplatVector(
17865 VT, DL, Op: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: SVT, Operand: N0->getOperand(Num: 0)));
17866 }
17867
17868 // Fold a series of buildvector, bitcast, and truncate if possible.
17869 // For example fold
17870 // (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to
17871 // (2xi32 (buildvector x, y)).
17872 if (Level == AfterLegalizeVectorOps && VT.isVector() &&
17873 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
17874 N0.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR &&
17875 N0.getOperand(i: 0).hasOneUse()) {
17876 SDValue BuildVect = N0.getOperand(i: 0);
17877 EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType();
17878 EVT TruncVecEltTy = VT.getVectorElementType();
17879
17880 // Check that the element types match.
17881 if (BuildVectEltTy == TruncVecEltTy) {
17882 // Now we only need to compute the offset of the truncated elements.
17883 unsigned BuildVecNumElts = BuildVect.getNumOperands();
17884 unsigned TruncVecNumElts = VT.getVectorNumElements();
17885 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
17886 unsigned FirstElt = isLE ? 0 : (TruncEltOffset - 1);
17887
17888 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
17889 "Invalid number of elements");
17890
17891 SmallVector<SDValue, 8> Opnds;
17892 for (unsigned i = FirstElt, e = BuildVecNumElts; i < e;
17893 i += TruncEltOffset)
17894 Opnds.push_back(Elt: BuildVect.getOperand(i));
17895
17896 return DAG.getBuildVector(VT, DL, Ops: Opnds);
17897 }
17898 }
17899
17900 // fold (truncate (load x)) -> (smaller load x)
17901 // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
17902 if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
17903 if (SDValue Reduced = reduceLoadWidth(N))
17904 return Reduced;
17905
17906 // Handle the case where the truncated result is at least as wide as the
17907 // loaded type.
17908 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N: N0.getNode())) {
17909 auto *LN0 = cast<LoadSDNode>(Val&: N0);
17910 if (LN0->isSimple() && LN0->getMemoryVT().bitsLE(VT)) {
17911 SDValue NewLoad = DAG.getExtLoad(
17912 ExtType: LN0->getExtensionType(), dl: SDLoc(LN0), VT, Chain: LN0->getChain(),
17913 Ptr: LN0->getBasePtr(), MemVT: LN0->getMemoryVT(), MMO: LN0->getMemOperand());
17914 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: NewLoad.getValue(R: 1));
17915 return NewLoad;
17916 }
17917 }
17918 }
17919
17920 // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)),
17921 // where ... are all 'undef'.
17922 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
17923 SmallVector<EVT, 8> VTs;
17924 SDValue V;
17925 unsigned Idx = 0;
17926 unsigned NumDefs = 0;
17927
17928 for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
17929 SDValue X = N0.getOperand(i);
17930 if (!X.isUndef()) {
17931 V = X;
17932 Idx = i;
17933 NumDefs++;
17934 }
17935 // Stop if more than one members are non-undef.
17936 if (NumDefs > 1)
17937 break;
17938
17939 VTs.push_back(Elt: EVT::getVectorVT(Context&: *DAG.getContext(),
17940 VT: VT.getVectorElementType(),
17941 EC: X.getValueType().getVectorElementCount()));
17942 }
17943
17944 if (NumDefs == 0)
17945 return DAG.getUNDEF(VT);
17946
17947 if (NumDefs == 1) {
17948 assert(V.getNode() && "The single defined operand is empty!");
17949 SmallVector<SDValue, 8> Opnds;
17950 for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
17951 if (i != Idx) {
17952 Opnds.push_back(Elt: DAG.getUNDEF(VT: VTs[i]));
17953 continue;
17954 }
17955 SDValue NV = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(V), VT: VTs[i], Operand: V);
17956 AddToWorklist(N: NV.getNode());
17957 Opnds.push_back(Elt: NV);
17958 }
17959 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: Opnds);
17960 }
17961 }
17962
17963 // Fold truncate of a bitcast of a vector to an extract of the low vector
17964 // element.
17965 //
17966 // e.g. trunc (i64 (bitcast v2i32:x)) -> extract_vector_elt v2i32:x, idx
17967 if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) {
17968 SDValue VecSrc = N0.getOperand(i: 0);
17969 EVT VecSrcVT = VecSrc.getValueType();
17970 if (VecSrcVT.isVectorOf(EltVT: VT) &&
17971 (!LegalOperations ||
17972 TLI.isOperationLegal(Op: ISD::EXTRACT_VECTOR_ELT, VT: VecSrcVT))) {
17973 unsigned Idx = isLE ? 0 : VecSrcVT.getVectorNumElements() - 1;
17974 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT, N1: VecSrc,
17975 N2: DAG.getVectorIdxConstant(Val: Idx, DL));
17976 }
17977 }
17978
17979 // Simplify the operands using demanded-bits information.
17980 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
17981 return SDValue(N, 0);
17982
17983 // fold (truncate (extract_subvector(ext x))) ->
17984 // (extract_subvector x)
17985 // TODO: This can be generalized to cover cases where the truncate and extract
17986 // do not fully cancel each other out.
17987 if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
17988 SDValue N00 = N0.getOperand(i: 0);
17989 if (N00.getOpcode() == ISD::SIGN_EXTEND ||
17990 N00.getOpcode() == ISD::ZERO_EXTEND ||
17991 N00.getOpcode() == ISD::ANY_EXTEND) {
17992 if (N00.getOperand(i: 0)->getValueType(ResNo: 0).getVectorElementType() ==
17993 VT.getVectorElementType())
17994 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N0->getOperand(Num: 0)), VT,
17995 N1: N00.getOperand(i: 0), N2: N0.getOperand(i: 1));
17996 }
17997 }
17998
17999 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(Cast: N))
18000 return NewVSel;
18001
18002 // Narrow a suitable binary operation with a non-opaque constant operand by
18003 // moving it ahead of the truncate. This is limited to pre-legalization
18004 // because targets may prefer a wider type during later combines and invert
18005 // this transform.
18006 switch (N0.getOpcode()) {
18007 case ISD::ADD:
18008 case ISD::SUB:
18009 case ISD::MUL:
18010 case ISD::AND:
18011 case ISD::OR:
18012 case ISD::XOR:
18013 if (!LegalOperations && N0.hasOneUse() &&
18014 (N0.getOperand(i: 0) == N0.getOperand(i: 1) ||
18015 isConstantOrConstantVector(N: N0.getOperand(i: 0), NoOpaques: true) ||
18016 isConstantOrConstantVector(N: N0.getOperand(i: 1), NoOpaques: true))) {
18017 // TODO: We already restricted this to pre-legalization, but for vectors
18018 // we are extra cautious to not create an unsupported operation.
18019 // Target-specific changes are likely needed to avoid regressions here.
18020 if (VT.isScalarInteger() || TLI.isOperationLegal(Op: N0.getOpcode(), VT)) {
18021 SDValue NarrowL = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0.getOperand(i: 0));
18022 SDValue NarrowR = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0.getOperand(i: 1));
18023 SDNodeFlags Flags;
18024 // Propagate nuw for sub.
18025 if (N0->getOpcode() == ISD::SUB && N0->getFlags().hasNoUnsignedWrap() &&
18026 DAG.MaskedValueIsZero(
18027 Op: N0->getOperand(Num: 0),
18028 Mask: APInt::getBitsSetFrom(numBits: SrcVT.getScalarSizeInBits(),
18029 loBit: VT.getScalarSizeInBits())))
18030 Flags.setNoUnsignedWrap(true);
18031 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT, N1: NarrowL, N2: NarrowR, Flags);
18032 }
18033 }
18034 break;
18035 case ISD::ADDE:
18036 case ISD::UADDO_CARRY:
18037 // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry)
18038 // (trunc uaddo_carry(X, Y, Carry)) ->
18039 // (uaddo_carry trunc(X), trunc(Y), Carry)
18040 // When the adde's carry is not used.
18041 // We only do for uaddo_carry before legalize operation
18042 if (((!LegalOperations && N0.getOpcode() == ISD::UADDO_CARRY) ||
18043 TLI.isOperationLegal(Op: N0.getOpcode(), VT)) &&
18044 N0.hasOneUse() && !N0->hasAnyUseOfValue(Value: 1)) {
18045 SDValue X = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0.getOperand(i: 0));
18046 SDValue Y = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0.getOperand(i: 1));
18047 SDVTList VTs = DAG.getVTList(VT1: VT, VT2: N0->getValueType(ResNo: 1));
18048 return DAG.getNode(Opcode: N0.getOpcode(), DL, VTList: VTs, N1: X, N2: Y, N3: N0.getOperand(i: 2));
18049 }
18050 break;
18051 case ISD::USUBSAT:
18052 // Truncate the USUBSAT only if LHS is a known zero-extension, its not
18053 // enough to know that the upper bits are zero we must ensure that we don't
18054 // introduce an extra truncate.
18055 if (!LegalOperations && N0.hasOneUse() &&
18056 N0.getOperand(i: 0).getOpcode() == ISD::ZERO_EXTEND &&
18057 N0.getOperand(i: 0).getOperand(i: 0).getScalarValueSizeInBits() <=
18058 VT.getScalarSizeInBits() &&
18059 hasOperation(Opcode: N0.getOpcode(), VT)) {
18060 return getTruncatedUSUBSAT(DstVT: VT, SrcVT, LHS: N0.getOperand(i: 0), RHS: N0.getOperand(i: 1),
18061 DAG, DL);
18062 }
18063 break;
18064 case ISD::AVGCEILS:
18065 case ISD::AVGCEILU:
18066 // trunc (avgceilu (sext (x), sext (y))) -> avgceils(x, y)
18067 // trunc (avgceils (zext (x), zext (y))) -> avgceilu(x, y)
18068 if (N0.hasOneUse()) {
18069 SDValue Op0 = N0.getOperand(i: 0);
18070 SDValue Op1 = N0.getOperand(i: 1);
18071 if (N0.getOpcode() == ISD::AVGCEILU) {
18072 if (TLI.isOperationLegalOrCustom(Op: ISD::AVGCEILS, VT) &&
18073 Op0.getOpcode() == ISD::SIGN_EXTEND &&
18074 Op1.getOpcode() == ISD::SIGN_EXTEND &&
18075 Op0.getOperand(i: 0).getValueType() == VT &&
18076 Op1.getOperand(i: 0).getValueType() == VT)
18077 return DAG.getNode(Opcode: ISD::AVGCEILS, DL, VT, N1: Op0.getOperand(i: 0),
18078 N2: Op1.getOperand(i: 0));
18079 } else {
18080 if (TLI.isOperationLegalOrCustom(Op: ISD::AVGCEILU, VT) &&
18081 Op0.getOpcode() == ISD::ZERO_EXTEND &&
18082 Op1.getOpcode() == ISD::ZERO_EXTEND &&
18083 Op0.getOperand(i: 0).getValueType() == VT &&
18084 Op1.getOperand(i: 0).getValueType() == VT)
18085 return DAG.getNode(Opcode: ISD::AVGCEILU, DL, VT, N1: Op0.getOperand(i: 0),
18086 N2: Op1.getOperand(i: 0));
18087 }
18088 }
18089 [[fallthrough]];
18090 case ISD::AVGFLOORS:
18091 case ISD::AVGFLOORU:
18092 case ISD::ABDS:
18093 case ISD::ABDU:
18094 // (trunc (avg a, b)) -> (avg (trunc a), (trunc b))
18095 // (trunc (abdu/abds a, b)) -> (abdu/abds (trunc a), (trunc b))
18096 if (!LegalOperations && N0.hasOneUse() &&
18097 TLI.isOperationLegal(Op: N0.getOpcode(), VT)) {
18098 EVT TruncVT = VT;
18099 unsigned SrcBits = SrcVT.getScalarSizeInBits();
18100 unsigned TruncBits = TruncVT.getScalarSizeInBits();
18101
18102 SDValue A = N0.getOperand(i: 0);
18103 SDValue B = N0.getOperand(i: 1);
18104 bool CanFold = false;
18105
18106 if (N0.getOpcode() == ISD::AVGFLOORU || N0.getOpcode() == ISD::AVGCEILU ||
18107 N0.getOpcode() == ISD::ABDU) {
18108 APInt UpperBits = APInt::getBitsSetFrom(numBits: SrcBits, loBit: TruncBits);
18109 CanFold = DAG.MaskedValueIsZero(Op: B, Mask: UpperBits) &&
18110 DAG.MaskedValueIsZero(Op: A, Mask: UpperBits);
18111 } else {
18112 unsigned NeededBits = SrcBits - TruncBits;
18113 CanFold = DAG.ComputeNumSignBits(Op: B) > NeededBits &&
18114 DAG.ComputeNumSignBits(Op: A) > NeededBits;
18115 }
18116
18117 if (CanFold) {
18118 SDValue NewA = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: TruncVT, Operand: A);
18119 SDValue NewB = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: TruncVT, Operand: B);
18120 return DAG.getNode(Opcode: N0.getOpcode(), DL, VT: TruncVT, N1: NewA, N2: NewB);
18121 }
18122 }
18123 break;
18124 }
18125
18126 return SDValue();
18127}
18128
18129static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
18130 SDValue Elt = N->getOperand(Num: i);
18131 if (Elt.getOpcode() != ISD::MERGE_VALUES)
18132 return Elt.getNode();
18133 return Elt.getOperand(i: Elt.getResNo()).getNode();
18134}
18135
18136/// build_pair (load, load) -> load
18137/// if load locations are consecutive.
18138SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
18139 assert(N->getOpcode() == ISD::BUILD_PAIR);
18140
18141 auto *LD1 = dyn_cast<LoadSDNode>(Val: getBuildPairElt(N, i: 0));
18142 auto *LD2 = dyn_cast<LoadSDNode>(Val: getBuildPairElt(N, i: 1));
18143
18144 // A BUILD_PAIR is always having the least significant part in elt 0 and the
18145 // most significant part in elt 1. So when combining into one large load, we
18146 // need to consider the endianness.
18147 if (DAG.getDataLayout().isBigEndian())
18148 std::swap(a&: LD1, b&: LD2);
18149
18150 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(N: LD1) || !ISD::isNON_EXTLoad(N: LD2) ||
18151 !LD1->hasOneUse() || !LD2->hasOneUse() ||
18152 LD1->getAddressSpace() != LD2->getAddressSpace())
18153 return SDValue();
18154
18155 unsigned LD1Fast = 0;
18156 EVT LD1VT = LD1->getValueType(ResNo: 0);
18157 unsigned LD1Bytes = LD1VT.getStoreSize();
18158 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::LOAD, VT)) &&
18159 DAG.areNonVolatileConsecutiveLoads(LD: LD2, Base: LD1, Bytes: LD1Bytes, Dist: 1) &&
18160 TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT,
18161 MMO: *LD1->getMemOperand(), Fast: &LD1Fast) && LD1Fast)
18162 return DAG.getLoad(VT, dl: SDLoc(N), Chain: LD1->getChain(), Ptr: LD1->getBasePtr(),
18163 PtrInfo: LD1->getPointerInfo(), Alignment: LD1->getAlign());
18164
18165 return SDValue();
18166}
18167
18168static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG) {
18169 // On little-endian machines, bitcasting from ppcf128 to i128 does swap the Hi
18170 // and Lo parts; on big-endian machines it doesn't.
18171 return DAG.getDataLayout().isBigEndian() ? 1 : 0;
18172}
18173
18174SDValue DAGCombiner::foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
18175 const TargetLowering &TLI) {
18176 // If this is not a bitcast to an FP type or if the target doesn't have
18177 // IEEE754-compliant FP logic, we're done.
18178 EVT VT = N->getValueType(ResNo: 0);
18179 SDValue N0 = N->getOperand(Num: 0);
18180 EVT SourceVT = N0.getValueType();
18181
18182 if (!VT.isFloatingPoint())
18183 return SDValue();
18184
18185 // TODO: Handle cases where the integer constant is a different scalar
18186 // bitwidth to the FP.
18187 if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits())
18188 return SDValue();
18189
18190 unsigned FPOpcode;
18191 APInt SignMask;
18192 switch (N0.getOpcode()) {
18193 case ISD::AND:
18194 FPOpcode = ISD::FABS;
18195 SignMask = ~APInt::getSignMask(BitWidth: SourceVT.getScalarSizeInBits());
18196 break;
18197 case ISD::XOR:
18198 FPOpcode = ISD::FNEG;
18199 SignMask = APInt::getSignMask(BitWidth: SourceVT.getScalarSizeInBits());
18200 break;
18201 case ISD::OR:
18202 FPOpcode = ISD::FABS;
18203 SignMask = APInt::getSignMask(BitWidth: SourceVT.getScalarSizeInBits());
18204 break;
18205 default:
18206 return SDValue();
18207 }
18208
18209 if (LegalOperations && !TLI.isOperationLegal(Op: FPOpcode, VT))
18210 return SDValue();
18211
18212 // This needs to be the inverse of logic in foldSignChangeInBitcast.
18213 // FIXME: I don't think looking for bitcast intrinsically makes sense, but
18214 // removing this would require more changes.
18215 auto IsBitCastOrFree = [&TLI, FPOpcode](SDValue Op, EVT VT) {
18216 if (sd_match(N: Op, P: m_BitCast(Op: m_SpecificVT(RefVT: VT))))
18217 return true;
18218
18219 return FPOpcode == ISD::FABS ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT);
18220 };
18221
18222 // Fold (bitcast int (and (bitcast fp X to int), 0x7fff...) to fp) -> fabs X
18223 // Fold (bitcast int (xor (bitcast fp X to int), 0x8000...) to fp) -> fneg X
18224 // Fold (bitcast int (or (bitcast fp X to int), 0x8000...) to fp) ->
18225 // fneg (fabs X)
18226 SDValue LogicOp0 = N0.getOperand(i: 0);
18227 ConstantSDNode *LogicOp1 = isConstOrConstSplat(N: N0.getOperand(i: 1), AllowUndefs: true);
18228 if (LogicOp1 && LogicOp1->getAPIntValue() == SignMask &&
18229 IsBitCastOrFree(LogicOp0, VT)) {
18230 SDValue CastOp0 = DAG.getNode(Opcode: ISD::BITCAST, DL: SDLoc(N), VT, Operand: LogicOp0);
18231 SDValue FPOp = DAG.getNode(Opcode: FPOpcode, DL: SDLoc(N), VT, Operand: CastOp0);
18232 NumFPLogicOpsConv++;
18233 if (N0.getOpcode() == ISD::OR)
18234 return DAG.getNode(Opcode: ISD::FNEG, DL: SDLoc(N), VT, Operand: FPOp);
18235 return FPOp;
18236 }
18237
18238 return SDValue();
18239}
18240
18241SDValue DAGCombiner::visitBITCAST(SDNode *N) {
18242 SDValue N0 = N->getOperand(Num: 0);
18243 EVT VT = N->getValueType(ResNo: 0);
18244
18245 if (N0.isUndef())
18246 return DAG.getUNDEF(VT);
18247
18248 // If the input is a BUILD_VECTOR with all constant elements, fold this now.
18249 // Only do this before legalize types, unless both types are integer and the
18250 // scalar type is legal. Only do this before legalize ops, since the target
18251 // maybe depending on the bitcast.
18252 // First check to see if this is all constant.
18253 // TODO: Support FP bitcasts after legalize types.
18254 if (VT.isVector() &&
18255 (!LegalTypes ||
18256 (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() &&
18257 TLI.isTypeLegal(VT: VT.getVectorElementType()))) &&
18258 N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() &&
18259 cast<BuildVectorSDNode>(Val&: N0)->isConstant())
18260 return DAG.FoldConstantBuildVector(BV: cast<BuildVectorSDNode>(Val&: N0), DL: SDLoc(N),
18261 DstEltVT: VT.getVectorElementType());
18262
18263 // If the input is a constant, let getNode fold it.
18264 if (isIntOrFPConstant(V: N0)) {
18265 // If we can't allow illegal operations, we need to check that this is just
18266 // a fp -> int or int -> conversion and that the resulting operation will
18267 // be legal.
18268 if (!LegalOperations ||
18269 (isa<ConstantSDNode>(Val: N0) && VT.isFloatingPoint() && !VT.isVector() &&
18270 TLI.isOperationLegal(Op: ISD::ConstantFP, VT)) ||
18271 (isa<ConstantFPSDNode>(Val: N0) && VT.isInteger() && !VT.isVector() &&
18272 TLI.isOperationLegal(Op: ISD::Constant, VT))) {
18273 SDValue C = DAG.getBitcast(VT, V: N0);
18274 if (C.getNode() != N)
18275 return C;
18276 }
18277 }
18278
18279 // (conv (conv x, t1), t2) -> (conv x, t2)
18280 if (N0.getOpcode() == ISD::BITCAST)
18281 return DAG.getBitcast(VT, V: N0.getOperand(i: 0));
18282
18283 // fold (conv (logicop (conv x), (c))) -> (logicop x, (conv c))
18284 // iff the current bitwise logicop type isn't legal
18285 if (ISD::isBitwiseLogicOp(Opcode: N0.getOpcode()) && VT.isInteger() &&
18286 !TLI.isTypeLegal(VT: N0.getOperand(i: 0).getValueType())) {
18287 auto IsFreeBitcast = [VT](SDValue V) {
18288 return (V.getOpcode() == ISD::BITCAST &&
18289 V.getOperand(i: 0).getValueType() == VT) ||
18290 (ISD::isBuildVectorOfConstantSDNodes(N: V.getNode()) &&
18291 V->hasOneUse());
18292 };
18293 if (IsFreeBitcast(N0.getOperand(i: 0)) && IsFreeBitcast(N0.getOperand(i: 1)))
18294 return DAG.getNode(Opcode: N0.getOpcode(), DL: SDLoc(N), VT,
18295 N1: DAG.getBitcast(VT, V: N0.getOperand(i: 0)),
18296 N2: DAG.getBitcast(VT, V: N0.getOperand(i: 1)));
18297 }
18298
18299 // fold (conv (load x)) -> (load (conv*)x)
18300 // fold (conv (freeze (load x))) -> (freeze (load (conv*)x))
18301 // If the resultant load doesn't need a higher alignment than the original!
18302 auto CastLoad = [this, &VT](SDValue N0, const SDLoc &DL) {
18303 // Peek through scalar_to_vector if the scalar is same size as VT - often a
18304 // leftover from legalization.
18305 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && N0.hasOneUse() &&
18306 N0.getOperand(i: 0).getValueSizeInBits() == VT.getSizeInBits())
18307 N0 = N0.getOperand(i: 0);
18308 if (N0.getOpcode() == ISD::AssertNoFPClass)
18309 N0 = N0.getOperand(i: 0);
18310 if (!ISD::isNormalLoad(N: N0.getNode()) || !N0.hasOneUse())
18311 return SDValue();
18312
18313 // Do not remove the cast if the types differ in endian layout.
18314 if (TLI.hasBigEndianPartOrdering(VT: N0.getValueType(), DL: DAG.getDataLayout()) !=
18315 TLI.hasBigEndianPartOrdering(VT, DL: DAG.getDataLayout()))
18316 return SDValue();
18317
18318 // If the load is volatile, we only want to change the load type if the
18319 // resulting load is legal. Otherwise we might increase the number of
18320 // memory accesses. We don't care if the original type was legal or not
18321 // as we assume software couldn't rely on the number of accesses of an
18322 // illegal type.
18323 auto *LN0 = cast<LoadSDNode>(Val&: N0);
18324 if ((LegalOperations || !LN0->isSimple()) &&
18325 !TLI.isOperationLegal(Op: ISD::LOAD, VT))
18326 return SDValue();
18327
18328 if (!TLI.isLoadBitCastBeneficial(LoadVT: N0.getValueType(), BitcastVT: VT, DAG,
18329 MMO: *LN0->getMemOperand()))
18330 return SDValue();
18331
18332 // If the range metadata type does not match the new memory
18333 // operation type, remove the range metadata.
18334 if (const MDNode *MD = LN0->getRanges()) {
18335 ConstantInt *Lower = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
18336 if (Lower->getBitWidth() != VT.getScalarSizeInBits() || !VT.isInteger()) {
18337 LN0->getMemOperand()->clearRanges();
18338 }
18339 }
18340 SDValue Load = DAG.getLoad(VT, dl: DL, Chain: LN0->getChain(), Ptr: LN0->getBasePtr(),
18341 MMO: LN0->getMemOperand());
18342 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: Load.getValue(R: 1));
18343 return Load;
18344 };
18345
18346 if (SDValue NewLd = CastLoad(N0, SDLoc(N)))
18347 return NewLd;
18348
18349 if (N0.getOpcode() == ISD::FREEZE && N0.hasOneUse())
18350 if (SDValue NewLd = CastLoad(N0.getOperand(i: 0), SDLoc(N)))
18351 return DAG.getFreeze(V: NewLd);
18352
18353 if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
18354 return V;
18355
18356 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
18357 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
18358 //
18359 // For ppc_fp128:
18360 // fold (bitcast (fneg x)) ->
18361 // flipbit = signbit
18362 // (xor (bitcast x) (build_pair flipbit, flipbit))
18363 //
18364 // fold (bitcast (fabs x)) ->
18365 // flipbit = (and (extract_element (bitcast x), 0), signbit)
18366 // (xor (bitcast x) (build_pair flipbit, flipbit))
18367 // This often reduces constant pool loads.
18368 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(VT: N0.getValueType())) ||
18369 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(VT: N0.getValueType()))) &&
18370 N0->hasOneUse() && VT.isInteger() && !VT.isVector() &&
18371 !N0.getValueType().isVector()) {
18372 SDValue NewConv = DAG.getBitcast(VT, V: N0.getOperand(i: 0));
18373 AddToWorklist(N: NewConv.getNode());
18374
18375 SDLoc DL(N);
18376 if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
18377 assert(VT.getSizeInBits() == 128);
18378 SDValue SignBit = DAG.getConstant(
18379 Val: APInt::getSignMask(BitWidth: VT.getSizeInBits() / 2), DL: SDLoc(N0), VT: MVT::i64);
18380 SDValue FlipBit;
18381 if (N0.getOpcode() == ISD::FNEG) {
18382 FlipBit = SignBit;
18383 AddToWorklist(N: FlipBit.getNode());
18384 } else {
18385 assert(N0.getOpcode() == ISD::FABS);
18386 SDValue Hi =
18387 DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(NewConv), VT: MVT::i64, N1: NewConv,
18388 N2: DAG.getIntPtrConstant(Val: getPPCf128HiElementSelector(DAG),
18389 DL: SDLoc(NewConv)));
18390 AddToWorklist(N: Hi.getNode());
18391 FlipBit = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N0), VT: MVT::i64, N1: Hi, N2: SignBit);
18392 AddToWorklist(N: FlipBit.getNode());
18393 }
18394 SDValue FlipBits =
18395 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N0), VT, N1: FlipBit, N2: FlipBit);
18396 AddToWorklist(N: FlipBits.getNode());
18397 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewConv, N2: FlipBits);
18398 }
18399 APInt SignBit = APInt::getSignMask(BitWidth: VT.getSizeInBits());
18400 if (N0.getOpcode() == ISD::FNEG)
18401 return DAG.getNode(Opcode: ISD::XOR, DL, VT,
18402 N1: NewConv, N2: DAG.getConstant(Val: SignBit, DL, VT));
18403 assert(N0.getOpcode() == ISD::FABS);
18404 return DAG.getNode(Opcode: ISD::AND, DL, VT,
18405 N1: NewConv, N2: DAG.getConstant(Val: ~SignBit, DL, VT));
18406 }
18407
18408 // fold (bitconvert (fcopysign cst, x)) ->
18409 // (or (and (bitconvert x), sign), (and cst, (not sign)))
18410 // Note that we don't handle (copysign x, cst) because this can always be
18411 // folded to an fneg or fabs.
18412 //
18413 // For ppc_fp128:
18414 // fold (bitcast (fcopysign cst, x)) ->
18415 // flipbit = (and (extract_element
18416 // (xor (bitcast cst), (bitcast x)), 0),
18417 // signbit)
18418 // (xor (bitcast cst) (build_pair flipbit, flipbit))
18419 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() &&
18420 isa<ConstantFPSDNode>(Val: N0.getOperand(i: 0)) && VT.isInteger() &&
18421 !VT.isVector()) {
18422 unsigned OrigXWidth = N0.getOperand(i: 1).getValueSizeInBits();
18423 EVT IntXVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OrigXWidth);
18424 if (isTypeLegal(VT: IntXVT)) {
18425 SDValue X = DAG.getBitcast(VT: IntXVT, V: N0.getOperand(i: 1));
18426 AddToWorklist(N: X.getNode());
18427
18428 // If X has a different width than the result/lhs, sext it or truncate it.
18429 unsigned VTWidth = VT.getSizeInBits();
18430 if (OrigXWidth < VTWidth) {
18431 X = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(N), VT, Operand: X);
18432 AddToWorklist(N: X.getNode());
18433 } else if (OrigXWidth > VTWidth) {
18434 // To get the sign bit in the right place, we have to shift it right
18435 // before truncating.
18436 SDLoc DL(X);
18437 X = DAG.getNode(Opcode: ISD::SRL, DL,
18438 VT: X.getValueType(), N1: X,
18439 N2: DAG.getConstant(Val: OrigXWidth-VTWidth, DL,
18440 VT: X.getValueType()));
18441 AddToWorklist(N: X.getNode());
18442 X = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(X), VT, Operand: X);
18443 AddToWorklist(N: X.getNode());
18444 }
18445
18446 if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
18447 APInt SignBit = APInt::getSignMask(BitWidth: VT.getSizeInBits() / 2);
18448 SDValue Cst = DAG.getBitcast(VT, V: N0.getOperand(i: 0));
18449 AddToWorklist(N: Cst.getNode());
18450 SDValue X = DAG.getBitcast(VT, V: N0.getOperand(i: 1));
18451 AddToWorklist(N: X.getNode());
18452 SDValue XorResult = DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N0), VT, N1: Cst, N2: X);
18453 AddToWorklist(N: XorResult.getNode());
18454 SDValue XorResult64 = DAG.getNode(
18455 Opcode: ISD::EXTRACT_ELEMENT, DL: SDLoc(XorResult), VT: MVT::i64, N1: XorResult,
18456 N2: DAG.getIntPtrConstant(Val: getPPCf128HiElementSelector(DAG),
18457 DL: SDLoc(XorResult)));
18458 AddToWorklist(N: XorResult64.getNode());
18459 SDValue FlipBit =
18460 DAG.getNode(Opcode: ISD::AND, DL: SDLoc(XorResult64), VT: MVT::i64, N1: XorResult64,
18461 N2: DAG.getConstant(Val: SignBit, DL: SDLoc(XorResult64), VT: MVT::i64));
18462 AddToWorklist(N: FlipBit.getNode());
18463 SDValue FlipBits =
18464 DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: SDLoc(N0), VT, N1: FlipBit, N2: FlipBit);
18465 AddToWorklist(N: FlipBits.getNode());
18466 return DAG.getNode(Opcode: ISD::XOR, DL: SDLoc(N), VT, N1: Cst, N2: FlipBits);
18467 }
18468 APInt SignBit = APInt::getSignMask(BitWidth: VT.getSizeInBits());
18469 X = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(X), VT,
18470 N1: X, N2: DAG.getConstant(Val: SignBit, DL: SDLoc(X), VT));
18471 AddToWorklist(N: X.getNode());
18472
18473 SDValue Cst = DAG.getBitcast(VT, V: N0.getOperand(i: 0));
18474 Cst = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(Cst), VT,
18475 N1: Cst, N2: DAG.getConstant(Val: ~SignBit, DL: SDLoc(Cst), VT));
18476 AddToWorklist(N: Cst.getNode());
18477
18478 return DAG.getNode(Opcode: ISD::OR, DL: SDLoc(N), VT, N1: X, N2: Cst);
18479 }
18480 }
18481
18482 // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
18483 if (N0.getOpcode() == ISD::BUILD_PAIR)
18484 if (SDValue CombineLD = CombineConsecutiveLoads(N: N0.getNode(), VT))
18485 return CombineLD;
18486
18487 // int_vt (bitcast (vec_vt (scalar_to_vector elt_vt:x)))
18488 // => int_vt (any_extend elt_vt:x)
18489 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && VT.isScalarInteger()) {
18490 SDValue SrcScalar = N0.getOperand(i: 0);
18491 EVT SrcVT = SrcScalar.getValueType();
18492 if (SrcVT.isScalarInteger() && VT.bitsGT(VT: SrcVT))
18493 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(N), VT, Operand: SrcScalar);
18494 }
18495
18496 // Remove double bitcasts from shuffles - this is often a legacy of
18497 // XformToShuffleWithZero being used to combine bitmaskings (of
18498 // float vectors bitcast to integer vectors) into shuffles.
18499 // bitcast(shuffle(bitcast(s0),bitcast(s1))) -> shuffle(s0,s1)
18500 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() &&
18501 N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() &&
18502 VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() &&
18503 !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) {
18504 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val&: N0);
18505
18506 // If operands are a bitcast, peek through if it casts the original VT.
18507 // If operands are a constant, just bitcast back to original VT.
18508 auto PeekThroughBitcast = [&](SDValue Op) {
18509 if (Op.getOpcode() == ISD::BITCAST &&
18510 Op.getOperand(i: 0).getValueType() == VT)
18511 return SDValue(Op.getOperand(i: 0));
18512 if (Op.isUndef() || isAnyConstantBuildVector(V: Op))
18513 return DAG.getBitcast(VT, V: Op);
18514 return SDValue();
18515 };
18516
18517 // FIXME: If either input vector is bitcast, try to convert the shuffle to
18518 // the result type of this bitcast. This would eliminate at least one
18519 // bitcast. See the transform in InstCombine.
18520 SDValue SV0 = PeekThroughBitcast(N0->getOperand(Num: 0));
18521 SDValue SV1 = PeekThroughBitcast(N0->getOperand(Num: 1));
18522 if (!(SV0 && SV1))
18523 return SDValue();
18524
18525 int MaskScale =
18526 VT.getVectorNumElements() / N0.getValueType().getVectorNumElements();
18527 SmallVector<int, 8> NewMask;
18528 for (int M : SVN->getMask())
18529 for (int i = 0; i != MaskScale; ++i)
18530 NewMask.push_back(Elt: M < 0 ? -1 : M * MaskScale + i);
18531
18532 SDValue LegalShuffle =
18533 TLI.buildLegalVectorShuffle(VT, DL: SDLoc(N), N0: SV0, N1: SV1, Mask: NewMask, DAG);
18534 if (LegalShuffle)
18535 return LegalShuffle;
18536 }
18537
18538 return SDValue();
18539}
18540
18541SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
18542 EVT VT = N->getValueType(ResNo: 0);
18543 return CombineConsecutiveLoads(N, VT);
18544}
18545
18546SDValue DAGCombiner::visitFREEZE(SDNode *N) {
18547 SDValue N0 = N->getOperand(Num: 0);
18548
18549 if (DAG.isGuaranteedNotToBeUndefOrPoison(Op: N0, Kind: UndefPoisonKind::UndefOrPoison))
18550 return N0;
18551
18552 // If we have frozen and unfrozen users of N0, update so everything uses N.
18553 if (!N0.isUndef() && !N0.hasOneUse()) {
18554 SDValue FrozenN0(N, 0);
18555 // Unfreeze all (possibly nested) uses of N to avoid double deleting N from
18556 // the CSE map.
18557 while (!N->use_empty())
18558 DAG.ReplaceAllUsesOfValueWith(From: FrozenN0, To: N0);
18559 DAG.ReplaceAllUsesOfValueWith(From: N0, To: FrozenN0);
18560 // ReplaceAllUsesOfValueWith will have also updated the use in N, thus
18561 // creating a cycle in a DAG. Let's undo that by mutating the freeze.
18562 assert(N->getOperand(0) == FrozenN0 && "Expected cycle in DAG");
18563 DAG.UpdateNodeOperands(N, Op: N0);
18564 // Revisit the node.
18565 AddToWorklist(N);
18566 return FrozenN0;
18567 }
18568
18569 // We currently avoid folding freeze over SRL, due to the problems seen
18570 // with (freeze (assert ext)) blocking simplifications of SRL. See for
18571 // example https://reviews.llvm.org/D136529#4120959.
18572 if (N0.getOpcode() == ISD::SRL)
18573 return SDValue();
18574
18575 // Fold freeze(op(x, ...)) -> op(freeze(x), ...).
18576 // Try to push freeze through instructions that propagate but don't produce
18577 // poison as far as possible. If an operand of freeze follows three
18578 // conditions 1) one-use, 2) does not produce poison, and 3) has all but one
18579 // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push
18580 // the freeze through to the operands that are not guaranteed non-poison.
18581 // NOTE: we will strip poison-generating flags, so ignore them here.
18582 if (DAG.canCreateUndefOrPoison(Op: N0, Kind: UndefPoisonKind::UndefOrPoison,
18583 /*ConsiderFlags*/ false) ||
18584 N0->getNumValues() != 1 || !N0->hasOneUse())
18585 return SDValue();
18586
18587 // TOOD: we should always allow multiple operands, however this increases the
18588 // likelihood of infinite loops due to the ReplaceAllUsesOfValueWith call
18589 // below causing later nodes that share frozen operands to fold again and no
18590 // longer being able to confirm other operands are not poison due to recursion
18591 // depth limits on isGuaranteedNotToBeUndefOrPoison.
18592 bool AllowMultipleMaybePoisonOperands =
18593 N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC ||
18594 N0.getOpcode() == ISD::BUILD_VECTOR ||
18595 N0.getOpcode() == ISD::INSERT_SUBVECTOR ||
18596 N0.getOpcode() == ISD::BUILD_PAIR ||
18597 N0.getOpcode() == ISD::VECTOR_SHUFFLE ||
18598 N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL;
18599
18600 // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all
18601 // ones" or "constant" into something that depends on FrozenUndef. We can
18602 // instead pick undef values to keep those properties, while at the same time
18603 // folding away the freeze.
18604 // If we implement a more general solution for folding away freeze(undef) in
18605 // the future, then this special handling can be removed.
18606 if (N0.getOpcode() == ISD::BUILD_VECTOR) {
18607 SDLoc DL(N0);
18608 EVT VT = N0.getValueType();
18609 if (llvm::ISD::isBuildVectorAllOnes(N: N0.getNode()) && VT.isInteger())
18610 return DAG.getAllOnesConstant(DL, VT);
18611 if (llvm::ISD::isBuildVectorOfConstantSDNodes(N: N0.getNode())) {
18612 SmallVector<SDValue, 8> NewVecC;
18613 for (const SDValue &Op : N0->op_values())
18614 NewVecC.push_back(
18615 Elt: Op.isUndef() ? DAG.getConstant(Val: 0, DL, VT: Op.getValueType()) : Op);
18616 return DAG.getBuildVector(VT, DL, Ops: NewVecC);
18617 }
18618 }
18619
18620 SmallSet<SDValue, 8> MaybePoisonOperands;
18621 SmallVector<unsigned, 8> MaybePoisonOperandNumbers;
18622 for (auto [OpNo, Op] : enumerate(First: N0->ops())) {
18623 if (DAG.isGuaranteedNotToBeUndefOrPoison(Op,
18624 Kind: UndefPoisonKind::UndefOrPoison))
18625 continue;
18626 bool HadMaybePoisonOperands = !MaybePoisonOperands.empty();
18627 bool IsNewMaybePoisonOperand = MaybePoisonOperands.insert(V: Op).second;
18628 if (IsNewMaybePoisonOperand)
18629 MaybePoisonOperandNumbers.push_back(Elt: OpNo);
18630 if (!HadMaybePoisonOperands)
18631 continue;
18632 if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
18633 // Multiple maybe-poison ops when not allowed - bail out.
18634 return SDValue();
18635 }
18636 }
18637 // NOTE: the whole op may be not guaranteed to not be undef or poison because
18638 // it could create undef or poison due to it's poison-generating flags.
18639 // So not finding any maybe-poison operands is fine.
18640
18641 for (unsigned OpNo : MaybePoisonOperandNumbers) {
18642 // N0 can mutate during iteration, so make sure to refetch the maybe poison
18643 // operands via the operand numbers. The typical scenario is that we have
18644 // something like this
18645 // t262: i32 = freeze t181
18646 // t150: i32 = ctlz_zero_poison t262
18647 // t184: i32 = ctlz_zero_poison t181
18648 // t268: i32 = select_cc t181, Constant:i32<0>, t184, t186, setne:ch
18649 // When freezing the t181 operand we get t262 back, and then the
18650 // ReplaceAllUsesOfValueWith call will not only replace t181 by t262, but
18651 // also recursively replace t184 by t150.
18652 SDValue MaybePoisonOperand = N->getOperand(Num: 0).getOperand(i: OpNo);
18653 // Don't replace every single UNDEF everywhere with frozen UNDEF, though.
18654 if (MaybePoisonOperand.isUndef())
18655 continue;
18656 // First, freeze each offending operand.
18657 SDValue FrozenMaybePoisonOperand = DAG.getFreeze(V: MaybePoisonOperand);
18658 // Then, change all other uses of unfrozen operand to use frozen operand.
18659 DAG.ReplaceAllUsesOfValueWith(From: MaybePoisonOperand, To: FrozenMaybePoisonOperand);
18660 if (FrozenMaybePoisonOperand.getOpcode() == ISD::FREEZE &&
18661 FrozenMaybePoisonOperand.getOperand(i: 0) == FrozenMaybePoisonOperand) {
18662 // But, that also updated the use in the freeze we just created, thus
18663 // creating a cycle in a DAG. Let's undo that by mutating the freeze.
18664 DAG.UpdateNodeOperands(N: FrozenMaybePoisonOperand.getNode(),
18665 Op: MaybePoisonOperand);
18666 }
18667
18668 // This node has been merged with another.
18669 if (N->getOpcode() == ISD::DELETED_NODE)
18670 return SDValue(N, 0);
18671 }
18672
18673 assert(N->getOpcode() != ISD::DELETED_NODE && "Node was deleted!");
18674
18675 // The whole node may have been updated, so the value we were holding
18676 // may no longer be valid. Re-fetch the operand we're `freeze`ing.
18677 N0 = N->getOperand(Num: 0);
18678
18679 // Finally, recreate the node, it's operands were updated to use
18680 // frozen operands, so we just need to use it's "original" operands.
18681 SmallVector<SDValue> Ops(N0->ops());
18682 // TODO: ISD::UNDEF and ISD::POISON should get separate handling, but best
18683 // leave for a future patch.
18684 for (SDValue &Op : Ops) {
18685 if (Op.isUndef())
18686 Op = DAG.getFreeze(V: Op);
18687 }
18688
18689 SDLoc DL(N0);
18690
18691 // Special case handling for ShuffleVectorSDNode nodes.
18692 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Val&: N0))
18693 return DAG.getVectorShuffle(VT: N0.getValueType(), dl: DL, N1: Ops[0], N2: Ops[1],
18694 Mask: SVN->getMask());
18695
18696 // NOTE: this strips poison generating flags.
18697 // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
18698 // ninf, nsz, or fast.
18699 // However, contract, reassoc, afn, and arcp should be preserved,
18700 // as these fast-math flags do not introduce poison values.
18701 SDNodeFlags SrcFlags = N0->getFlags();
18702 SDNodeFlags SafeFlags;
18703 SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
18704 SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
18705 SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
18706 SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
18707 return DAG.getNode(Opcode: N0.getOpcode(), DL, VTList: N0->getVTList(), Ops, Flags: SafeFlags);
18708}
18709
18710// Returns true if floating point contraction is allowed on the FMUL-SDValue
18711// `N`
18712static bool isContractableFMUL(SDValue N) {
18713 assert(N.getOpcode() == ISD::FMUL);
18714
18715 return N->getFlags().hasAllowContract();
18716}
18717
18718/// Try to perform FMA combining on a given FADD node.
18719SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
18720 SDValue N0 = N->getOperand(Num: 0);
18721 SDValue N1 = N->getOperand(Num: 1);
18722 EVT VT = N->getValueType(ResNo: 0);
18723 SDLoc SL(N);
18724
18725 // Floating-point multiply-add with intermediate rounding.
18726 bool HasFMAD = (LegalOperations && TLI.isFMADLegal(DAG, N));
18727
18728 // Floating-point multiply-add without intermediate rounding.
18729 bool HasFMA =
18730 (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::FMA, VT)) &&
18731 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT);
18732
18733 // No valid opcode, do not combine.
18734 if (!HasFMAD && !HasFMA)
18735 return SDValue();
18736
18737 // FMAD (with intermediate rounding) is always safe to form; FMA requires the
18738 // contract fast-math flag.
18739 bool AllowFusionGlobally = HasFMAD;
18740 // If the addition is not contractable, do not combine.
18741 if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
18742 return SDValue();
18743
18744 // Folding fadd (fmul x, y), (fmul x, y) -> fma x, y, (fmul x, y) is never
18745 // beneficial. It does not reduce latency. It increases register pressure. It
18746 // replaces an fadd with an fma which is a more complex instruction, so is
18747 // likely to have a larger encoding, use more functional units, etc.
18748 if (N0 == N1)
18749 return SDValue();
18750
18751 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel))
18752 return SDValue();
18753
18754 // Always prefer FMAD to FMA for precision.
18755 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
18756 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
18757
18758 auto isFusedOp = [&](SDValue N) {
18759 unsigned Opcode = N.getOpcode();
18760 return Opcode == ISD::FMA || Opcode == ISD::FMAD;
18761 };
18762
18763 // Is the node an FMUL and contractable either due to global flags or
18764 // SDNodeFlags.
18765 auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
18766 if (N.getOpcode() != ISD::FMUL)
18767 return false;
18768 return AllowFusionGlobally || N->getFlags().hasAllowContract();
18769 };
18770 // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
18771 // prefer to fold the multiply with fewer uses.
18772 if (Aggressive && isContractableFMUL(N0) && isContractableFMUL(N1)) {
18773 if (N0->use_size() > N1->use_size())
18774 std::swap(a&: N0, b&: N1);
18775 }
18776
18777 // fold (fadd (fmul x, y), z) -> (fma x, y, z)
18778 if (isContractableFMUL(N0) && (Aggressive || N0->hasOneUse())) {
18779 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: N0.getOperand(i: 0),
18780 N2: N0.getOperand(i: 1), N3: N1);
18781 }
18782
18783 // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
18784 // Note: Commutes FADD operands.
18785 if (isContractableFMUL(N1) && (Aggressive || N1->hasOneUse())) {
18786 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: N1.getOperand(i: 0),
18787 N2: N1.getOperand(i: 1), N3: N0);
18788 }
18789
18790 // fadd (fma A, B, (fmul C, D)), E --> fma A, B, (fma C, D, E)
18791 // fadd E, (fma A, B, (fmul C, D)) --> fma A, B, (fma C, D, E)
18792 // This also works with nested fma instructions:
18793 // fadd (fma A, B, (fma (C, D, (fmul (E, F))))), G -->
18794 // fma A, B, (fma C, D, fma (E, F, G))
18795 // fadd (G, (fma A, B, (fma (C, D, (fmul (E, F)))))) -->
18796 // fma A, B, (fma C, D, fma (E, F, G)).
18797 // This requires reassociation because it changes the order of operations.
18798 bool CanReassociate = N->getFlags().hasAllowReassociation();
18799 if (CanReassociate) {
18800 SDValue FMA, E;
18801 if (isFusedOp(N0) && N0.hasOneUse()) {
18802 FMA = N0;
18803 E = N1;
18804 } else if (isFusedOp(N1) && N1.hasOneUse()) {
18805 FMA = N1;
18806 E = N0;
18807 }
18808
18809 SDValue TmpFMA = FMA;
18810 while (E && isFusedOp(TmpFMA) && TmpFMA.hasOneUse()) {
18811 SDValue FMul = TmpFMA->getOperand(Num: 2);
18812 if (FMul.getOpcode() == ISD::FMUL && FMul.hasOneUse()) {
18813 SDValue C = FMul.getOperand(i: 0);
18814 SDValue D = FMul.getOperand(i: 1);
18815 SDValue CDE = DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: C, N2: D, N3: E);
18816 DAG.ReplaceAllUsesOfValueWith(From: FMul, To: CDE);
18817 // Replacing the inner FMul could cause the outer FMA to be simplified
18818 // away.
18819 return FMA.getOpcode() == ISD::DELETED_NODE ? SDValue(N, 0) : FMA;
18820 }
18821
18822 TmpFMA = TmpFMA->getOperand(Num: 2);
18823 }
18824 }
18825
18826 // Look through FP_EXTEND nodes to do more combining.
18827
18828 // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
18829 if (N0.getOpcode() == ISD::FP_EXTEND) {
18830 SDValue N00 = N0.getOperand(i: 0);
18831 if (isContractableFMUL(N00) &&
18832 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
18833 SrcVT: N00.getValueType())) {
18834 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
18835 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N00.getOperand(i: 0)),
18836 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N00.getOperand(i: 1)),
18837 N3: N1);
18838 }
18839 }
18840
18841 // fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
18842 // Note: Commutes FADD operands.
18843 if (N1.getOpcode() == ISD::FP_EXTEND) {
18844 SDValue N10 = N1.getOperand(i: 0);
18845 if (isContractableFMUL(N10) &&
18846 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
18847 SrcVT: N10.getValueType())) {
18848 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
18849 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N10.getOperand(i: 0)),
18850 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N10.getOperand(i: 1)),
18851 N3: N0);
18852 }
18853 }
18854
18855 // More folding opportunities when target permits.
18856 if (Aggressive) {
18857 // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
18858 // -> (fma x, y, (fma (fpext u), (fpext v), z))
18859 auto FoldFAddFMAFPExtFMul = [&](SDValue X, SDValue Y, SDValue U, SDValue V,
18860 SDValue Z) {
18861 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: X, N2: Y,
18862 N3: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
18863 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: U),
18864 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: V),
18865 N3: Z));
18866 };
18867 if (isFusedOp(N0)) {
18868 SDValue N02 = N0.getOperand(i: 2);
18869 if (N02.getOpcode() == ISD::FP_EXTEND) {
18870 SDValue N020 = N02.getOperand(i: 0);
18871 if (isContractableFMUL(N020) &&
18872 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
18873 SrcVT: N020.getValueType())) {
18874 return FoldFAddFMAFPExtFMul(N0.getOperand(i: 0), N0.getOperand(i: 1),
18875 N020.getOperand(i: 0), N020.getOperand(i: 1),
18876 N1);
18877 }
18878 }
18879 }
18880
18881 // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
18882 // -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
18883 // FIXME: This turns two single-precision and one double-precision
18884 // operation into two double-precision operations, which might not be
18885 // interesting for all targets, especially GPUs.
18886 auto FoldFAddFPExtFMAFMul = [&](SDValue X, SDValue Y, SDValue U, SDValue V,
18887 SDValue Z) {
18888 return DAG.getNode(
18889 Opcode: PreferredFusedOpcode, DL: SL, VT, N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: X),
18890 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: Y),
18891 N3: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
18892 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: U),
18893 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: V), N3: Z));
18894 };
18895 if (N0.getOpcode() == ISD::FP_EXTEND) {
18896 SDValue N00 = N0.getOperand(i: 0);
18897 if (isFusedOp(N00)) {
18898 SDValue N002 = N00.getOperand(i: 2);
18899 if (isContractableFMUL(N002) &&
18900 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
18901 SrcVT: N00.getValueType())) {
18902 return FoldFAddFPExtFMAFMul(N00.getOperand(i: 0), N00.getOperand(i: 1),
18903 N002.getOperand(i: 0), N002.getOperand(i: 1),
18904 N1);
18905 }
18906 }
18907 }
18908
18909 // fold (fadd x, (fma y, z, (fpext (fmul u, v)))
18910 // -> (fma y, z, (fma (fpext u), (fpext v), x))
18911 if (isFusedOp(N1)) {
18912 SDValue N12 = N1.getOperand(i: 2);
18913 if (N12.getOpcode() == ISD::FP_EXTEND) {
18914 SDValue N120 = N12.getOperand(i: 0);
18915 if (isContractableFMUL(N120) &&
18916 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
18917 SrcVT: N120.getValueType())) {
18918 return FoldFAddFMAFPExtFMul(N1.getOperand(i: 0), N1.getOperand(i: 1),
18919 N120.getOperand(i: 0), N120.getOperand(i: 1),
18920 N0);
18921 }
18922 }
18923 }
18924
18925 // fold (fadd x, (fpext (fma y, z, (fmul u, v)))
18926 // -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
18927 // FIXME: This turns two single-precision and one double-precision
18928 // operation into two double-precision operations, which might not be
18929 // interesting for all targets, especially GPUs.
18930 if (N1.getOpcode() == ISD::FP_EXTEND) {
18931 SDValue N10 = N1.getOperand(i: 0);
18932 if (isFusedOp(N10)) {
18933 SDValue N102 = N10.getOperand(i: 2);
18934 if (isContractableFMUL(N102) &&
18935 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
18936 SrcVT: N10.getValueType())) {
18937 return FoldFAddFPExtFMAFMul(N10.getOperand(i: 0), N10.getOperand(i: 1),
18938 N102.getOperand(i: 0), N102.getOperand(i: 1),
18939 N0);
18940 }
18941 }
18942 }
18943 }
18944
18945 return SDValue();
18946}
18947
18948/// Try to perform FMA combining on a given FSUB node.
18949SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
18950 SDValue N0 = N->getOperand(Num: 0);
18951 SDValue N1 = N->getOperand(Num: 1);
18952 EVT VT = N->getValueType(ResNo: 0);
18953 SDLoc SL(N);
18954
18955 // Floating-point multiply-add with intermediate rounding.
18956 bool HasFMAD = (LegalOperations && TLI.isFMADLegal(DAG, N));
18957
18958 // Floating-point multiply-add without intermediate rounding.
18959 bool HasFMA =
18960 (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::FMA, VT)) &&
18961 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT);
18962
18963 // No valid opcode, do not combine.
18964 if (!HasFMAD && !HasFMA)
18965 return SDValue();
18966
18967 const SDNodeFlags Flags = N->getFlags();
18968 // FMAD (with intermediate rounding) is always safe to form; FMA requires the
18969 // contract fast-math flag.
18970 bool AllowFusionGlobally = HasFMAD;
18971
18972 // If the subtraction is not contractable, do not combine.
18973 if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
18974 return SDValue();
18975
18976 if (TLI.generateFMAsInMachineCombiner(VT, OptLevel))
18977 return SDValue();
18978
18979 // Always prefer FMAD to FMA for precision.
18980 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
18981 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
18982 bool NoSignedZero = Flags.hasNoSignedZeros();
18983
18984 // Is the node an FMUL and contractable either due to global flags or
18985 // SDNodeFlags.
18986 auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
18987 if (N.getOpcode() != ISD::FMUL)
18988 return false;
18989 return AllowFusionGlobally || N->getFlags().hasAllowContract();
18990 };
18991
18992 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
18993 auto tryToFoldXYSubZ = [&](SDValue XY, SDValue Z) {
18994 if (isContractableFMUL(XY) && (Aggressive || XY->hasOneUse())) {
18995 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: XY.getOperand(i: 0),
18996 N2: XY.getOperand(i: 1), N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Z));
18997 }
18998 return SDValue();
18999 };
19000
19001 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
19002 // Note: Commutes FSUB operands.
19003 auto tryToFoldXSubYZ = [&](SDValue X, SDValue YZ) {
19004 if (isContractableFMUL(YZ) && (Aggressive || YZ->hasOneUse())) {
19005 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19006 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: YZ.getOperand(i: 0)),
19007 N2: YZ.getOperand(i: 1), N3: X);
19008 }
19009 return SDValue();
19010 };
19011
19012 // If we have two choices trying to fold (fsub (fmul u, v), (fmul x, y)),
19013 // prefer to fold the multiply with fewer uses.
19014 if (isContractableFMUL(N0) && isContractableFMUL(N1) &&
19015 (N0->use_size() > N1->use_size())) {
19016 // fold (fsub (fmul a, b), (fmul c, d)) -> (fma (fneg c), d, (fmul a, b))
19017 if (SDValue V = tryToFoldXSubYZ(N0, N1))
19018 return V;
19019 // fold (fsub (fmul a, b), (fmul c, d)) -> (fma a, b, (fneg (fmul c, d)))
19020 if (SDValue V = tryToFoldXYSubZ(N0, N1))
19021 return V;
19022 } else {
19023 // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
19024 if (SDValue V = tryToFoldXYSubZ(N0, N1))
19025 return V;
19026 // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
19027 if (SDValue V = tryToFoldXSubYZ(N0, N1))
19028 return V;
19029 }
19030
19031 // fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
19032 if (N0.getOpcode() == ISD::FNEG && isContractableFMUL(N0.getOperand(i: 0)) &&
19033 (Aggressive || (N0->hasOneUse() && N0.getOperand(i: 0).hasOneUse()))) {
19034 SDValue N00 = N0.getOperand(i: 0).getOperand(i: 0);
19035 SDValue N01 = N0.getOperand(i: 0).getOperand(i: 1);
19036 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19037 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N00), N2: N01,
19038 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N1));
19039 }
19040
19041 // Look through FP_EXTEND nodes to do more combining.
19042
19043 // fold (fsub (fpext (fmul x, y)), z)
19044 // -> (fma (fpext x), (fpext y), (fneg z))
19045 if (N0.getOpcode() == ISD::FP_EXTEND) {
19046 SDValue N00 = N0.getOperand(i: 0);
19047 if (isContractableFMUL(N00) &&
19048 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19049 SrcVT: N00.getValueType())) {
19050 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19051 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N00.getOperand(i: 0)),
19052 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N00.getOperand(i: 1)),
19053 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N1));
19054 }
19055 }
19056
19057 // fold (fsub x, (fpext (fmul y, z)))
19058 // -> (fma (fneg (fpext y)), (fpext z), x)
19059 // Note: Commutes FSUB operands.
19060 if (N1.getOpcode() == ISD::FP_EXTEND) {
19061 SDValue N10 = N1.getOperand(i: 0);
19062 if (isContractableFMUL(N10) &&
19063 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19064 SrcVT: N10.getValueType())) {
19065 return DAG.getNode(
19066 Opcode: PreferredFusedOpcode, DL: SL, VT,
19067 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT,
19068 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N10.getOperand(i: 0))),
19069 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N10.getOperand(i: 1)), N3: N0);
19070 }
19071 }
19072
19073 // fold (fsub (fpext (fneg (fmul, x, y))), z)
19074 // -> (fneg (fma (fpext x), (fpext y), z))
19075 // Note: This could be removed with appropriate canonicalization of the
19076 // input expression into (fneg (fadd (fpext (fmul, x, y)), z)). However, the
19077 // command line flag -fp-contract=fast and fast-math flag contract prevent
19078 // from implementing the canonicalization in visitFSUB.
19079 if (N0.getOpcode() == ISD::FP_EXTEND) {
19080 SDValue N00 = N0.getOperand(i: 0);
19081 if (N00.getOpcode() == ISD::FNEG) {
19082 SDValue N000 = N00.getOperand(i: 0);
19083 if (isContractableFMUL(N000) &&
19084 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19085 SrcVT: N00.getValueType())) {
19086 return DAG.getNode(
19087 Opcode: ISD::FNEG, DL: SL, VT,
19088 Operand: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19089 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N000.getOperand(i: 0)),
19090 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N000.getOperand(i: 1)),
19091 N3: N1));
19092 }
19093 }
19094 }
19095
19096 // fold (fsub (fneg (fpext (fmul, x, y))), z)
19097 // -> (fneg (fma (fpext x)), (fpext y), z)
19098 // Note: This could be removed with appropriate canonicalization of the
19099 // input expression into (fneg (fadd (fpext (fmul, x, y)), z). However, the
19100 // command line flag -fp-contract=fast and fast-math flag contract prevent
19101 // from implementing the canonicalization in visitFSUB.
19102 if (N0.getOpcode() == ISD::FNEG) {
19103 SDValue N00 = N0.getOperand(i: 0);
19104 if (N00.getOpcode() == ISD::FP_EXTEND) {
19105 SDValue N000 = N00.getOperand(i: 0);
19106 if (isContractableFMUL(N000) &&
19107 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19108 SrcVT: N000.getValueType())) {
19109 return DAG.getNode(
19110 Opcode: ISD::FNEG, DL: SL, VT,
19111 Operand: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19112 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N000.getOperand(i: 0)),
19113 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N000.getOperand(i: 1)),
19114 N3: N1));
19115 }
19116 }
19117 }
19118
19119 auto isContractableAndReassociableFMUL = [&isContractableFMUL](SDValue N) {
19120 return isContractableFMUL(N) && N->getFlags().hasAllowReassociation();
19121 };
19122
19123 auto isFusedOp = [&](SDValue N) {
19124 unsigned Opcode = N.getOpcode();
19125 return Opcode == ISD::FMA || Opcode == ISD::FMAD;
19126 };
19127
19128 // More folding opportunities when target permits.
19129 if (Aggressive && N->getFlags().hasAllowReassociation()) {
19130 bool CanFuse = N->getFlags().hasAllowContract();
19131 // fold (fsub (fma x, y, (fmul u, v)), z)
19132 // -> (fma x, y (fma u, v, (fneg z)))
19133 if (CanFuse && isFusedOp(N0) &&
19134 isContractableAndReassociableFMUL(N0.getOperand(i: 2)) &&
19135 N0->hasOneUse() && N0.getOperand(i: 2)->hasOneUse()) {
19136 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: N0.getOperand(i: 0),
19137 N2: N0.getOperand(i: 1),
19138 N3: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19139 N1: N0.getOperand(i: 2).getOperand(i: 0),
19140 N2: N0.getOperand(i: 2).getOperand(i: 1),
19141 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N1)));
19142 }
19143
19144 // fold (fsub x, (fma y, z, (fmul u, v)))
19145 // -> (fma (fneg y), z, (fma (fneg u), v, x))
19146 if (CanFuse && isFusedOp(N1) &&
19147 isContractableAndReassociableFMUL(N1.getOperand(i: 2)) &&
19148 N1->hasOneUse() && NoSignedZero) {
19149 SDValue N20 = N1.getOperand(i: 2).getOperand(i: 0);
19150 SDValue N21 = N1.getOperand(i: 2).getOperand(i: 1);
19151 return DAG.getNode(
19152 Opcode: PreferredFusedOpcode, DL: SL, VT,
19153 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N1.getOperand(i: 0)), N2: N1.getOperand(i: 1),
19154 N3: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19155 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N20), N2: N21, N3: N0));
19156 }
19157
19158 // fold (fsub (fma x, y, (fpext (fmul u, v))), z)
19159 // -> (fma x, y (fma (fpext u), (fpext v), (fneg z)))
19160 if (isFusedOp(N0) && N0->hasOneUse()) {
19161 SDValue N02 = N0.getOperand(i: 2);
19162 if (N02.getOpcode() == ISD::FP_EXTEND) {
19163 SDValue N020 = N02.getOperand(i: 0);
19164 if (isContractableAndReassociableFMUL(N020) &&
19165 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19166 SrcVT: N020.getValueType())) {
19167 return DAG.getNode(
19168 Opcode: PreferredFusedOpcode, DL: SL, VT, N1: N0.getOperand(i: 0), N2: N0.getOperand(i: 1),
19169 N3: DAG.getNode(
19170 Opcode: PreferredFusedOpcode, DL: SL, VT,
19171 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N020.getOperand(i: 0)),
19172 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N020.getOperand(i: 1)),
19173 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N1)));
19174 }
19175 }
19176 }
19177
19178 // fold (fsub (fpext (fma x, y, (fmul u, v))), z)
19179 // -> (fma (fpext x), (fpext y),
19180 // (fma (fpext u), (fpext v), (fneg z)))
19181 // FIXME: This turns two single-precision and one double-precision
19182 // operation into two double-precision operations, which might not be
19183 // interesting for all targets, especially GPUs.
19184 if (N0.getOpcode() == ISD::FP_EXTEND) {
19185 SDValue N00 = N0.getOperand(i: 0);
19186 if (isFusedOp(N00)) {
19187 SDValue N002 = N00.getOperand(i: 2);
19188 if (isContractableAndReassociableFMUL(N002) &&
19189 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19190 SrcVT: N00.getValueType())) {
19191 return DAG.getNode(
19192 Opcode: PreferredFusedOpcode, DL: SL, VT,
19193 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N00.getOperand(i: 0)),
19194 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N00.getOperand(i: 1)),
19195 N3: DAG.getNode(
19196 Opcode: PreferredFusedOpcode, DL: SL, VT,
19197 N1: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N002.getOperand(i: 0)),
19198 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N002.getOperand(i: 1)),
19199 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N1)));
19200 }
19201 }
19202 }
19203
19204 // fold (fsub x, (fma y, z, (fpext (fmul u, v))))
19205 // -> (fma (fneg y), z, (fma (fneg (fpext u)), (fpext v), x))
19206 if (isFusedOp(N1) && N1.getOperand(i: 2).getOpcode() == ISD::FP_EXTEND &&
19207 N1->hasOneUse()) {
19208 SDValue N120 = N1.getOperand(i: 2).getOperand(i: 0);
19209 if (isContractableAndReassociableFMUL(N120) &&
19210 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19211 SrcVT: N120.getValueType())) {
19212 SDValue N1200 = N120.getOperand(i: 0);
19213 SDValue N1201 = N120.getOperand(i: 1);
19214 return DAG.getNode(
19215 Opcode: PreferredFusedOpcode, DL: SL, VT,
19216 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: N1.getOperand(i: 0)), N2: N1.getOperand(i: 1),
19217 N3: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19218 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT,
19219 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N1200)),
19220 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N1201), N3: N0));
19221 }
19222 }
19223
19224 // fold (fsub x, (fpext (fma y, z, (fmul u, v))))
19225 // -> (fma (fneg (fpext y)), (fpext z),
19226 // (fma (fneg (fpext u)), (fpext v), x))
19227 // FIXME: This turns two single-precision and one double-precision
19228 // operation into two double-precision operations, which might not be
19229 // interesting for all targets, especially GPUs.
19230 if (N1.getOpcode() == ISD::FP_EXTEND && isFusedOp(N1.getOperand(i: 0))) {
19231 SDValue CvtSrc = N1.getOperand(i: 0);
19232 SDValue N100 = CvtSrc.getOperand(i: 0);
19233 SDValue N101 = CvtSrc.getOperand(i: 1);
19234 SDValue N102 = CvtSrc.getOperand(i: 2);
19235 if (isContractableAndReassociableFMUL(N102) &&
19236 TLI.isFPExtFoldable(DAG, Opcode: PreferredFusedOpcode, DestVT: VT,
19237 SrcVT: CvtSrc.getValueType())) {
19238 SDValue N1020 = N102.getOperand(i: 0);
19239 SDValue N1021 = N102.getOperand(i: 1);
19240 return DAG.getNode(
19241 Opcode: PreferredFusedOpcode, DL: SL, VT,
19242 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT,
19243 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N100)),
19244 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N101),
19245 N3: DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19246 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT,
19247 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N1020)),
19248 N2: DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SL, VT, Operand: N1021), N3: N0));
19249 }
19250 }
19251 }
19252
19253 return SDValue();
19254}
19255
19256/// Try to perform FMA combining on a given FMUL node based on the distributive
19257/// law x * (y + 1) = x * y + x and variants thereof (commuted versions,
19258/// subtraction instead of addition).
19259SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
19260 SDValue N0 = N->getOperand(Num: 0);
19261 SDValue N1 = N->getOperand(Num: 1);
19262 EVT VT = N->getValueType(ResNo: 0);
19263 SDLoc SL(N);
19264
19265 assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation");
19266
19267 // The transforms below are incorrect when x == 0 and y == inf, because the
19268 // intermediate multiplication produces a nan.
19269 SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1;
19270 if (!FAdd->getFlags().hasNoInfs())
19271 return SDValue();
19272
19273 // Floating-point multiply-add without intermediate rounding.
19274 bool HasFMA =
19275 isContractableFMUL(N: SDValue(N, 0)) &&
19276 (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::FMA, VT)) &&
19277 TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT);
19278
19279 // Floating-point multiply-add with intermediate rounding. This can result
19280 // in a less precise result due to the changed rounding order.
19281 bool HasFMAD = LegalOperations && TLI.isFMADLegal(DAG, N);
19282
19283 // No valid opcode, do not combine.
19284 if (!HasFMAD && !HasFMA)
19285 return SDValue();
19286
19287 // Always prefer FMAD to FMA for precision.
19288 unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
19289 bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
19290
19291 // fold (fmul (fadd x0, +1.0), y) -> (fma x0, y, y)
19292 // fold (fmul (fadd x0, -1.0), y) -> (fma x0, y, (fneg y))
19293 auto FuseFADD = [&](SDValue X, SDValue Y) {
19294 if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) {
19295 if (auto *C = isConstOrConstSplatFP(N: X.getOperand(i: 1), AllowUndefs: true)) {
19296 if (C->isOne())
19297 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: X.getOperand(i: 0), N2: Y,
19298 N3: Y);
19299 if (C->isMinusOne())
19300 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: X.getOperand(i: 0), N2: Y,
19301 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y));
19302 }
19303 }
19304 return SDValue();
19305 };
19306
19307 if (SDValue FMA = FuseFADD(N0, N1))
19308 return FMA;
19309 if (SDValue FMA = FuseFADD(N1, N0))
19310 return FMA;
19311
19312 // fold (fmul (fsub +1.0, x1), y) -> (fma (fneg x1), y, y)
19313 // fold (fmul (fsub -1.0, x1), y) -> (fma (fneg x1), y, (fneg y))
19314 // fold (fmul (fsub x0, +1.0), y) -> (fma x0, y, (fneg y))
19315 // fold (fmul (fsub x0, -1.0), y) -> (fma x0, y, y)
19316 auto FuseFSUB = [&](SDValue X, SDValue Y) {
19317 if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) {
19318 if (auto *C0 = isConstOrConstSplatFP(N: X.getOperand(i: 0), AllowUndefs: true)) {
19319 if (C0->isOne())
19320 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19321 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: X.getOperand(i: 1)), N2: Y,
19322 N3: Y);
19323 if (C0->isMinusOne())
19324 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT,
19325 N1: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: X.getOperand(i: 1)), N2: Y,
19326 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y));
19327 }
19328 if (auto *C1 = isConstOrConstSplatFP(N: X.getOperand(i: 1), AllowUndefs: true)) {
19329 if (C1->isOne())
19330 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: X.getOperand(i: 0), N2: Y,
19331 N3: DAG.getNode(Opcode: ISD::FNEG, DL: SL, VT, Operand: Y));
19332 if (C1->isMinusOne())
19333 return DAG.getNode(Opcode: PreferredFusedOpcode, DL: SL, VT, N1: X.getOperand(i: 0), N2: Y,
19334 N3: Y);
19335 }
19336 }
19337 return SDValue();
19338 };
19339
19340 if (SDValue FMA = FuseFSUB(N0, N1))
19341 return FMA;
19342 if (SDValue FMA = FuseFSUB(N1, N0))
19343 return FMA;
19344
19345 return SDValue();
19346}
19347
19348SDValue DAGCombiner::visitFADD(SDNode *N) {
19349 SDValue N0 = N->getOperand(Num: 0);
19350 SDValue N1 = N->getOperand(Num: 1);
19351 bool N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N: N0);
19352 bool N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N: N1);
19353 EVT VT = N->getValueType(ResNo: 0);
19354 SDLoc DL(N);
19355 SDNodeFlags Flags = N->getFlags();
19356 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
19357
19358 if (SDValue R = DAG.simplifyFPBinop(Opcode: N->getOpcode(), X: N0, Y: N1, Flags))
19359 return R;
19360
19361 // fold (fadd c1, c2) -> c1 + c2
19362 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FADD, DL, VT, Ops: {N0, N1}))
19363 return C;
19364
19365 // canonicalize constant to RHS
19366 if (N0CFP && !N1CFP)
19367 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1, N2: N0);
19368
19369 // fold vector ops
19370 if (VT.isVector())
19371 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
19372 return FoldedVOp;
19373
19374 // N0 + -0.0 --> N0 (also allowed with +0.0 and fast-math)
19375 ConstantFPSDNode *N1C = isConstOrConstSplatFP(N: N1, AllowUndefs: true);
19376 if (N1C && N1C->isZero())
19377 if (N1C->isNegative() || DAG.canIgnoreSignBitOfZero(Op: SDValue(N, 0)))
19378 return N0;
19379
19380 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
19381 return NewSel;
19382
19383 // fold (fadd A, (fneg B)) -> (fsub A, B)
19384 if (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::FSUB, VT))
19385 if (SDValue NegN1 = TLI.getCheaperNegatedExpression(
19386 Op: N1, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize))
19387 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: N0, N2: NegN1);
19388
19389 // fold (fadd (fneg A), B) -> (fsub B, A)
19390 if (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::FSUB, VT))
19391 if (SDValue NegN0 = TLI.getCheaperNegatedExpression(
19392 Op: N0, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize))
19393 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1, N2: NegN0);
19394
19395 auto isFMulNegTwo = [](SDValue FMul) {
19396 if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL)
19397 return false;
19398 auto *C = isConstOrConstSplatFP(N: FMul.getOperand(i: 1), AllowUndefs: true);
19399 return C && C->isExactlyValue(V: -2.0);
19400 };
19401
19402 // fadd (fmul B, -2.0), A --> fsub A, (fadd B, B)
19403 if (isFMulNegTwo(N0)) {
19404 SDValue B = N0.getOperand(i: 0);
19405 SDValue Add = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: B, N2: B);
19406 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1, N2: Add);
19407 }
19408 // fadd A, (fmul B, -2.0) --> fsub A, (fadd B, B)
19409 if (isFMulNegTwo(N1)) {
19410 SDValue B = N1.getOperand(i: 0);
19411 SDValue Add = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: B, N2: B);
19412 return DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: N0, N2: Add);
19413 }
19414
19415 // No FP constant should be created after legalization as Instruction
19416 // Selection pass has a hard time dealing with FP constants.
19417 bool AllowNewConst = (Level < AfterLegalizeDAG);
19418
19419 // If nnan is enabled, fold lots of things.
19420 if (Flags.hasNoNaNs() && AllowNewConst) {
19421 // If allowed, fold (fadd (fneg x), x) -> 0.0
19422 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(i: 0) == N1)
19423 return DAG.getConstantFP(Val: 0.0, DL, VT);
19424
19425 // If allowed, fold (fadd x, (fneg x)) -> 0.0
19426 if (N1.getOpcode() == ISD::FNEG && N1.getOperand(i: 0) == N0)
19427 return DAG.getConstantFP(Val: 0.0, DL, VT);
19428 }
19429
19430 // If reassoc and nsz, fold lots of things.
19431 // TODO: break out portions of the transformations below for which Unsafe is
19432 // considered and which do not require both nsz and reassoc
19433 if (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros() &&
19434 AllowNewConst) {
19435 // fadd (fadd x, c1), c2 -> fadd x, c1 + c2
19436 if (N1CFP && N0.getOpcode() == ISD::FADD &&
19437 DAG.isConstantFPBuildVectorOrConstantFP(N: N0.getOperand(i: 1))) {
19438 SDValue NewC = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0.getOperand(i: 1), N2: N1);
19439 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0.getOperand(i: 0), N2: NewC);
19440 }
19441
19442 // We can fold chains of FADD's of the same value into multiplications.
19443 // This transform is not safe in general because we are reducing the number
19444 // of rounding steps.
19445 if ((!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT)) &&
19446 !N0CFP && !N1CFP) {
19447 if (N0.getOpcode() == ISD::FMUL) {
19448 bool CFP00 = DAG.isConstantFPBuildVectorOrConstantFP(N: N0.getOperand(i: 0));
19449 bool CFP01 = DAG.isConstantFPBuildVectorOrConstantFP(N: N0.getOperand(i: 1));
19450
19451 // (fadd (fmul x, c), x) -> (fmul x, c+1)
19452 if (CFP01 && !CFP00 && N0.getOperand(i: 0) == N1) {
19453 SDValue NewCFP = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0.getOperand(i: 1),
19454 N2: DAG.getConstantFP(Val: 1.0, DL, VT));
19455 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1, N2: NewCFP);
19456 }
19457
19458 // (fadd (fmul x, c), (fadd x, x)) -> (fmul x, c+2)
19459 if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD &&
19460 N1.getOperand(i: 0) == N1.getOperand(i: 1) &&
19461 N0.getOperand(i: 0) == N1.getOperand(i: 0)) {
19462 SDValue NewCFP = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0.getOperand(i: 1),
19463 N2: DAG.getConstantFP(Val: 2.0, DL, VT));
19464 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0.getOperand(i: 0), N2: NewCFP);
19465 }
19466 }
19467
19468 if (N1.getOpcode() == ISD::FMUL) {
19469 bool CFP10 = DAG.isConstantFPBuildVectorOrConstantFP(N: N1.getOperand(i: 0));
19470 bool CFP11 = DAG.isConstantFPBuildVectorOrConstantFP(N: N1.getOperand(i: 1));
19471
19472 // (fadd x, (fmul x, c)) -> (fmul x, c+1)
19473 if (CFP11 && !CFP10 && N1.getOperand(i: 0) == N0) {
19474 SDValue NewCFP = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N1.getOperand(i: 1),
19475 N2: DAG.getConstantFP(Val: 1.0, DL, VT));
19476 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0, N2: NewCFP);
19477 }
19478
19479 // (fadd (fadd x, x), (fmul x, c)) -> (fmul x, c+2)
19480 if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD &&
19481 N0.getOperand(i: 0) == N0.getOperand(i: 1) &&
19482 N1.getOperand(i: 0) == N0.getOperand(i: 0)) {
19483 SDValue NewCFP = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N1.getOperand(i: 1),
19484 N2: DAG.getConstantFP(Val: 2.0, DL, VT));
19485 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N1.getOperand(i: 0), N2: NewCFP);
19486 }
19487 }
19488
19489 if (N0.getOpcode() == ISD::FADD) {
19490 bool CFP00 = DAG.isConstantFPBuildVectorOrConstantFP(N: N0.getOperand(i: 0));
19491 // (fadd (fadd x, x), x) -> (fmul x, 3.0)
19492 if (!CFP00 && N0.getOperand(i: 0) == N0.getOperand(i: 1) &&
19493 (N0.getOperand(i: 0) == N1)) {
19494 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1,
19495 N2: DAG.getConstantFP(Val: 3.0, DL, VT));
19496 }
19497 }
19498
19499 if (N1.getOpcode() == ISD::FADD) {
19500 bool CFP10 = DAG.isConstantFPBuildVectorOrConstantFP(N: N1.getOperand(i: 0));
19501 // (fadd x, (fadd x, x)) -> (fmul x, 3.0)
19502 if (!CFP10 && N1.getOperand(i: 0) == N1.getOperand(i: 1) &&
19503 N1.getOperand(i: 0) == N0) {
19504 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0,
19505 N2: DAG.getConstantFP(Val: 3.0, DL, VT));
19506 }
19507 }
19508
19509 // (fadd (fadd x, x), (fadd x, x)) -> (fmul x, 4.0)
19510 if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD &&
19511 N0.getOperand(i: 0) == N0.getOperand(i: 1) &&
19512 N1.getOperand(i: 0) == N1.getOperand(i: 1) &&
19513 N0.getOperand(i: 0) == N1.getOperand(i: 0)) {
19514 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0.getOperand(i: 0),
19515 N2: DAG.getConstantFP(Val: 4.0, DL, VT));
19516 }
19517 }
19518 } // reassoc && nsz && AllowNewConst
19519
19520 if (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros()) {
19521 // Fold fadd(vecreduce(x), vecreduce(y)) -> vecreduce(fadd(x, y))
19522 if (SDValue SD = reassociateReduction(RedOpc: ISD::VECREDUCE_FADD, Opc: ISD::FADD, DL,
19523 VT, N0, N1, Flags))
19524 return SD;
19525 }
19526
19527 // FADD -> FMA combines:
19528 if (SDValue Fused = visitFADDForFMACombine(N)) {
19529 if (Fused.getOpcode() != ISD::DELETED_NODE)
19530 AddToWorklist(N: Fused.getNode());
19531 return Fused;
19532 }
19533 return SDValue();
19534}
19535
19536SDValue DAGCombiner::visitSTRICT_FADD(SDNode *N) {
19537 SDValue Chain = N->getOperand(Num: 0);
19538 SDValue N0 = N->getOperand(Num: 1);
19539 SDValue N1 = N->getOperand(Num: 2);
19540 EVT VT = N->getValueType(ResNo: 0);
19541 EVT ChainVT = N->getValueType(ResNo: 1);
19542 SDLoc DL(N);
19543 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
19544
19545 // fold (strict_fadd A, (fneg B)) -> (strict_fsub A, B)
19546 if (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FSUB, VT))
19547 if (SDValue NegN1 = TLI.getCheaperNegatedExpression(
19548 Op: N1, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize)) {
19549 return DAG.getNode(Opcode: ISD::STRICT_FSUB, DL, VTList: DAG.getVTList(VT1: VT, VT2: ChainVT),
19550 Ops: {Chain, N0, NegN1});
19551 }
19552
19553 // fold (strict_fadd (fneg A), B) -> (strict_fsub B, A)
19554 if (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::STRICT_FSUB, VT))
19555 if (SDValue NegN0 = TLI.getCheaperNegatedExpression(
19556 Op: N0, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize)) {
19557 return DAG.getNode(Opcode: ISD::STRICT_FSUB, DL, VTList: DAG.getVTList(VT1: VT, VT2: ChainVT),
19558 Ops: {Chain, N1, NegN0});
19559 }
19560 return SDValue();
19561}
19562
19563SDValue DAGCombiner::visitFSUB(SDNode *N) {
19564 SDValue N0 = N->getOperand(Num: 0);
19565 SDValue N1 = N->getOperand(Num: 1);
19566 ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N: N0, AllowUndefs: true);
19567 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, AllowUndefs: true);
19568 EVT VT = N->getValueType(ResNo: 0);
19569 SDLoc DL(N);
19570 const SDNodeFlags Flags = N->getFlags();
19571 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
19572
19573 if (SDValue R = DAG.simplifyFPBinop(Opcode: N->getOpcode(), X: N0, Y: N1, Flags))
19574 return R;
19575
19576 // fold (fsub c1, c2) -> c1-c2
19577 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FSUB, DL, VT, Ops: {N0, N1}))
19578 return C;
19579
19580 // fold vector ops
19581 if (VT.isVector())
19582 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
19583 return FoldedVOp;
19584
19585 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
19586 return NewSel;
19587
19588 // (fsub A, 0) -> A
19589 if (N1CFP && N1CFP->isZero()) {
19590 if (!N1CFP->isNegative() || DAG.canIgnoreSignBitOfZero(Op: SDValue(N, 0))) {
19591 return N0;
19592 }
19593 }
19594
19595 if (N0 == N1) {
19596 // (fsub x, x) -> 0.0
19597 if (Flags.hasNoNaNs())
19598 return DAG.getConstantFP(Val: 0.0f, DL, VT);
19599 }
19600
19601 // (fsub -0.0, N1) -> -N1
19602 if (N0CFP && N0CFP->isZero()) {
19603 if (N0CFP->isNegative() || DAG.canIgnoreSignBitOfZero(Op: SDValue(N, 0))) {
19604 // We cannot replace an FSUB(+-0.0,X) with FNEG(X) when denormals are
19605 // flushed to zero, unless all users treat denorms as zero (DAZ).
19606 // FIXME: This transform will change the sign of a NaN and the behavior
19607 // of a signaling NaN. It is only valid when a NoNaN flag is present.
19608 DenormalMode DenormMode = DAG.getDenormalMode(VT);
19609 if (DenormMode == DenormalMode::getIEEE()) {
19610 if (SDValue NegN1 =
19611 TLI.getNegatedExpression(Op: N1, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize))
19612 return NegN1;
19613 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::FNEG, VT))
19614 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: N1);
19615 }
19616 }
19617 }
19618
19619 if (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros() &&
19620 N1.getOpcode() == ISD::FADD) {
19621 // X - (X + Y) -> -Y
19622 if (N0 == N1->getOperand(Num: 0))
19623 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: N1->getOperand(Num: 1));
19624 // X - (Y + X) -> -Y
19625 if (N0 == N1->getOperand(Num: 1))
19626 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: N1->getOperand(Num: 0));
19627 }
19628
19629 // fold (fsub A, (fneg B)) -> (fadd A, B)
19630 if (SDValue NegN1 =
19631 TLI.getNegatedExpression(Op: N1, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize))
19632 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0, N2: NegN1);
19633
19634 // FSUB -> FMA combines:
19635 if (SDValue Fused = visitFSUBForFMACombine(N)) {
19636 AddToWorklist(N: Fused.getNode());
19637 return Fused;
19638 }
19639
19640 return SDValue();
19641}
19642
19643// Transform IEEE Floats:
19644// (fmul C, (uitofp Pow2))
19645// -> (bitcast_to_FP (add (bitcast_to_INT C), Log2(Pow2) << mantissa))
19646// (fdiv C, (uitofp Pow2))
19647// -> (bitcast_to_FP (sub (bitcast_to_INT C), Log2(Pow2) << mantissa))
19648//
19649// The rationale is fmul/fdiv by a power of 2 is just change the exponent, so
19650// there is no need for more than an add/sub.
19651//
19652// This is valid under the following circumstances:
19653// 1) We are dealing with IEEE floats
19654// 2) C is normal
19655// 3) The fmul/fdiv add/sub will not go outside of min/max exponent bounds.
19656// TODO: Much of this could also be used for generating `ldexp` on targets the
19657// prefer it.
19658SDValue DAGCombiner::combineFMulOrFDivWithIntPow2(SDNode *N) {
19659 EVT VT = N->getValueType(ResNo: 0);
19660 if (!APFloat::isIEEELikeFP(VT.getFltSemantics()))
19661 return SDValue();
19662
19663 SDValue ConstOp, Pow2Op;
19664
19665 std::optional<int> Mantissa;
19666 auto GetConstAndPow2Ops = [&](unsigned ConstOpIdx) {
19667 if (ConstOpIdx == 1 && N->getOpcode() == ISD::FDIV)
19668 return false;
19669
19670 ConstOp = peekThroughBitcasts(V: N->getOperand(Num: ConstOpIdx));
19671 Pow2Op = N->getOperand(Num: 1 - ConstOpIdx);
19672 unsigned Pow2Opc = Pow2Op.getOpcode();
19673 if (Pow2Opc != ISD::UINT_TO_FP && Pow2Opc != ISD::SINT_TO_FP)
19674 return false;
19675
19676 Pow2Op = Pow2Op.getOperand(i: 0);
19677
19678 KnownBits Pow2OpKnownBits = DAG.computeKnownBits(Op: Pow2Op);
19679 if (Pow2Opc == ISD::SINT_TO_FP && !Pow2OpKnownBits.isNonNegative())
19680 return false;
19681
19682 int MaxExpChange = Pow2OpKnownBits.countMaxActiveBits();
19683
19684 auto IsFPConstValid = [N, MaxExpChange, &Mantissa](ConstantFPSDNode *CFP) {
19685 if (CFP == nullptr)
19686 return false;
19687
19688 const APFloat &APF = CFP->getValueAPF();
19689
19690 // Make sure we have normal constant.
19691 if (!APF.isNormal())
19692 return false;
19693
19694 // Make sure the floats exponent is within the bounds that this transform
19695 // produces bitwise equals value.
19696 int CurExp = ilogb(Arg: APF);
19697 // FMul by pow2 will only increase exponent.
19698 int MinExp =
19699 N->getOpcode() == ISD::FMUL ? CurExp : (CurExp - MaxExpChange);
19700 // FDiv by pow2 will only decrease exponent.
19701 int MaxExp =
19702 N->getOpcode() == ISD::FDIV ? CurExp : (CurExp + MaxExpChange);
19703 if (MinExp <= APFloat::semanticsMinExponent(APF.getSemantics()) ||
19704 MaxExp >= APFloat::semanticsMaxExponent(APF.getSemantics()))
19705 return false;
19706
19707 // Finally make sure we actually know the mantissa for the float type.
19708 int ThisMantissa = APFloat::semanticsPrecision(APF.getSemantics()) - 1;
19709 if (!Mantissa)
19710 Mantissa = ThisMantissa;
19711
19712 return *Mantissa == ThisMantissa && ThisMantissa > 0;
19713 };
19714
19715 // TODO: We may be able to include undefs.
19716 return ISD::matchUnaryFpPredicate(Op: ConstOp, Match: IsFPConstValid);
19717 };
19718
19719 if (!GetConstAndPow2Ops(0) && !GetConstAndPow2Ops(1))
19720 return SDValue();
19721
19722 if (!TLI.optimizeFMulOrFDivAsShiftAddBitcast(N, FPConst: ConstOp, IntPow2: Pow2Op))
19723 return SDValue();
19724
19725 // Get log2 after all other checks have taken place. This is because
19726 // BuildLogBase2 may create a new node.
19727 SDLoc DL(N);
19728 // Get Log2 type with same bitwidth as the float type (VT).
19729 EVT NewIntVT = VT.changeElementType(
19730 Context&: *DAG.getContext(),
19731 EltVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VT.getScalarSizeInBits()));
19732
19733 SDValue Log2 = BuildLogBase2(V: Pow2Op, DL, KnownNeverZero: DAG.isKnownNeverZero(Op: Pow2Op),
19734 /*InexpensiveOnly*/ true, OutVT: NewIntVT);
19735 if (!Log2)
19736 return SDValue();
19737
19738 // Perform actual transform.
19739 SDValue MantissaShiftCnt =
19740 DAG.getShiftAmountConstant(Val: *Mantissa, VT: NewIntVT, DL);
19741 // TODO: Sometimes Log2 is of form `(X + C)`. `(X + C) << C1` should fold to
19742 // `(X << C1) + (C << C1)`, but that isn't always the case because of the
19743 // cast. We could implement that by handle here to handle the casts.
19744 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT: NewIntVT, N1: Log2, N2: MantissaShiftCnt);
19745 SDValue ResAsInt =
19746 DAG.getNode(Opcode: N->getOpcode() == ISD::FMUL ? ISD::ADD : ISD::SUB, DL,
19747 VT: NewIntVT, N1: DAG.getBitcast(VT: NewIntVT, V: ConstOp), N2: Shift);
19748 SDValue ResAsFP = DAG.getBitcast(VT, V: ResAsInt);
19749 return ResAsFP;
19750}
19751
19752SDValue DAGCombiner::visitFMUL(SDNode *N) {
19753 SDValue N0 = N->getOperand(Num: 0);
19754 SDValue N1 = N->getOperand(Num: 1);
19755 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1, AllowUndefs: true);
19756 EVT VT = N->getValueType(ResNo: 0);
19757 SDLoc DL(N);
19758 const SDNodeFlags Flags = N->getFlags();
19759 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
19760
19761 if (SDValue R = DAG.simplifyFPBinop(Opcode: N->getOpcode(), X: N0, Y: N1, Flags))
19762 return R;
19763
19764 // fold (fmul c1, c2) -> c1*c2
19765 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FMUL, DL, VT, Ops: {N0, N1}))
19766 return C;
19767
19768 // canonicalize constant to RHS
19769 if (DAG.isConstantFPBuildVectorOrConstantFP(N: N0) &&
19770 !DAG.isConstantFPBuildVectorOrConstantFP(N: N1))
19771 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1, N2: N0);
19772
19773 // fold vector ops
19774 if (VT.isVector())
19775 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
19776 return FoldedVOp;
19777
19778 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
19779 return NewSel;
19780
19781 if (Flags.hasAllowReassociation()) {
19782 // fmul (fmul X, C1), C2 -> fmul X, C1 * C2
19783 if (DAG.isConstantFPBuildVectorOrConstantFP(N: N1) &&
19784 N0.getOpcode() == ISD::FMUL) {
19785 SDValue N00 = N0.getOperand(i: 0);
19786 SDValue N01 = N0.getOperand(i: 1);
19787 // Avoid an infinite loop by making sure that N00 is not a constant
19788 // (the inner multiply has not been constant folded yet).
19789 if (DAG.isConstantFPBuildVectorOrConstantFP(N: N01) &&
19790 !DAG.isConstantFPBuildVectorOrConstantFP(N: N00)) {
19791 SDValue MulConsts = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N01, N2: N1);
19792 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N00, N2: MulConsts);
19793 }
19794 }
19795
19796 // Match a special-case: we convert X * 2.0 into fadd.
19797 // fmul (fadd X, X), C -> fmul X, 2.0 * C
19798 if (N0.getOpcode() == ISD::FADD && N0.hasOneUse() &&
19799 N0.getOperand(i: 0) == N0.getOperand(i: 1)) {
19800 const SDValue Two = DAG.getConstantFP(Val: 2.0, DL, VT);
19801 SDValue MulConsts = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Two, N2: N1);
19802 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0.getOperand(i: 0), N2: MulConsts);
19803 }
19804
19805 // Fold fmul(vecreduce(x), vecreduce(y)) -> vecreduce(fmul(x, y))
19806 if (SDValue SD = reassociateReduction(RedOpc: ISD::VECREDUCE_FMUL, Opc: ISD::FMUL, DL,
19807 VT, N0, N1, Flags))
19808 return SD;
19809 }
19810
19811 // fold (fmul X, 2.0) -> (fadd X, X)
19812 if (N1CFP && N1CFP->isExactlyValue(V: +2.0))
19813 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0, N2: N0);
19814
19815 // fold (fmul X, -1.0) -> (fsub -0.0, X)
19816 if (N1CFP && N1CFP->isMinusOne()) {
19817 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::FSUB, VT)) {
19818 return DAG.getNode(Opcode: ISD::FSUB, DL, VT,
19819 N1: DAG.getConstantFP(Val: -0.0, DL, VT), N2: N0, Flags);
19820 }
19821 }
19822
19823 // -N0 * -N1 --> N0 * N1
19824 TargetLowering::NegatibleCost CostN0 =
19825 TargetLowering::NegatibleCost::Expensive;
19826 TargetLowering::NegatibleCost CostN1 =
19827 TargetLowering::NegatibleCost::Expensive;
19828 SDValue NegN0 =
19829 TLI.getNegatedExpression(Op: N0, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize, Cost&: CostN0);
19830 if (NegN0) {
19831 HandleSDNode NegN0Handle(NegN0);
19832 SDValue NegN1 =
19833 TLI.getNegatedExpression(Op: N1, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize, Cost&: CostN1);
19834 if (NegN1 && (CostN0 == TargetLowering::NegatibleCost::Cheaper ||
19835 CostN1 == TargetLowering::NegatibleCost::Cheaper))
19836 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: NegN0, N2: NegN1);
19837 }
19838
19839 // fold (fmul X, (select (fcmp X > 0.0), -1.0, 1.0)) -> (fneg (fabs X))
19840 // fold (fmul X, (select (fcmp X > 0.0), 1.0, -1.0)) -> (fabs X)
19841 if (Flags.hasNoNaNs() && Flags.hasNoSignedZeros() &&
19842 (N0.getOpcode() == ISD::SELECT || N1.getOpcode() == ISD::SELECT) &&
19843 TLI.isOperationLegal(Op: ISD::FABS, VT)) {
19844 SDValue Select = N0, X = N1;
19845 if (Select.getOpcode() != ISD::SELECT)
19846 std::swap(a&: Select, b&: X);
19847
19848 SDValue Cond = Select.getOperand(i: 0);
19849 auto TrueOpnd = dyn_cast<ConstantFPSDNode>(Val: Select.getOperand(i: 1));
19850 auto FalseOpnd = dyn_cast<ConstantFPSDNode>(Val: Select.getOperand(i: 2));
19851
19852 if (TrueOpnd && FalseOpnd && Cond.getOpcode() == ISD::SETCC &&
19853 Cond.getOperand(i: 0) == X && isa<ConstantFPSDNode>(Val: Cond.getOperand(i: 1)) &&
19854 cast<ConstantFPSDNode>(Val: Cond.getOperand(i: 1))->isPosZero()) {
19855 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
19856 switch (CC) {
19857 default: break;
19858 case ISD::SETOLT:
19859 case ISD::SETULT:
19860 case ISD::SETOLE:
19861 case ISD::SETULE:
19862 case ISD::SETLT:
19863 case ISD::SETLE:
19864 std::swap(a&: TrueOpnd, b&: FalseOpnd);
19865 [[fallthrough]];
19866 case ISD::SETOGT:
19867 case ISD::SETUGT:
19868 case ISD::SETOGE:
19869 case ISD::SETUGE:
19870 case ISD::SETGT:
19871 case ISD::SETGE:
19872 if (TrueOpnd->isMinusOne() && FalseOpnd->isOne() &&
19873 TLI.isOperationLegal(Op: ISD::FNEG, VT))
19874 return DAG.getNode(Opcode: ISD::FNEG, DL, VT,
19875 Operand: DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: X));
19876 if (TrueOpnd->isOne() && FalseOpnd->isMinusOne())
19877 return DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: X);
19878
19879 break;
19880 }
19881 }
19882 }
19883
19884 // FMUL -> FMA combines:
19885 if (SDValue Fused = visitFMULForFMADistributiveCombine(N)) {
19886 AddToWorklist(N: Fused.getNode());
19887 return Fused;
19888 }
19889
19890 // Don't do `combineFMulOrFDivWithIntPow2` until after FMUL -> FMA has been
19891 // able to run.
19892 if (SDValue R = combineFMulOrFDivWithIntPow2(N))
19893 return R;
19894
19895 return SDValue();
19896}
19897
19898SDValue DAGCombiner::visitFMA(SDNode *N) {
19899 SDValue N0 = N->getOperand(Num: 0);
19900 SDValue N1 = N->getOperand(Num: 1);
19901 SDValue N2 = N->getOperand(Num: 2);
19902 ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Val&: N0);
19903 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(Val&: N1);
19904 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(Val&: N2);
19905 EVT VT = N->getValueType(ResNo: 0);
19906 SDLoc DL(N);
19907 // FMA nodes have flags that propagate to the created nodes.
19908 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
19909
19910 // Constant fold FMA.
19911 if (SDValue C =
19912 DAG.FoldConstantArithmetic(Opcode: N->getOpcode(), DL, VT, Ops: {N0, N1, N2}))
19913 return C;
19914
19915 // (-N0 * -N1) + N2 --> (N0 * N1) + N2
19916 TargetLowering::NegatibleCost CostN0 =
19917 TargetLowering::NegatibleCost::Expensive;
19918 TargetLowering::NegatibleCost CostN1 =
19919 TargetLowering::NegatibleCost::Expensive;
19920 SDValue NegN0 =
19921 TLI.getNegatedExpression(Op: N0, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize, Cost&: CostN0);
19922 if (NegN0) {
19923 HandleSDNode NegN0Handle(NegN0);
19924 SDValue NegN1 =
19925 TLI.getNegatedExpression(Op: N1, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize, Cost&: CostN1);
19926 if (NegN1 && (CostN0 == TargetLowering::NegatibleCost::Cheaper ||
19927 CostN1 == TargetLowering::NegatibleCost::Cheaper))
19928 return DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: NegN0, N2: NegN1, N3: N2);
19929 }
19930
19931 if (N->getFlags().hasNoNaNs() && N->getFlags().hasNoInfs()) {
19932 if (N->getFlags().hasNoSignedZeros() || (N2CFP && !N2CFP->isNegZero())) {
19933 if (N0CFP && N0CFP->isZero())
19934 return N2;
19935 if (N1CFP && N1CFP->isZero())
19936 return N2;
19937 }
19938 }
19939
19940 if (N0CFP && N0CFP->isOne())
19941 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1, N2);
19942 if (N1CFP && N1CFP->isOne())
19943 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0, N2);
19944
19945 // Canonicalize (fma c, x, y) -> (fma x, c, y)
19946 if (DAG.isConstantFPBuildVectorOrConstantFP(N: N0) &&
19947 !DAG.isConstantFPBuildVectorOrConstantFP(N: N1))
19948 return DAG.getNode(Opcode: ISD::FMA, DL, VT, N1, N2: N0, N3: N2);
19949
19950 bool CanReassociate = N->getFlags().hasAllowReassociation();
19951 if (CanReassociate) {
19952 // (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2)
19953 if (N2.getOpcode() == ISD::FMUL && N0 == N2.getOperand(i: 0) &&
19954 DAG.isConstantFPBuildVectorOrConstantFP(N: N1) &&
19955 DAG.isConstantFPBuildVectorOrConstantFP(N: N2.getOperand(i: 1))) {
19956 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0,
19957 N2: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1, N2: N2.getOperand(i: 1)));
19958 }
19959
19960 // (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y)
19961 if (N0.getOpcode() == ISD::FMUL &&
19962 DAG.isConstantFPBuildVectorOrConstantFP(N: N1) &&
19963 DAG.isConstantFPBuildVectorOrConstantFP(N: N0.getOperand(i: 1))) {
19964 return DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: N0.getOperand(i: 0),
19965 N2: DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1, N2: N0.getOperand(i: 1)),
19966 N3: N2);
19967 }
19968 }
19969
19970 // (fma x, -1, y) -> (fadd (fneg x), y)
19971 if (N1CFP) {
19972 if (N1CFP->isOne())
19973 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N0, N2);
19974
19975 if (N1CFP->isMinusOne() &&
19976 (!LegalOperations || TLI.isOperationLegal(Op: ISD::FNEG, VT))) {
19977 SDValue RHSNeg = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: N0);
19978 AddToWorklist(N: RHSNeg.getNode());
19979 return DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: N2, N2: RHSNeg);
19980 }
19981
19982 // fma (fneg x), K, y -> fma x -K, y
19983 if (N0.getOpcode() == ISD::FNEG &&
19984 (TLI.isOperationLegal(Op: ISD::ConstantFP, VT) ||
19985 (N1.hasOneUse() &&
19986 !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT, ForCodeSize)))) {
19987 return DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: N0.getOperand(i: 0),
19988 N2: DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: N1), N3: N2);
19989 }
19990 }
19991
19992 if (CanReassociate) {
19993 // (fma x, c, x) -> (fmul x, (c+1))
19994 if (N1CFP && N0 == N2) {
19995 return DAG.getNode(
19996 Opcode: ISD::FMUL, DL, VT, N1: N0,
19997 N2: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1, N2: DAG.getConstantFP(Val: 1.0, DL, VT)));
19998 }
19999
20000 // (fma x, c, (fneg x)) -> (fmul x, (c-1))
20001 if (N1CFP && N2.getOpcode() == ISD::FNEG && N2.getOperand(i: 0) == N0) {
20002 return DAG.getNode(
20003 Opcode: ISD::FMUL, DL, VT, N1: N0,
20004 N2: DAG.getNode(Opcode: ISD::FADD, DL, VT, N1, N2: DAG.getConstantFP(Val: -1.0, DL, VT)));
20005 }
20006 }
20007
20008 // fold ((fma (fneg X), Y, (fneg Z)) -> fneg (fma X, Y, Z))
20009 // fold ((fma X, (fneg Y), (fneg Z)) -> fneg (fma X, Y, Z))
20010 if (!TLI.isFNegFree(VT))
20011 if (SDValue Neg = TLI.getCheaperNegatedExpression(
20012 Op: SDValue(N, 0), DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize))
20013 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Neg);
20014 return SDValue();
20015}
20016
20017SDValue DAGCombiner::visitFMAD(SDNode *N) {
20018 SDValue N0 = N->getOperand(Num: 0);
20019 SDValue N1 = N->getOperand(Num: 1);
20020 SDValue N2 = N->getOperand(Num: 2);
20021 EVT VT = N->getValueType(ResNo: 0);
20022 SDLoc DL(N);
20023
20024 // Constant fold FMAD.
20025 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FMAD, DL, VT, Ops: {N0, N1, N2}))
20026 return C;
20027
20028 return SDValue();
20029}
20030
20031SDValue DAGCombiner::visitFMULADD(SDNode *N) {
20032 SDValue N0 = N->getOperand(Num: 0);
20033 SDValue N1 = N->getOperand(Num: 1);
20034 SDValue N2 = N->getOperand(Num: 2);
20035 EVT VT = N->getValueType(ResNo: 0);
20036 SDLoc DL(N);
20037
20038 // Constant fold FMULADD.
20039 if (SDValue C =
20040 DAG.FoldConstantArithmetic(Opcode: ISD::FMULADD, DL, VT, Ops: {N0, N1, N2}))
20041 return C;
20042
20043 return SDValue();
20044}
20045
20046// Combine multiple FDIVs with the same divisor into multiple FMULs by the
20047// reciprocal.
20048// E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip)
20049// Notice that this is not always beneficial. One reason is different targets
20050// may have different costs for FDIV and FMUL, so sometimes the cost of two
20051// FDIVs may be lower than the cost of one FDIV and two FMULs. Another reason
20052// is the critical path is increased from "one FDIV" to "one FDIV + one FMUL".
20053SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
20054 // TODO: Limit this transform based on optsize/minsize - it always creates at
20055 // least 1 extra instruction. But the perf win may be substantial enough
20056 // that only minsize should restrict this.
20057 const SDNodeFlags Flags = N->getFlags();
20058 if (LegalDAG || !Flags.hasAllowReciprocal())
20059 return SDValue();
20060
20061 // Skip if current node is a reciprocal/fneg-reciprocal.
20062 SDValue N0 = N->getOperand(Num: 0), N1 = N->getOperand(Num: 1);
20063 ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N: N0, /* AllowUndefs */ true);
20064 if (N0CFP && (N0CFP->isOne() || N0CFP->isMinusOne()))
20065 return SDValue();
20066
20067 // Exit early if the target does not want this transform or if there can't
20068 // possibly be enough uses of the divisor to make the transform worthwhile.
20069 unsigned MinUses = TLI.combineRepeatedFPDivisors();
20070
20071 // For splat vectors, scale the number of uses by the splat factor. If we can
20072 // convert the division into a scalar op, that will likely be much faster.
20073 unsigned NumElts = 1;
20074 EVT VT = N->getValueType(ResNo: 0);
20075 if (VT.isVector() && DAG.isSplatValue(V: N1))
20076 NumElts = VT.getVectorMinNumElements();
20077
20078 if (!MinUses || (N1->use_size() * NumElts) < MinUses)
20079 return SDValue();
20080
20081 // Find all FDIV users of the same divisor.
20082 // Use a set because duplicates may be present in the user list.
20083 SetVector<SDNode *> Users;
20084 for (auto *U : N1->users()) {
20085 if (U->getOpcode() == ISD::FDIV && U->getOperand(Num: 1) == N1) {
20086 // Skip X/sqrt(X) that has not been simplified to sqrt(X) yet.
20087 if (U->getOperand(Num: 1).getOpcode() == ISD::FSQRT &&
20088 U->getOperand(Num: 0) == U->getOperand(Num: 1).getOperand(i: 0) &&
20089 U->getFlags().hasAllowReassociation() &&
20090 U->getFlags().hasNoSignedZeros())
20091 continue;
20092
20093 // This division is eligible for optimization only if global unsafe math
20094 // is enabled or if this division allows reciprocal formation.
20095 if (U->getFlags().hasAllowReciprocal())
20096 Users.insert(X: U);
20097 }
20098 }
20099
20100 // Now that we have the actual number of divisor uses, make sure it meets
20101 // the minimum threshold specified by the target.
20102 if ((Users.size() * NumElts) < MinUses)
20103 return SDValue();
20104
20105 SDLoc DL(N);
20106 SDValue FPOne = DAG.getConstantFP(Val: 1.0, DL, VT);
20107 SDValue Reciprocal = DAG.getNode(Opcode: ISD::FDIV, DL, VT, N1: FPOne, N2: N1, Flags);
20108
20109 // Dividend / Divisor -> Dividend * Reciprocal
20110 for (auto *U : Users) {
20111 SDValue Dividend = U->getOperand(Num: 0);
20112 if (Dividend != FPOne) {
20113 SDValue NewNode = DAG.getNode(Opcode: ISD::FMUL, DL: SDLoc(U), VT, N1: Dividend,
20114 N2: Reciprocal, Flags);
20115 CombineTo(N: U, Res: NewNode);
20116 } else if (U != Reciprocal.getNode()) {
20117 // In the absence of fast-math-flags, this user node is always the
20118 // same node as Reciprocal, but with FMF they may be different nodes.
20119 CombineTo(N: U, Res: Reciprocal);
20120 }
20121 }
20122 return SDValue(N, 0); // N was replaced.
20123}
20124
20125SDValue DAGCombiner::visitFDIV(SDNode *N) {
20126 SDValue N0 = N->getOperand(Num: 0);
20127 SDValue N1 = N->getOperand(Num: 1);
20128 EVT VT = N->getValueType(ResNo: 0);
20129 SDLoc DL(N);
20130 SDNodeFlags Flags = N->getFlags();
20131 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
20132
20133 if (SDValue R = DAG.simplifyFPBinop(Opcode: N->getOpcode(), X: N0, Y: N1, Flags))
20134 return R;
20135
20136 // fold (fdiv c1, c2) -> c1/c2
20137 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FDIV, DL, VT, Ops: {N0, N1}))
20138 return C;
20139
20140 // fold vector ops
20141 if (VT.isVector())
20142 if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
20143 return FoldedVOp;
20144
20145 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
20146 return NewSel;
20147
20148 if (SDValue V = combineRepeatedFPDivisors(N))
20149 return V;
20150
20151 // fold (fdiv X, c2) -> (fmul X, 1/c2) if there is no loss in precision, or
20152 // the loss is acceptable with AllowReciprocal.
20153 if (auto *N1CFP = isConstOrConstSplatFP(N: N1, AllowUndefs: true)) {
20154 // Compute the reciprocal 1.0 / c2.
20155 const APFloat &N1APF = N1CFP->getValueAPF();
20156 APFloat Recip = APFloat::getOne(Sem: N1APF.getSemantics());
20157 APFloat::opStatus st = Recip.divide(RHS: N1APF, RM: APFloat::rmNearestTiesToEven);
20158 // Only do the transform if the reciprocal is a legal fp immediate that
20159 // isn't too nasty (eg NaN, denormal, ...).
20160 if (((st == APFloat::opOK && !Recip.isDenormal()) ||
20161 (st == APFloat::opInexact && Flags.hasAllowReciprocal())) &&
20162 (!LegalOperations ||
20163 // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
20164 // backend)... we should handle this gracefully after Legalize.
20165 // TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT) ||
20166 TLI.isOperationLegal(Op: ISD::ConstantFP, VT) ||
20167 TLI.isFPImmLegal(Recip, VT, ForCodeSize)))
20168 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0,
20169 N2: DAG.getConstantFP(Val: Recip, DL, VT));
20170 }
20171
20172 if (Flags.hasAllowReciprocal()) {
20173 // If this FDIV is part of a reciprocal square root, it may be folded
20174 // into a target-specific square root estimate instruction.
20175 bool N1AllowReciprocal = N1->getFlags().hasAllowReciprocal();
20176 if (N1.getOpcode() == ISD::FSQRT) {
20177 if (SDValue RV = buildRsqrtEstimate(Op: N1.getOperand(i: 0), Flags: N1->getFlags()))
20178 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0, N2: RV);
20179 } else if (N1.getOpcode() == ISD::FP_EXTEND &&
20180 N1.getOperand(i: 0).getOpcode() == ISD::FSQRT &&
20181 N1AllowReciprocal) {
20182 if (SDValue RV = buildRsqrtEstimate(Op: N1.getOperand(i: 0).getOperand(i: 0),
20183 Flags: N1.getOperand(i: 0)->getFlags())) {
20184 RV = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(N1), VT, Operand: RV);
20185 AddToWorklist(N: RV.getNode());
20186 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0, N2: RV);
20187 }
20188 } else if (N1.getOpcode() == ISD::FP_ROUND &&
20189 N1.getOperand(i: 0).getOpcode() == ISD::FSQRT) {
20190 if (SDValue RV = buildRsqrtEstimate(Op: N1.getOperand(i: 0).getOperand(i: 0),
20191 Flags: N1.getOperand(i: 0)->getFlags())) {
20192 RV = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N1), VT, N1: RV, N2: N1.getOperand(i: 1));
20193 AddToWorklist(N: RV.getNode());
20194 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0, N2: RV);
20195 }
20196 } else if (N1.getOpcode() == ISD::FMUL) {
20197 // Look through an FMUL. Even though this won't remove the FDIV directly,
20198 // it's still worthwhile to get rid of the FSQRT if possible.
20199 SDValue Sqrt, Y;
20200 if (N1.getOperand(i: 0).getOpcode() == ISD::FSQRT) {
20201 Sqrt = N1.getOperand(i: 0);
20202 Y = N1.getOperand(i: 1);
20203 } else if (N1.getOperand(i: 1).getOpcode() == ISD::FSQRT) {
20204 Sqrt = N1.getOperand(i: 1);
20205 Y = N1.getOperand(i: 0);
20206 }
20207 if (Sqrt.getNode()) {
20208 // If the other multiply operand is known positive, pull it into the
20209 // sqrt. That will eliminate the division if we convert to an estimate.
20210 if (Flags.hasAllowReassociation() && N1.hasOneUse() &&
20211 N1->getFlags().hasAllowReassociation() && Sqrt.hasOneUse()) {
20212 SDValue A;
20213 if (Y.getOpcode() == ISD::FABS && Y.hasOneUse())
20214 A = Y.getOperand(i: 0);
20215 else if (Y == Sqrt.getOperand(i: 0))
20216 A = Y;
20217 if (A) {
20218 // X / (fabs(A) * sqrt(Z)) --> X / sqrt(A*A*Z) --> X * rsqrt(A*A*Z)
20219 // X / (A * sqrt(A)) --> X / sqrt(A*A*A) --> X * rsqrt(A*A*A)
20220 SDValue AA = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: A, N2: A);
20221 SDValue AAZ =
20222 DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: AA, N2: Sqrt.getOperand(i: 0));
20223 if (SDValue Rsqrt = buildRsqrtEstimate(Op: AAZ, Flags: Sqrt->getFlags()))
20224 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0, N2: Rsqrt);
20225
20226 // Estimate creation failed. Clean up speculatively created nodes.
20227 recursivelyDeleteUnusedNodes(N: AAZ.getNode());
20228 }
20229 }
20230
20231 // We found a FSQRT, so try to make this fold:
20232 // X / (Y * sqrt(Z)) -> X * (rsqrt(Z) / Y)
20233 if (SDValue Rsqrt =
20234 buildRsqrtEstimate(Op: Sqrt.getOperand(i: 0), Flags: Sqrt->getFlags())) {
20235 SDValue Div = DAG.getNode(Opcode: ISD::FDIV, DL: SDLoc(N1), VT, N1: Rsqrt, N2: Y);
20236 AddToWorklist(N: Div.getNode());
20237 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N0, N2: Div);
20238 }
20239 }
20240 }
20241
20242 // Fold into a reciprocal estimate and multiply instead of a real divide.
20243 if (Flags.hasNoInfs())
20244 if (SDValue RV = BuildDivEstimate(N: N0, Op: N1, Flags))
20245 return RV;
20246 }
20247
20248 // Fold X/Sqrt(X) -> Sqrt(X)
20249 if (DAG.canIgnoreSignBitOfZero(Op: SDValue(N, 0)) &&
20250 Flags.hasAllowReassociation())
20251 if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(i: 0))
20252 return N1;
20253
20254 // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y)
20255 TargetLowering::NegatibleCost CostN0 =
20256 TargetLowering::NegatibleCost::Expensive;
20257 TargetLowering::NegatibleCost CostN1 =
20258 TargetLowering::NegatibleCost::Expensive;
20259 SDValue NegN0 =
20260 TLI.getNegatedExpression(Op: N0, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize, Cost&: CostN0);
20261 if (NegN0) {
20262 HandleSDNode NegN0Handle(NegN0);
20263 SDValue NegN1 =
20264 TLI.getNegatedExpression(Op: N1, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize, Cost&: CostN1);
20265 if (NegN1 && (CostN0 == TargetLowering::NegatibleCost::Cheaper ||
20266 CostN1 == TargetLowering::NegatibleCost::Cheaper))
20267 return DAG.getNode(Opcode: ISD::FDIV, DL, VT, N1: NegN0, N2: NegN1);
20268 }
20269
20270 if (SDValue R = combineFMulOrFDivWithIntPow2(N))
20271 return R;
20272
20273 return SDValue();
20274}
20275
20276SDValue DAGCombiner::visitFREM(SDNode *N) {
20277 SDValue N0 = N->getOperand(Num: 0);
20278 SDValue N1 = N->getOperand(Num: 1);
20279 EVT VT = N->getValueType(ResNo: 0);
20280 SDNodeFlags Flags = N->getFlags();
20281 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
20282 SDLoc DL(N);
20283
20284 if (SDValue R = DAG.simplifyFPBinop(Opcode: N->getOpcode(), X: N0, Y: N1, Flags))
20285 return R;
20286
20287 // fold (frem c1, c2) -> fmod(c1,c2)
20288 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FREM, DL, VT, Ops: {N0, N1}))
20289 return C;
20290
20291 if (SDValue NewSel = foldBinOpIntoSelect(BO: N))
20292 return NewSel;
20293
20294 // Lower frem N0, N1 => x - trunc(N0 / N1) * N1, providing N1 is an integer
20295 // power of 2.
20296 if (!TLI.isOperationLegal(Op: ISD::FREM, VT) &&
20297 TLI.isOperationLegalOrCustom(Op: ISD::FMUL, VT) &&
20298 TLI.isOperationLegalOrCustom(Op: ISD::FDIV, VT) &&
20299 TLI.isOperationLegalOrCustom(Op: ISD::FTRUNC, VT) &&
20300 DAG.isKnownToBeAPowerOfTwoFP(Val: N1)) {
20301 bool NeedsCopySign = !DAG.canIgnoreSignBitOfZero(Op: SDValue(N, 0)) &&
20302 !DAG.cannotBeOrderedNegativeFP(Op: N0);
20303 SDValue Div = DAG.getNode(Opcode: ISD::FDIV, DL, VT, N1: N0, N2: N1);
20304 SDValue Rnd = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT, Operand: Div);
20305 SDValue MLA;
20306 if (TLI.isFMAFasterThanFMulAndFAdd(MF: DAG.getMachineFunction(), VT)) {
20307 MLA = DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Rnd),
20308 N2: N1, N3: N0);
20309 } else {
20310 SDValue Mul = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Rnd, N2: N1);
20311 MLA = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: N0, N2: Mul);
20312 }
20313 return NeedsCopySign ? DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT, N1: MLA, N2: N0) : MLA;
20314 }
20315
20316 return SDValue();
20317}
20318
20319SDValue DAGCombiner::visitFSQRT(SDNode *N) {
20320 SDNodeFlags Flags = N->getFlags();
20321
20322 // Require 'ninf' flag since sqrt(+Inf) = +Inf, but the estimation goes as:
20323 // sqrt(+Inf) == rsqrt(+Inf) * +Inf = 0 * +Inf = NaN
20324 if (!Flags.hasApproximateFuncs() || !Flags.hasNoInfs())
20325 return SDValue();
20326
20327 SDValue N0 = N->getOperand(Num: 0);
20328 if (TLI.isFsqrtCheap(X: N0, DAG))
20329 return SDValue();
20330
20331 // FSQRT nodes have flags that propagate to the created nodes.
20332 SelectionDAG::FlagInserter FlagInserter(DAG, Flags);
20333 // TODO: If this is N0/sqrt(N0), and we reach this node before trying to
20334 // transform the fdiv, we may produce a sub-optimal estimate sequence
20335 // because the reciprocal calculation may not have to filter out a
20336 // 0.0 input.
20337 return buildSqrtEstimate(Op: N0, Flags);
20338}
20339
20340/// copysign(x, fp_extend(y)) -> copysign(x, y)
20341/// copysign(x, fp_round(y)) -> copysign(x, y)
20342/// Operands to the functions are the type of X and Y respectively.
20343static inline bool CanCombineFCOPYSIGN_EXTEND_ROUND(EVT XTy, EVT YTy) {
20344 // Always fold no-op FP casts.
20345 if (XTy == YTy)
20346 return true;
20347
20348 // Do not optimize out type conversion of f128 type yet.
20349 // For some targets like x86_64, configuration is changed to keep one f128
20350 // value in one SSE register, but instruction selection cannot handle
20351 // FCOPYSIGN on SSE registers yet.
20352 if (YTy == MVT::f128)
20353 return false;
20354
20355 // Avoid mismatched vector operand types, for better instruction selection.
20356 return !YTy.isVector();
20357}
20358
20359static inline bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N) {
20360 SDValue N1 = N->getOperand(Num: 1);
20361 if (N1.getOpcode() != ISD::FP_EXTEND &&
20362 N1.getOpcode() != ISD::FP_ROUND)
20363 return false;
20364 EVT N1VT = N1->getValueType(ResNo: 0);
20365 EVT N1Op0VT = N1->getOperand(Num: 0).getValueType();
20366 return CanCombineFCOPYSIGN_EXTEND_ROUND(XTy: N1VT, YTy: N1Op0VT);
20367}
20368
20369SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
20370 SDValue N0 = N->getOperand(Num: 0);
20371 SDValue N1 = N->getOperand(Num: 1);
20372 EVT VT = N->getValueType(ResNo: 0);
20373 SDLoc DL(N);
20374
20375 // fold (fcopysign c1, c2) -> fcopysign(c1,c2)
20376 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FCOPYSIGN, DL, VT, Ops: {N0, N1}))
20377 return C;
20378
20379 // copysign(x, fp_extend(y)) -> copysign(x, y)
20380 // copysign(x, fp_round(y)) -> copysign(x, y)
20381 if (CanCombineFCOPYSIGN_EXTEND_ROUND(N))
20382 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT, N1: N0, N2: N1.getOperand(i: 0));
20383
20384 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
20385 return SDValue(N, 0);
20386
20387 if (VT != N1.getValueType())
20388 return SDValue();
20389
20390 // If this is equivalent to a disjoint or, replace it with one. This can
20391 // happen if the sign operand is a sign mask (i.e., x << sign_bit_position).
20392 if (DAG.SignBitIsZeroFP(Op: N0) &&
20393 DAG.computeKnownBits(Op: N1).Zero.isMaxSignedValue()) {
20394 // TODO: Just directly match the shift pattern. computeKnownBits is heavy
20395 // for a such a narrowly targeted case.
20396 EVT IntVT = VT.changeTypeToInteger();
20397 // TODO: It appears to be profitable in some situations to unconditionally
20398 // emit a fabs(n0) to perform this combine.
20399 SDValue CastSrc0 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: N0);
20400 SDValue CastSrc1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IntVT, Operand: N1);
20401
20402 SDValue SignOr = DAG.getNode(Opcode: ISD::OR, DL, VT: IntVT, N1: CastSrc0, N2: CastSrc1,
20403 Flags: SDNodeFlags::Disjoint);
20404 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: SignOr);
20405 }
20406
20407 return SDValue();
20408}
20409
20410SDValue DAGCombiner::visitFPOW(SDNode *N) {
20411 ConstantFPSDNode *ExponentC = isConstOrConstSplatFP(N: N->getOperand(Num: 1));
20412 if (!ExponentC)
20413 return SDValue();
20414 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
20415
20416 // Try to convert x ** (1/3) into cube root.
20417 // TODO: Handle the various flavors of long double.
20418 // TODO: Since we're approximating, we don't need an exact 1/3 exponent.
20419 // Some range near 1/3 should be fine.
20420 EVT VT = N->getValueType(ResNo: 0);
20421 EVT ScalarVT = VT.getScalarType();
20422 if ((ScalarVT == MVT::f32 &&
20423 ExponentC->getValueAPF().isExactlyValue(V: 1.0f / 3.0f)) ||
20424 (ScalarVT == MVT::f64 &&
20425 ExponentC->getValueAPF().isExactlyValue(V: 1.0 / 3.0))) {
20426 // pow(-0.0, 1/3) = +0.0; cbrt(-0.0) = -0.0.
20427 // pow(-inf, 1/3) = +inf; cbrt(-inf) = -inf.
20428 // pow(-val, 1/3) = nan; cbrt(-val) = -num.
20429 // For regular numbers, rounding may cause the results to differ.
20430 // Therefore, we require { nsz ninf nnan afn } for this transform.
20431 // TODO: We could select out the special cases if we don't have nsz/ninf.
20432 SDNodeFlags Flags = N->getFlags();
20433 if (!Flags.hasNoSignedZeros() || !Flags.hasNoInfs() || !Flags.hasNoNaNs() ||
20434 !Flags.hasApproximateFuncs())
20435 return SDValue();
20436
20437 // Do not create a cbrt() libcall if the target does not have it, and do not
20438 // turn a pow that has lowering support into a cbrt() libcall.
20439 RTLIB::Libcall LC = RTLIB::getCBRT(VT);
20440 bool HasLibCall =
20441 DAG.getLibcalls().getLibcallImpl(Call: LC) != RTLIB::Unsupported;
20442 if (!HasLibCall ||
20443 (!DAG.getTargetLoweringInfo().isOperationExpand(Op: ISD::FPOW, VT) &&
20444 DAG.getTargetLoweringInfo().isOperationExpand(Op: ISD::FCBRT, VT)))
20445 return SDValue();
20446
20447 return DAG.getNode(Opcode: ISD::FCBRT, DL: SDLoc(N), VT, Operand: N->getOperand(Num: 0));
20448 }
20449
20450 // Try to convert x ** (1/4) and x ** (3/4) into square roots.
20451 // x ** (1/2) is canonicalized to sqrt, so we do not bother with that case.
20452 // TODO: This could be extended (using a target hook) to handle smaller
20453 // power-of-2 fractional exponents.
20454 bool ExponentIs025 = ExponentC->getValueAPF().isExactlyValue(V: 0.25);
20455 bool ExponentIs075 = ExponentC->getValueAPF().isExactlyValue(V: 0.75);
20456 if (ExponentIs025 || ExponentIs075) {
20457 // pow(-0.0, 0.25) = +0.0; sqrt(sqrt(-0.0)) = -0.0.
20458 // pow(-inf, 0.25) = +inf; sqrt(sqrt(-inf)) = NaN.
20459 // pow(-0.0, 0.75) = +0.0; sqrt(-0.0) * sqrt(sqrt(-0.0)) = +0.0.
20460 // pow(-inf, 0.75) = +inf; sqrt(-inf) * sqrt(sqrt(-inf)) = NaN.
20461 // For regular numbers, rounding may cause the results to differ.
20462 // Therefore, we require { nsz ninf afn } for this transform.
20463 // TODO: We could select out the special cases if we don't have nsz/ninf.
20464 SDNodeFlags Flags = N->getFlags();
20465
20466 // We only need no signed zeros for the 0.25 case.
20467 if ((!Flags.hasNoSignedZeros() && ExponentIs025) || !Flags.hasNoInfs() ||
20468 !Flags.hasApproximateFuncs())
20469 return SDValue();
20470
20471 // Don't double the number of libcalls. We are trying to inline fast code.
20472 if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(Op: ISD::FSQRT, VT))
20473 return SDValue();
20474
20475 // Assume that libcalls are the smallest code.
20476 // TODO: This restriction should probably be lifted for vectors.
20477 if (ForCodeSize)
20478 return SDValue();
20479
20480 // pow(X, 0.25) --> sqrt(sqrt(X))
20481 SDLoc DL(N);
20482 SDValue Sqrt = DAG.getNode(Opcode: ISD::FSQRT, DL, VT, Operand: N->getOperand(Num: 0));
20483 SDValue SqrtSqrt = DAG.getNode(Opcode: ISD::FSQRT, DL, VT, Operand: Sqrt);
20484 if (ExponentIs025)
20485 return SqrtSqrt;
20486 // pow(X, 0.75) --> sqrt(X) * sqrt(sqrt(X))
20487 return DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Sqrt, N2: SqrtSqrt);
20488 }
20489
20490 return SDValue();
20491}
20492
20493static SDValue foldFPToIntToFP(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
20494 const TargetLowering &TLI) {
20495 // We can fold the fpto[us]i -> [us]itofp pattern into a single ftrunc.
20496 // Additionally, if there are clamps ([us]min or [us]max) around
20497 // the fpto[us]i, we can fold those into fminnum/fmaxnum around the ftrunc.
20498 // If NoSignedZerosFPMath is enabled, this is a direct replacement.
20499 // Otherwise, for strict math, we must handle edge cases:
20500 // 1. For unsigned conversions, use FABS to handle negative cases. Take -0.0
20501 // as example, it first becomes integer 0, and is converted back to +0.0.
20502 // FTRUNC on its own could produce -0.0.
20503
20504 // FIXME: We should be able to use node-level FMF here.
20505 EVT VT = N->getValueType(ResNo: 0);
20506 if (!TLI.isOperationLegalOrCustom(Op: ISD::FTRUNC, VT))
20507 return SDValue();
20508
20509 bool IsUnsigned = N->getOpcode() == ISD::UINT_TO_FP;
20510 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP;
20511 assert(IsSigned || IsUnsigned);
20512
20513 // Don't fold if the individual cast operations are already legal,
20514 // as FTRUNC may have a more expensive custom expansion.
20515 EVT IntVT = N->getOperand(Num: 0).getValueType();
20516 EVT LegalIntVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: IntVT);
20517 unsigned FPToIntOp = IsUnsigned ? ISD::FP_TO_UINT : ISD::FP_TO_SINT;
20518 unsigned IntToFPOp = N->getOpcode(); // UINT_TO_FP or SINT_TO_FP
20519 if (!TLI.isOperationLegal(Op: ISD::FTRUNC, VT) &&
20520 TLI.isOperationLegal(Op: FPToIntOp, VT: LegalIntVT) &&
20521 TLI.isOperationLegal(Op: IntToFPOp, VT))
20522 return SDValue();
20523
20524 bool IsSignedZeroSafe = DAG.canIgnoreSignBitOfZero(Op: SDValue(N, 0));
20525 // For signed conversions: The optimization changes signed zero behavior.
20526 if (IsSigned && !IsSignedZeroSafe)
20527 return SDValue();
20528 // For unsigned conversions, we need FABS to canonicalize -0.0 to +0.0
20529 // (unless outputting a signed zero is OK).
20530 if (IsUnsigned && !IsSignedZeroSafe && !TLI.isFAbsFree(VT))
20531 return SDValue();
20532
20533 // Collect potential clamp operations (outermost to innermost) and peel.
20534 struct ClampInfo {
20535 bool IsMin;
20536 SDValue Constant;
20537 };
20538 constexpr unsigned MaxClamps = 2;
20539 SmallVector<ClampInfo, MaxClamps> Clamps;
20540 unsigned MinOp = IsUnsigned ? ISD::UMIN : ISD::SMIN;
20541 unsigned MaxOp = IsUnsigned ? ISD::UMAX : ISD::SMAX;
20542 SDValue IntVal = N->getOperand(Num: 0);
20543 for (unsigned Level = 0; Level < MaxClamps; ++Level) {
20544 if (!IntVal.hasOneUse() ||
20545 (IntVal.getOpcode() != MinOp && IntVal.getOpcode() != MaxOp))
20546 break;
20547 SDValue RHS = IntVal.getOperand(i: 1);
20548 APInt IntConst;
20549 if (auto *IntConstNode = dyn_cast<ConstantSDNode>(Val&: RHS))
20550 IntConst = IntConstNode->getAPIntValue();
20551 else if (!ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: IntConst))
20552 return SDValue();
20553 APFloat FPConst(VT.getFltSemantics());
20554 FPConst.convertFromAPInt(Input: IntConst, IsSigned, RM: APFloat::rmNearestTiesToEven);
20555 // Verify roundtrip exactness.
20556 APSInt RoundTrip(IntConst.getBitWidth(), IsUnsigned);
20557 bool IsExact;
20558 if (FPConst.convertToInteger(Result&: RoundTrip, RM: APFloat::rmTowardZero, IsExact: &IsExact) !=
20559 APFloat::opOK ||
20560 !IsExact || static_cast<const APInt &>(RoundTrip) != IntConst)
20561 return SDValue();
20562 bool IsMin = IntVal.getOpcode() == MinOp;
20563 Clamps.push_back(Elt: {.IsMin: IsMin, .Constant: DAG.getConstantFP(Val: FPConst, DL, VT)});
20564 IntVal = IntVal.getOperand(i: 0);
20565 }
20566
20567 // Check that the sequence ends with the correct kind of fpto[us]i.
20568 if (IntVal.getOpcode() != FPToIntOp ||
20569 IntVal.getOperand(i: 0).getValueType() != VT)
20570 return SDValue();
20571
20572 SDValue Result = IntVal.getOperand(i: 0);
20573 if (IsUnsigned && !IsSignedZeroSafe && TLI.isFAbsFree(VT))
20574 Result = DAG.getNode(Opcode: ISD::FABS, DL, VT, Operand: Result);
20575 Result = DAG.getNode(Opcode: ISD::FTRUNC, DL, VT, Operand: Result);
20576 // Apply clamps, if any, in reverse order (innermost first).
20577 for (const ClampInfo &Clamp : reverse(C&: Clamps)) {
20578 unsigned FPClampOp =
20579 getMinMaxOpcodeForClamp(IsMin: Clamp.IsMin, Operand1: Result, Operand2: Clamp.Constant, DAG, TLI);
20580 if (FPClampOp == ISD::DELETED_NODE)
20581 return SDValue();
20582 Result = DAG.getNode(Opcode: FPClampOp, DL, VT, N1: Result, N2: Clamp.Constant);
20583 }
20584 return Result;
20585}
20586
20587SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
20588 SDValue N0 = N->getOperand(Num: 0);
20589 EVT VT = N->getValueType(ResNo: 0);
20590 EVT OpVT = N0.getValueType();
20591 SDLoc DL(N);
20592
20593 // [us]itofp(undef) = 0, because the result value is bounded.
20594 if (N0.isUndef())
20595 return DAG.getConstantFP(Val: 0.0, DL, VT);
20596
20597 // fold (sint_to_fp c1) -> c1fp
20598 // ...but only if the target supports immediate floating-point values
20599 if ((!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::ConstantFP, VT)))
20600 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::SINT_TO_FP, DL, VT, Ops: {N0}))
20601 return C;
20602
20603 // If the input is a legal type, and SINT_TO_FP is not legal on this target,
20604 // but UINT_TO_FP is legal on this target, try to convert.
20605 if (!hasOperation(Opcode: ISD::SINT_TO_FP, VT: OpVT) &&
20606 hasOperation(Opcode: ISD::UINT_TO_FP, VT: OpVT)) {
20607 // If the sign bit is known to be zero, we can change this to UINT_TO_FP.
20608 if (DAG.SignBitIsZero(Op: N0))
20609 return DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT, Operand: N0);
20610 }
20611
20612 // The next optimizations are desirable only if SELECT_CC can be lowered.
20613 // fold (sint_to_fp (setcc x, y, cc)) -> (select (setcc x, y, cc), -1.0, 0.0)
20614 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 &&
20615 !VT.isVector() &&
20616 (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::ConstantFP, VT)))
20617 return DAG.getSelect(DL, VT, Cond: N0, LHS: DAG.getConstantFP(Val: -1.0, DL, VT),
20618 RHS: DAG.getConstantFP(Val: 0.0, DL, VT));
20619
20620 // fold (sint_to_fp (zext (setcc x, y, cc))) ->
20621 // (select (setcc x, y, cc), 1.0, 0.0)
20622 if (N0.getOpcode() == ISD::ZERO_EXTEND &&
20623 N0.getOperand(i: 0).getOpcode() == ISD::SETCC && !VT.isVector() &&
20624 (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::ConstantFP, VT)))
20625 return DAG.getSelect(DL, VT, Cond: N0.getOperand(i: 0),
20626 LHS: DAG.getConstantFP(Val: 1.0, DL, VT),
20627 RHS: DAG.getConstantFP(Val: 0.0, DL, VT));
20628
20629 if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
20630 return FTrunc;
20631
20632 // fold (sint_to_fp (trunc nsw x)) -> (sint_to_fp x)
20633 if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoSignedWrap() &&
20634 TLI.isTypeDesirableForOp(ISD::SINT_TO_FP,
20635 VT: N0.getOperand(i: 0).getValueType()))
20636 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: N0.getOperand(i: 0));
20637
20638 return SDValue();
20639}
20640
20641SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
20642 SDValue N0 = N->getOperand(Num: 0);
20643 EVT VT = N->getValueType(ResNo: 0);
20644 EVT OpVT = N0.getValueType();
20645 SDLoc DL(N);
20646
20647 // [us]itofp(undef) = 0, because the result value is bounded.
20648 if (N0.isUndef())
20649 return DAG.getConstantFP(Val: 0.0, DL, VT);
20650
20651 // fold (uint_to_fp c1) -> c1fp
20652 // ...but only if the target supports immediate floating-point values
20653 if ((!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::ConstantFP, VT)))
20654 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::UINT_TO_FP, DL, VT, Ops: {N0}))
20655 return C;
20656
20657 // If the input is a legal type, and UINT_TO_FP is not legal on this target,
20658 // but SINT_TO_FP is legal on this target, try to convert.
20659 if (!hasOperation(Opcode: ISD::UINT_TO_FP, VT: OpVT) &&
20660 hasOperation(Opcode: ISD::SINT_TO_FP, VT: OpVT)) {
20661 // If the sign bit is known to be zero, we can change this to SINT_TO_FP.
20662 if (DAG.SignBitIsZero(Op: N0))
20663 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: N0);
20664 }
20665
20666 // fold (uint_to_fp (setcc x, y, cc)) -> (select (setcc x, y, cc), 1.0, 0.0)
20667 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() &&
20668 (!LegalOperations || TLI.isOperationLegalOrCustom(Op: ISD::ConstantFP, VT)))
20669 return DAG.getSelect(DL, VT, Cond: N0, LHS: DAG.getConstantFP(Val: 1.0, DL, VT),
20670 RHS: DAG.getConstantFP(Val: 0.0, DL, VT));
20671
20672 if (SDValue FTrunc = foldFPToIntToFP(N, DL, DAG, TLI))
20673 return FTrunc;
20674
20675 // fold (uint_to_fp (trunc nuw x)) -> (uint_to_fp x)
20676 if (N0.getOpcode() == ISD::TRUNCATE && N0->getFlags().hasNoUnsignedWrap() &&
20677 TLI.isTypeDesirableForOp(ISD::UINT_TO_FP,
20678 VT: N0.getOperand(i: 0).getValueType()))
20679 return DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT, Operand: N0.getOperand(i: 0));
20680
20681 return SDValue();
20682}
20683
20684// Fold (fp_to_{s/u}int ({s/u}int_to_fpx)) -> zext x, sext x, trunc x, or x
20685static SDValue FoldIntToFPToInt(SDNode *N, const SDLoc &DL, SelectionDAG &DAG) {
20686 SDValue N0 = N->getOperand(Num: 0);
20687 EVT VT = N->getValueType(ResNo: 0);
20688
20689 if (N0.getOpcode() != ISD::UINT_TO_FP && N0.getOpcode() != ISD::SINT_TO_FP)
20690 return SDValue();
20691
20692 SDValue Src = N0.getOperand(i: 0);
20693 EVT SrcVT = Src.getValueType();
20694 bool IsInputSigned = N0.getOpcode() == ISD::SINT_TO_FP;
20695 bool IsOutputSigned = N->getOpcode() == ISD::FP_TO_SINT;
20696
20697 // We can safely assume the conversion won't overflow the output range,
20698 // because (for example) (uint8_t)18293.f is undefined behavior.
20699
20700 // Since we can assume the conversion won't overflow, our decision as to
20701 // whether the input will fit in the float should depend on the minimum
20702 // of the input range and output range.
20703
20704 // This means this is also safe for a signed input and unsigned output, since
20705 // a negative input would lead to undefined behavior.
20706 unsigned InputSize = (int)SrcVT.getScalarSizeInBits() - IsInputSigned;
20707 unsigned OutputSize = (int)VT.getScalarSizeInBits();
20708 unsigned ActualSize = std::min(a: InputSize, b: OutputSize);
20709 const fltSemantics &Sem = N0.getValueType().getFltSemantics();
20710
20711 // We can only fold away the float conversion if the input range can be
20712 // represented exactly in the float range.
20713 if (APFloat::semanticsPrecision(Sem) >= ActualSize) {
20714 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits()) {
20715 unsigned ExtOp =
20716 IsInputSigned && IsOutputSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
20717 return DAG.getNode(Opcode: ExtOp, DL, VT, Operand: Src);
20718 }
20719 if (VT.getScalarSizeInBits() < SrcVT.getScalarSizeInBits())
20720 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Src);
20721 return DAG.getBitcast(VT, V: Src);
20722 }
20723 return SDValue();
20724}
20725
20726SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) {
20727 SDValue N0 = N->getOperand(Num: 0);
20728 EVT VT = N->getValueType(ResNo: 0);
20729 SDLoc DL(N);
20730
20731 // fold (fp_to_sint undef) -> undef
20732 if (N0.isUndef())
20733 return DAG.getUNDEF(VT);
20734
20735 // fold (fp_to_sint c1fp) -> c1
20736 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FP_TO_SINT, DL, VT, Ops: {N0}))
20737 return C;
20738
20739 return FoldIntToFPToInt(N, DL, DAG);
20740}
20741
20742SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
20743 SDValue N0 = N->getOperand(Num: 0);
20744 EVT VT = N->getValueType(ResNo: 0);
20745 SDLoc DL(N);
20746
20747 // fold (fp_to_uint undef) -> undef
20748 if (N0.isUndef())
20749 return DAG.getUNDEF(VT);
20750
20751 // fold (fp_to_uint c1fp) -> c1
20752 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FP_TO_UINT, DL, VT, Ops: {N0}))
20753 return C;
20754
20755 return FoldIntToFPToInt(N, DL, DAG);
20756}
20757
20758SDValue DAGCombiner::visitXROUND(SDNode *N) {
20759 SDValue N0 = N->getOperand(Num: 0);
20760 EVT VT = N->getValueType(ResNo: 0);
20761
20762 // fold (lrint|llrint undef) -> undef
20763 // fold (lround|llround undef) -> undef
20764 if (N0.isUndef())
20765 return DAG.getUNDEF(VT);
20766
20767 // fold (lrint|llrint c1fp) -> c1
20768 // fold (lround|llround c1fp) -> c1
20769 if (SDValue C =
20770 DAG.FoldConstantArithmetic(Opcode: N->getOpcode(), DL: SDLoc(N), VT, Ops: {N0}))
20771 return C;
20772
20773 return SDValue();
20774}
20775
20776SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
20777 SDValue N0 = N->getOperand(Num: 0);
20778 SDValue N1 = N->getOperand(Num: 1);
20779 EVT VT = N->getValueType(ResNo: 0);
20780 SDLoc DL(N);
20781
20782 // fold (fp_round c1fp) -> c1fp
20783 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FP_ROUND, DL, VT, Ops: {N0, N1}))
20784 return C;
20785
20786 // fold (fp_round (fp_extend x)) -> x
20787 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(i: 0).getValueType())
20788 return N0.getOperand(i: 0);
20789
20790 // fold (fp_round (fp_round x)) -> (fp_round x)
20791 if (N0.getOpcode() == ISD::FP_ROUND) {
20792 const bool NIsTrunc = N->getConstantOperandVal(Num: 1) == 1;
20793 const bool N0IsTrunc = N0.getConstantOperandVal(i: 1) == 1;
20794
20795 // Avoid folding legal fp_rounds into non-legal ones.
20796 if (!hasOperation(Opcode: ISD::FP_ROUND, VT))
20797 return SDValue();
20798
20799 // Skip this folding if it results in an fp_round from f80 to f16.
20800 //
20801 // f80 to f16 always generates an expensive (and as yet, unimplemented)
20802 // libcall to __truncxfhf2 instead of selecting native f16 conversion
20803 // instructions from f32 or f64. Moreover, the first (value-preserving)
20804 // fp_round from f80 to either f32 or f64 may become a NOP in platforms like
20805 // x86.
20806 if (N0.getOperand(i: 0).getValueType() == MVT::f80 && VT == MVT::f16)
20807 return SDValue();
20808
20809 // If the first fp_round isn't a value preserving truncation, it might
20810 // introduce a tie in the second fp_round, that wouldn't occur in the
20811 // single-step fp_round we want to fold to.
20812 // In other words, double rounding isn't the same as rounding.
20813 // Also, this is a value preserving truncation iff both fp_round's are.
20814 if ((N->getFlags().hasAllowContract() &&
20815 N0->getFlags().hasAllowContract()) ||
20816 N0IsTrunc)
20817 return DAG.getNode(
20818 Opcode: ISD::FP_ROUND, DL, VT, N1: N0.getOperand(i: 0),
20819 N2: DAG.getIntPtrConstant(Val: NIsTrunc && N0IsTrunc, DL, /*isTarget=*/true));
20820 }
20821
20822 // fold (fp_round (copysign X, Y)) -> (copysign (fp_round X), Y)
20823 // Note: From a legality perspective, this is a two step transform. First,
20824 // we duplicate the fp_round to the arguments of the copysign, then we
20825 // eliminate the fp_round on Y. The second step requires an additional
20826 // predicate to match the implementation above.
20827 if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() &&
20828 CanCombineFCOPYSIGN_EXTEND_ROUND(XTy: VT,
20829 YTy: N0.getValueType())) {
20830 SDValue Tmp = DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N0), VT,
20831 N1: N0.getOperand(i: 0), N2: N1);
20832 AddToWorklist(N: Tmp.getNode());
20833 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT, N1: Tmp, N2: N0.getOperand(i: 1));
20834 }
20835
20836 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(Cast: N))
20837 return NewVSel;
20838
20839 return SDValue();
20840}
20841
20842// Eliminate a floating-point widening of a narrowed value if the fast math
20843// flags allow it.
20844static SDValue eliminateFPCastPair(SDNode *N) {
20845 SDValue N0 = N->getOperand(Num: 0);
20846 EVT VT = N->getValueType(ResNo: 0);
20847
20848 unsigned NarrowingOp;
20849 switch (N->getOpcode()) {
20850 case ISD::FP16_TO_FP:
20851 NarrowingOp = ISD::FP_TO_FP16;
20852 break;
20853 case ISD::BF16_TO_FP:
20854 NarrowingOp = ISD::FP_TO_BF16;
20855 break;
20856 case ISD::FP_EXTEND:
20857 NarrowingOp = ISD::FP_ROUND;
20858 break;
20859 default:
20860 llvm_unreachable("Expected widening FP cast");
20861 }
20862
20863 if (N0.getOpcode() == NarrowingOp && N0.getOperand(i: 0).getValueType() == VT) {
20864 const SDNodeFlags NarrowFlags = N0->getFlags();
20865 const SDNodeFlags WidenFlags = N->getFlags();
20866 // Narrowing can introduce inf and change the encoding of a nan, so the
20867 // widen must have the nnan and ninf flags to indicate that we don't need to
20868 // care about that. We are also removing a rounding step, and that requires
20869 // both the narrow and widen to allow contraction.
20870 if (WidenFlags.hasNoNaNs() && WidenFlags.hasNoInfs() &&
20871 NarrowFlags.hasAllowContract() && WidenFlags.hasAllowContract()) {
20872 return N0.getOperand(i: 0);
20873 }
20874 }
20875
20876 return SDValue();
20877}
20878
20879SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
20880 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
20881 SDValue N0 = N->getOperand(Num: 0);
20882 EVT VT = N->getValueType(ResNo: 0);
20883 SDLoc DL(N);
20884
20885 if (VT.isVector())
20886 if (SDValue FoldedVOp = SimplifyVCastOp(N, DL))
20887 return FoldedVOp;
20888
20889 // If this is fp_round(fpextend), don't fold it, allow ourselves to be folded.
20890 if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::FP_ROUND)
20891 return SDValue();
20892
20893 // fold (fp_extend c1fp) -> c1fp
20894 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FP_EXTEND, DL, VT, Ops: {N0}))
20895 return C;
20896
20897 // fold (fp_extend (fp16_to_fp op)) -> (fp16_to_fp op)
20898 if (N0.getOpcode() == ISD::FP16_TO_FP &&
20899 TLI.getOperationAction(Op: ISD::FP16_TO_FP, VT) == TargetLowering::Legal)
20900 return DAG.getNode(Opcode: ISD::FP16_TO_FP, DL, VT, Operand: N0.getOperand(i: 0));
20901
20902 // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the
20903 // value of X.
20904 if (N0.getOpcode() == ISD::FP_ROUND && N0.getConstantOperandVal(i: 1) == 1) {
20905 SDValue In = N0.getOperand(i: 0);
20906 if (In.getValueType() == VT) return In;
20907 if (VT.bitsLT(VT: In.getValueType()))
20908 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT, N1: In, N2: N0.getOperand(i: 1));
20909 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: In);
20910 }
20911
20912 // fold (fpext (load x)) -> (fpext (fptrunc (extload x)))
20913 if (ISD::isNormalLoad(N: N0.getNode()) && N0.hasOneUse()) {
20914 LoadSDNode *LN0 = cast<LoadSDNode>(Val&: N0);
20915 if (TLI.isLoadLegalOrCustom(ValVT: VT, MemVT: N0.getValueType(), Alignment: LN0->getAlign(),
20916 AddrSpace: LN0->getAddressSpace(), ExtType: ISD::EXTLOAD, Atomic: false)) {
20917 SDValue ExtLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT, Chain: LN0->getChain(),
20918 Ptr: LN0->getBasePtr(), MemVT: N0.getValueType(),
20919 MMO: LN0->getMemOperand());
20920 CombineTo(N, Res: ExtLoad);
20921 CombineTo(
20922 N: N0.getNode(),
20923 Res0: DAG.getNode(Opcode: ISD::FP_ROUND, DL: SDLoc(N0), VT: N0.getValueType(), N1: ExtLoad,
20924 N2: DAG.getIntPtrConstant(Val: 1, DL: SDLoc(N0), /*isTarget=*/true)),
20925 Res1: ExtLoad.getValue(R: 1));
20926 return SDValue(N, 0); // Return N so it doesn't get rechecked!
20927 }
20928 }
20929
20930 if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(Cast: N))
20931 return NewVSel;
20932
20933 if (SDValue CastEliminated = eliminateFPCastPair(N))
20934 return CastEliminated;
20935
20936 return SDValue();
20937}
20938
20939SDValue DAGCombiner::visitFCEIL(SDNode *N) {
20940 SDValue N0 = N->getOperand(Num: 0);
20941 EVT VT = N->getValueType(ResNo: 0);
20942
20943 // fold (fceil c1) -> fceil(c1)
20944 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FCEIL, DL: SDLoc(N), VT, Ops: {N0}))
20945 return C;
20946
20947 return SDValue();
20948}
20949
20950SDValue DAGCombiner::visitFTRUNC(SDNode *N) {
20951 SDValue N0 = N->getOperand(Num: 0);
20952 EVT VT = N->getValueType(ResNo: 0);
20953
20954 // fold (ftrunc c1) -> ftrunc(c1)
20955 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FTRUNC, DL: SDLoc(N), VT, Ops: {N0}))
20956 return C;
20957
20958 // fold ftrunc (known rounded int x) -> x
20959 // ftrunc is a part of fptosi/fptoui expansion on some targets, so this is
20960 // likely to be generated to extract integer from a rounded floating value.
20961 switch (N0.getOpcode()) {
20962 default: break;
20963 case ISD::FRINT:
20964 case ISD::FTRUNC:
20965 case ISD::FNEARBYINT:
20966 case ISD::FROUND:
20967 case ISD::FROUNDEVEN:
20968 case ISD::FFLOOR:
20969 case ISD::FCEIL:
20970 return N0;
20971 }
20972
20973 return SDValue();
20974}
20975
20976SDValue DAGCombiner::visitFFREXP(SDNode *N) {
20977 SDValue N0 = N->getOperand(Num: 0);
20978
20979 // fold (ffrexp c1) -> ffrexp(c1)
20980 if (DAG.isConstantFPBuildVectorOrConstantFP(N: N0))
20981 return DAG.getNode(Opcode: ISD::FFREXP, DL: SDLoc(N), VTList: N->getVTList(), N: N0);
20982 return SDValue();
20983}
20984
20985SDValue DAGCombiner::visitFFLOOR(SDNode *N) {
20986 SDValue N0 = N->getOperand(Num: 0);
20987 EVT VT = N->getValueType(ResNo: 0);
20988
20989 // fold (ffloor c1) -> ffloor(c1)
20990 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FFLOOR, DL: SDLoc(N), VT, Ops: {N0}))
20991 return C;
20992
20993 return SDValue();
20994}
20995
20996SDValue DAGCombiner::visitFNEG(SDNode *N) {
20997 SDValue N0 = N->getOperand(Num: 0);
20998 EVT VT = N->getValueType(ResNo: 0);
20999 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
21000
21001 // Constant fold FNEG.
21002 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FNEG, DL: SDLoc(N), VT, Ops: {N0}))
21003 return C;
21004
21005 if (SDValue NegN0 =
21006 TLI.getNegatedExpression(Op: N0, DAG, LegalOps: LegalOperations, OptForSize: ForCodeSize))
21007 return NegN0;
21008
21009 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0
21010 // FIXME: This is duplicated in getNegatibleCost, but getNegatibleCost doesn't
21011 // know it was called from a context with a nsz flag if the input fsub does
21012 // not.
21013 if (N0.getOpcode() == ISD::FSUB && N->getFlags().hasNoSignedZeros() &&
21014 N0.hasOneUse()) {
21015 return DAG.getNode(Opcode: ISD::FSUB, DL: SDLoc(N), VT, N1: N0.getOperand(i: 1),
21016 N2: N0.getOperand(i: 0));
21017 }
21018
21019 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
21020 return SDValue(N, 0);
21021
21022 if (SDValue Cast = foldSignChangeInBitcast(N))
21023 return Cast;
21024
21025 return SDValue();
21026}
21027
21028SDValue DAGCombiner::visitFMinMax(SDNode *N) {
21029 SDValue N0 = N->getOperand(Num: 0);
21030 SDValue N1 = N->getOperand(Num: 1);
21031 EVT VT = N->getValueType(ResNo: 0);
21032 const SDNodeFlags Flags = N->getFlags();
21033 unsigned Opc = N->getOpcode();
21034 bool PropAllNaNsToQNaNs = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
21035 bool PropOnlySNaNsToQNaNs = Opc == ISD::FMINNUM || Opc == ISD::FMAXNUM;
21036 bool IsMin =
21037 Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM || Opc == ISD::FMINIMUMNUM;
21038 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
21039
21040 // Constant fold.
21041 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: Opc, DL: SDLoc(N), VT, Ops: {N0, N1}))
21042 return C;
21043
21044 // Canonicalize to constant on RHS.
21045 if (DAG.isConstantFPBuildVectorOrConstantFP(N: N0) &&
21046 !DAG.isConstantFPBuildVectorOrConstantFP(N: N1))
21047 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, N1, N2: N0);
21048
21049 if (const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N: N1)) {
21050 const APFloat &AF = N1CFP->getValueAPF();
21051
21052 // minnum(X, qnan) -> X
21053 // maxnum(X, qnan) -> X
21054 // minnum(X, snan) -> qnan
21055 // maxnum(X, snan) -> qnan
21056 // minimum(X, nan) -> qnan
21057 // maximum(X, nan) -> qnan
21058 // minimumnum(X, nan) -> X
21059 // maximumnum(X, nan) -> X
21060 if (AF.isNaN()) {
21061 if (PropAllNaNsToQNaNs || (AF.isSignaling() && PropOnlySNaNsToQNaNs)) {
21062 if (AF.isSignaling())
21063 return DAG.getConstantFP(Val: AF.makeQuiet(), DL: SDLoc(N), VT);
21064 return N->getOperand(Num: 1);
21065 }
21066 return N->getOperand(Num: 0);
21067 }
21068
21069 // In the following folds, inf can be replaced with the largest finite
21070 // float, if the ninf flag is set.
21071 if (AF.isInfinity() || (Flags.hasNoInfs() && AF.isLargest())) {
21072 // minnum(X, -inf) -> -inf (ignoring sNaN -> qNaN propagation)
21073 // maxnum(X, +inf) -> +inf (ignoring sNaN -> qNaN propagation)
21074 // minimum(X, -inf) -> -inf if nnan
21075 // maximum(X, +inf) -> +inf if nnan
21076 // minimumnum(X, -inf) -> -inf
21077 // maximumnum(X, +inf) -> +inf
21078 if (IsMin == AF.isNegative() &&
21079 (!PropAllNaNsToQNaNs || Flags.hasNoNaNs()))
21080 return N->getOperand(Num: 1);
21081
21082 // minnum(X, +inf) -> X if nnan
21083 // maxnum(X, -inf) -> X if nnan
21084 // minimum(X, +inf) -> X (ignoring quieting of sNaNs)
21085 // maximum(X, -inf) -> X (ignoring quieting of sNaNs)
21086 // minimumnum(X, +inf) -> X if nnan
21087 // maximumnum(X, -inf) -> X if nnan
21088 if (IsMin != AF.isNegative() && (PropAllNaNsToQNaNs || Flags.hasNoNaNs()))
21089 return N->getOperand(Num: 0);
21090 }
21091 }
21092
21093 unsigned ReduceOpc;
21094 if (PropAllNaNsToQNaNs)
21095 ReduceOpc = IsMin ? ISD::VECREDUCE_FMINIMUM : ISD::VECREDUCE_FMAXIMUM;
21096 else if (PropOnlySNaNsToQNaNs)
21097 ReduceOpc = IsMin ? ISD::VECREDUCE_FMIN : ISD::VECREDUCE_FMAX;
21098 else
21099 ReduceOpc = IsMin ? ISD::VECREDUCE_FMINIMUMNUM : ISD::VECREDUCE_FMAXIMUMNUM;
21100
21101 if (SDValue SD =
21102 reassociateReduction(RedOpc: ReduceOpc, Opc, DL: SDLoc(N), VT, N0, N1, Flags))
21103 return SD;
21104
21105 return SDValue();
21106}
21107
21108SDValue DAGCombiner::visitFABS(SDNode *N) {
21109 SDValue N0 = N->getOperand(Num: 0);
21110 EVT VT = N->getValueType(ResNo: 0);
21111 SDLoc DL(N);
21112
21113 // fold (fabs c1) -> fabs(c1)
21114 if (SDValue C = DAG.FoldConstantArithmetic(Opcode: ISD::FABS, DL, VT, Ops: {N0}))
21115 return C;
21116
21117 if (SimplifyDemandedBits(Op: SDValue(N, 0)))
21118 return SDValue(N, 0);
21119
21120 if (SDValue Cast = foldSignChangeInBitcast(N))
21121 return Cast;
21122
21123 return SDValue();
21124}
21125
21126SDValue DAGCombiner::visitBRCOND(SDNode *N) {
21127 SDValue Chain = N->getOperand(Num: 0);
21128 SDValue N1 = N->getOperand(Num: 1);
21129 SDValue N2 = N->getOperand(Num: 2);
21130
21131 // BRCOND(FREEZE(cond)) is equivalent to BRCOND(cond) (both are
21132 // nondeterministic jumps).
21133 if (N1->getOpcode() == ISD::FREEZE && N1.hasOneUse()) {
21134 return DAG.getNode(Opcode: ISD::BRCOND, DL: SDLoc(N), VT: MVT::Other, N1: Chain,
21135 N2: N1->getOperand(Num: 0), N3: N2, Flags: N->getFlags());
21136 }
21137
21138 // Variant of the previous fold where there is a SETCC in between:
21139 // BRCOND(SETCC(FREEZE(X), CONST, Cond))
21140 // =>
21141 // BRCOND(FREEZE(SETCC(X, CONST, Cond)))
21142 // =>
21143 // BRCOND(SETCC(X, CONST, Cond))
21144 // This is correct if FREEZE(X) has one use and SETCC(FREEZE(X), CONST, Cond)
21145 // isn't equivalent to true or false.
21146 // For example, SETCC(FREEZE(X), -128, SETULT) cannot be folded to
21147 // FREEZE(SETCC(X, -128, SETULT)) because X can be poison.
21148 if (N1->getOpcode() == ISD::SETCC && N1.hasOneUse()) {
21149 SDValue S0 = N1->getOperand(Num: 0), S1 = N1->getOperand(Num: 1);
21150 ISD::CondCode Cond = cast<CondCodeSDNode>(Val: N1->getOperand(Num: 2))->get();
21151 ConstantSDNode *S0C = dyn_cast<ConstantSDNode>(Val&: S0);
21152 ConstantSDNode *S1C = dyn_cast<ConstantSDNode>(Val&: S1);
21153 bool Updated = false;
21154
21155 // Is 'X Cond C' always true or false?
21156 auto IsAlwaysTrueOrFalse = [](ISD::CondCode Cond, ConstantSDNode *C) {
21157 bool False = (Cond == ISD::SETULT && C->isZero()) ||
21158 (Cond == ISD::SETLT && C->isMinSignedValue()) ||
21159 (Cond == ISD::SETUGT && C->isAllOnes()) ||
21160 (Cond == ISD::SETGT && C->isMaxSignedValue());
21161 bool True = (Cond == ISD::SETULE && C->isAllOnes()) ||
21162 (Cond == ISD::SETLE && C->isMaxSignedValue()) ||
21163 (Cond == ISD::SETUGE && C->isZero()) ||
21164 (Cond == ISD::SETGE && C->isMinSignedValue());
21165 return True || False;
21166 };
21167
21168 if (S0->getOpcode() == ISD::FREEZE && S0.hasOneUse() && S1C) {
21169 if (!IsAlwaysTrueOrFalse(Cond, S1C)) {
21170 S0 = S0->getOperand(Num: 0);
21171 Updated = true;
21172 }
21173 }
21174 if (S1->getOpcode() == ISD::FREEZE && S1.hasOneUse() && S0C) {
21175 if (!IsAlwaysTrueOrFalse(ISD::getSetCCSwappedOperands(Operation: Cond), S0C)) {
21176 S1 = S1->getOperand(Num: 0);
21177 Updated = true;
21178 }
21179 }
21180
21181 if (Updated)
21182 return DAG.getNode(
21183 Opcode: ISD::BRCOND, DL: SDLoc(N), VT: MVT::Other, N1: Chain,
21184 N2: DAG.getSetCC(DL: SDLoc(N1), VT: N1->getValueType(ResNo: 0), LHS: S0, RHS: S1, Cond), N3: N2,
21185 Flags: N->getFlags());
21186 }
21187
21188 // If N is a constant we could fold this into a fallthrough or unconditional
21189 // branch. However that doesn't happen very often in normal code, because
21190 // Instcombine/SimplifyCFG should have handled the available opportunities.
21191 // If we did this folding here, it would be necessary to update the
21192 // MachineBasicBlock CFG, which is awkward.
21193
21194 // fold a brcond with a setcc condition into a BR_CC node if BR_CC is legal
21195 // on the target, also copy fast math flags.
21196 if (N1.getOpcode() == ISD::SETCC &&
21197 TLI.isOperationLegalOrCustom(Op: ISD::BR_CC,
21198 VT: N1.getOperand(i: 0).getValueType())) {
21199 return DAG.getNode(Opcode: ISD::BR_CC, DL: SDLoc(N), VT: MVT::Other, N1: Chain,
21200 N2: N1.getOperand(i: 2), N3: N1.getOperand(i: 0), N4: N1.getOperand(i: 1), N5: N2,
21201 Flags: N1->getFlags());
21202 }
21203
21204 if (N1.hasOneUse()) {
21205 // rebuildSetCC calls visitXor which may change the Chain when there is a
21206 // STRICT_FSETCC/STRICT_FSETCCS involved. Use a handle to track changes.
21207 HandleSDNode ChainHandle(Chain);
21208 if (SDValue NewN1 = rebuildSetCC(N: N1))
21209 return DAG.getNode(Opcode: ISD::BRCOND, DL: SDLoc(N), VT: MVT::Other,
21210 N1: ChainHandle.getValue(), N2: NewN1, N3: N2, Flags: N->getFlags());
21211 }
21212
21213 return SDValue();
21214}
21215
21216SDValue DAGCombiner::rebuildSetCC(SDValue N) {
21217 if (N.getOpcode() == ISD::SRL ||
21218 (N.getOpcode() == ISD::TRUNCATE &&
21219 (N.getOperand(i: 0).hasOneUse() &&
21220 N.getOperand(i: 0).getOpcode() == ISD::SRL))) {
21221 // Look pass the truncate.
21222 if (N.getOpcode() == ISD::TRUNCATE)
21223 N = N.getOperand(i: 0);
21224
21225 // Match this pattern so that we can generate simpler code:
21226 //
21227 // %a = ...
21228 // %b = and i32 %a, 2
21229 // %c = srl i32 %b, 1
21230 // brcond i32 %c ...
21231 //
21232 // into
21233 //
21234 // %a = ...
21235 // %b = and i32 %a, 2
21236 // %c = setcc eq %b, 0
21237 // brcond %c ...
21238 //
21239 // This applies only when the AND constant value has one bit set and the
21240 // SRL constant is equal to the log2 of the AND constant. The back-end is
21241 // smart enough to convert the result into a TEST/JMP sequence.
21242 SDValue Op0 = N.getOperand(i: 0);
21243 SDValue Op1 = N.getOperand(i: 1);
21244
21245 if (Op0.getOpcode() == ISD::AND && Op1.getOpcode() == ISD::Constant) {
21246 SDValue AndOp1 = Op0.getOperand(i: 1);
21247
21248 if (AndOp1.getOpcode() == ISD::Constant) {
21249 const APInt &AndConst = AndOp1->getAsAPIntVal();
21250
21251 if (AndConst.isPowerOf2() &&
21252 Op1->getAsAPIntVal() == AndConst.logBase2()) {
21253 SDLoc DL(N);
21254 return DAG.getSetCC(DL, VT: getSetCCResultType(VT: Op0.getValueType()),
21255 LHS: Op0, RHS: DAG.getConstant(Val: 0, DL, VT: Op0.getValueType()),
21256 Cond: ISD::SETNE);
21257 }
21258 }
21259 }
21260 }
21261
21262 // Transform (brcond (xor x, y)) -> (brcond (setcc, x, y, ne))
21263 // Transform (brcond (xor (xor x, y), -1)) -> (brcond (setcc, x, y, eq))
21264 if (N.getOpcode() == ISD::XOR) {
21265 // Because we may call this on a speculatively constructed
21266 // SimplifiedSetCC Node, we need to simplify this node first.
21267 // Ideally this should be folded into SimplifySetCC and not
21268 // here. For now, grab a handle to N so we don't lose it from
21269 // replacements interal to the visit.
21270 while (N.getOpcode() == ISD::XOR) {
21271 HandleSDNode XORHandle(N);
21272 SDValue Tmp = visitXOR(N: N.getNode());
21273 // No simplification done.
21274 if (!Tmp.getNode())
21275 break;
21276 // Returning N is form in-visit replacement that may invalidated
21277 // N. Grab value from Handle.
21278 if (Tmp.getNode() == N.getNode())
21279 N = XORHandle.getValue();
21280 else // Node simplified. Try simplifying again.
21281 N = Tmp;
21282 }
21283
21284 if (N.getOpcode() != ISD::XOR)
21285 return N;
21286
21287 SDValue Op0 = N->getOperand(Num: 0);
21288 SDValue Op1 = N->getOperand(Num: 1);
21289
21290 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) {
21291 bool Equal = false;
21292 // (brcond (xor (xor x, y), -1)) -> (brcond (setcc x, y, eq))
21293 if (isBitwiseNot(V: N) && Op0.hasOneUse() && Op0.getOpcode() == ISD::XOR &&
21294 Op0.getValueType() == MVT::i1) {
21295 N = Op0;
21296 Op0 = N->getOperand(Num: 0);
21297 Op1 = N->getOperand(Num: 1);
21298 Equal = true;
21299 }
21300
21301 EVT SetCCVT = N.getValueType();
21302 if (LegalTypes)
21303 SetCCVT = getSetCCResultType(VT: SetCCVT);
21304 // Replace the uses of XOR with SETCC. Note, avoid this transformation if
21305 // it would introduce illegal operations post-legalization as this can
21306 // result in infinite looping between converting xor->setcc here, and
21307 // expanding setcc->xor in LegalizeSetCCCondCode if requested.
21308 const ISD::CondCode CC = Equal ? ISD::SETEQ : ISD::SETNE;
21309 if (!LegalOperations || TLI.isCondCodeLegal(CC, VT: Op0.getSimpleValueType()))
21310 return DAG.getSetCC(DL: SDLoc(N), VT: SetCCVT, LHS: Op0, RHS: Op1, Cond: CC);
21311 }
21312 }
21313
21314 return SDValue();
21315}
21316
21317// Operand List for BR_CC: Chain, CondCC, CondLHS, CondRHS, DestBB.
21318//
21319SDValue DAGCombiner::visitBR_CC(SDNode *N) {
21320 CondCodeSDNode *CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1));
21321 SDValue CondLHS = N->getOperand(Num: 2), CondRHS = N->getOperand(Num: 3);
21322
21323 // If N is a constant we could fold this into a fallthrough or unconditional
21324 // branch. However that doesn't happen very often in normal code, because
21325 // Instcombine/SimplifyCFG should have handled the available opportunities.
21326 // If we did this folding here, it would be necessary to update the
21327 // MachineBasicBlock CFG, which is awkward.
21328
21329 // Use SimplifySetCC to simplify SETCC's.
21330 SDValue Simp = SimplifySetCC(VT: getSetCCResultType(VT: CondLHS.getValueType()),
21331 N0: CondLHS, N1: CondRHS, Cond: CC->get(), DL: SDLoc(N),
21332 foldBooleans: false);
21333 if (Simp.getNode()) AddToWorklist(N: Simp.getNode());
21334
21335 // fold to a simpler setcc
21336 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC)
21337 return DAG.getNode(Opcode: ISD::BR_CC, DL: SDLoc(N), VT: MVT::Other,
21338 N1: N->getOperand(Num: 0), N2: Simp.getOperand(i: 2),
21339 N3: Simp.getOperand(i: 0), N4: Simp.getOperand(i: 1),
21340 N5: N->getOperand(Num: 4));
21341
21342 return SDValue();
21343}
21344
21345static bool getCombineLoadStoreParts(SDNode *N, unsigned Inc, unsigned Dec,
21346 bool &IsLoad, bool &IsMasked, SDValue &Ptr,
21347 const TargetLowering &TLI) {
21348 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
21349 if (LD->isIndexed())
21350 return false;
21351 EVT VT = LD->getMemoryVT();
21352 if (!TLI.isIndexedLoadLegal(IdxMode: Inc, VT) && !TLI.isIndexedLoadLegal(IdxMode: Dec, VT))
21353 return false;
21354 Ptr = LD->getBasePtr();
21355 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
21356 if (ST->isIndexed())
21357 return false;
21358 EVT VT = ST->getMemoryVT();
21359 if (!TLI.isIndexedStoreLegal(IdxMode: Inc, VT) && !TLI.isIndexedStoreLegal(IdxMode: Dec, VT))
21360 return false;
21361 Ptr = ST->getBasePtr();
21362 IsLoad = false;
21363 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Val: N)) {
21364 if (LD->isIndexed())
21365 return false;
21366 EVT VT = LD->getMemoryVT();
21367 if (!TLI.isIndexedMaskedLoadLegal(IdxMode: Inc, VT) &&
21368 !TLI.isIndexedMaskedLoadLegal(IdxMode: Dec, VT))
21369 return false;
21370 Ptr = LD->getBasePtr();
21371 IsMasked = true;
21372 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Val: N)) {
21373 if (ST->isIndexed())
21374 return false;
21375 EVT VT = ST->getMemoryVT();
21376 if (!TLI.isIndexedMaskedStoreLegal(IdxMode: Inc, VT) &&
21377 !TLI.isIndexedMaskedStoreLegal(IdxMode: Dec, VT))
21378 return false;
21379 Ptr = ST->getBasePtr();
21380 IsLoad = false;
21381 IsMasked = true;
21382 } else {
21383 return false;
21384 }
21385 return true;
21386}
21387
21388/// Try turning a load/store into a pre-indexed load/store when the base
21389/// pointer is an add or subtract and it has other uses besides the load/store.
21390/// After the transformation, the new indexed load/store has effectively folded
21391/// the add/subtract in and all of its other uses are redirected to the
21392/// new load/store.
21393bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
21394 if (Level < AfterLegalizeDAG)
21395 return false;
21396
21397 bool IsLoad = true;
21398 bool IsMasked = false;
21399 SDValue Ptr;
21400 if (!getCombineLoadStoreParts(N, Inc: ISD::PRE_INC, Dec: ISD::PRE_DEC, IsLoad, IsMasked,
21401 Ptr, TLI))
21402 return false;
21403
21404 // If the pointer is not an add/sub, or if it doesn't have multiple uses, bail
21405 // out. There is no reason to make this a preinc/predec.
21406 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) ||
21407 Ptr->hasOneUse())
21408 return false;
21409
21410 // Ask the target to do addressing mode selection.
21411 SDValue BasePtr;
21412 SDValue Offset;
21413 ISD::MemIndexedMode AM = ISD::UNINDEXED;
21414 if (!TLI.getPreIndexedAddressParts(N, BasePtr, Offset, AM, DAG))
21415 return false;
21416
21417 // Backends without true r+i pre-indexed forms may need to pass a
21418 // constant base with a variable offset so that constant coercion
21419 // will work with the patterns in canonical form.
21420 bool Swapped = false;
21421 if (isa<ConstantSDNode>(Val: BasePtr)) {
21422 std::swap(a&: BasePtr, b&: Offset);
21423 Swapped = true;
21424 }
21425
21426 // Don't create a indexed load / store with zero offset.
21427 if (isNullConstant(V: Offset))
21428 return false;
21429
21430 // Try turning it into a pre-indexed load / store except when:
21431 // 1) The new base ptr is a frame index.
21432 // 2) If N is a store and the new base ptr is either the same as or is a
21433 // predecessor of the value being stored.
21434 // 3) Another use of old base ptr is a predecessor of N. If ptr is folded
21435 // that would create a cycle.
21436 // 4) All uses are load / store ops that use it as old base ptr.
21437
21438 // Check #1. Preinc'ing a frame index would require copying the stack pointer
21439 // (plus the implicit offset) to a register to preinc anyway.
21440 if (isa<FrameIndexSDNode>(Val: BasePtr) || isa<RegisterSDNode>(Val: BasePtr))
21441 return false;
21442
21443 // Check #2.
21444 if (!IsLoad) {
21445 SDValue Val = IsMasked ? cast<MaskedStoreSDNode>(Val: N)->getValue()
21446 : cast<StoreSDNode>(Val: N)->getValue();
21447
21448 // Would require a copy.
21449 if (Val == BasePtr)
21450 return false;
21451
21452 // Would create a cycle.
21453 if (Val == Ptr || Ptr->isPredecessorOf(N: Val.getNode()))
21454 return false;
21455 }
21456
21457 // Caches for hasPredecessorHelper.
21458 SmallPtrSet<const SDNode *, 32> Visited;
21459 SmallVector<const SDNode *, 16> Worklist;
21460 Worklist.push_back(Elt: N);
21461
21462 // If the offset is a constant, there may be other adds of constants that
21463 // can be folded with this one. We should do this to avoid having to keep
21464 // a copy of the original base pointer.
21465 SmallVector<SDNode *, 16> OtherUses;
21466 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
21467 if (isa<ConstantSDNode>(Val: Offset))
21468 for (SDUse &Use : BasePtr->uses()) {
21469 // Skip the use that is Ptr and uses of other results from BasePtr's
21470 // node (important for nodes that return multiple results).
21471 if (Use.getUser() == Ptr.getNode() || Use != BasePtr)
21472 continue;
21473
21474 if (SDNode::hasPredecessorHelper(N: Use.getUser(), Visited, Worklist,
21475 MaxSteps))
21476 continue;
21477
21478 if (Use.getUser()->getOpcode() != ISD::ADD &&
21479 Use.getUser()->getOpcode() != ISD::SUB) {
21480 OtherUses.clear();
21481 break;
21482 }
21483
21484 SDValue Op1 = Use.getUser()->getOperand(Num: (Use.getOperandNo() + 1) & 1);
21485 if (!isa<ConstantSDNode>(Val: Op1)) {
21486 OtherUses.clear();
21487 break;
21488 }
21489
21490 // FIXME: In some cases, we can be smarter about this.
21491 if (Op1.getValueType() != Offset.getValueType()) {
21492 OtherUses.clear();
21493 break;
21494 }
21495
21496 OtherUses.push_back(Elt: Use.getUser());
21497 }
21498
21499 if (Swapped)
21500 std::swap(a&: BasePtr, b&: Offset);
21501
21502 // Now check for #3 and #4.
21503 bool RealUse = false;
21504
21505 for (SDNode *User : Ptr->users()) {
21506 if (User == N)
21507 continue;
21508 if (SDNode::hasPredecessorHelper(N: User, Visited, Worklist, MaxSteps))
21509 return false;
21510
21511 // If Ptr may be folded in addressing mode of other use, then it's
21512 // not profitable to do this transformation.
21513 if (!canFoldInAddressingMode(N: Ptr.getNode(), Use: User, DAG, TLI))
21514 RealUse = true;
21515 }
21516
21517 if (!RealUse)
21518 return false;
21519
21520 SDValue Result;
21521 if (!IsMasked) {
21522 if (IsLoad)
21523 Result = DAG.getIndexedLoad(OrigLoad: SDValue(N, 0), dl: SDLoc(N), Base: BasePtr, Offset, AM);
21524 else
21525 Result =
21526 DAG.getIndexedStore(OrigStore: SDValue(N, 0), dl: SDLoc(N), Base: BasePtr, Offset, AM);
21527 } else {
21528 if (IsLoad)
21529 Result = DAG.getIndexedMaskedLoad(OrigLoad: SDValue(N, 0), dl: SDLoc(N), Base: BasePtr,
21530 Offset, AM);
21531 else
21532 Result = DAG.getIndexedMaskedStore(OrigStore: SDValue(N, 0), dl: SDLoc(N), Base: BasePtr,
21533 Offset, AM);
21534 }
21535 ++PreIndexedNodes;
21536 ++NodesCombined;
21537 LLVM_DEBUG(dbgs() << "\nReplacing.4 "; N->dump(&DAG); dbgs() << "\nWith: ";
21538 Result.dump(&DAG); dbgs() << '\n');
21539 WorklistRemover DeadNodes(*this);
21540 if (IsLoad) {
21541 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result.getValue(R: 0));
21542 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Result.getValue(R: 2));
21543 } else {
21544 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result.getValue(R: 1));
21545 }
21546
21547 // Finally, since the node is now dead, remove it from the graph.
21548 deleteAndRecombine(N);
21549
21550 if (Swapped)
21551 std::swap(a&: BasePtr, b&: Offset);
21552
21553 // Replace other uses of BasePtr that can be updated to use Ptr
21554 for (SDNode *OtherUse : OtherUses) {
21555 unsigned OffsetIdx = 1;
21556 if (OtherUse->getOperand(Num: OffsetIdx).getNode() == BasePtr.getNode())
21557 OffsetIdx = 0;
21558 assert(OtherUse->getOperand(!OffsetIdx).getNode() == BasePtr.getNode() &&
21559 "Expected BasePtr operand");
21560
21561 // We need to replace ptr0 in the following expression:
21562 // x0 * offset0 + y0 * ptr0 = t0
21563 // knowing that
21564 // x1 * offset1 + y1 * ptr0 = t1 (the indexed load/store)
21565 //
21566 // where x0, x1, y0 and y1 in {-1, 1} are given by the types of the
21567 // indexed load/store and the expression that needs to be re-written.
21568 //
21569 // Therefore, we have:
21570 // t0 = (x0 * offset0 - x1 * y0 * y1 *offset1) + (y0 * y1) * t1
21571
21572 auto *CN = cast<ConstantSDNode>(Val: OtherUse->getOperand(Num: OffsetIdx));
21573 const APInt &Offset0 = CN->getAPIntValue();
21574 const APInt &Offset1 = Offset->getAsAPIntVal();
21575 int X0 = (OtherUse->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1;
21576 int Y0 = (OtherUse->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1;
21577 int X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1;
21578 int Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1;
21579
21580 unsigned Opcode = (Y0 * Y1 < 0) ? ISD::SUB : ISD::ADD;
21581
21582 APInt CNV = Offset0;
21583 if (X0 < 0) CNV = -CNV;
21584 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
21585 else CNV = CNV - Offset1;
21586
21587 SDLoc DL(OtherUse);
21588
21589 // We can now generate the new expression.
21590 SDValue NewOp1 = DAG.getConstant(Val: CNV, DL, VT: CN->getValueType(ResNo: 0));
21591 SDValue NewOp2 = Result.getValue(R: IsLoad ? 1 : 0);
21592
21593 SDValue NewUse =
21594 DAG.getNode(Opcode, DL, VT: OtherUse->getValueType(ResNo: 0), N1: NewOp1, N2: NewOp2);
21595 DAG.ReplaceAllUsesOfValueWith(From: SDValue(OtherUse, 0), To: NewUse);
21596 deleteAndRecombine(N: OtherUse);
21597 }
21598
21599 // Replace the uses of Ptr with uses of the updated base value.
21600 DAG.ReplaceAllUsesOfValueWith(From: Ptr, To: Result.getValue(R: IsLoad ? 1 : 0));
21601 deleteAndRecombine(N: Ptr.getNode());
21602 AddToWorklist(N: Result.getNode());
21603
21604 return true;
21605}
21606
21607static bool shouldCombineToPostInc(SDNode *N, SDValue Ptr, SDNode *PtrUse,
21608 SDValue &BasePtr, SDValue &Offset,
21609 ISD::MemIndexedMode &AM,
21610 SelectionDAG &DAG,
21611 const TargetLowering &TLI) {
21612 if (PtrUse == N ||
21613 (PtrUse->getOpcode() != ISD::ADD && PtrUse->getOpcode() != ISD::SUB))
21614 return false;
21615
21616 if (!TLI.getPostIndexedAddressParts(N, PtrUse, BasePtr, Offset, AM, DAG))
21617 return false;
21618
21619 // Don't create a indexed load / store with zero offset.
21620 if (isNullConstant(V: Offset))
21621 return false;
21622
21623 if (isa<FrameIndexSDNode>(Val: BasePtr) || isa<RegisterSDNode>(Val: BasePtr))
21624 return false;
21625
21626 SmallPtrSet<const SDNode *, 32> Visited;
21627 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
21628 for (SDNode *User : BasePtr->users()) {
21629 if (User == Ptr.getNode())
21630 continue;
21631
21632 // No if there's a later user which could perform the index instead.
21633 if (isa<MemSDNode>(Val: User)) {
21634 bool IsLoad = true;
21635 bool IsMasked = false;
21636 SDValue OtherPtr;
21637 if (getCombineLoadStoreParts(N: User, Inc: ISD::POST_INC, Dec: ISD::POST_DEC, IsLoad,
21638 IsMasked, Ptr&: OtherPtr, TLI)) {
21639 SmallVector<const SDNode *, 2> Worklist;
21640 Worklist.push_back(Elt: User);
21641 if (SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps))
21642 return false;
21643 }
21644 }
21645
21646 // If all the uses are load / store addresses, then don't do the
21647 // transformation.
21648 if (User->getOpcode() == ISD::ADD || User->getOpcode() == ISD::SUB) {
21649 for (SDNode *UserUser : User->users())
21650 if (canFoldInAddressingMode(N: User, Use: UserUser, DAG, TLI))
21651 return false;
21652 }
21653 }
21654 return true;
21655}
21656
21657static SDNode *getPostIndexedLoadStoreOp(SDNode *N, bool &IsLoad,
21658 bool &IsMasked, SDValue &Ptr,
21659 SDValue &BasePtr, SDValue &Offset,
21660 ISD::MemIndexedMode &AM,
21661 SelectionDAG &DAG,
21662 const TargetLowering &TLI) {
21663 if (!getCombineLoadStoreParts(N, Inc: ISD::POST_INC, Dec: ISD::POST_DEC, IsLoad,
21664 IsMasked, Ptr, TLI) ||
21665 Ptr->hasOneUse())
21666 return nullptr;
21667
21668 // Try turning it into a post-indexed load / store except when
21669 // 1) All uses are load / store ops that use it as base ptr (and
21670 // it may be folded as addressing mmode).
21671 // 2) Op must be independent of N, i.e. Op is neither a predecessor
21672 // nor a successor of N. Otherwise, if Op is folded that would
21673 // create a cycle.
21674 unsigned MaxSteps = SelectionDAG::getHasPredecessorMaxSteps();
21675 for (SDUse &U : Ptr->uses()) {
21676 if (U.getResNo() != Ptr.getResNo())
21677 continue;
21678
21679 // Check for #1.
21680 SDNode *Op = U.getUser();
21681 if (!shouldCombineToPostInc(N, Ptr, PtrUse: Op, BasePtr, Offset, AM, DAG, TLI))
21682 continue;
21683
21684 // Check for #2.
21685 SmallPtrSet<const SDNode *, 32> Visited;
21686 SmallVector<const SDNode *, 8> Worklist;
21687 // Ptr is predecessor to both N and Op.
21688 Visited.insert(Ptr: Ptr.getNode());
21689 Worklist.push_back(Elt: N);
21690 Worklist.push_back(Elt: Op);
21691 if (!SDNode::hasPredecessorHelper(N, Visited, Worklist, MaxSteps) &&
21692 !SDNode::hasPredecessorHelper(N: Op, Visited, Worklist, MaxSteps))
21693 return Op;
21694 }
21695 return nullptr;
21696}
21697
21698/// Try to combine a load/store with a add/sub of the base pointer node into a
21699/// post-indexed load/store. The transformation folded the add/subtract into the
21700/// new indexed load/store effectively and all of its uses are redirected to the
21701/// new load/store.
21702bool DAGCombiner::CombineToPostIndexedLoadStore(SDNode *N) {
21703 if (Level < AfterLegalizeDAG)
21704 return false;
21705
21706 bool IsLoad = true;
21707 bool IsMasked = false;
21708 SDValue Ptr;
21709 SDValue BasePtr;
21710 SDValue Offset;
21711 ISD::MemIndexedMode AM = ISD::UNINDEXED;
21712 SDNode *Op = getPostIndexedLoadStoreOp(N, IsLoad, IsMasked, Ptr, BasePtr,
21713 Offset, AM, DAG, TLI);
21714 if (!Op)
21715 return false;
21716
21717 SDValue Result;
21718 if (!IsMasked)
21719 Result = IsLoad ? DAG.getIndexedLoad(OrigLoad: SDValue(N, 0), dl: SDLoc(N), Base: BasePtr,
21720 Offset, AM)
21721 : DAG.getIndexedStore(OrigStore: SDValue(N, 0), dl: SDLoc(N),
21722 Base: BasePtr, Offset, AM);
21723 else
21724 Result = IsLoad ? DAG.getIndexedMaskedLoad(OrigLoad: SDValue(N, 0), dl: SDLoc(N),
21725 Base: BasePtr, Offset, AM)
21726 : DAG.getIndexedMaskedStore(OrigStore: SDValue(N, 0), dl: SDLoc(N),
21727 Base: BasePtr, Offset, AM);
21728 ++PostIndexedNodes;
21729 ++NodesCombined;
21730 LLVM_DEBUG(dbgs() << "\nReplacing.5 "; N->dump(&DAG); dbgs() << "\nWith: ";
21731 Result.dump(&DAG); dbgs() << '\n');
21732 WorklistRemover DeadNodes(*this);
21733 if (IsLoad) {
21734 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result.getValue(R: 0));
21735 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Result.getValue(R: 2));
21736 } else {
21737 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Result.getValue(R: 1));
21738 }
21739
21740 // Finally, since the node is now dead, remove it from the graph.
21741 deleteAndRecombine(N);
21742
21743 // Replace the uses of Use with uses of the updated base value.
21744 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Op, 0),
21745 To: Result.getValue(R: IsLoad ? 1 : 0));
21746 deleteAndRecombine(N: Op);
21747 return true;
21748}
21749
21750/// Return the base-pointer arithmetic from an indexed \p LD.
21751SDValue DAGCombiner::SplitIndexingFromLoad(LoadSDNode *LD) {
21752 ISD::MemIndexedMode AM = LD->getAddressingMode();
21753 assert(AM != ISD::UNINDEXED);
21754 SDValue BP = LD->getOperand(Num: 1);
21755 SDValue Inc = LD->getOperand(Num: 2);
21756
21757 // Some backends use TargetConstants for load offsets, but don't expect
21758 // TargetConstants in general ADD nodes. We can convert these constants into
21759 // regular Constants (if the constant is not opaque).
21760 assert((Inc.getOpcode() != ISD::TargetConstant ||
21761 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
21762 "Cannot split out indexing using opaque target constants");
21763 if (Inc.getOpcode() == ISD::TargetConstant) {
21764 ConstantSDNode *ConstInc = cast<ConstantSDNode>(Val&: Inc);
21765 Inc = DAG.getConstant(Val: *ConstInc->getConstantIntValue(), DL: SDLoc(Inc),
21766 VT: ConstInc->getValueType(ResNo: 0));
21767 }
21768
21769 unsigned Opc =
21770 (AM == ISD::PRE_INC || AM == ISD::POST_INC ? ISD::ADD : ISD::SUB);
21771 return DAG.getNode(Opcode: Opc, DL: SDLoc(LD), VT: BP.getSimpleValueType(), N1: BP, N2: Inc);
21772}
21773
21774static inline ElementCount numVectorEltsOrZero(EVT T) {
21775 return T.isVector() ? T.getVectorElementCount() : ElementCount::getFixed(MinVal: 0);
21776}
21777
21778bool DAGCombiner::getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val) {
21779 EVT STType = Val.getValueType();
21780 EVT STMemType = ST->getMemoryVT();
21781 if (STType == STMemType)
21782 return true;
21783 if (isTypeLegal(VT: STMemType))
21784 return false; // fail.
21785 if (STType.isFloatingPoint() && STMemType.isFloatingPoint() &&
21786 TLI.isOperationLegal(Op: ISD::FTRUNC, VT: STMemType)) {
21787 Val = DAG.getNode(Opcode: ISD::FTRUNC, DL: SDLoc(ST), VT: STMemType, Operand: Val);
21788 return true;
21789 }
21790 if (numVectorEltsOrZero(T: STType) == numVectorEltsOrZero(T: STMemType) &&
21791 STType.isInteger() && STMemType.isInteger()) {
21792 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(ST), VT: STMemType, Operand: Val);
21793 return true;
21794 }
21795 if (STType.getSizeInBits() == STMemType.getSizeInBits()) {
21796 Val = DAG.getBitcast(VT: STMemType, V: Val);
21797 return true;
21798 }
21799 return false; // fail.
21800}
21801
21802bool DAGCombiner::extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val) {
21803 EVT LDMemType = LD->getMemoryVT();
21804 EVT LDType = LD->getValueType(ResNo: 0);
21805 assert(Val.getValueType() == LDMemType &&
21806 "Attempting to extend value of non-matching type");
21807 if (LDType == LDMemType)
21808 return true;
21809 if (LDMemType.isInteger() && LDType.isInteger()) {
21810 switch (LD->getExtensionType()) {
21811 case ISD::NON_EXTLOAD:
21812 Val = DAG.getBitcast(VT: LDType, V: Val);
21813 return true;
21814 case ISD::EXTLOAD:
21815 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: SDLoc(LD), VT: LDType, Operand: Val);
21816 return true;
21817 case ISD::SEXTLOAD:
21818 Val = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: SDLoc(LD), VT: LDType, Operand: Val);
21819 return true;
21820 case ISD::ZEXTLOAD:
21821 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(LD), VT: LDType, Operand: Val);
21822 return true;
21823 }
21824 }
21825 return false;
21826}
21827
21828StoreSDNode *DAGCombiner::getUniqueStoreFeeding(LoadSDNode *LD,
21829 int64_t &Offset) {
21830 SDValue Chain = LD->getOperand(Num: 0);
21831
21832 // Look through CALLSEQ_START.
21833 if (Chain.getOpcode() == ISD::CALLSEQ_START)
21834 Chain = Chain->getOperand(Num: 0);
21835
21836 StoreSDNode *ST = nullptr;
21837 SmallVector<SDValue, 8> Aliases;
21838 if (Chain.getOpcode() == ISD::TokenFactor) {
21839 // Look for unique store within the TokenFactor.
21840 for (SDValue Op : Chain->ops()) {
21841 StoreSDNode *Store = dyn_cast<StoreSDNode>(Val: Op.getNode());
21842 if (!Store)
21843 continue;
21844 BaseIndexOffset BasePtrLD = BaseIndexOffset::match(N: LD, DAG);
21845 BaseIndexOffset BasePtrST = BaseIndexOffset::match(N: Store, DAG);
21846 if (!BasePtrST.equalBaseIndex(Other: BasePtrLD, DAG, Off&: Offset))
21847 continue;
21848 // Make sure the store is not aliased with any nodes in TokenFactor.
21849 GatherAllAliases(N: Store, OriginalChain: Chain, Aliases);
21850 if (Aliases.empty() ||
21851 (Aliases.size() == 1 && Aliases.front().getNode() == Store))
21852 ST = Store;
21853 break;
21854 }
21855 } else {
21856 StoreSDNode *Store = dyn_cast<StoreSDNode>(Val: Chain.getNode());
21857 if (Store) {
21858 BaseIndexOffset BasePtrLD = BaseIndexOffset::match(N: LD, DAG);
21859 BaseIndexOffset BasePtrST = BaseIndexOffset::match(N: Store, DAG);
21860 if (BasePtrST.equalBaseIndex(Other: BasePtrLD, DAG, Off&: Offset))
21861 ST = Store;
21862 }
21863 }
21864
21865 return ST;
21866}
21867
21868SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) {
21869 if (OptLevel == CodeGenOptLevel::None || !LD->isSimple())
21870 return SDValue();
21871 SDValue Chain = LD->getOperand(Num: 0);
21872 int64_t Offset;
21873
21874 StoreSDNode *ST = getUniqueStoreFeeding(LD, Offset);
21875 // TODO: Relax this restriction for unordered atomics (see D66309)
21876 if (!ST || !ST->isSimple() || ST->getAddressSpace() != LD->getAddressSpace())
21877 return SDValue();
21878
21879 EVT LDType = LD->getValueType(ResNo: 0);
21880 EVT LDMemType = LD->getMemoryVT();
21881 EVT STMemType = ST->getMemoryVT();
21882 EVT STType = ST->getValue().getValueType();
21883
21884 // There are two cases to consider here:
21885 // 1. The store is fixed width and the load is scalable. In this case we
21886 // don't know at compile time if the store completely envelops the load
21887 // so we abandon the optimisation.
21888 // 2. The store is scalable and the load is fixed width. We could
21889 // potentially support a limited number of cases here, but there has been
21890 // no cost-benefit analysis to prove it's worth it.
21891 bool LdStScalable = LDMemType.isScalableVT();
21892 if (LdStScalable != STMemType.isScalableVT())
21893 return SDValue();
21894
21895 // If we are dealing with scalable vectors on a big endian platform the
21896 // calculation of offsets below becomes trickier, since we do not know at
21897 // compile time the absolute size of the vector. Until we've done more
21898 // analysis on big-endian platforms it seems better to bail out for now.
21899 if (LdStScalable && DAG.getDataLayout().isBigEndian())
21900 return SDValue();
21901
21902 // Normalize for Endianness. After this Offset=0 will denote that the least
21903 // significant bit in the loaded value maps to the least significant bit in
21904 // the stored value). With Offset=n (for n > 0) the loaded value starts at the
21905 // n:th least significant byte of the stored value.
21906 int64_t OrigOffset = Offset;
21907 if (DAG.getDataLayout().isBigEndian())
21908 Offset = ((int64_t)STMemType.getStoreSizeInBits().getFixedValue() -
21909 (int64_t)LDMemType.getStoreSizeInBits().getFixedValue()) /
21910 8 -
21911 Offset;
21912
21913 // Check that the stored value cover all bits that are loaded.
21914 bool STCoversLD;
21915
21916 TypeSize LdMemSize = LDMemType.getSizeInBits();
21917 TypeSize StMemSize = STMemType.getSizeInBits();
21918 if (LdStScalable)
21919 STCoversLD = (Offset == 0) && LdMemSize == StMemSize;
21920 else
21921 STCoversLD = (Offset >= 0) && (Offset * 8 + LdMemSize.getFixedValue() <=
21922 StMemSize.getFixedValue());
21923
21924 auto ReplaceLd = [&](LoadSDNode *LD, SDValue Val, SDValue Chain) -> SDValue {
21925 if (LD->isIndexed()) {
21926 // Cannot handle opaque target constants and we must respect the user's
21927 // request not to split indexes from loads.
21928 if (!canSplitIdx(LD))
21929 return SDValue();
21930 SDValue Idx = SplitIndexingFromLoad(LD);
21931 SDValue Ops[] = {Val, Idx, Chain};
21932 return CombineTo(N: LD, To: Ops, NumTo: 3);
21933 }
21934 return CombineTo(N: LD, Res0: Val, Res1: Chain);
21935 };
21936
21937 if (!STCoversLD)
21938 return SDValue();
21939
21940 // Memory as copy space (potentially masked).
21941 if (Offset == 0 && LDType == STType && STMemType == LDMemType) {
21942 // Simple case: Direct non-truncating forwarding
21943 if (LDType.getSizeInBits() == LdMemSize)
21944 return ReplaceLd(LD, ST->getValue(), Chain);
21945 // Can we model the truncate and extension with an and mask?
21946 if (STType.isInteger() && LDMemType.isInteger() && !STType.isVector() &&
21947 !LDMemType.isVector() && LD->getExtensionType() != ISD::SEXTLOAD) {
21948 // Mask to size of LDMemType
21949 auto Mask =
21950 DAG.getConstant(Val: APInt::getLowBitsSet(numBits: STType.getFixedSizeInBits(),
21951 loBitsSet: StMemSize.getFixedValue()),
21952 DL: SDLoc(ST), VT: STType);
21953 auto Val = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(LD), VT: LDType, N1: ST->getValue(), N2: Mask);
21954 return ReplaceLd(LD, Val, Chain);
21955 }
21956 }
21957
21958 // Handle some cases for big-endian that would be Offset 0 and handled for
21959 // little-endian.
21960 SDValue Val = ST->getValue();
21961 if (DAG.getDataLayout().isBigEndian() && Offset > 0 && OrigOffset == 0) {
21962 if (STType.isInteger() && !STType.isVector() && LDType.isInteger() &&
21963 !LDType.isVector() && isTypeLegal(VT: STType) &&
21964 TLI.isOperationLegal(Op: ISD::SRL, VT: STType)) {
21965 Val = DAG.getNode(
21966 Opcode: ISD::SRL, DL: SDLoc(LD), VT: STType, N1: Val,
21967 N2: DAG.getShiftAmountConstant(Val: Offset * 8, VT: STType, DL: SDLoc(LD)));
21968 Offset = 0;
21969 }
21970 }
21971
21972 // TODO: Deal with nonzero offset.
21973 if (LD->getBasePtr().isUndef() || Offset != 0)
21974 return SDValue();
21975 // Model necessary truncations / extenstions.
21976 // Truncate Value To Stored Memory Size.
21977 do {
21978 if (!getTruncatedStoreValue(ST, Val))
21979 break;
21980 if (!isTypeLegal(VT: LDMemType))
21981 break;
21982 if (STMemType != LDMemType) {
21983 if (LdMemSize == StMemSize) {
21984 if (TLI.isOperationLegal(Op: ISD::BITCAST, VT: LDMemType) &&
21985 isTypeLegal(VT: LDMemType) &&
21986 TLI.isOperationLegal(Op: ISD::BITCAST, VT: STMemType) &&
21987 isTypeLegal(VT: STMemType) &&
21988 TLI.isLoadBitCastBeneficial(LoadVT: LDMemType, BitcastVT: STMemType, DAG,
21989 MMO: *LD->getMemOperand()))
21990 Val = DAG.getBitcast(VT: LDMemType, V: Val);
21991 else
21992 break;
21993 } else if (LDMemType.isVector() && isTypeLegal(VT: STMemType)) {
21994 EVT EltVT = LDMemType.getVectorElementType();
21995 TypeSize EltSize = EltVT.getSizeInBits();
21996
21997 if (!StMemSize.isKnownMultipleOf(RHS: EltSize))
21998 break;
21999
22000 EVT InterVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT,
22001 NumElements: StMemSize.divideCoefficientBy(RHS: EltSize));
22002 if (!TLI.isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: LDMemType) ||
22003 !TLI.isTypeLegal(VT: InterVT))
22004 break;
22005
22006 // In case of big-endian the offset is normalized to zero, denoting
22007 // the last bit. For big-endian we need to transform the extraction
22008 // to the last sub-vector.
22009 unsigned ExtIdx = 0;
22010 if (DAG.getDataLayout().isBigEndian()) {
22011 ExtIdx =
22012 InterVT.getVectorNumElements() - LDMemType.getVectorNumElements();
22013 }
22014
22015 if (TLI.getExtractSubvectorCost(ResVT: LDMemType, SrcVT: InterVT, Index: ExtIdx) >
22016 TargetLowering::ExtractSubvectorCost::Cheap)
22017 break;
22018 Val = DAG.getExtractSubvector(DL: SDLoc(LD), VT: LDMemType,
22019 Vec: DAG.getBitcast(VT: InterVT, V: Val), Idx: ExtIdx);
22020 } else if (!STMemType.isVector() && !LDMemType.isVector() &&
22021 STMemType.isInteger() && LDMemType.isInteger())
22022 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(LD), VT: LDMemType, Operand: Val);
22023 else
22024 break;
22025 }
22026 if (!extendLoadedValueToExtension(LD, Val))
22027 break;
22028 return ReplaceLd(LD, Val, Chain);
22029 } while (false);
22030
22031 // On failure, cleanup dead nodes we may have created.
22032 if (Val->use_empty())
22033 deleteAndRecombine(N: Val.getNode());
22034 return SDValue();
22035}
22036
22037SDValue DAGCombiner::visitLOAD(SDNode *N) {
22038 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
22039 SDValue Chain = LD->getChain();
22040 SDValue Ptr = LD->getBasePtr();
22041
22042 // If load is not volatile and there are no uses of the loaded value (and
22043 // the updated indexed value in case of indexed loads), change uses of the
22044 // chain value into uses of the chain input (i.e. delete the dead load).
22045 // TODO: Allow this for unordered atomics (see D66309)
22046 if (LD->isSimple()) {
22047 if (N->getValueType(ResNo: 1) == MVT::Other) {
22048 // Unindexed loads.
22049 if (!N->hasAnyUseOfValue(Value: 0)) {
22050 // It's not safe to use the two value CombineTo variant here. e.g.
22051 // v1, chain2 = load chain1, loc
22052 // v2, chain3 = load chain2, loc
22053 // v3 = add v2, c
22054 // Now we replace use of chain2 with chain1. This makes the second load
22055 // isomorphic to the one we are deleting, and thus makes this load live.
22056 LLVM_DEBUG(dbgs() << "\nReplacing.6 "; N->dump(&DAG);
22057 dbgs() << "\nWith chain: "; Chain.dump(&DAG);
22058 dbgs() << "\n");
22059 WorklistRemover DeadNodes(*this);
22060 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
22061 AddUsersToWorklist(N: Chain.getNode());
22062 if (N->use_empty())
22063 deleteAndRecombine(N);
22064
22065 return SDValue(N, 0); // Return N so it doesn't get rechecked!
22066 }
22067 } else {
22068 // Indexed loads.
22069 assert(N->getValueType(2) == MVT::Other && "Malformed indexed loads?");
22070
22071 // If this load has an opaque TargetConstant offset, then we cannot split
22072 // the indexing into an add/sub directly (that TargetConstant may not be
22073 // valid for a different type of node, and we cannot convert an opaque
22074 // target constant into a regular constant).
22075 bool CanSplitIdx = canSplitIdx(LD);
22076
22077 if (!N->hasAnyUseOfValue(Value: 0) && (CanSplitIdx || !N->hasAnyUseOfValue(Value: 1))) {
22078 SDValue Poison = DAG.getPOISON(VT: N->getValueType(ResNo: 0));
22079 SDValue Index;
22080 if (N->hasAnyUseOfValue(Value: 1) && CanSplitIdx) {
22081 Index = SplitIndexingFromLoad(LD);
22082 // Try to fold the base pointer arithmetic into subsequent loads and
22083 // stores.
22084 AddUsersToWorklist(N);
22085 } else
22086 Index = DAG.getPOISON(VT: N->getValueType(ResNo: 1));
22087 LLVM_DEBUG(dbgs() << "\nReplacing.7 "; N->dump(&DAG);
22088 dbgs() << "\nWith: "; Poison.dump(&DAG);
22089 dbgs() << " and 2 other values\n");
22090 WorklistRemover DeadNodes(*this);
22091 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 0), To: Poison);
22092 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Index);
22093 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 2), To: Chain);
22094 deleteAndRecombine(N);
22095 return SDValue(N, 0); // Return N so it doesn't get rechecked!
22096 }
22097 }
22098 }
22099
22100 // If this load is directly stored, replace the load value with the stored
22101 // value.
22102 if (auto V = ForwardStoreValueToDirectLoad(LD))
22103 return V;
22104
22105 // Try to infer better alignment information than the load already has.
22106 if (OptLevel != CodeGenOptLevel::None && LD->isUnindexed() &&
22107 !LD->isAtomic()) {
22108 if (MaybeAlign Alignment = DAG.InferPtrAlign(Ptr)) {
22109 if (*Alignment > LD->getAlign() &&
22110 isAligned(Lhs: *Alignment, SizeInBytes: LD->getSrcValueOffset())) {
22111 SDValue NewLoad = DAG.getLoad(
22112 AM: LD->getAddressingMode(), ExtType: LD->getExtensionType(),
22113 VT: LD->getValueType(ResNo: 0), dl: SDLoc(N), Chain, Ptr, Offset: LD->getOffset(),
22114 PtrInfo: LD->getPointerInfo(), MemVT: LD->getMemoryVT(), Alignment: *Alignment,
22115 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
22116 // NewLoad will always be N as we are only refining the alignment
22117 assert(NewLoad.getNode() == N);
22118 (void)NewLoad;
22119 }
22120 }
22121 }
22122
22123 if (LD->isUnindexed()) {
22124 // Walk up chain skipping non-aliasing memory nodes.
22125 SDValue BetterChain = FindBetterChain(N: LD, Chain);
22126
22127 // If there is a better chain.
22128 if (Chain != BetterChain) {
22129 SDValue ReplLoad;
22130
22131 // Replace the chain to void dependency.
22132 if (LD->getExtensionType() == ISD::NON_EXTLOAD) {
22133 ReplLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl: SDLoc(LD),
22134 Chain: BetterChain, Ptr, MMO: LD->getMemOperand());
22135 } else {
22136 ReplLoad = DAG.getExtLoad(ExtType: LD->getExtensionType(), dl: SDLoc(LD),
22137 VT: LD->getValueType(ResNo: 0),
22138 Chain: BetterChain, Ptr, MemVT: LD->getMemoryVT(),
22139 MMO: LD->getMemOperand());
22140 }
22141
22142 // Create token factor to keep old chain connected.
22143 SDValue Token = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(N),
22144 VT: MVT::Other, N1: Chain, N2: ReplLoad.getValue(R: 1));
22145
22146 // Replace uses with load result and token factor
22147 return CombineTo(N, Res0: ReplLoad.getValue(R: 0), Res1: Token);
22148 }
22149 }
22150
22151 // Try transforming N to an indexed load.
22152 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
22153 return SDValue(N, 0);
22154
22155 // Try to slice up N to more direct loads if the slices are mapped to
22156 // different register banks or pairing can take place.
22157 if (SliceUpLoad(N))
22158 return SDValue(N, 0);
22159
22160 return SDValue();
22161}
22162
22163namespace {
22164
22165/// Helper structure used to slice a load in smaller loads.
22166/// Basically a slice is obtained from the following sequence:
22167/// Origin = load Ty1, Base
22168/// Shift = srl Ty1 Origin, CstTy Amount
22169/// Inst = trunc Shift to Ty2
22170///
22171/// Then, it will be rewritten into:
22172/// Slice = load SliceTy, Base + SliceOffset
22173/// [Inst = zext Slice to Ty2], only if SliceTy <> Ty2
22174///
22175/// SliceTy is deduced from the number of bits that are actually used to
22176/// build Inst.
22177struct LoadedSlice {
22178 /// Helper structure used to compute the cost of a slice.
22179 struct Cost {
22180 /// Are we optimizing for code size.
22181 bool ForCodeSize = false;
22182
22183 /// Various cost.
22184 unsigned Loads = 0;
22185 unsigned Truncates = 0;
22186 unsigned CrossRegisterBanksCopies = 0;
22187 unsigned ZExts = 0;
22188 unsigned Shift = 0;
22189
22190 explicit Cost(bool ForCodeSize) : ForCodeSize(ForCodeSize) {}
22191
22192 /// Get the cost of one isolated slice.
22193 Cost(const LoadedSlice &LS, bool ForCodeSize)
22194 : ForCodeSize(ForCodeSize), Loads(1) {
22195 EVT TruncType = LS.Inst->getValueType(ResNo: 0);
22196 EVT LoadedType = LS.getLoadedType();
22197 if (TruncType != LoadedType &&
22198 !LS.DAG->getTargetLoweringInfo().isZExtFree(FromTy: LoadedType, ToTy: TruncType))
22199 ZExts = 1;
22200 }
22201
22202 /// Account for slicing gain in the current cost.
22203 /// Slicing provide a few gains like removing a shift or a
22204 /// truncate. This method allows to grow the cost of the original
22205 /// load with the gain from this slice.
22206 void addSliceGain(const LoadedSlice &LS) {
22207 // Each slice saves a truncate.
22208 const TargetLowering &TLI = LS.DAG->getTargetLoweringInfo();
22209 if (!TLI.isTruncateFree(Val: LS.Inst->getOperand(Num: 0), VT2: LS.Inst->getValueType(ResNo: 0)))
22210 ++Truncates;
22211 // If there is a shift amount, this slice gets rid of it.
22212 if (LS.Shift)
22213 ++Shift;
22214 // If this slice can merge a cross register bank copy, account for it.
22215 if (LS.canMergeExpensiveCrossRegisterBankCopy())
22216 ++CrossRegisterBanksCopies;
22217 }
22218
22219 Cost &operator+=(const Cost &RHS) {
22220 Loads += RHS.Loads;
22221 Truncates += RHS.Truncates;
22222 CrossRegisterBanksCopies += RHS.CrossRegisterBanksCopies;
22223 ZExts += RHS.ZExts;
22224 Shift += RHS.Shift;
22225 return *this;
22226 }
22227
22228 bool operator==(const Cost &RHS) const {
22229 return Loads == RHS.Loads && Truncates == RHS.Truncates &&
22230 CrossRegisterBanksCopies == RHS.CrossRegisterBanksCopies &&
22231 ZExts == RHS.ZExts && Shift == RHS.Shift;
22232 }
22233
22234 bool operator!=(const Cost &RHS) const { return !(*this == RHS); }
22235
22236 bool operator<(const Cost &RHS) const {
22237 // Assume cross register banks copies are as expensive as loads.
22238 // FIXME: Do we want some more target hooks?
22239 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
22240 unsigned ExpensiveOpsRHS = RHS.Loads + RHS.CrossRegisterBanksCopies;
22241 // Unless we are optimizing for code size, consider the
22242 // expensive operation first.
22243 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
22244 return ExpensiveOpsLHS < ExpensiveOpsRHS;
22245 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
22246 (RHS.Truncates + RHS.ZExts + RHS.Shift + ExpensiveOpsRHS);
22247 }
22248
22249 bool operator>(const Cost &RHS) const { return RHS < *this; }
22250
22251 bool operator<=(const Cost &RHS) const { return !(RHS < *this); }
22252
22253 bool operator>=(const Cost &RHS) const { return !(*this < RHS); }
22254 };
22255
22256 // The last instruction that represent the slice. This should be a
22257 // truncate instruction.
22258 SDNode *Inst;
22259
22260 // The original load instruction.
22261 LoadSDNode *Origin;
22262
22263 // The right shift amount in bits from the original load.
22264 unsigned Shift;
22265
22266 // The DAG from which Origin came from.
22267 // This is used to get some contextual information about legal types, etc.
22268 SelectionDAG *DAG;
22269
22270 LoadedSlice(SDNode *Inst = nullptr, LoadSDNode *Origin = nullptr,
22271 unsigned Shift = 0, SelectionDAG *DAG = nullptr)
22272 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
22273
22274 /// Get the bits used in a chunk of bits \p BitWidth large.
22275 /// \return Result is \p BitWidth and has used bits set to 1 and
22276 /// not used bits set to 0.
22277 APInt getUsedBits() const {
22278 // Reproduce the trunc(lshr) sequence:
22279 // - Start from the truncated value.
22280 // - Zero extend to the desired bit width.
22281 // - Shift left.
22282 assert(Origin && "No original load to compare against.");
22283 unsigned BitWidth = Origin->getValueSizeInBits(ResNo: 0);
22284 assert(Inst && "This slice is not bound to an instruction");
22285 assert(Inst->getValueSizeInBits(0) <= BitWidth &&
22286 "Extracted slice is bigger than the whole type!");
22287 APInt UsedBits(Inst->getValueSizeInBits(ResNo: 0), 0);
22288 UsedBits.setAllBits();
22289 UsedBits = UsedBits.zext(width: BitWidth);
22290 UsedBits <<= Shift;
22291 return UsedBits;
22292 }
22293
22294 /// Get the size of the slice to be loaded in bytes.
22295 unsigned getLoadedSize() const {
22296 unsigned SliceSize = getUsedBits().popcount();
22297 assert(!(SliceSize & 0x7) && "Size is not a multiple of a byte.");
22298 return SliceSize / 8;
22299 }
22300
22301 /// Get the type that will be loaded for this slice.
22302 /// Note: This may not be the final type for the slice.
22303 EVT getLoadedType() const {
22304 assert(DAG && "Missing context");
22305 LLVMContext &Ctxt = *DAG->getContext();
22306 return EVT::getIntegerVT(Context&: Ctxt, BitWidth: getLoadedSize() * 8);
22307 }
22308
22309 /// Get the alignment of the load used for this slice.
22310 Align getAlign() const {
22311 Align Alignment = Origin->getAlign();
22312 uint64_t Offset = getOffsetFromBase();
22313 if (Offset != 0)
22314 Alignment = commonAlignment(A: Alignment, Offset: Alignment.value() + Offset);
22315 return Alignment;
22316 }
22317
22318 /// Check if this slice can be rewritten with legal operations.
22319 bool isLegal() const {
22320 // An invalid slice is not legal.
22321 if (!Origin || !Inst || !DAG)
22322 return false;
22323
22324 // Offsets are for indexed load only, we do not handle that.
22325 if (!Origin->getOffset().isUndef())
22326 return false;
22327
22328 const TargetLowering &TLI = DAG->getTargetLoweringInfo();
22329
22330 // Check that the type is legal.
22331 EVT SliceType = getLoadedType();
22332 if (!TLI.isTypeLegal(VT: SliceType))
22333 return false;
22334
22335 // Check that the load is legal for this type.
22336 if (!TLI.isOperationLegal(Op: ISD::LOAD, VT: SliceType))
22337 return false;
22338
22339 // Check that the offset can be computed.
22340 // 1. Check its type.
22341 EVT PtrType = Origin->getBasePtr().getValueType();
22342 if (PtrType == MVT::Untyped || PtrType.isExtended())
22343 return false;
22344
22345 // 2. Check that it fits in the immediate.
22346 if (!TLI.isLegalAddImmediate(getOffsetFromBase()))
22347 return false;
22348
22349 // 3. Check that the computation is legal.
22350 if (!TLI.isOperationLegal(Op: ISD::ADD, VT: PtrType))
22351 return false;
22352
22353 // Check that the zext is legal if it needs one.
22354 EVT TruncateType = Inst->getValueType(ResNo: 0);
22355 if (TruncateType != SliceType &&
22356 !TLI.isOperationLegal(Op: ISD::ZERO_EXTEND, VT: TruncateType))
22357 return false;
22358
22359 return true;
22360 }
22361
22362 /// Get the offset in bytes of this slice in the original chunk of
22363 /// bits.
22364 /// \pre DAG != nullptr.
22365 uint64_t getOffsetFromBase() const {
22366 assert(DAG && "Missing context.");
22367 bool IsBigEndian = DAG->getDataLayout().isBigEndian();
22368 assert(!(Shift & 0x7) && "Shifts not aligned on Bytes are not supported.");
22369 uint64_t Offset = Shift / 8;
22370 unsigned TySizeInBytes = Origin->getValueSizeInBits(ResNo: 0) / 8;
22371 assert(!(Origin->getValueSizeInBits(0) & 0x7) &&
22372 "The size of the original loaded type is not a multiple of a"
22373 " byte.");
22374 // If Offset is bigger than TySizeInBytes, it means we are loading all
22375 // zeros. This should have been optimized before in the process.
22376 assert(TySizeInBytes > Offset &&
22377 "Invalid shift amount for given loaded size");
22378 if (IsBigEndian)
22379 Offset = TySizeInBytes - Offset - getLoadedSize();
22380 return Offset;
22381 }
22382
22383 /// Generate the sequence of instructions to load the slice
22384 /// represented by this object and redirect the uses of this slice to
22385 /// this new sequence of instructions.
22386 /// \pre this->Inst && this->Origin are valid Instructions and this
22387 /// object passed the legal check: LoadedSlice::isLegal returned true.
22388 /// \return The last instruction of the sequence used to load the slice.
22389 SDValue loadSlice() const {
22390 assert(Inst && Origin && "Unable to replace a non-existing slice.");
22391 const SDValue &OldBaseAddr = Origin->getBasePtr();
22392 SDValue BaseAddr = OldBaseAddr;
22393 // Get the offset in that chunk of bytes w.r.t. the endianness.
22394 int64_t Offset = static_cast<int64_t>(getOffsetFromBase());
22395 assert(Offset >= 0 && "Offset too big to fit in int64_t!");
22396 if (Offset) {
22397 // BaseAddr = BaseAddr + Offset.
22398 EVT ArithType = BaseAddr.getValueType();
22399 SDLoc DL(Origin);
22400 BaseAddr = DAG->getNode(Opcode: ISD::ADD, DL, VT: ArithType, N1: BaseAddr,
22401 N2: DAG->getConstant(Val: Offset, DL, VT: ArithType));
22402 }
22403
22404 // Create the type of the loaded slice according to its size.
22405 EVT SliceType = getLoadedType();
22406
22407 // Create the load for the slice.
22408 SDValue LastInst =
22409 DAG->getLoad(VT: SliceType, dl: SDLoc(Origin), Chain: Origin->getChain(), Ptr: BaseAddr,
22410 PtrInfo: Origin->getPointerInfo().getWithOffset(O: Offset), Alignment: getAlign(),
22411 MMOFlags: Origin->getMemOperand()->getFlags());
22412 // If the final type is not the same as the loaded type, this means that
22413 // we have to pad with zero. Create a zero extend for that.
22414 EVT FinalType = Inst->getValueType(ResNo: 0);
22415 if (SliceType != FinalType)
22416 LastInst =
22417 DAG->getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(LastInst), VT: FinalType, Operand: LastInst);
22418 return LastInst;
22419 }
22420
22421 /// Check if this slice can be merged with an expensive cross register
22422 /// bank copy. E.g.,
22423 /// i = load i32
22424 /// f = bitcast i32 i to float
22425 bool canMergeExpensiveCrossRegisterBankCopy() const {
22426 if (!Inst || !Inst->hasOneUse())
22427 return false;
22428 SDNode *User = *Inst->user_begin();
22429 if (User->getOpcode() != ISD::BITCAST)
22430 return false;
22431 assert(DAG && "Missing context");
22432 const TargetLowering &TLI = DAG->getTargetLoweringInfo();
22433 EVT ResVT = User->getValueType(ResNo: 0);
22434 const TargetRegisterClass *ResRC =
22435 TLI.getRegClassFor(VT: ResVT.getSimpleVT(), isDivergent: User->isDivergent());
22436 const TargetRegisterClass *ArgRC =
22437 TLI.getRegClassFor(VT: User->getOperand(Num: 0).getValueType().getSimpleVT(),
22438 isDivergent: User->getOperand(Num: 0)->isDivergent());
22439 if (ArgRC == ResRC || !TLI.isOperationLegal(Op: ISD::LOAD, VT: ResVT))
22440 return false;
22441
22442 // At this point, we know that we perform a cross-register-bank copy.
22443 // Check if it is expensive.
22444 const TargetRegisterInfo *TRI = DAG->getSubtarget().getRegisterInfo();
22445 // Assume bitcasts are cheap, unless both register classes do not
22446 // explicitly share a common sub class.
22447 if (!TRI || TRI->getCommonSubClass(A: ArgRC, B: ResRC))
22448 return false;
22449
22450 // Check if it will be merged with the load.
22451 // 1. Check the alignment / fast memory access constraint.
22452 unsigned IsFast = 0;
22453 if (!TLI.allowsMemoryAccess(Context&: *DAG->getContext(), DL: DAG->getDataLayout(), VT: ResVT,
22454 AddrSpace: Origin->getAddressSpace(), Alignment: getAlign(),
22455 Flags: Origin->getMemOperand()->getFlags(), Fast: &IsFast) ||
22456 !IsFast)
22457 return false;
22458
22459 // 2. Check that the load is a legal operation for that type.
22460 if (!TLI.isOperationLegal(Op: ISD::LOAD, VT: ResVT))
22461 return false;
22462
22463 // 3. Check that we do not have a zext in the way.
22464 if (Inst->getValueType(ResNo: 0) != getLoadedType())
22465 return false;
22466
22467 return true;
22468 }
22469};
22470
22471} // end anonymous namespace
22472
22473/// Check that all bits set in \p UsedBits form a dense region, i.e.,
22474/// \p UsedBits looks like 0..0 1..1 0..0.
22475static bool areUsedBitsDense(const APInt &UsedBits) {
22476 // If all the bits are one, this is dense!
22477 if (UsedBits.isAllOnes())
22478 return true;
22479
22480 // Get rid of the unused bits on the right.
22481 APInt NarrowedUsedBits = UsedBits.lshr(shiftAmt: UsedBits.countr_zero());
22482 // Get rid of the unused bits on the left.
22483 if (NarrowedUsedBits.countl_zero())
22484 NarrowedUsedBits = NarrowedUsedBits.trunc(width: NarrowedUsedBits.getActiveBits());
22485 // Check that the chunk of bits is completely used.
22486 return NarrowedUsedBits.isAllOnes();
22487}
22488
22489/// Check whether or not \p First and \p Second are next to each other
22490/// in memory. This means that there is no hole between the bits loaded
22491/// by \p First and the bits loaded by \p Second.
22492static bool areSlicesNextToEachOther(const LoadedSlice &First,
22493 const LoadedSlice &Second) {
22494 assert(First.Origin == Second.Origin && First.Origin &&
22495 "Unable to match different memory origins.");
22496 APInt UsedBits = First.getUsedBits();
22497 assert((UsedBits & Second.getUsedBits()) == 0 &&
22498 "Slices are not supposed to overlap.");
22499 UsedBits |= Second.getUsedBits();
22500 return areUsedBitsDense(UsedBits);
22501}
22502
22503/// Adjust the \p GlobalLSCost according to the target
22504/// paring capabilities and the layout of the slices.
22505/// \pre \p GlobalLSCost should account for at least as many loads as
22506/// there is in the slices in \p LoadedSlices.
22507static void adjustCostForPairing(SmallVectorImpl<LoadedSlice> &LoadedSlices,
22508 LoadedSlice::Cost &GlobalLSCost) {
22509 unsigned NumberOfSlices = LoadedSlices.size();
22510 // If there is less than 2 elements, no pairing is possible.
22511 if (NumberOfSlices < 2)
22512 return;
22513
22514 // Sort the slices so that elements that are likely to be next to each
22515 // other in memory are next to each other in the list.
22516 llvm::sort(C&: LoadedSlices, Comp: [](const LoadedSlice &LHS, const LoadedSlice &RHS) {
22517 assert(LHS.Origin == RHS.Origin && "Different bases not implemented.");
22518 return LHS.getOffsetFromBase() < RHS.getOffsetFromBase();
22519 });
22520 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
22521 // First (resp. Second) is the first (resp. Second) potentially candidate
22522 // to be placed in a paired load.
22523 const LoadedSlice *First = nullptr;
22524 const LoadedSlice *Second = nullptr;
22525 for (unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
22526 // Set the beginning of the pair.
22527 First = Second) {
22528 Second = &LoadedSlices[CurrSlice];
22529
22530 // If First is NULL, it means we start a new pair.
22531 // Get to the next slice.
22532 if (!First)
22533 continue;
22534
22535 EVT LoadedType = First->getLoadedType();
22536
22537 // If the types of the slices are different, we cannot pair them.
22538 if (LoadedType != Second->getLoadedType())
22539 continue;
22540
22541 // Check if the target supplies paired loads for this type.
22542 Align RequiredAlignment;
22543 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
22544 // move to the next pair, this type is hopeless.
22545 Second = nullptr;
22546 continue;
22547 }
22548 // Check if we meet the alignment requirement.
22549 if (First->getAlign() < RequiredAlignment)
22550 continue;
22551
22552 // Check that both loads are next to each other in memory.
22553 if (!areSlicesNextToEachOther(First: *First, Second: *Second))
22554 continue;
22555
22556 assert(GlobalLSCost.Loads > 0 && "We save more loads than we created!");
22557 --GlobalLSCost.Loads;
22558 // Move to the next pair.
22559 Second = nullptr;
22560 }
22561}
22562
22563/// Check the profitability of all involved LoadedSlice.
22564/// Currently, it is considered profitable if there is exactly two
22565/// involved slices (1) which are (2) next to each other in memory, and
22566/// whose cost (\see LoadedSlice::Cost) is smaller than the original load (3).
22567///
22568/// Note: The order of the elements in \p LoadedSlices may be modified, but not
22569/// the elements themselves.
22570///
22571/// FIXME: When the cost model will be mature enough, we can relax
22572/// constraints (1) and (2).
22573static bool isSlicingProfitable(SmallVectorImpl<LoadedSlice> &LoadedSlices,
22574 const APInt &UsedBits, bool ForCodeSize) {
22575 unsigned NumberOfSlices = LoadedSlices.size();
22576 if (StressLoadSlicing)
22577 return NumberOfSlices > 1;
22578
22579 // Check (1).
22580 if (NumberOfSlices != 2)
22581 return false;
22582
22583 // Check (2).
22584 if (!areUsedBitsDense(UsedBits))
22585 return false;
22586
22587 // Check (3).
22588 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
22589 // The original code has one big load.
22590 OrigCost.Loads = 1;
22591 for (unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
22592 const LoadedSlice &LS = LoadedSlices[CurrSlice];
22593 // Accumulate the cost of all the slices.
22594 LoadedSlice::Cost SliceCost(LS, ForCodeSize);
22595 GlobalSlicingCost += SliceCost;
22596
22597 // Account as cost in the original configuration the gain obtained
22598 // with the current slices.
22599 OrigCost.addSliceGain(LS);
22600 }
22601
22602 // If the target supports paired load, adjust the cost accordingly.
22603 adjustCostForPairing(LoadedSlices, GlobalLSCost&: GlobalSlicingCost);
22604 return OrigCost > GlobalSlicingCost;
22605}
22606
22607/// If the given load, \p LI, is used only by trunc or trunc(lshr)
22608/// operations, split it in the various pieces being extracted.
22609///
22610/// This sort of thing is introduced by SROA.
22611/// This slicing takes care not to insert overlapping loads.
22612/// \pre LI is a simple load (i.e., not an atomic or volatile load).
22613bool DAGCombiner::SliceUpLoad(SDNode *N) {
22614 if (Level < AfterLegalizeDAG)
22615 return false;
22616
22617 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
22618 if (!LD->isSimple() || !ISD::isNormalLoad(N: LD) ||
22619 !LD->getValueType(ResNo: 0).isInteger())
22620 return false;
22621
22622 // The algorithm to split up a load of a scalable vector into individual
22623 // elements currently requires knowing the length of the loaded type,
22624 // so will need adjusting to work on scalable vectors.
22625 if (LD->getValueType(ResNo: 0).isScalableVector())
22626 return false;
22627
22628 // Keep track of already used bits to detect overlapping values.
22629 // In that case, we will just abort the transformation.
22630 APInt UsedBits(LD->getValueSizeInBits(ResNo: 0), 0);
22631
22632 SmallVector<LoadedSlice, 4> LoadedSlices;
22633
22634 // Check if this load is used as several smaller chunks of bits.
22635 // Basically, look for uses in trunc or trunc(lshr) and record a new chain
22636 // of computation for each trunc.
22637 for (SDUse &U : LD->uses()) {
22638 // Skip the uses of the chain.
22639 if (U.getResNo() != 0)
22640 continue;
22641
22642 SDNode *User = U.getUser();
22643 unsigned Shift = 0;
22644
22645 // Check if this is a trunc(lshr).
22646 if (User->getOpcode() == ISD::SRL && User->hasOneUse() &&
22647 isa<ConstantSDNode>(Val: User->getOperand(Num: 1))) {
22648 Shift = User->getConstantOperandVal(Num: 1);
22649 User = *User->user_begin();
22650 }
22651
22652 // At this point, User is a Truncate, iff we encountered, trunc or
22653 // trunc(lshr).
22654 if (User->getOpcode() != ISD::TRUNCATE)
22655 return false;
22656
22657 // The width of the type must be a power of 2 and greater than 8-bits.
22658 // Otherwise the load cannot be represented in LLVM IR.
22659 // Moreover, if we shifted with a non-8-bits multiple, the slice
22660 // will be across several bytes. We do not support that.
22661 unsigned Width = User->getValueSizeInBits(ResNo: 0);
22662 if (Width < 8 || !isPowerOf2_32(Value: Width) || (Shift & 0x7))
22663 return false;
22664
22665 // Build the slice for this chain of computations.
22666 LoadedSlice LS(User, LD, Shift, &DAG);
22667 APInt CurrentUsedBits = LS.getUsedBits();
22668
22669 // Check if this slice overlaps with another.
22670 if ((CurrentUsedBits & UsedBits) != 0)
22671 return false;
22672 // Update the bits used globally.
22673 UsedBits |= CurrentUsedBits;
22674
22675 // Check if the new slice would be legal.
22676 if (!LS.isLegal())
22677 return false;
22678
22679 // Record the slice.
22680 LoadedSlices.push_back(Elt: LS);
22681 }
22682
22683 // Abort slicing if it does not seem to be profitable.
22684 if (!isSlicingProfitable(LoadedSlices, UsedBits, ForCodeSize))
22685 return false;
22686
22687 ++SlicedLoads;
22688
22689 // Rewrite each chain to use an independent load.
22690 // By construction, each chain can be represented by a unique load.
22691
22692 // Prepare the argument for the new token factor for all the slices.
22693 SmallVector<SDValue, 8> ArgChains;
22694 for (const LoadedSlice &LS : LoadedSlices) {
22695 SDValue SliceInst = LS.loadSlice();
22696 CombineTo(N: LS.Inst, Res: SliceInst, AddTo: true);
22697 if (SliceInst.getOpcode() != ISD::LOAD)
22698 SliceInst = SliceInst.getOperand(i: 0);
22699 assert(SliceInst->getOpcode() == ISD::LOAD &&
22700 "It takes more than a zext to get to the loaded slice!!");
22701 ArgChains.push_back(Elt: SliceInst.getValue(R: 1));
22702 }
22703
22704 SDValue Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: SDLoc(LD), VT: MVT::Other,
22705 Ops: ArgChains);
22706 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: Chain);
22707 AddToWorklist(N: Chain.getNode());
22708 return true;
22709}
22710
22711/// Check to see if V is (and load (ptr), imm), where the load is having
22712/// specific bytes cleared out. If so, return the byte size being masked out
22713/// and the shift amount.
22714static std::pair<unsigned, unsigned>
22715CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) {
22716 std::pair<unsigned, unsigned> Result(0, 0);
22717
22718 // Check for the structure we're looking for.
22719 if (V->getOpcode() != ISD::AND ||
22720 !isa<ConstantSDNode>(Val: V->getOperand(Num: 1)) ||
22721 !ISD::isNormalLoad(N: V->getOperand(Num: 0).getNode()))
22722 return Result;
22723
22724 // Check the chain and pointer.
22725 LoadSDNode *LD = cast<LoadSDNode>(Val: V->getOperand(Num: 0));
22726 if (LD->getBasePtr() != Ptr) return Result; // Not from same pointer.
22727
22728 // This only handles simple types.
22729 if (V.getValueType() != MVT::i16 &&
22730 V.getValueType() != MVT::i32 &&
22731 V.getValueType() != MVT::i64)
22732 return Result;
22733
22734 // Check the constant mask. Invert it so that the bits being masked out are
22735 // 0 and the bits being kept are 1. Use getSExtValue so that leading bits
22736 // follow the sign bit for uniformity.
22737 uint64_t NotMask = ~cast<ConstantSDNode>(Val: V->getOperand(Num: 1))->getSExtValue();
22738 unsigned NotMaskLZ = llvm::countl_zero(Val: NotMask);
22739 if (NotMaskLZ & 7) return Result; // Must be multiple of a byte.
22740 unsigned NotMaskTZ = llvm::countr_zero(Val: NotMask);
22741 if (NotMaskTZ & 7) return Result; // Must be multiple of a byte.
22742 if (NotMaskLZ == 64) return Result; // All zero mask.
22743
22744 // See if we have a continuous run of bits. If so, we have 0*1+0*
22745 if (llvm::countr_one(Value: NotMask >> NotMaskTZ) + NotMaskTZ + NotMaskLZ != 64)
22746 return Result;
22747
22748 // Adjust NotMaskLZ down to be from the actual size of the int instead of i64.
22749 if (V.getValueType() != MVT::i64 && NotMaskLZ)
22750 NotMaskLZ -= 64-V.getValueSizeInBits();
22751
22752 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8;
22753 switch (MaskedBytes) {
22754 case 1:
22755 case 2:
22756 case 4: break;
22757 default: return Result; // All one mask, or 5-byte mask.
22758 }
22759
22760 // Verify that the first bit starts at a multiple of mask so that the access
22761 // is aligned the same as the access width.
22762 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes) return Result;
22763
22764 // For narrowing to be valid, it must be the case that the load the
22765 // immediately preceding memory operation before the store.
22766 if (LD == Chain.getNode())
22767 ; // ok.
22768 else if (Chain->getOpcode() == ISD::TokenFactor &&
22769 SDValue(LD, 1).hasOneUse()) {
22770 // LD has only 1 chain use so they are no indirect dependencies.
22771 if (!LD->isOperandOf(N: Chain.getNode()))
22772 return Result;
22773 } else
22774 return Result; // Fail.
22775
22776 Result.first = MaskedBytes;
22777 Result.second = NotMaskTZ/8;
22778 return Result;
22779}
22780
22781/// Check to see if IVal is something that provides a value as specified by
22782/// MaskInfo. If so, replace the specified store with a narrower store of
22783/// truncated IVal.
22784static SDValue
22785ShrinkLoadReplaceStoreWithStore(const std::pair<unsigned, unsigned> &MaskInfo,
22786 SDValue IVal, StoreSDNode *St,
22787 DAGCombiner *DC) {
22788 unsigned NumBytes = MaskInfo.first;
22789 unsigned ByteShift = MaskInfo.second;
22790 SelectionDAG &DAG = DC->getDAG();
22791
22792 // Check to see if IVal is all zeros in the part being masked in by the 'or'
22793 // that uses this. If not, this is not a replacement.
22794 APInt Mask = ~APInt::getBitsSet(numBits: IVal.getValueSizeInBits(),
22795 loBit: ByteShift*8, hiBit: (ByteShift+NumBytes)*8);
22796 if (!DAG.MaskedValueIsZero(Op: IVal, Mask)) return SDValue();
22797
22798 // Check that it is legal on the target to do this. It is legal if the new
22799 // VT we're shrinking to (i8/i16/i32) is legal or we're still before type
22800 // legalization. If the source type is legal, but the store type isn't, see
22801 // if we can use a truncating store.
22802 MVT VT = MVT::getIntegerVT(BitWidth: NumBytes * 8);
22803 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
22804 bool UseTruncStore;
22805 if (DC->isTypeLegal(VT))
22806 UseTruncStore = false;
22807 else if (TLI.isTypeLegal(VT: IVal.getValueType()) &&
22808 TLI.isTruncStoreLegal(ValVT: IVal.getValueType(), MemVT: VT, Alignment: St->getAlign(),
22809 AddrSpace: St->getAddressSpace()))
22810 UseTruncStore = true;
22811 else
22812 return SDValue();
22813
22814 // Can't do this for indexed stores.
22815 if (St->isIndexed())
22816 return SDValue();
22817
22818 // Check that the target doesn't think this is a bad idea.
22819 if (St->getMemOperand() &&
22820 !TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT,
22821 MMO: *St->getMemOperand()))
22822 return SDValue();
22823
22824 // Okay, we can do this! Replace the 'St' store with a store of IVal that is
22825 // shifted by ByteShift and truncated down to NumBytes.
22826 if (ByteShift) {
22827 SDLoc DL(IVal);
22828 IVal = DAG.getNode(
22829 Opcode: ISD::SRL, DL, VT: IVal.getValueType(), N1: IVal,
22830 N2: DAG.getShiftAmountConstant(Val: ByteShift * 8, VT: IVal.getValueType(), DL));
22831 }
22832
22833 // Figure out the offset for the store and the alignment of the access.
22834 unsigned StOffset;
22835 if (DAG.getDataLayout().isLittleEndian())
22836 StOffset = ByteShift;
22837 else
22838 StOffset = IVal.getValueType().getStoreSize() - ByteShift - NumBytes;
22839
22840 SDValue Ptr = St->getBasePtr();
22841 if (StOffset) {
22842 SDLoc DL(IVal);
22843 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: StOffset), DL);
22844 }
22845
22846 ++OpsNarrowed;
22847 if (UseTruncStore)
22848 return DAG.getTruncStore(Chain: St->getChain(), dl: SDLoc(St), Val: IVal, Ptr,
22849 PtrInfo: St->getPointerInfo().getWithOffset(O: StOffset), SVT: VT,
22850 Alignment: St->getBaseAlign());
22851
22852 // Truncate down to the new size.
22853 IVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(IVal), VT, Operand: IVal);
22854
22855 return DAG.getStore(Chain: St->getChain(), dl: SDLoc(St), Val: IVal, Ptr,
22856 PtrInfo: St->getPointerInfo().getWithOffset(O: StOffset),
22857 Alignment: St->getBaseAlign());
22858}
22859
22860/// Look for sequence of load / op / store where op is one of 'or', 'xor', and
22861/// 'and' of immediates. If 'op' is only touching some of the loaded bits, try
22862/// narrowing the load and store if it would end up being a win for performance
22863/// or code size.
22864SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
22865 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
22866 if (!ST->isSimple())
22867 return SDValue();
22868
22869 SDValue Chain = ST->getChain();
22870 SDValue Value = ST->getValue();
22871 SDValue Ptr = ST->getBasePtr();
22872 EVT VT = Value.getValueType();
22873
22874 if (ST->isTruncatingStore() || VT.isVector())
22875 return SDValue();
22876
22877 unsigned Opc = Value.getOpcode();
22878
22879 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) ||
22880 !Value.hasOneUse())
22881 return SDValue();
22882
22883 // If this is "store (or X, Y), P" and X is "(and (load P), cst)", where cst
22884 // is a byte mask indicating a consecutive number of bytes, check to see if
22885 // Y is known to provide just those bytes. If so, we try to replace the
22886 // load + replace + store sequence with a single (narrower) store, which makes
22887 // the load dead.
22888 if (Opc == ISD::OR && EnableShrinkLoadReplaceStoreWithStore) {
22889 std::pair<unsigned, unsigned> MaskedLoad;
22890 MaskedLoad = CheckForMaskedLoad(V: Value.getOperand(i: 0), Ptr, Chain);
22891 if (MaskedLoad.first)
22892 if (SDValue NewST = ShrinkLoadReplaceStoreWithStore(MaskInfo: MaskedLoad,
22893 IVal: Value.getOperand(i: 1), St: ST,DC: this))
22894 return NewST;
22895
22896 // Or is commutative, so try swapping X and Y.
22897 MaskedLoad = CheckForMaskedLoad(V: Value.getOperand(i: 1), Ptr, Chain);
22898 if (MaskedLoad.first)
22899 if (SDValue NewST = ShrinkLoadReplaceStoreWithStore(MaskInfo: MaskedLoad,
22900 IVal: Value.getOperand(i: 0), St: ST,DC: this))
22901 return NewST;
22902 }
22903
22904 if (!EnableReduceLoadOpStoreWidth)
22905 return SDValue();
22906
22907 if (Value.getOperand(i: 1).getOpcode() != ISD::Constant)
22908 return SDValue();
22909
22910 SDValue N0 = Value.getOperand(i: 0);
22911 if (ISD::isNormalLoad(N: N0.getNode()) && N0.hasOneUse() &&
22912 Chain == SDValue(N0.getNode(), 1)) {
22913 LoadSDNode *LD = cast<LoadSDNode>(Val&: N0);
22914 if (LD->getBasePtr() != Ptr ||
22915 LD->getPointerInfo().getAddrSpace() !=
22916 ST->getPointerInfo().getAddrSpace())
22917 return SDValue();
22918
22919 // Find the type NewVT to narrow the load / op / store to.
22920 SDValue N1 = Value.getOperand(i: 1);
22921 unsigned BitWidth = N1.getValueSizeInBits();
22922 APInt Imm = N1->getAsAPIntVal();
22923 if (Opc == ISD::AND)
22924 Imm.flipAllBits();
22925 if (Imm == 0 || Imm.isAllOnes())
22926 return SDValue();
22927 // Find least/most significant bit that need to be part of the narrowed
22928 // operation. We assume target will need to address/access full bytes, so
22929 // we make sure to align LSB and MSB at byte boundaries.
22930 unsigned BitsPerByteMask = 7u;
22931 unsigned LSB = Imm.countr_zero() & ~BitsPerByteMask;
22932 unsigned MSB = (Imm.getActiveBits() - 1) | BitsPerByteMask;
22933 unsigned NewBW = NextPowerOf2(A: MSB - LSB);
22934 EVT NewVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewBW);
22935 // The narrowing should be profitable, the load/store operation should be
22936 // legal (or custom) and the store size should be equal to the NewVT width.
22937 while (NewBW < BitWidth &&
22938 (NewVT.getStoreSizeInBits() != NewBW ||
22939 !TLI.isOperationLegalOrCustom(Op: Opc, VT: NewVT) ||
22940 (!ReduceLoadOpStoreWidthForceNarrowingProfitable &&
22941 !TLI.isNarrowingProfitable(N, SrcVT: VT, DestVT: NewVT)))) {
22942 NewBW = NextPowerOf2(A: NewBW);
22943 NewVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewBW);
22944 }
22945 if (NewBW >= BitWidth)
22946 return SDValue();
22947
22948 // If we come this far NewVT/NewBW reflect a power-of-2 sized type that is
22949 // large enough to cover all bits that should be modified. This type might
22950 // however be larger than really needed (such as i32 while we actually only
22951 // need to modify one byte). Now we need to find our how to align the memory
22952 // accesses to satisfy preferred alignments as well as avoiding to access
22953 // memory outside the store size of the orignal access.
22954
22955 unsigned VTStoreSize = VT.getStoreSizeInBits().getFixedValue();
22956
22957 // Let ShAmt denote amount of bits to skip, counted from the least
22958 // significant bits of Imm. And let PtrOff how much the pointer needs to be
22959 // offsetted (in bytes) for the new access.
22960 unsigned ShAmt = 0;
22961 uint64_t PtrOff = 0;
22962 for (; ShAmt + NewBW <= VTStoreSize; ShAmt += 8) {
22963 // Make sure the range [ShAmt, ShAmt+NewBW) cover both LSB and MSB.
22964 if (ShAmt > LSB)
22965 return SDValue();
22966 if (ShAmt + NewBW < MSB)
22967 continue;
22968
22969 // Calculate PtrOff.
22970 unsigned PtrAdjustmentInBits = DAG.getDataLayout().isBigEndian()
22971 ? VTStoreSize - NewBW - ShAmt
22972 : ShAmt;
22973 PtrOff = PtrAdjustmentInBits / 8;
22974
22975 // Now check if narrow access is allowed and fast, considering alignments.
22976 unsigned IsFast = 0;
22977 Align NewAlign = commonAlignment(A: LD->getAlign(), Offset: PtrOff);
22978 if (TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: NewVT,
22979 AddrSpace: LD->getAddressSpace(), Alignment: NewAlign,
22980 Flags: LD->getMemOperand()->getFlags(), Fast: &IsFast) &&
22981 IsFast)
22982 break;
22983 }
22984 // If loop above did not find any accepted ShAmt we need to exit here.
22985 if (ShAmt + NewBW > VTStoreSize)
22986 return SDValue();
22987
22988 APInt NewImm = Imm.lshr(shiftAmt: ShAmt).trunc(width: NewBW);
22989 if (Opc == ISD::AND)
22990 NewImm.flipAllBits();
22991 Align NewAlign = commonAlignment(A: LD->getAlign(), Offset: PtrOff);
22992 SDValue NewPtr =
22993 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: PtrOff), DL: SDLoc(LD));
22994 SDValue NewLD =
22995 DAG.getLoad(VT: NewVT, dl: SDLoc(N0), Chain: LD->getChain(), Ptr: NewPtr,
22996 PtrInfo: LD->getPointerInfo().getWithOffset(O: PtrOff), Alignment: NewAlign,
22997 MMOFlags: LD->getMemOperand()->getFlags(), Metadata: LD->getAAInfo());
22998 SDValue NewVal = DAG.getNode(Opcode: Opc, DL: SDLoc(Value), VT: NewVT, N1: NewLD,
22999 N2: DAG.getConstant(Val: NewImm, DL: SDLoc(Value), VT: NewVT));
23000 SDValue NewST =
23001 DAG.getStore(Chain, dl: SDLoc(N), Val: NewVal, Ptr: NewPtr,
23002 PtrInfo: ST->getPointerInfo().getWithOffset(O: PtrOff), Alignment: NewAlign);
23003
23004 AddToWorklist(N: NewPtr.getNode());
23005 AddToWorklist(N: NewLD.getNode());
23006 AddToWorklist(N: NewVal.getNode());
23007 WorklistRemover DeadNodes(*this);
23008 DAG.ReplaceAllUsesOfValueWith(From: N0.getValue(R: 1), To: NewLD.getValue(R: 1));
23009 ++OpsNarrowed;
23010 return NewST;
23011 }
23012
23013 return SDValue();
23014}
23015
23016/// For a given floating point load / store pair, if the load value isn't used
23017/// by any other operations, then consider transforming the pair to integer
23018/// load / store operations if the target deems the transformation profitable.
23019SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
23020 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
23021 SDValue Value = ST->getValue();
23022 if (ISD::isNormalStore(N: ST) && ISD::isNormalLoad(N: Value.getNode()) &&
23023 Value.hasOneUse()) {
23024 LoadSDNode *LD = cast<LoadSDNode>(Val&: Value);
23025 EVT VT = LD->getMemoryVT();
23026 if (!VT.isSimple() || !VT.isFloatingPoint() || VT != ST->getMemoryVT() ||
23027 LD->isNonTemporal() || ST->isNonTemporal() ||
23028 LD->getPointerInfo().getAddrSpace() != 0 ||
23029 ST->getPointerInfo().getAddrSpace() != 0)
23030 return SDValue();
23031
23032 TypeSize VTSize = VT.getSizeInBits();
23033
23034 // We don't know the size of scalable types at compile time so we cannot
23035 // create an integer of the equivalent size.
23036 if (VTSize.isScalable())
23037 return SDValue();
23038
23039 unsigned FastLD = 0, FastST = 0;
23040 EVT IntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: VTSize.getFixedValue());
23041 if (!TLI.isOperationLegal(Op: ISD::LOAD, VT: IntVT) ||
23042 !TLI.isOperationLegal(Op: ISD::STORE, VT: IntVT) ||
23043 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) ||
23044 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT) ||
23045 !TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: IntVT,
23046 MMO: *LD->getMemOperand(), Fast: &FastLD) ||
23047 !TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: IntVT,
23048 MMO: *ST->getMemOperand(), Fast: &FastST) ||
23049 !FastLD || !FastST)
23050 return SDValue();
23051
23052 SDValue NewLD = DAG.getLoad(VT: IntVT, dl: SDLoc(Value), Chain: LD->getChain(),
23053 Ptr: LD->getBasePtr(), MMO: LD->getMemOperand());
23054
23055 SDValue NewST = DAG.getStore(Chain: ST->getChain(), dl: SDLoc(N), Val: NewLD,
23056 Ptr: ST->getBasePtr(), MMO: ST->getMemOperand());
23057
23058 AddToWorklist(N: NewLD.getNode());
23059 AddToWorklist(N: NewST.getNode());
23060 WorklistRemover DeadNodes(*this);
23061 DAG.ReplaceAllUsesOfValueWith(From: Value.getValue(R: 1), To: NewLD.getValue(R: 1));
23062 ++LdStFP2Int;
23063 return NewST;
23064 }
23065
23066 return SDValue();
23067}
23068
23069// This is a helper function for visitMUL to check the profitability
23070// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
23071// MulNode is the original multiply, AddNode is (add x, c1),
23072// and ConstNode is c2.
23073//
23074// If the (add x, c1) has multiple uses, we could increase
23075// the number of adds if we make this transformation.
23076// It would only be worth doing this if we can remove a
23077// multiply in the process. Check for that here.
23078// To illustrate:
23079// (A + c1) * c3
23080// (A + c2) * c3
23081// We're checking for cases where we have common "c3 * A" expressions.
23082bool DAGCombiner::isMulAddWithConstProfitable(SDNode *MulNode, SDValue AddNode,
23083 SDValue ConstNode) {
23084 // If the add only has one use, and the target thinks the folding is
23085 // profitable or does not lead to worse code, this would be OK to do.
23086 if (AddNode->hasOneUse() &&
23087 TLI.isMulAddWithConstProfitable(AddNode, ConstNode))
23088 return true;
23089
23090 // Walk all the users of the constant with which we're multiplying.
23091 for (SDNode *User : ConstNode->users()) {
23092 if (User == MulNode) // This use is the one we're on right now. Skip it.
23093 continue;
23094
23095 if (User->getOpcode() == ISD::MUL) { // We have another multiply use.
23096 SDNode *OtherOp;
23097 SDNode *MulVar = AddNode.getOperand(i: 0).getNode();
23098
23099 // OtherOp is what we're multiplying against the constant.
23100 if (User->getOperand(Num: 0) == ConstNode)
23101 OtherOp = User->getOperand(Num: 1).getNode();
23102 else
23103 OtherOp = User->getOperand(Num: 0).getNode();
23104
23105 // Check to see if multiply is with the same operand of our "add".
23106 //
23107 // ConstNode = CONST
23108 // User = ConstNode * A <-- visiting User. OtherOp is A.
23109 // ...
23110 // AddNode = (A + c1) <-- MulVar is A.
23111 // = AddNode * ConstNode <-- current visiting instruction.
23112 //
23113 // If we make this transformation, we will have a common
23114 // multiply (ConstNode * A) that we can save.
23115 if (OtherOp == MulVar)
23116 return true;
23117
23118 // Now check to see if a future expansion will give us a common
23119 // multiply.
23120 //
23121 // ConstNode = CONST
23122 // AddNode = (A + c1)
23123 // ... = AddNode * ConstNode <-- current visiting instruction.
23124 // ...
23125 // OtherOp = (A + c2)
23126 // User = OtherOp * ConstNode <-- visiting User.
23127 //
23128 // If we make this transformation, we will have a common
23129 // multiply (CONST * A) after we also do the same transformation
23130 // to the "t2" instruction.
23131 if (OtherOp->getOpcode() == ISD::ADD &&
23132 DAG.isConstantIntBuildVectorOrConstantInt(N: OtherOp->getOperand(Num: 1)) &&
23133 OtherOp->getOperand(Num: 0).getNode() == MulVar)
23134 return true;
23135 }
23136 }
23137
23138 // Didn't find a case where this would be profitable.
23139 return false;
23140}
23141
23142SDValue DAGCombiner::getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
23143 unsigned NumStores) {
23144 SmallVector<SDValue, 8> Chains;
23145 SmallPtrSet<const SDNode *, 8> Visited;
23146 SDLoc StoreDL(StoreNodes[0].MemNode);
23147
23148 for (unsigned i = 0; i < NumStores; ++i) {
23149 Visited.insert(Ptr: StoreNodes[i].MemNode);
23150 }
23151
23152 // don't include nodes that are children or repeated nodes.
23153 for (unsigned i = 0; i < NumStores; ++i) {
23154 if (Visited.insert(Ptr: StoreNodes[i].MemNode->getChain().getNode()).second)
23155 Chains.push_back(Elt: StoreNodes[i].MemNode->getChain());
23156 }
23157
23158 assert(!Chains.empty() && "Chain should have generated a chain");
23159 return DAG.getTokenFactor(DL: StoreDL, Vals&: Chains);
23160}
23161
23162bool DAGCombiner::hasSameUnderlyingObj(ArrayRef<MemOpLink> StoreNodes) {
23163 const Value *UnderlyingObj = nullptr;
23164 for (const auto &MemOp : StoreNodes) {
23165 const MachineMemOperand *MMO = MemOp.MemNode->getMemOperand();
23166 // Pseudo value like stack frame has its own frame index and size, should
23167 // not use the first store's frame index for other frames.
23168 if (MMO->getPseudoValue())
23169 return false;
23170
23171 if (!MMO->getValue())
23172 return false;
23173
23174 const Value *Obj = getUnderlyingObject(V: MMO->getValue());
23175
23176 if (UnderlyingObj && UnderlyingObj != Obj)
23177 return false;
23178
23179 if (!UnderlyingObj)
23180 UnderlyingObj = Obj;
23181 }
23182
23183 return true;
23184}
23185
23186bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
23187 SmallVectorImpl<MemOpLink> &StoreNodes, EVT MemVT, unsigned NumStores,
23188 bool IsConstantSrc, bool UseVector, bool UseTrunc) {
23189 // Make sure we have something to merge.
23190 if (NumStores < 2)
23191 return false;
23192
23193 assert((!UseTrunc || !UseVector) &&
23194 "This optimization cannot emit a vector truncating store");
23195
23196 // The latest Node in the DAG.
23197 SDLoc DL(StoreNodes[0].MemNode);
23198
23199 TypeSize ElementSizeBits = MemVT.getStoreSizeInBits();
23200 unsigned SizeInBits = NumStores * ElementSizeBits;
23201 unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
23202
23203 std::optional<MachineMemOperand::Flags> Flags;
23204 AAMDNodes AAInfo;
23205 for (unsigned I = 0; I != NumStores; ++I) {
23206 StoreSDNode *St = cast<StoreSDNode>(Val: StoreNodes[I].MemNode);
23207 if (!Flags) {
23208 Flags = St->getMemOperand()->getFlags();
23209 AAInfo = St->getAAInfo();
23210 continue;
23211 }
23212 // Skip merging if there's an inconsistent flag.
23213 if (Flags != St->getMemOperand()->getFlags())
23214 return false;
23215 // Concatenate AA metadata.
23216 AAInfo = AAInfo.concat(Other: St->getAAInfo());
23217 }
23218
23219 EVT StoreTy;
23220 if (UseVector) {
23221 unsigned Elts = NumStores * NumMemElts;
23222 // Get the type for the merged vector store.
23223 StoreTy = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getScalarType(), NumElements: Elts);
23224 } else
23225 StoreTy = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: SizeInBits);
23226
23227 SDValue StoredVal;
23228 if (UseVector) {
23229 if (IsConstantSrc) {
23230 SmallVector<SDValue, 8> BuildVector;
23231 for (unsigned I = 0; I != NumStores; ++I) {
23232 StoreSDNode *St = cast<StoreSDNode>(Val: StoreNodes[I].MemNode);
23233 SDValue Val = St->getValue();
23234 // If constant is of the wrong type, convert it now. This comes up
23235 // when one of our stores was truncating.
23236 if (MemVT != Val.getValueType()) {
23237 Val = peekThroughBitcasts(V: Val);
23238 // Deal with constants of wrong size.
23239 if (ElementSizeBits != Val.getValueSizeInBits()) {
23240 auto *C = dyn_cast<ConstantSDNode>(Val);
23241 if (!C)
23242 // Not clear how to truncate FP values.
23243 // TODO: Handle truncation of build_vector constants
23244 return false;
23245
23246 EVT IntMemVT =
23247 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits());
23248 Val = DAG.getConstant(Val: C->getAPIntValue()
23249 .zextOrTrunc(width: Val.getValueSizeInBits())
23250 .zextOrTrunc(width: ElementSizeBits),
23251 DL: SDLoc(C), VT: IntMemVT);
23252 }
23253 // Make sure correctly size type is the correct type.
23254 Val = DAG.getBitcast(VT: MemVT, V: Val);
23255 }
23256 BuildVector.push_back(Elt: Val);
23257 }
23258 StoredVal = DAG.getNode(Opcode: MemVT.isVector() ? ISD::CONCAT_VECTORS
23259 : ISD::BUILD_VECTOR,
23260 DL, VT: StoreTy, Ops: BuildVector);
23261 } else {
23262 SmallVector<SDValue, 8> Ops;
23263 for (unsigned i = 0; i < NumStores; ++i) {
23264 StoreSDNode *St = cast<StoreSDNode>(Val: StoreNodes[i].MemNode);
23265 SDValue Val = peekThroughBitcasts(V: St->getValue());
23266 // All operands of BUILD_VECTOR / CONCAT_VECTOR must be of
23267 // type MemVT. If the underlying value is not the correct
23268 // type, but it is an extraction of an appropriate vector we
23269 // can recast Val to be of the correct type. This may require
23270 // converting between EXTRACT_VECTOR_ELT and
23271 // EXTRACT_SUBVECTOR.
23272 if ((MemVT != Val.getValueType()) &&
23273 (Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
23274 Val.getOpcode() == ISD::EXTRACT_SUBVECTOR)) {
23275 EVT MemVTScalarTy = MemVT.getScalarType();
23276 // We may need to add a bitcast here to get types to line up.
23277 if (MemVTScalarTy != Val.getValueType().getScalarType()) {
23278 Val = DAG.getBitcast(VT: MemVT, V: Val);
23279 } else if (MemVT.isVector() &&
23280 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
23281 Val = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MemVT, Operand: Val);
23282 } else {
23283 unsigned OpC = MemVT.isVector() ? ISD::EXTRACT_SUBVECTOR
23284 : ISD::EXTRACT_VECTOR_ELT;
23285 SDValue Vec = Val.getOperand(i: 0);
23286 SDValue Idx = Val.getOperand(i: 1);
23287 Val = DAG.getNode(Opcode: OpC, DL: SDLoc(Val), VT: MemVT, N1: Vec, N2: Idx);
23288 }
23289 }
23290 Ops.push_back(Elt: Val);
23291 }
23292
23293 // Build the extracted vector elements back into a vector.
23294 StoredVal = DAG.getNode(Opcode: MemVT.isVector() ? ISD::CONCAT_VECTORS
23295 : ISD::BUILD_VECTOR,
23296 DL, VT: StoreTy, Ops);
23297 }
23298 } else {
23299 // We should always use a vector store when merging extracted vector
23300 // elements, so this path implies a store of constants.
23301 assert(IsConstantSrc && "Merged vector elements should use vector store");
23302
23303 APInt StoreInt(SizeInBits, 0);
23304
23305 // Construct a single integer constant which is made of the smaller
23306 // constant inputs.
23307 bool IsLE = DAG.getDataLayout().isLittleEndian();
23308 for (unsigned i = 0; i < NumStores; ++i) {
23309 unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
23310 StoreSDNode *St = cast<StoreSDNode>(Val: StoreNodes[Idx].MemNode);
23311
23312 SDValue Val = St->getValue();
23313 Val = peekThroughBitcasts(V: Val);
23314 StoreInt <<= ElementSizeBits;
23315 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val)) {
23316 StoreInt |= C->getAPIntValue()
23317 .zextOrTrunc(width: ElementSizeBits)
23318 .zextOrTrunc(width: SizeInBits);
23319 } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val)) {
23320 StoreInt |= C->getValueAPF()
23321 .bitcastToAPInt()
23322 .zextOrTrunc(width: ElementSizeBits)
23323 .zextOrTrunc(width: SizeInBits);
23324 // If fp truncation is necessary give up for now.
23325 if (MemVT.getSizeInBits() != ElementSizeBits)
23326 return false;
23327 } else if (ISD::isBuildVectorOfConstantSDNodes(N: Val.getNode()) ||
23328 ISD::isBuildVectorOfConstantFPSDNodes(N: Val.getNode())) {
23329 // Not yet handled
23330 return false;
23331 } else {
23332 llvm_unreachable("Invalid constant element type");
23333 }
23334 }
23335
23336 // Create the new Load and Store operations.
23337 StoredVal = DAG.getConstant(Val: StoreInt, DL, VT: StoreTy);
23338 }
23339
23340 LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
23341 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
23342 bool CanReusePtrInfo = hasSameUnderlyingObj(StoreNodes);
23343
23344 // make sure we use trunc store if it's necessary to be legal.
23345 // When generate the new widen store, if the first store's pointer info can
23346 // not be reused, discard the pointer info except the address space because
23347 // now the widen store can not be represented by the original pointer info
23348 // which is for the narrow memory object.
23349 SDValue NewStore;
23350 if (!UseTrunc) {
23351 NewStore = DAG.getStore(
23352 Chain: NewChain, dl: DL, Val: StoredVal, Ptr: FirstInChain->getBasePtr(),
23353 PtrInfo: CanReusePtrInfo
23354 ? FirstInChain->getPointerInfo()
23355 : MachinePointerInfo(FirstInChain->getPointerInfo().getAddrSpace()),
23356 Alignment: FirstInChain->getAlign(), MMOFlags: *Flags, Metadata: AAInfo);
23357 } else { // Must be realized as a trunc store
23358 EVT LegalizedStoredValTy =
23359 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: StoredVal.getValueType());
23360 unsigned LegalizedStoreSize = LegalizedStoredValTy.getSizeInBits();
23361 ConstantSDNode *C = cast<ConstantSDNode>(Val&: StoredVal);
23362 SDValue ExtendedStoreVal =
23363 DAG.getConstant(Val: C->getAPIntValue().zextOrTrunc(width: LegalizedStoreSize), DL,
23364 VT: LegalizedStoredValTy);
23365 NewStore = DAG.getTruncStore(
23366 Chain: NewChain, dl: DL, Val: ExtendedStoreVal, Ptr: FirstInChain->getBasePtr(),
23367 PtrInfo: CanReusePtrInfo
23368 ? FirstInChain->getPointerInfo()
23369 : MachinePointerInfo(FirstInChain->getPointerInfo().getAddrSpace()),
23370 SVT: StoredVal.getValueType() /*TVT*/, Alignment: FirstInChain->getAlign(), MMOFlags: *Flags,
23371 Metadata: AAInfo);
23372 }
23373
23374 // Replace all merged stores with the new store.
23375 for (unsigned i = 0; i < NumStores; ++i)
23376 CombineTo(N: StoreNodes[i].MemNode, Res: NewStore);
23377
23378 AddToWorklist(N: NewChain.getNode());
23379 return true;
23380}
23381
23382SDNode *
23383DAGCombiner::getStoreMergeCandidates(StoreSDNode *St,
23384 SmallVectorImpl<MemOpLink> &StoreNodes) {
23385 // This holds the base pointer, index, and the offset in bytes from the base
23386 // pointer. We must have a base and an offset. Do not handle stores to undef
23387 // base pointers.
23388 BaseIndexOffset BasePtr = BaseIndexOffset::match(N: St, DAG);
23389 if (!BasePtr.getBase().getNode() || BasePtr.getBase().isUndef())
23390 return nullptr;
23391
23392 SDValue Val = peekThroughBitcasts(V: St->getValue());
23393 StoreSource StoreSrc = getStoreSource(StoreVal: Val);
23394 assert(StoreSrc != StoreSource::Unknown && "Expected known source for store");
23395
23396 // Match on loadbaseptr if relevant.
23397 EVT MemVT = St->getMemoryVT();
23398 BaseIndexOffset LBasePtr;
23399 EVT LoadVT;
23400 if (StoreSrc == StoreSource::Load) {
23401 auto *Ld = cast<LoadSDNode>(Val);
23402 LBasePtr = BaseIndexOffset::match(N: Ld, DAG);
23403 LoadVT = Ld->getMemoryVT();
23404 // Load and store should be the same type.
23405 if (MemVT != LoadVT)
23406 return nullptr;
23407 // Loads must only have one use.
23408 if (!Ld->hasNUsesOfValue(NUses: 1, Value: 0))
23409 return nullptr;
23410 // The memory operands must not be volatile/indexed/atomic.
23411 // TODO: May be able to relax for unordered atomics (see D66309)
23412 if (!Ld->isSimple() || Ld->isIndexed())
23413 return nullptr;
23414 }
23415 auto CandidateMatch = [&](StoreSDNode *Other, BaseIndexOffset &Ptr,
23416 int64_t &Offset) -> bool {
23417 // The memory operands must not be volatile/indexed/atomic.
23418 // TODO: May be able to relax for unordered atomics (see D66309)
23419 if (!Other->isSimple() || Other->isIndexed())
23420 return false;
23421 // Don't mix temporal stores with non-temporal stores.
23422 if (St->isNonTemporal() != Other->isNonTemporal())
23423 return false;
23424 if (!TLI.areTwoSDNodeTargetMMOFlagsMergeable(NodeX: *St, NodeY: *Other))
23425 return false;
23426 SDValue OtherBC = peekThroughBitcasts(V: Other->getValue());
23427 // Allow merging constants of different types as integers.
23428 bool NoTypeMatch = (MemVT.isInteger()) ? !MemVT.bitsEq(VT: Other->getMemoryVT())
23429 : Other->getMemoryVT() != MemVT;
23430 switch (StoreSrc) {
23431 case StoreSource::Load: {
23432 if (NoTypeMatch)
23433 return false;
23434 // The Load's Base Ptr must also match.
23435 auto *OtherLd = dyn_cast<LoadSDNode>(Val&: OtherBC);
23436 if (!OtherLd)
23437 return false;
23438 BaseIndexOffset LPtr = BaseIndexOffset::match(N: OtherLd, DAG);
23439 if (LoadVT != OtherLd->getMemoryVT())
23440 return false;
23441 // Loads must only have one use.
23442 if (!OtherLd->hasNUsesOfValue(NUses: 1, Value: 0))
23443 return false;
23444 // The memory operands must not be volatile/indexed/atomic.
23445 // TODO: May be able to relax for unordered atomics (see D66309)
23446 if (!OtherLd->isSimple() || OtherLd->isIndexed())
23447 return false;
23448 // Don't mix temporal loads with non-temporal loads.
23449 if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal())
23450 return false;
23451 if (!TLI.areTwoSDNodeTargetMMOFlagsMergeable(NodeX: *cast<LoadSDNode>(Val),
23452 NodeY: *OtherLd))
23453 return false;
23454 if (!(LBasePtr.equalBaseIndex(Other: LPtr, DAG)))
23455 return false;
23456 break;
23457 }
23458 case StoreSource::Constant:
23459 if (NoTypeMatch)
23460 return false;
23461 if (getStoreSource(StoreVal: OtherBC) != StoreSource::Constant)
23462 return false;
23463 break;
23464 case StoreSource::Extract:
23465 // Do not merge truncated stores here.
23466 if (Other->isTruncatingStore())
23467 return false;
23468 if (!MemVT.bitsEq(VT: OtherBC.getValueType()))
23469 return false;
23470 if (OtherBC.getOpcode() != ISD::EXTRACT_VECTOR_ELT &&
23471 OtherBC.getOpcode() != ISD::EXTRACT_SUBVECTOR)
23472 return false;
23473 break;
23474 default:
23475 llvm_unreachable("Unhandled store source for merging");
23476 }
23477 Ptr = BaseIndexOffset::match(N: Other, DAG);
23478 return (BasePtr.equalBaseIndex(Other: Ptr, DAG, Off&: Offset));
23479 };
23480
23481 // We are looking for a root node which is an ancestor to all mergable
23482 // stores. We search up through a load, to our root and then down
23483 // through all children. For instance we will find Store{1,2,3} if
23484 // St is Store1, Store2. or Store3 where the root is not a load
23485 // which always true for nonvolatile ops. TODO: Expand
23486 // the search to find all valid candidates through multiple layers of loads.
23487 //
23488 // Root
23489 // |-------|-------|
23490 // Load Load Store3
23491 // | |
23492 // Store1 Store2
23493 //
23494 // FIXME: We should be able to climb and
23495 // descend TokenFactors to find candidates as well.
23496
23497 SDNode *RootNode = St->getChain().getNode();
23498 // Bail out if we already analyzed this root node and found nothing.
23499 if (ChainsWithoutMergeableStores.contains(Ptr: RootNode))
23500 return nullptr;
23501
23502 // Check if the pair of StoreNode and the RootNode already bail out many
23503 // times which is over the limit in dependence check.
23504 auto OverLimitInDependenceCheck = [&](SDNode *StoreNode,
23505 SDNode *RootNode) -> bool {
23506 auto RootCount = StoreRootCountMap.find(Val: StoreNode);
23507 return RootCount != StoreRootCountMap.end() &&
23508 RootCount->second.first == RootNode &&
23509 RootCount->second.second > StoreMergeDependenceLimit;
23510 };
23511
23512 auto TryToAddCandidate = [&](SDUse &Use) {
23513 // This must be a chain use.
23514 if (Use.getOperandNo() != 0)
23515 return;
23516 if (auto *OtherStore = dyn_cast<StoreSDNode>(Val: Use.getUser())) {
23517 BaseIndexOffset Ptr;
23518 int64_t PtrDiff;
23519 if (CandidateMatch(OtherStore, Ptr, PtrDiff) &&
23520 !OverLimitInDependenceCheck(OtherStore, RootNode))
23521 StoreNodes.push_back(Elt: MemOpLink(OtherStore, PtrDiff));
23522 }
23523 };
23524
23525 unsigned NumNodesExplored = 0;
23526 const unsigned MaxSearchNodes = 1024;
23527 if (auto *Ldn = dyn_cast<LoadSDNode>(Val: RootNode)) {
23528 RootNode = Ldn->getChain().getNode();
23529 // Bail out if we already analyzed this root node and found nothing.
23530 if (ChainsWithoutMergeableStores.contains(Ptr: RootNode))
23531 return nullptr;
23532 for (auto I = RootNode->use_begin(), E = RootNode->use_end();
23533 I != E && NumNodesExplored < MaxSearchNodes; ++I, ++NumNodesExplored) {
23534 SDNode *User = I->getUser();
23535 if (I->getOperandNo() == 0 && isa<LoadSDNode>(Val: User)) { // walk down chain
23536 for (SDUse &U2 : User->uses())
23537 TryToAddCandidate(U2);
23538 }
23539 // Check stores that depend on the root (e.g. Store 3 in the chart above).
23540 if (I->getOperandNo() == 0 && isa<StoreSDNode>(Val: User)) {
23541 TryToAddCandidate(*I);
23542 }
23543 }
23544 } else {
23545 for (auto I = RootNode->use_begin(), E = RootNode->use_end();
23546 I != E && NumNodesExplored < MaxSearchNodes; ++I, ++NumNodesExplored)
23547 TryToAddCandidate(*I);
23548 }
23549
23550 return RootNode;
23551}
23552
23553// We need to check that merging these stores does not cause a loop in the
23554// DAG. Any store candidate may depend on another candidate indirectly through
23555// its operands. Check in parallel by searching up from operands of candidates.
23556bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
23557 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
23558 SDNode *RootNode) {
23559 // FIXME: We should be able to truncate a full search of
23560 // predecessors by doing a BFS and keeping tabs the originating
23561 // stores from which worklist nodes come from in a similar way to
23562 // TokenFactor simplfication.
23563
23564 SmallPtrSet<const SDNode *, 32> Visited;
23565 SmallVector<const SDNode *, 8> Worklist;
23566
23567 // RootNode is a predecessor to all candidates so we need not search
23568 // past it. Add RootNode (peeking through TokenFactors). Do not count
23569 // these towards size check.
23570
23571 Worklist.push_back(Elt: RootNode);
23572 while (!Worklist.empty()) {
23573 auto N = Worklist.pop_back_val();
23574 if (!Visited.insert(Ptr: N).second)
23575 continue; // Already present in Visited.
23576 if (N->getOpcode() == ISD::TokenFactor) {
23577 for (SDValue Op : N->ops())
23578 Worklist.push_back(Elt: Op.getNode());
23579 }
23580 }
23581
23582 // Don't count pruning nodes towards max.
23583 unsigned int Max = 1024 + Visited.size();
23584 // Search Ops of store candidates.
23585 for (unsigned i = 0; i < NumStores; ++i) {
23586 SDNode *N = StoreNodes[i].MemNode;
23587 // Of the 4 Store Operands:
23588 // * Chain (Op 0) -> We have already considered these
23589 // in candidate selection, but only by following the
23590 // chain dependencies. We could still have a chain
23591 // dependency to a load, that has a non-chain dep to
23592 // another load, that depends on a store, etc. So it is
23593 // possible to have dependencies that consist of a mix
23594 // of chain and non-chain deps, and we need to include
23595 // chain operands in the analysis here..
23596 // * Value (Op 1) -> Cycles may happen (e.g. through load chains)
23597 // * Address (Op 2) -> Merged addresses may only vary by a fixed constant,
23598 // but aren't necessarily fromt the same base node, so
23599 // cycles possible (e.g. via indexed store).
23600 // * (Op 3) -> Represents the pre or post-indexing offset (or undef for
23601 // non-indexed stores). Not constant on all targets (e.g. ARM)
23602 // and so can participate in a cycle.
23603 for (const SDValue &Op : N->op_values())
23604 Worklist.push_back(Elt: Op.getNode());
23605 }
23606 // Search through DAG. We can stop early if we find a store node.
23607 for (unsigned i = 0; i < NumStores; ++i)
23608 if (SDNode::hasPredecessorHelper(N: StoreNodes[i].MemNode, Visited, Worklist,
23609 MaxSteps: Max)) {
23610 // If the searching bail out, record the StoreNode and RootNode in the
23611 // StoreRootCountMap. If we have seen the pair many times over a limit,
23612 // we won't add the StoreNode into StoreNodes set again.
23613 if (Visited.size() >= Max) {
23614 auto &RootCount = StoreRootCountMap[StoreNodes[i].MemNode];
23615 if (RootCount.first == RootNode)
23616 RootCount.second++;
23617 else
23618 RootCount = {RootNode, 1};
23619 }
23620 return false;
23621 }
23622 return true;
23623}
23624
23625bool DAGCombiner::hasCallInLdStChain(StoreSDNode *St, LoadSDNode *Ld) {
23626 SmallPtrSet<const SDNode *, 32> Visited;
23627 SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
23628 Worklist.emplace_back(Args: St->getChain().getNode(), Args: false);
23629
23630 while (!Worklist.empty()) {
23631 auto [Node, FoundCall] = Worklist.pop_back_val();
23632 if (!Visited.insert(Ptr: Node).second || Node->getNumOperands() == 0)
23633 continue;
23634
23635 switch (Node->getOpcode()) {
23636 case ISD::CALLSEQ_END:
23637 Worklist.emplace_back(Args: Node->getOperand(Num: 0).getNode(), Args: true);
23638 break;
23639 case ISD::TokenFactor:
23640 for (SDValue Op : Node->ops())
23641 Worklist.emplace_back(Args: Op.getNode(), Args&: FoundCall);
23642 break;
23643 case ISD::LOAD:
23644 if (Node == Ld)
23645 return FoundCall;
23646 [[fallthrough]];
23647 default:
23648 assert(Node->getOperand(0).getValueType() == MVT::Other &&
23649 "Invalid chain type");
23650 Worklist.emplace_back(Args: Node->getOperand(Num: 0).getNode(), Args&: FoundCall);
23651 break;
23652 }
23653 }
23654 return false;
23655}
23656
23657unsigned
23658DAGCombiner::getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
23659 int64_t ElementSizeBytes) const {
23660 while (true) {
23661 // Find a store past the width of the first store.
23662 size_t StartIdx = 0;
23663 while ((StartIdx + 1 < StoreNodes.size()) &&
23664 StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
23665 StoreNodes[StartIdx + 1].OffsetFromBase)
23666 ++StartIdx;
23667
23668 // Bail if we don't have enough candidates to merge.
23669 if (StartIdx + 1 >= StoreNodes.size())
23670 return 0;
23671
23672 // Trim stores that overlapped with the first store.
23673 if (StartIdx)
23674 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + StartIdx);
23675
23676 // Scan the memory operations on the chain and find the first
23677 // non-consecutive store memory address.
23678 unsigned NumConsecutiveStores = 1;
23679 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
23680 // Check that the addresses are consecutive starting from the second
23681 // element in the list of stores.
23682 for (unsigned i = 1, e = StoreNodes.size(); i < e; ++i) {
23683 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
23684 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
23685 break;
23686 NumConsecutiveStores = i + 1;
23687 }
23688 if (NumConsecutiveStores > 1)
23689 return NumConsecutiveStores;
23690
23691 // There are no consecutive stores at the start of the list.
23692 // Remove the first store and try again.
23693 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + 1);
23694 }
23695}
23696
23697bool DAGCombiner::tryStoreMergeOfConstants(
23698 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumConsecutiveStores,
23699 EVT MemVT, SDNode *RootNode, bool AllowVectors) {
23700 LLVMContext &Context = *DAG.getContext();
23701 const DataLayout &DL = DAG.getDataLayout();
23702 int64_t ElementSizeBytes = MemVT.getStoreSize();
23703 unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
23704 bool MadeChange = false;
23705
23706 // Store the constants into memory as one consecutive store.
23707 while (NumConsecutiveStores >= 2) {
23708 LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
23709 unsigned FirstStoreAS = FirstInChain->getAddressSpace();
23710 Align FirstStoreAlign = FirstInChain->getAlign();
23711 unsigned LastLegalType = 1;
23712 unsigned LastLegalVectorType = 1;
23713 bool LastIntegerTrunc = false;
23714 bool NonZero = false;
23715 unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
23716 for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
23717 StoreSDNode *ST = cast<StoreSDNode>(Val: StoreNodes[i].MemNode);
23718 SDValue StoredVal = ST->getValue();
23719 bool IsElementZero = false;
23720 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: StoredVal))
23721 IsElementZero = C->isZero();
23722 else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: StoredVal))
23723 IsElementZero = C->getConstantFPValue()->isNullValue();
23724 else if (ISD::isBuildVectorAllZeros(N: StoredVal.getNode()))
23725 IsElementZero = true;
23726 if (IsElementZero) {
23727 if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
23728 FirstZeroAfterNonZero = i;
23729 }
23730 NonZero |= !IsElementZero;
23731
23732 // Find a legal type for the constant store.
23733 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
23734 EVT StoreTy = EVT::getIntegerVT(Context, BitWidth: SizeInBits);
23735 unsigned IsFast = 0;
23736
23737 // Break early when size is too large to be legal.
23738 if (StoreTy.getSizeInBits() > TLI.getMaximumLegalStoreInBits())
23739 break;
23740
23741 if (TLI.isTypeLegal(VT: StoreTy) &&
23742 TLI.canMergeStoresTo(AS: FirstStoreAS, MemVT: StoreTy,
23743 MF: DAG.getMachineFunction()) &&
23744 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
23745 MMO: *FirstInChain->getMemOperand(), Fast: &IsFast) &&
23746 IsFast) {
23747 LastIntegerTrunc = false;
23748 LastLegalType = i + 1;
23749 // Or check whether a truncstore is legal.
23750 } else if (TLI.getTypeAction(Context, VT: StoreTy) ==
23751 TargetLowering::TypePromoteInteger) {
23752 EVT LegalizedStoredValTy =
23753 TLI.getTypeToTransformTo(Context, VT: StoredVal.getValueType());
23754 if (TLI.isTruncStoreLegal(ValVT: LegalizedStoredValTy, MemVT: StoreTy,
23755 Alignment: FirstStoreAlign, AddrSpace: FirstStoreAS) &&
23756 TLI.canMergeStoresTo(AS: FirstStoreAS, MemVT: LegalizedStoredValTy,
23757 MF: DAG.getMachineFunction()) &&
23758 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
23759 MMO: *FirstInChain->getMemOperand(), Fast: &IsFast) &&
23760 IsFast) {
23761 LastIntegerTrunc = true;
23762 LastLegalType = i + 1;
23763 }
23764 }
23765
23766 // We only use vectors if the target allows it and the function is not
23767 // marked with the noimplicitfloat attribute.
23768 if (TLI.storeOfVectorConstantIsCheap(IsZero: !NonZero, MemVT, NumElem: i + 1, AddrSpace: FirstStoreAS) &&
23769 AllowVectors) {
23770 // Find a legal type for the vector store.
23771 unsigned Elts = (i + 1) * NumMemElts;
23772 EVT Ty = EVT::getVectorVT(Context, VT: MemVT.getScalarType(), NumElements: Elts);
23773 if (TLI.isTypeLegal(VT: Ty) && TLI.isTypeLegal(VT: MemVT) &&
23774 TLI.canMergeStoresTo(AS: FirstStoreAS, MemVT: Ty, MF: DAG.getMachineFunction()) &&
23775 TLI.allowsMemoryAccess(Context, DL, VT: Ty,
23776 MMO: *FirstInChain->getMemOperand(), Fast: &IsFast) &&
23777 IsFast)
23778 LastLegalVectorType = i + 1;
23779 }
23780 }
23781
23782 bool UseVector = (LastLegalVectorType > LastLegalType) && AllowVectors;
23783 unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
23784 bool UseTrunc = LastIntegerTrunc && !UseVector;
23785
23786 // Check if we found a legal integer type that creates a meaningful
23787 // merge.
23788 if (NumElem < 2) {
23789 // We know that candidate stores are in order and of correct
23790 // shape. While there is no mergeable sequence from the
23791 // beginning one may start later in the sequence. The only
23792 // reason a merge of size N could have failed where another of
23793 // the same size would not have, is if the alignment has
23794 // improved or we've dropped a non-zero value. Drop as many
23795 // candidates as we can here.
23796 unsigned NumSkip = 1;
23797 while ((NumSkip < NumConsecutiveStores) &&
23798 (NumSkip < FirstZeroAfterNonZero) &&
23799 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
23800 NumSkip++;
23801
23802 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumSkip);
23803 NumConsecutiveStores -= NumSkip;
23804 continue;
23805 }
23806
23807 // Check that we can merge these candidates without causing a cycle.
23808 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStores: NumElem,
23809 RootNode)) {
23810 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumElem);
23811 NumConsecutiveStores -= NumElem;
23812 continue;
23813 }
23814
23815 MadeChange |= mergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumStores: NumElem,
23816 /*IsConstantSrc*/ true,
23817 UseVector, UseTrunc);
23818
23819 // Remove merged stores for next iteration.
23820 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumElem);
23821 NumConsecutiveStores -= NumElem;
23822 }
23823 return MadeChange;
23824}
23825
23826bool DAGCombiner::tryStoreMergeOfExtracts(
23827 SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumConsecutiveStores,
23828 EVT MemVT, SDNode *RootNode) {
23829 LLVMContext &Context = *DAG.getContext();
23830 const DataLayout &DL = DAG.getDataLayout();
23831 unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
23832 bool MadeChange = false;
23833
23834 // Loop on Consecutive Stores on success.
23835 while (NumConsecutiveStores >= 2) {
23836 LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
23837 unsigned FirstStoreAS = FirstInChain->getAddressSpace();
23838 Align FirstStoreAlign = FirstInChain->getAlign();
23839 unsigned NumStoresToMerge = 1;
23840 for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
23841 // Find a legal type for the vector store.
23842 unsigned Elts = (i + 1) * NumMemElts;
23843 EVT Ty = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT.getScalarType(), NumElements: Elts);
23844 unsigned IsFast = 0;
23845
23846 // Break early when size is too large to be legal.
23847 if (Ty.getSizeInBits() > TLI.getMaximumLegalStoreInBits())
23848 break;
23849
23850 if (TLI.isTypeLegal(VT: Ty) &&
23851 TLI.canMergeStoresTo(AS: FirstStoreAS, MemVT: Ty, MF: DAG.getMachineFunction()) &&
23852 TLI.allowsMemoryAccess(Context, DL, VT: Ty,
23853 MMO: *FirstInChain->getMemOperand(), Fast: &IsFast) &&
23854 IsFast)
23855 NumStoresToMerge = i + 1;
23856 }
23857
23858 // Check if we found a legal integer type creating a meaningful
23859 // merge.
23860 if (NumStoresToMerge < 2) {
23861 // We know that candidate stores are in order and of correct
23862 // shape. While there is no mergeable sequence from the
23863 // beginning one may start later in the sequence. The only
23864 // reason a merge of size N could have failed where another of
23865 // the same size would not have, is if the alignment has
23866 // improved. Drop as many candidates as we can here.
23867 unsigned NumSkip = 1;
23868 while ((NumSkip < NumConsecutiveStores) &&
23869 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
23870 NumSkip++;
23871
23872 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumSkip);
23873 NumConsecutiveStores -= NumSkip;
23874 continue;
23875 }
23876
23877 // Check that we can merge these candidates without causing a cycle.
23878 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStores: NumStoresToMerge,
23879 RootNode)) {
23880 StoreNodes.erase(CS: StoreNodes.begin(),
23881 CE: StoreNodes.begin() + NumStoresToMerge);
23882 NumConsecutiveStores -= NumStoresToMerge;
23883 continue;
23884 }
23885
23886 MadeChange |= mergeStoresOfConstantsOrVecElts(
23887 StoreNodes, MemVT, NumStores: NumStoresToMerge, /*IsConstantSrc*/ false,
23888 /*UseVector*/ true, /*UseTrunc*/ false);
23889
23890 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumStoresToMerge);
23891 NumConsecutiveStores -= NumStoresToMerge;
23892 }
23893 return MadeChange;
23894}
23895
23896bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
23897 unsigned NumConsecutiveStores, EVT MemVT,
23898 SDNode *RootNode, bool AllowVectors,
23899 bool IsNonTemporalStore,
23900 bool IsNonTemporalLoad) {
23901 LLVMContext &Context = *DAG.getContext();
23902 const DataLayout &DL = DAG.getDataLayout();
23903 int64_t ElementSizeBytes = MemVT.getStoreSize();
23904 unsigned NumMemElts = MemVT.isVector() ? MemVT.getVectorNumElements() : 1;
23905 bool MadeChange = false;
23906
23907 // Look for load nodes which are used by the stored values.
23908 SmallVector<MemOpLink, 8> LoadNodes;
23909
23910 // Find acceptable loads. Loads need to have the same chain (token factor),
23911 // must not be zext, volatile, indexed, and they must be consecutive.
23912 BaseIndexOffset LdBasePtr;
23913
23914 for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
23915 StoreSDNode *St = cast<StoreSDNode>(Val: StoreNodes[i].MemNode);
23916 SDValue Val = peekThroughBitcasts(V: St->getValue());
23917 LoadSDNode *Ld = cast<LoadSDNode>(Val);
23918
23919 BaseIndexOffset LdPtr = BaseIndexOffset::match(N: Ld, DAG);
23920 // If this is not the first ptr that we check.
23921 int64_t LdOffset = 0;
23922 if (LdBasePtr.getBase().getNode()) {
23923 // The base ptr must be the same.
23924 if (!LdBasePtr.equalBaseIndex(Other: LdPtr, DAG, Off&: LdOffset))
23925 break;
23926 } else {
23927 // Check that all other base pointers are the same as this one.
23928 LdBasePtr = LdPtr;
23929 }
23930
23931 // We found a potential memory operand to merge.
23932 LoadNodes.push_back(Elt: MemOpLink(Ld, LdOffset));
23933 }
23934
23935 while (NumConsecutiveStores >= 2 && LoadNodes.size() >= 2) {
23936 Align RequiredAlignment;
23937 bool NeedRotate = false;
23938 if (LoadNodes.size() == 2) {
23939 // If we have load/store pair instructions and we only have two values,
23940 // don't bother merging.
23941 if (TLI.hasPairedLoad(MemVT, RequiredAlignment) &&
23942 StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) {
23943 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + 2);
23944 LoadNodes.erase(CS: LoadNodes.begin(), CE: LoadNodes.begin() + 2);
23945 break;
23946 }
23947 // If the loads are reversed, see if we can rotate the halves into place.
23948 int64_t Offset0 = LoadNodes[0].OffsetFromBase;
23949 int64_t Offset1 = LoadNodes[1].OffsetFromBase;
23950 EVT PairVT = EVT::getIntegerVT(Context, BitWidth: ElementSizeBytes * 8 * 2);
23951 if (Offset0 - Offset1 == ElementSizeBytes &&
23952 (hasOperation(Opcode: ISD::ROTL, VT: PairVT) ||
23953 hasOperation(Opcode: ISD::ROTR, VT: PairVT))) {
23954 std::swap(a&: LoadNodes[0], b&: LoadNodes[1]);
23955 NeedRotate = true;
23956 }
23957 }
23958 LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
23959 unsigned FirstStoreAS = FirstInChain->getAddressSpace();
23960 Align FirstStoreAlign = FirstInChain->getAlign();
23961 LoadSDNode *FirstLoad = cast<LoadSDNode>(Val: LoadNodes[0].MemNode);
23962
23963 // Scan the memory operations on the chain and find the first
23964 // non-consecutive load memory address. These variables hold the index in
23965 // the store node array.
23966
23967 unsigned LastConsecutiveLoad = 1;
23968
23969 // This variable refers to the size and not index in the array.
23970 unsigned LastLegalVectorType = 1;
23971 unsigned LastLegalIntegerType = 1;
23972 bool isDereferenceable = true;
23973 bool DoIntegerTruncate = false;
23974 int64_t StartAddress = LoadNodes[0].OffsetFromBase;
23975 SDValue LoadChain = FirstLoad->getChain();
23976 for (unsigned i = 1; i < LoadNodes.size(); ++i) {
23977 // All loads must share the same chain.
23978 if (LoadNodes[i].MemNode->getChain() != LoadChain)
23979 break;
23980
23981 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
23982 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
23983 break;
23984 LastConsecutiveLoad = i;
23985
23986 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
23987 isDereferenceable = false;
23988
23989 // Find a legal type for the vector store.
23990 unsigned Elts = (i + 1) * NumMemElts;
23991 EVT StoreTy = EVT::getVectorVT(Context, VT: MemVT.getScalarType(), NumElements: Elts);
23992
23993 // Break early when size is too large to be legal.
23994 if (StoreTy.getSizeInBits() > TLI.getMaximumLegalStoreInBits())
23995 break;
23996
23997 unsigned IsFastSt = 0;
23998 unsigned IsFastLd = 0;
23999 // Don't try vector types if we need a rotate. We may still fail the
24000 // legality checks for the integer type, but we can't handle the rotate
24001 // case with vectors.
24002 // FIXME: We could use a shuffle in place of the rotate.
24003 if (!NeedRotate && TLI.isTypeLegal(VT: StoreTy) &&
24004 TLI.canMergeStoresTo(AS: FirstStoreAS, MemVT: StoreTy,
24005 MF: DAG.getMachineFunction()) &&
24006 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
24007 MMO: *FirstInChain->getMemOperand(), Fast: &IsFastSt) &&
24008 IsFastSt &&
24009 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
24010 MMO: *FirstLoad->getMemOperand(), Fast: &IsFastLd) &&
24011 IsFastLd) {
24012 LastLegalVectorType = i + 1;
24013 }
24014
24015 // Find a legal type for the integer store.
24016 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
24017 StoreTy = EVT::getIntegerVT(Context, BitWidth: SizeInBits);
24018 if (TLI.isTypeLegal(VT: StoreTy) &&
24019 TLI.canMergeStoresTo(AS: FirstStoreAS, MemVT: StoreTy,
24020 MF: DAG.getMachineFunction()) &&
24021 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
24022 MMO: *FirstInChain->getMemOperand(), Fast: &IsFastSt) &&
24023 IsFastSt &&
24024 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
24025 MMO: *FirstLoad->getMemOperand(), Fast: &IsFastLd) &&
24026 IsFastLd) {
24027 LastLegalIntegerType = i + 1;
24028 DoIntegerTruncate = false;
24029 // Or check whether a truncstore and extload is legal.
24030 } else if (TLI.getTypeAction(Context, VT: StoreTy) ==
24031 TargetLowering::TypePromoteInteger) {
24032 EVT LegalizedStoredValTy = TLI.getTypeToTransformTo(Context, VT: StoreTy);
24033 if (TLI.isTruncStoreLegal(ValVT: LegalizedStoredValTy, MemVT: StoreTy,
24034 Alignment: FirstStoreAlign, AddrSpace: FirstStoreAS) &&
24035 TLI.canMergeStoresTo(AS: FirstStoreAS, MemVT: LegalizedStoredValTy,
24036 MF: DAG.getMachineFunction()) &&
24037 TLI.isLoadLegal(ValVT: LegalizedStoredValTy, MemVT: StoreTy,
24038 Alignment: FirstLoad->getAlign(), AddrSpace: FirstLoad->getAddressSpace(),
24039 ExtType: ISD::ZEXTLOAD, Atomic: false) &&
24040 TLI.isLoadLegal(ValVT: LegalizedStoredValTy, MemVT: StoreTy,
24041 Alignment: FirstLoad->getAlign(), AddrSpace: FirstLoad->getAddressSpace(),
24042 ExtType: ISD::SEXTLOAD, Atomic: false) &&
24043 TLI.isLoadLegal(ValVT: LegalizedStoredValTy, MemVT: StoreTy,
24044 Alignment: FirstLoad->getAlign(), AddrSpace: FirstLoad->getAddressSpace(),
24045 ExtType: ISD::EXTLOAD, Atomic: false) &&
24046 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
24047 MMO: *FirstInChain->getMemOperand(), Fast: &IsFastSt) &&
24048 IsFastSt &&
24049 TLI.allowsMemoryAccess(Context, DL, VT: StoreTy,
24050 MMO: *FirstLoad->getMemOperand(), Fast: &IsFastLd) &&
24051 IsFastLd) {
24052 LastLegalIntegerType = i + 1;
24053 DoIntegerTruncate = true;
24054 }
24055 }
24056 }
24057
24058 // Only use vector types if the vector type is larger than the integer
24059 // type. If they are the same, use integers.
24060 bool UseVectorTy =
24061 LastLegalVectorType > LastLegalIntegerType && AllowVectors;
24062 unsigned LastLegalType =
24063 std::max(a: LastLegalVectorType, b: LastLegalIntegerType);
24064
24065 // We add +1 here because the LastXXX variables refer to location while
24066 // the NumElem refers to array/index size.
24067 unsigned NumElem = std::min(a: NumConsecutiveStores, b: LastConsecutiveLoad + 1);
24068 NumElem = std::min(a: LastLegalType, b: NumElem);
24069 Align FirstLoadAlign = FirstLoad->getAlign();
24070
24071 if (NumElem < 2) {
24072 // We know that candidate stores are in order and of correct
24073 // shape. While there is no mergeable sequence from the
24074 // beginning one may start later in the sequence. The only
24075 // reason a merge of size N could have failed where another of
24076 // the same size would not have is if the alignment or either
24077 // the load or store has improved. Drop as many candidates as we
24078 // can here.
24079 unsigned NumSkip = 1;
24080 while ((NumSkip < LoadNodes.size()) &&
24081 (LoadNodes[NumSkip].MemNode->getAlign() <= FirstLoadAlign) &&
24082 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
24083 NumSkip++;
24084 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumSkip);
24085 LoadNodes.erase(CS: LoadNodes.begin(), CE: LoadNodes.begin() + NumSkip);
24086 NumConsecutiveStores -= NumSkip;
24087 continue;
24088 }
24089
24090 // Check that we can merge these candidates without causing a cycle.
24091 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStores: NumElem,
24092 RootNode)) {
24093 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumElem);
24094 LoadNodes.erase(CS: LoadNodes.begin(), CE: LoadNodes.begin() + NumElem);
24095 NumConsecutiveStores -= NumElem;
24096 continue;
24097 }
24098
24099 // Find if it is better to use vectors or integers to load and store
24100 // to memory.
24101 EVT JointMemOpVT;
24102 if (UseVectorTy) {
24103 // Find a legal type for the vector store.
24104 unsigned Elts = NumElem * NumMemElts;
24105 JointMemOpVT = EVT::getVectorVT(Context, VT: MemVT.getScalarType(), NumElements: Elts);
24106 } else {
24107 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
24108 JointMemOpVT = EVT::getIntegerVT(Context, BitWidth: SizeInBits);
24109 }
24110
24111 // Check if there is a call in the load/store chain.
24112 if (!TLI.shouldMergeStoreOfLoadsOverCall(MemVT, JointMemOpVT) &&
24113 hasCallInLdStChain(St: cast<StoreSDNode>(Val: StoreNodes[0].MemNode),
24114 Ld: cast<LoadSDNode>(Val: LoadNodes[0].MemNode))) {
24115 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumElem);
24116 LoadNodes.erase(CS: LoadNodes.begin(), CE: LoadNodes.begin() + NumElem);
24117 NumConsecutiveStores -= NumElem;
24118 continue;
24119 }
24120
24121 SDLoc LoadDL(LoadNodes[0].MemNode);
24122 SDLoc StoreDL(StoreNodes[0].MemNode);
24123
24124 // The merged loads are required to have the same incoming chain, so
24125 // using the first's chain is acceptable.
24126
24127 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumStores: NumElem);
24128 bool CanReusePtrInfo = hasSameUnderlyingObj(StoreNodes);
24129 AddToWorklist(N: NewStoreChain.getNode());
24130
24131 MachineMemOperand::Flags LdMMOFlags =
24132 isDereferenceable ? MachineMemOperand::MODereferenceable
24133 : MachineMemOperand::MONone;
24134 if (IsNonTemporalLoad)
24135 LdMMOFlags |= MachineMemOperand::MONonTemporal;
24136
24137 LdMMOFlags |= TLI.getTargetMMOFlags(Node: *FirstLoad);
24138
24139 MachineMemOperand::Flags StMMOFlags = IsNonTemporalStore
24140 ? MachineMemOperand::MONonTemporal
24141 : MachineMemOperand::MONone;
24142
24143 StMMOFlags |= TLI.getTargetMMOFlags(Node: *StoreNodes[0].MemNode);
24144
24145 SDValue NewLoad, NewStore;
24146 if (UseVectorTy || !DoIntegerTruncate) {
24147 NewLoad = DAG.getLoad(
24148 VT: JointMemOpVT, dl: LoadDL, Chain: FirstLoad->getChain(), Ptr: FirstLoad->getBasePtr(),
24149 PtrInfo: FirstLoad->getPointerInfo(), Alignment: FirstLoadAlign, MMOFlags: LdMMOFlags);
24150 SDValue StoreOp = NewLoad;
24151 if (NeedRotate) {
24152 unsigned LoadWidth = ElementSizeBytes * 8 * 2;
24153 assert(JointMemOpVT == EVT::getIntegerVT(Context, LoadWidth) &&
24154 "Unexpected type for rotate-able load pair");
24155 SDValue RotAmt =
24156 DAG.getShiftAmountConstant(Val: LoadWidth / 2, VT: JointMemOpVT, DL: LoadDL);
24157 // Target can convert to the identical ROTR if it does not have ROTL.
24158 StoreOp = DAG.getNode(Opcode: ISD::ROTL, DL: LoadDL, VT: JointMemOpVT, N1: NewLoad, N2: RotAmt);
24159 }
24160 NewStore = DAG.getStore(
24161 Chain: NewStoreChain, dl: StoreDL, Val: StoreOp, Ptr: FirstInChain->getBasePtr(),
24162 PtrInfo: CanReusePtrInfo ? FirstInChain->getPointerInfo()
24163 : MachinePointerInfo(FirstStoreAS),
24164 Alignment: FirstStoreAlign, MMOFlags: StMMOFlags);
24165 } else { // This must be the truncstore/extload case
24166 EVT ExtendedTy =
24167 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: JointMemOpVT);
24168 NewLoad = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: LoadDL, VT: ExtendedTy,
24169 Chain: FirstLoad->getChain(), Ptr: FirstLoad->getBasePtr(),
24170 PtrInfo: FirstLoad->getPointerInfo(), MemVT: JointMemOpVT,
24171 Alignment: FirstLoadAlign, MMOFlags: LdMMOFlags);
24172 NewStore = DAG.getTruncStore(
24173 Chain: NewStoreChain, dl: StoreDL, Val: NewLoad, Ptr: FirstInChain->getBasePtr(),
24174 PtrInfo: CanReusePtrInfo ? FirstInChain->getPointerInfo()
24175 : MachinePointerInfo(FirstStoreAS),
24176 SVT: JointMemOpVT, Alignment: FirstInChain->getAlign(),
24177 MMOFlags: FirstInChain->getMemOperand()->getFlags());
24178 }
24179
24180 // Transfer chain users from old loads to the new load.
24181 for (unsigned i = 0; i < NumElem; ++i) {
24182 LoadSDNode *Ld = cast<LoadSDNode>(Val: LoadNodes[i].MemNode);
24183 DAG.ReplaceAllUsesOfValueWith(From: SDValue(Ld, 1),
24184 To: SDValue(NewLoad.getNode(), 1));
24185 }
24186
24187 // Replace all stores with the new store. Recursively remove corresponding
24188 // values if they are no longer used.
24189 for (unsigned i = 0; i < NumElem; ++i) {
24190 SDValue Val = StoreNodes[i].MemNode->getOperand(Num: 1);
24191 CombineTo(N: StoreNodes[i].MemNode, Res: NewStore);
24192 if (Val->use_empty())
24193 recursivelyDeleteUnusedNodes(N: Val.getNode());
24194 }
24195
24196 MadeChange = true;
24197 StoreNodes.erase(CS: StoreNodes.begin(), CE: StoreNodes.begin() + NumElem);
24198 LoadNodes.erase(CS: LoadNodes.begin(), CE: LoadNodes.begin() + NumElem);
24199 NumConsecutiveStores -= NumElem;
24200 }
24201 return MadeChange;
24202}
24203
24204bool DAGCombiner::mergeConsecutiveStores(StoreSDNode *St) {
24205 if (OptLevel == CodeGenOptLevel::None || !EnableStoreMerging)
24206 return false;
24207
24208 // TODO: Extend this function to merge stores of scalable vectors.
24209 // (i.e. two <vscale x 8 x i8> stores can be merged to one <vscale x 16 x i8>
24210 // store since we know <vscale x 16 x i8> is exactly twice as large as
24211 // <vscale x 8 x i8>). Until then, bail out for scalable vectors.
24212 EVT MemVT = St->getMemoryVT();
24213 if (MemVT.isScalableVT())
24214 return false;
24215 if (!MemVT.isSimple() ||
24216 MemVT.getSizeInBits() * 2 > TLI.getMaximumLegalStoreInBits())
24217 return false;
24218
24219 // This function cannot currently deal with non-byte-sized memory sizes.
24220 int64_t ElementSizeBytes = MemVT.getStoreSize();
24221 if (ElementSizeBytes * 8 != (int64_t)MemVT.getSizeInBits())
24222 return false;
24223
24224 // Do not bother looking at stored values that are not constants, loads, or
24225 // extracted vector elements.
24226 SDValue StoredVal = peekThroughBitcasts(V: St->getValue());
24227 const StoreSource StoreSrc = getStoreSource(StoreVal: StoredVal);
24228 if (StoreSrc == StoreSource::Unknown)
24229 return false;
24230
24231 SmallVector<MemOpLink, 8> StoreNodes;
24232 // Find potential store merge candidates by searching through chain sub-DAG
24233 SDNode *RootNode = getStoreMergeCandidates(St, StoreNodes);
24234
24235 // Check if there is anything to merge.
24236 if (StoreNodes.size() < 2)
24237 return false;
24238
24239 // Sort the memory operands according to their distance from the
24240 // base pointer.
24241 llvm::sort(C&: StoreNodes, Comp: [](MemOpLink LHS, MemOpLink RHS) {
24242 return LHS.OffsetFromBase < RHS.OffsetFromBase;
24243 });
24244
24245 bool AllowVectors = !DAG.getMachineFunction().getFunction().hasFnAttribute(
24246 Kind: Attribute::NoImplicitFloat);
24247 bool IsNonTemporalStore = St->isNonTemporal();
24248 bool IsNonTemporalLoad = StoreSrc == StoreSource::Load &&
24249 cast<LoadSDNode>(Val&: StoredVal)->isNonTemporal();
24250
24251 // Store Merge attempts to merge the lowest stores. This generally
24252 // works out as if successful, as the remaining stores are checked
24253 // after the first collection of stores is merged. However, in the
24254 // case that a non-mergeable store is found first, e.g., {p[-2],
24255 // p[0], p[1], p[2], p[3]}, we would fail and miss the subsequent
24256 // mergeable cases. To prevent this, we prune such stores from the
24257 // front of StoreNodes here.
24258 bool MadeChange = false;
24259 while (StoreNodes.size() > 1) {
24260 unsigned NumConsecutiveStores =
24261 getConsecutiveStores(StoreNodes, ElementSizeBytes);
24262 // There are no more stores in the list to examine.
24263 if (NumConsecutiveStores == 0)
24264 return MadeChange;
24265
24266 // We have at least 2 consecutive stores. Try to merge them.
24267 assert(NumConsecutiveStores >= 2 && "Expected at least 2 stores");
24268 switch (StoreSrc) {
24269 case StoreSource::Constant:
24270 MadeChange |= tryStoreMergeOfConstants(StoreNodes, NumConsecutiveStores,
24271 MemVT, RootNode, AllowVectors);
24272 break;
24273
24274 case StoreSource::Extract:
24275 MadeChange |= tryStoreMergeOfExtracts(StoreNodes, NumConsecutiveStores,
24276 MemVT, RootNode);
24277 break;
24278
24279 case StoreSource::Load:
24280 MadeChange |= tryStoreMergeOfLoads(StoreNodes, NumConsecutiveStores,
24281 MemVT, RootNode, AllowVectors,
24282 IsNonTemporalStore, IsNonTemporalLoad);
24283 break;
24284
24285 default:
24286 llvm_unreachable("Unhandled store source type");
24287 }
24288 }
24289
24290 // Remember if we failed to optimize, to save compile time.
24291 if (!MadeChange)
24292 ChainsWithoutMergeableStores.insert(Ptr: RootNode);
24293
24294 return MadeChange;
24295}
24296
24297SDValue DAGCombiner::replaceStoreChain(StoreSDNode *ST, SDValue BetterChain) {
24298 SDLoc SL(ST);
24299 SDValue ReplStore;
24300
24301 // Replace the chain to avoid dependency.
24302 if (ST->isTruncatingStore()) {
24303 ReplStore = DAG.getTruncStore(Chain: BetterChain, dl: SL, Val: ST->getValue(),
24304 Ptr: ST->getBasePtr(), SVT: ST->getMemoryVT(),
24305 MMO: ST->getMemOperand());
24306 } else {
24307 ReplStore = DAG.getStore(Chain: BetterChain, dl: SL, Val: ST->getValue(), Ptr: ST->getBasePtr(),
24308 MMO: ST->getMemOperand());
24309 }
24310
24311 // Create token to keep both nodes around.
24312 SDValue Token = DAG.getNode(Opcode: ISD::TokenFactor, DL: SL,
24313 VT: MVT::Other, N1: ST->getChain(), N2: ReplStore);
24314
24315 // Make sure the new and old chains are cleaned up.
24316 AddToWorklist(N: Token.getNode());
24317
24318 // Don't add users to work list.
24319 return CombineTo(N: ST, Res: Token, AddTo: false);
24320}
24321
24322SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) {
24323 SDValue Value = ST->getValue();
24324 if (Value.getOpcode() == ISD::TargetConstantFP)
24325 return SDValue();
24326
24327 if (!ISD::isNormalStore(N: ST))
24328 return SDValue();
24329
24330 SDLoc DL(ST);
24331
24332 SDValue Chain = ST->getChain();
24333 SDValue Ptr = ST->getBasePtr();
24334
24335 const ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Val&: Value);
24336
24337 // NOTE: If the original store is volatile, this transform must not increase
24338 // the number of stores. For example, on x86-32 an f64 can be stored in one
24339 // processor operation but an i64 (which is not legal) requires two. So the
24340 // transform should not be done in this case.
24341
24342 SDValue Tmp;
24343 switch (CFP->getSimpleValueType(ResNo: 0).SimpleTy) {
24344 default:
24345 llvm_unreachable("Unknown FP type");
24346 case MVT::f16: // We don't do this for these yet.
24347 case MVT::bf16:
24348 case MVT::f80:
24349 case MVT::f128:
24350 case MVT::ppcf128:
24351 return SDValue();
24352 case MVT::f32:
24353 if ((isTypeLegal(VT: MVT::i32) && !LegalOperations && ST->isSimple()) ||
24354 TLI.isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::i32)) {
24355 Tmp = DAG.getConstant(Val: (uint32_t)CFP->getValueAPF().
24356 bitcastToAPInt().getZExtValue(), DL: SDLoc(CFP),
24357 VT: MVT::i32);
24358 return DAG.getStore(Chain, dl: DL, Val: Tmp, Ptr, MMO: ST->getMemOperand());
24359 }
24360
24361 return SDValue();
24362 case MVT::f64:
24363 if ((TLI.isTypeLegal(VT: MVT::i64) && !LegalOperations &&
24364 ST->isSimple()) ||
24365 TLI.isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::i64)) {
24366 Tmp = DAG.getConstant(Val: CFP->getValueAPF().bitcastToAPInt().
24367 getZExtValue(), DL: SDLoc(CFP), VT: MVT::i64);
24368 return DAG.getStore(Chain, dl: DL, Val: Tmp,
24369 Ptr, MMO: ST->getMemOperand());
24370 }
24371
24372 if (ST->isSimple() && TLI.isOperationLegalOrCustom(Op: ISD::STORE, VT: MVT::i32) &&
24373 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
24374 // Many FP stores are not made apparent until after legalize, e.g. for
24375 // argument passing. Since this is so common, custom legalize the
24376 // 64-bit integer store into two 32-bit stores.
24377 uint64_t Val = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
24378 SDValue Lo = DAG.getConstant(Val: Val & 0xFFFFFFFF, DL: SDLoc(CFP), VT: MVT::i32);
24379 SDValue Hi = DAG.getConstant(Val: Val >> 32, DL: SDLoc(CFP), VT: MVT::i32);
24380 if (DAG.getDataLayout().isBigEndian())
24381 std::swap(a&: Lo, b&: Hi);
24382
24383 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
24384 AAMDNodes AAInfo = ST->getAAInfo();
24385
24386 SDValue St0 = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
24387 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
24388 Ptr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: 4), DL);
24389 SDValue St1 = DAG.getStore(Chain, dl: DL, Val: Hi, Ptr,
24390 PtrInfo: ST->getPointerInfo().getWithOffset(O: 4),
24391 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
24392 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
24393 N1: St0, N2: St1);
24394 }
24395
24396 return SDValue();
24397 }
24398}
24399
24400// (store (insert_vector_elt (load p), x, i), p) -> (store x, p+offset)
24401//
24402// If a store of a load with an element inserted into it has no other
24403// uses in between the chain, then we can consider the vector store
24404// dead and replace it with just the single scalar element store.
24405SDValue DAGCombiner::replaceStoreOfInsertLoad(StoreSDNode *ST) {
24406 SDLoc DL(ST);
24407 SDValue Value = ST->getValue();
24408 SDValue Ptr = ST->getBasePtr();
24409 SDValue Chain = ST->getChain();
24410 if (Value.getOpcode() != ISD::INSERT_VECTOR_ELT || !Value.hasOneUse())
24411 return SDValue();
24412
24413 SDValue Elt = Value.getOperand(i: 1);
24414 SDValue Idx = Value.getOperand(i: 2);
24415
24416 // If the element isn't byte sized or is implicitly truncated then we can't
24417 // compute an offset.
24418 EVT EltVT = Elt.getValueType();
24419 if (!EltVT.isByteSized() ||
24420 EltVT != Value.getOperand(i: 0).getValueType().getVectorElementType())
24421 return SDValue();
24422
24423 auto *Ld = dyn_cast<LoadSDNode>(Val: Value.getOperand(i: 0));
24424 if (!Ld || Ld->getBasePtr() != Ptr ||
24425 ST->getMemoryVT() != Ld->getMemoryVT() || !ST->isSimple() ||
24426 !ISD::isNormalStore(N: ST) ||
24427 Ld->getAddressSpace() != ST->getAddressSpace() ||
24428 !Chain.reachesChainWithoutSideEffects(Dest: SDValue(Ld, 1)))
24429 return SDValue();
24430
24431 unsigned IsFast;
24432 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
24433 VT: Elt.getValueType(), AddrSpace: ST->getAddressSpace(),
24434 Alignment: ST->getAlign(), Flags: ST->getMemOperand()->getFlags(),
24435 Fast: &IsFast) ||
24436 !IsFast)
24437 return SDValue();
24438
24439 MachinePointerInfo PointerInfo(ST->getAddressSpace());
24440 Align NewAlign;
24441
24442 // If the offset is a known constant then try to recover the pointer
24443 // info
24444 SDValue NewPtr;
24445 if (auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx)) {
24446 unsigned COffset = CIdx->getSExtValue() * EltVT.getFixedSizeInBits() / 8;
24447 NewPtr = DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: COffset), DL);
24448 PointerInfo = ST->getPointerInfo().getWithOffset(O: COffset);
24449 NewAlign = ST->getAlign();
24450 } else {
24451 // The original DAG loaded the entire vector from memory, so arithmetic
24452 // within it must be inbounds.
24453 NewPtr = TLI.getInboundsVectorElementPointer(DAG, VecPtr: Ptr, VecVT: Value.getValueType(),
24454 Index: Idx);
24455 // MachinePointerInfo can't represent a variable offset, so use a generic
24456 // MachinePointerInfo and recompute the alignment.
24457 NewAlign = commonAlignment(A: ST->getAlign(), Offset: EltVT.getFixedSizeInBits() / 8);
24458 }
24459
24460 return DAG.getStore(Chain, dl: DL, Val: Elt, Ptr: NewPtr, PtrInfo: PointerInfo, Alignment: NewAlign,
24461 MMOFlags: ST->getMemOperand()->getFlags());
24462}
24463
24464SDValue DAGCombiner::visitATOMIC_STORE(SDNode *N) {
24465 AtomicSDNode *ST = cast<AtomicSDNode>(Val: N);
24466 SDValue Val = ST->getVal();
24467 EVT VT = Val.getValueType();
24468 EVT MemVT = ST->getMemoryVT();
24469
24470 if (MemVT.bitsLT(VT)) { // Is truncating store
24471 APInt TruncDemandedBits = APInt::getLowBitsSet(numBits: VT.getScalarSizeInBits(),
24472 loBitsSet: MemVT.getScalarSizeInBits());
24473 // See if we can simplify the operation with SimplifyDemandedBits, which
24474 // only works if the value has a single use.
24475 if (SimplifyDemandedBits(Op: Val, DemandedBits: TruncDemandedBits))
24476 return SDValue(N, 0);
24477 }
24478
24479 return SDValue();
24480}
24481
24482static SDValue foldToMaskedStore(StoreSDNode *Store, SelectionDAG &DAG,
24483 const SDLoc &Dl) {
24484 if (!Store->isSimple() || !ISD::isNormalStore(N: Store))
24485 return SDValue();
24486
24487 SDValue StoredVal = Store->getValue();
24488 SDValue StorePtr = Store->getBasePtr();
24489 SDValue StoreOffset = Store->getOffset();
24490 EVT VT = Store->getMemoryVT();
24491
24492 // Skip this combine for non-vector types and for <1 x ty> vectors, as they
24493 // will be scalarized later.
24494 if (!VT.isVector() || VT.isScalableVector() || VT.getVectorNumElements() == 1)
24495 return SDValue();
24496
24497 unsigned AddrSpace = Store->getAddressSpace();
24498 Align Alignment = Store->getAlign();
24499 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24500
24501 // A legal masked store can still be slower than the original sequence,
24502 // e.g. on pre-AVX-512 Zen, which we avoid by checking isTypeDesirableForOp.
24503 if (!TLI.isOperationLegalOrCustom(Op: ISD::MSTORE, VT) ||
24504 !TLI.isTypeDesirableForOp(ISD::MSTORE, VT) ||
24505 !TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment))
24506 return SDValue();
24507
24508 SDValue Mask, OtherVec, LoadCh;
24509 unsigned LoadPos;
24510 if (sd_match(N: StoredVal,
24511 P: m_VSelect(Cond: m_Value(N&: Mask), T: m_Value(N&: OtherVec),
24512 F: m_Load(Ch: m_Value(N&: LoadCh), Ptr: m_Specific(N: StorePtr),
24513 Offset: m_Specific(N: StoreOffset))))) {
24514 LoadPos = 2;
24515 } else if (sd_match(N: StoredVal,
24516 P: m_VSelect(Cond: m_Value(N&: Mask),
24517 T: m_Load(Ch: m_Value(N&: LoadCh), Ptr: m_Specific(N: StorePtr),
24518 Offset: m_Specific(N: StoreOffset)),
24519 F: m_Value(N&: OtherVec)))) {
24520 LoadPos = 1;
24521 } else {
24522 return SDValue();
24523 }
24524
24525 auto *Load = cast<LoadSDNode>(Val: StoredVal.getOperand(i: LoadPos));
24526 if (!Load->isSimple() || !ISD::isNormalLoad(N: Load) ||
24527 Load->getAddressSpace() != AddrSpace)
24528 return SDValue();
24529
24530 if (!Store->getChain().reachesChainWithoutSideEffects(Dest: LoadCh))
24531 return SDValue();
24532
24533 if (LoadPos == 1)
24534 Mask = DAG.getNOT(DL: Dl, Val: Mask, VT: Mask.getValueType());
24535
24536 // A masked store follows the IR convention of a vXi1 mask (one bit per
24537 // element). A vselect condition may instead be a wider boolean vector, e.g.
24538 // a vXi32/vXi64 comparison result produced on AVX512 targets without VLX.
24539 // When the matching vXi1 type is legal, narrow the mask to it so that targets
24540 // expecting a vXi1 mask lower it correctly. Targets where vXi1 is illegal
24541 // (e.g. AVX/AVX2) keep the wide mask and lower it as a blend/vmaskmov.
24542 EVT MaskVT = Mask.getValueType();
24543 if (MaskVT.getVectorElementType() != MVT::i1) {
24544 EVT BoolVT = MaskVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i1);
24545 if (TLI.isTypeLegal(VT: BoolVT))
24546 Mask = DAG.getNode(Opcode: ISD::TRUNCATE, DL: Dl, VT: BoolVT, Operand: Mask);
24547 }
24548
24549 return DAG.getMaskedStore(Chain: Store->getChain(), dl: Dl, Val: OtherVec, Base: StorePtr,
24550 Offset: StoreOffset, Mask, MemVT: VT, MMO: Store->getMemOperand(),
24551 AM: Store->getAddressingMode());
24552}
24553
24554// store(concat_vector(truncate, truncate))
24555// --> store(truncate)
24556// store(truncate)
24557SDValue DAGCombiner::combineStoreConcatTruncVector(StoreSDNode *ST) {
24558 if (!LegalTypes)
24559 return SDValue();
24560
24561 if (!ST->isSimple() || ST->isTruncatingStore() || ST->isIndexed())
24562 return SDValue();
24563
24564 SDValue Chain = ST->getChain();
24565 SDValue ConcatVec = ST->getValue();
24566
24567 if (ConcatVec.getOpcode() != ISD::CONCAT_VECTORS ||
24568 ConcatVec.getNumOperands() != 2 || !ConcatVec->hasOneUse())
24569 return SDValue();
24570
24571 SDValue T1 = ConcatVec.getOperand(i: 0);
24572 SDValue T2 = ConcatVec.getOperand(i: 1);
24573 if (T1.getOpcode() != ISD::TRUNCATE || T2.getOpcode() != ISD::TRUNCATE)
24574 return SDValue();
24575
24576 EVT LoMemVT = T1.getValueType();
24577 EVT HiMemVT = T2.getValueType();
24578 if (!LoMemVT.isFixedLengthVector())
24579 return SDValue();
24580
24581 if (!T1.hasOneUse() || !T2.hasOneUse())
24582 return SDValue();
24583
24584 unsigned LoBytes = LoMemVT.getStoreSize();
24585 unsigned HiBytes = HiMemVT.getStoreSize();
24586 Align LoAlign = ST->getAlign();
24587 Align HiAlign = commonAlignment(A: LoAlign, Offset: LoBytes);
24588
24589 if (!TLI.canCombineTruncStore(ValVT: T1.getOperand(i: 0).getValueType(), MemVT: LoMemVT,
24590 Alignment: LoAlign, AddrSpace: ST->getAddressSpace(),
24591 LegalOnly: LegalOperations) ||
24592 !TLI.canCombineTruncStore(ValVT: T2.getOperand(i: 0).getValueType(), MemVT: HiMemVT,
24593 Alignment: HiAlign, AddrSpace: ST->getAddressSpace(),
24594 LegalOnly: LegalOperations))
24595 return SDValue();
24596
24597 SDLoc DL(ST);
24598 SDValue LoPtr = ST->getBasePtr();
24599 SDValue HiPtr =
24600 DAG.getObjectPtrOffset(SL: DL, Ptr: LoPtr, Offset: TypeSize::getFixed(ExactSize: LoBytes));
24601
24602 MachineFunction &MF = DAG.getMachineFunction();
24603 MachineMemOperand *LoMMO =
24604 MF.getMachineMemOperand(MMO: ST->getMemOperand(), Offset: 0, Size: LoBytes);
24605 MachineMemOperand *HiMMO =
24606 MF.getMachineMemOperand(MMO: ST->getMemOperand(), Offset: LoBytes, Size: HiBytes);
24607
24608 SDValue LoSt =
24609 DAG.getTruncStore(Chain, dl: DL, Val: T1.getOperand(i: 0), Ptr: LoPtr, SVT: LoMemVT, MMO: LoMMO);
24610 SDValue HiSt =
24611 DAG.getTruncStore(Chain, dl: DL, Val: T2.getOperand(i: 0), Ptr: HiPtr, SVT: HiMemVT, MMO: HiMMO);
24612
24613 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: LoSt, N2: HiSt);
24614}
24615
24616SDValue DAGCombiner::visitSTORE(SDNode *N) {
24617 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
24618 SDValue Chain = ST->getChain();
24619 SDValue Value = ST->getValue();
24620 SDValue Ptr = ST->getBasePtr();
24621
24622 // If this is a store of a bit convert, store the input value if the
24623 // resultant store does not need a higher alignment than the original.
24624 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() &&
24625 ST->isUnindexed()) {
24626 EVT SVT = Value.getOperand(i: 0).getValueType();
24627 // If the store is volatile, we only want to change the store type if the
24628 // resulting store is legal. Otherwise we might increase the number of
24629 // memory accesses. We don't care if the original type was legal or not
24630 // as we assume software couldn't rely on the number of accesses of an
24631 // illegal type.
24632 // TODO: May be able to relax for unordered atomics (see D66309)
24633 if (((!LegalOperations && ST->isSimple()) ||
24634 TLI.isOperationLegal(Op: ISD::STORE, VT: SVT)) &&
24635 TLI.isStoreBitCastBeneficial(StoreVT: Value.getValueType(), BitcastVT: SVT,
24636 DAG, MMO: *ST->getMemOperand())) {
24637 return DAG.getStore(Chain, dl: SDLoc(N), Val: Value.getOperand(i: 0), Ptr,
24638 MMO: ST->getMemOperand());
24639 }
24640 }
24641
24642 // Turn 'store undef, Ptr' -> nothing.
24643 if (Value.isUndef() && ST->isUnindexed() && !ST->isVolatile())
24644 return Chain;
24645
24646 // Try to infer better alignment information than the store already has.
24647 if (OptLevel != CodeGenOptLevel::None && ST->isUnindexed() &&
24648 !ST->isAtomic()) {
24649 if (MaybeAlign Alignment = DAG.InferPtrAlign(Ptr)) {
24650 if (*Alignment > ST->getAlign() &&
24651 isAligned(Lhs: *Alignment, SizeInBytes: ST->getSrcValueOffset())) {
24652 SDValue NewStore = DAG.getTruncStore(
24653 Chain, dl: SDLoc(N), Val: Value, Ptr, Offset: ST->getOffset(), PtrInfo: ST->getPointerInfo(),
24654 SVT: ST->getMemoryVT(), Alignment: *Alignment, MMOFlags: ST->getMemOperand()->getFlags(),
24655 Metadata: ST->getAAInfo());
24656 // NewStore will always be N as we are only refining the alignment
24657 assert(NewStore.getNode() == N);
24658 (void)NewStore;
24659 }
24660 }
24661 }
24662
24663 // Try transforming a pair floating point load / store ops to integer
24664 // load / store ops.
24665 if (SDValue NewST = TransformFPLoadStorePair(N))
24666 return NewST;
24667
24668 // Try transforming several stores into STORE (BSWAP).
24669 if (SDValue Store = mergeTruncStores(N: ST))
24670 return Store;
24671
24672 if (ST->isUnindexed()) {
24673 // Walk up chain skipping non-aliasing memory nodes, on this store and any
24674 // adjacent stores.
24675 if (findBetterNeighborChains(St: ST)) {
24676 // replaceStoreChain uses CombineTo, which handled all of the worklist
24677 // manipulation. Return the original node to not do anything else.
24678 return SDValue(ST, 0);
24679 }
24680 Chain = ST->getChain();
24681 }
24682
24683 if (SDValue R = combineStoreConcatTruncVector(ST))
24684 return R;
24685
24686 // FIXME: is there such a thing as a truncating indexed store?
24687 if (ST->isTruncatingStore() && ST->isUnindexed() &&
24688 Value.getValueType().isInteger() &&
24689 (!isa<ConstantSDNode>(Val: Value) ||
24690 !cast<ConstantSDNode>(Val&: Value)->isOpaque())) {
24691 // Convert a truncating store of a extension into a standard store.
24692 if ((Value.getOpcode() == ISD::ZERO_EXTEND ||
24693 Value.getOpcode() == ISD::SIGN_EXTEND ||
24694 Value.getOpcode() == ISD::ANY_EXTEND) &&
24695 Value.getOperand(i: 0).getValueType() == ST->getMemoryVT() &&
24696 TLI.isOperationLegalOrCustom(Op: ISD::STORE, VT: ST->getMemoryVT()))
24697 return DAG.getStore(Chain, dl: SDLoc(N), Val: Value.getOperand(i: 0), Ptr,
24698 MMO: ST->getMemOperand());
24699
24700 APInt TruncDemandedBits =
24701 APInt::getLowBitsSet(numBits: Value.getScalarValueSizeInBits(),
24702 loBitsSet: ST->getMemoryVT().getScalarSizeInBits());
24703
24704 // See if we can simplify the operation with SimplifyDemandedBits, which
24705 // only works if the value has a single use.
24706 AddToWorklist(N: Value.getNode());
24707 if (SimplifyDemandedBits(Op: Value, DemandedBits: TruncDemandedBits)) {
24708 // Re-visit the store if anything changed and the store hasn't been merged
24709 // with another node (N is deleted) SimplifyDemandedBits will add Value's
24710 // node back to the worklist if necessary, but we also need to re-visit
24711 // the Store node itself.
24712 if (N->getOpcode() != ISD::DELETED_NODE)
24713 AddToWorklist(N);
24714 return SDValue(N, 0);
24715 }
24716
24717 // Otherwise, see if we can simplify the input to this truncstore with
24718 // knowledge that only the low bits are being used. For example:
24719 // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8"
24720 if (SDValue Shorter =
24721 TLI.SimplifyMultipleUseDemandedBits(Op: Value, DemandedBits: TruncDemandedBits, DAG))
24722 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Shorter, Ptr, SVT: ST->getMemoryVT(),
24723 MMO: ST->getMemOperand());
24724
24725 // If we're storing a truncated constant, see if we can simplify it.
24726 // TODO: Move this to targetShrinkDemandedConstant?
24727 if (auto *Cst = dyn_cast<ConstantSDNode>(Val&: Value))
24728 if (!Cst->isOpaque()) {
24729 const APInt &CValue = Cst->getAPIntValue();
24730 APInt NewVal = CValue & TruncDemandedBits;
24731 if (NewVal != CValue) {
24732 SDValue Shorter =
24733 DAG.getConstant(Val: NewVal, DL: SDLoc(N), VT: Value.getValueType());
24734 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Shorter, Ptr,
24735 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
24736 }
24737 }
24738 }
24739
24740 // If this is a load followed by a store to the same location, then the store
24741 // is dead/noop. Peek through any truncates if canCombineTruncStore failed.
24742 // TODO: Add big-endian truncate support with test coverage.
24743 // TODO: Can relax for unordered atomics (see D66309)
24744 SDValue TruncVal = DAG.getDataLayout().isLittleEndian()
24745 ? peekThroughTruncates(V: Value)
24746 : Value;
24747 if (auto *Ld = dyn_cast<LoadSDNode>(Val&: TruncVal)) {
24748 if (Ld->getBasePtr() == Ptr && ST->getMemoryVT() == Ld->getMemoryVT() &&
24749 ST->isUnindexed() && ST->isSimple() &&
24750 Ld->getAddressSpace() == ST->getAddressSpace() &&
24751 // There can't be any side effects between the load and store, such as
24752 // a call or store.
24753 Chain.reachesChainWithoutSideEffects(Dest: SDValue(Ld, 1))) {
24754 // The store is dead, remove it.
24755 return Chain;
24756 }
24757 }
24758
24759 // Try scalarizing vector stores of loads where we only change one element
24760 if (SDValue NewST = replaceStoreOfInsertLoad(ST))
24761 return NewST;
24762
24763 // TODO: Can relax for unordered atomics (see D66309)
24764 if (StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Val&: Chain)) {
24765 if (ST->isUnindexed() && ST->isSimple() &&
24766 ST1->isUnindexed() && ST1->isSimple()) {
24767 if (OptLevel != CodeGenOptLevel::None && ST1->getBasePtr() == Ptr &&
24768 ST1->getValue() == Value && ST->getMemoryVT() == ST1->getMemoryVT() &&
24769 ST->getAddressSpace() == ST1->getAddressSpace()) {
24770 // If this is a store followed by a store with the same value to the
24771 // same location, then the store is dead/noop.
24772 return Chain;
24773 }
24774
24775 if (OptLevel != CodeGenOptLevel::None && ST1->hasOneUse() &&
24776 !ST1->getBasePtr().isUndef() &&
24777 ST->getAddressSpace() == ST1->getAddressSpace()) {
24778 // If we consider two stores and one smaller in size is a scalable
24779 // vector type and another one a bigger size store with a fixed type,
24780 // then we could not allow the scalable store removal because we don't
24781 // know its final size in the end.
24782 if (ST->getMemoryVT().isScalableVector() ||
24783 ST1->getMemoryVT().isScalableVector()) {
24784 if (ST1->getBasePtr() == Ptr &&
24785 TypeSize::isKnownLE(LHS: ST1->getMemoryVT().getStoreSize(),
24786 RHS: ST->getMemoryVT().getStoreSize())) {
24787 CombineTo(N: ST1, Res: ST1->getChain());
24788 return SDValue(N, 0);
24789 }
24790 } else {
24791 const BaseIndexOffset STBase = BaseIndexOffset::match(N: ST, DAG);
24792 const BaseIndexOffset ChainBase = BaseIndexOffset::match(N: ST1, DAG);
24793 // If this is a store who's preceding store to a subset of the current
24794 // location and no one other node is chained to that store we can
24795 // effectively drop the store. Do not remove stores to undef as they
24796 // may be used as data sinks.
24797 if (STBase.contains(DAG, BitSize: ST->getMemoryVT().getFixedSizeInBits(),
24798 Other: ChainBase,
24799 OtherBitSize: ST1->getMemoryVT().getFixedSizeInBits())) {
24800 CombineTo(N: ST1, Res: ST1->getChain());
24801 return SDValue(N, 0);
24802 }
24803 }
24804 }
24805 }
24806 }
24807
24808 // If this is an FP_ROUND or TRUNC followed by a store, fold this into a
24809 // truncating store. We can do this even if this is already a truncstore.
24810 if ((Value.getOpcode() == ISD::FP_ROUND ||
24811 Value.getOpcode() == ISD::TRUNCATE) &&
24812 Value->hasOneUse() && ST->isUnindexed() &&
24813 TLI.canCombineTruncStore(ValVT: Value.getOperand(i: 0).getValueType(),
24814 MemVT: ST->getMemoryVT(), Alignment: ST->getAlign(),
24815 AddrSpace: ST->getAddressSpace(), LegalOnly: LegalOperations)) {
24816 return DAG.getTruncStore(Chain, dl: SDLoc(N), Val: Value.getOperand(i: 0), Ptr,
24817 SVT: ST->getMemoryVT(), MMO: ST->getMemOperand());
24818 }
24819
24820 // Always perform this optimization before types are legal. If the target
24821 // prefers, also try this after legalization to catch stores that were created
24822 // by intrinsics or other nodes.
24823 if (!LegalTypes || (TLI.mergeStoresAfterLegalization(MemVT: ST->getMemoryVT()))) {
24824 while (true) {
24825 // There can be multiple store sequences on the same chain.
24826 // Keep trying to merge store sequences until we are unable to do so
24827 // or until we merge the last store on the chain.
24828 bool Changed = mergeConsecutiveStores(St: ST);
24829 if (!Changed) break;
24830 // Return N as merge only uses CombineTo and no worklist clean
24831 // up is necessary.
24832 if (N->getOpcode() == ISD::DELETED_NODE || !isa<StoreSDNode>(Val: N))
24833 return SDValue(N, 0);
24834 }
24835 }
24836
24837 // Try transforming N to an indexed store.
24838 if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
24839 return SDValue(N, 0);
24840
24841 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
24842 //
24843 // Make sure to do this only after attempting to merge stores in order to
24844 // avoid changing the types of some subset of stores due to visit order,
24845 // preventing their merging.
24846 if (isa<ConstantFPSDNode>(Val: ST->getValue())) {
24847 if (SDValue NewSt = replaceStoreOfFPConstant(ST))
24848 return NewSt;
24849 }
24850
24851 if (SDValue NewSt = splitMergedValStore(ST))
24852 return NewSt;
24853
24854 if (SDValue MaskedStore = foldToMaskedStore(Store: ST, DAG, Dl: SDLoc(N)))
24855 return MaskedStore;
24856
24857 return ReduceLoadOpStoreWidth(N);
24858}
24859
24860SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) {
24861 const auto *LifetimeEnd = cast<LifetimeSDNode>(Val: N);
24862 const BaseIndexOffset LifetimeEndBase(N->getOperand(Num: 1), SDValue(), 0, false);
24863
24864 // We walk up the chains to find stores.
24865 SmallVector<SDValue, 8> Chains = {N->getOperand(Num: 0)};
24866 while (!Chains.empty()) {
24867 SDValue Chain = Chains.pop_back_val();
24868 if (!Chain.hasOneUse())
24869 continue;
24870 switch (Chain.getOpcode()) {
24871 case ISD::TokenFactor:
24872 for (unsigned Nops = Chain.getNumOperands(); Nops;)
24873 Chains.push_back(Elt: Chain.getOperand(i: --Nops));
24874 break;
24875 case ISD::LIFETIME_START:
24876 case ISD::LIFETIME_END:
24877 // We can forward past any lifetime start/end that can be proven not to
24878 // alias the node.
24879 if (!mayAlias(Op0: Chain.getNode(), Op1: N))
24880 Chains.push_back(Elt: Chain.getOperand(i: 0));
24881 break;
24882 case ISD::STORE: {
24883 StoreSDNode *ST = dyn_cast<StoreSDNode>(Val&: Chain);
24884 // TODO: Can relax for unordered atomics (see D66309)
24885 if (!ST->isSimple() || ST->isIndexed())
24886 continue;
24887 const TypeSize StoreSize = ST->getMemoryVT().getStoreSize();
24888 // The bounds of a scalable store are not known until runtime, so this
24889 // store cannot be elided.
24890 if (StoreSize.isScalable())
24891 continue;
24892 const BaseIndexOffset StoreBase = BaseIndexOffset::match(N: ST, DAG);
24893 // If we store purely within object bounds just before its lifetime ends,
24894 // we can remove the store.
24895 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
24896 if (LifetimeEndBase.contains(
24897 DAG, BitSize: MFI.getObjectSize(ObjectIdx: LifetimeEnd->getFrameIndex()) * 8,
24898 Other: StoreBase, OtherBitSize: StoreSize.getFixedValue() * 8)) {
24899 LLVM_DEBUG(dbgs() << "\nRemoving store:"; StoreBase.dump();
24900 dbgs() << "\nwithin LIFETIME_END of : ";
24901 LifetimeEndBase.dump(); dbgs() << "\n");
24902 CombineTo(N: ST, Res: ST->getChain());
24903 return SDValue(N, 0);
24904 }
24905 }
24906 }
24907 }
24908 return SDValue();
24909}
24910
24911/// For the instruction sequence of store below, F and I values
24912/// are bundled together as an i64 value before being stored into memory.
24913/// Sometimes it is more efficent to generate separate stores for F and I,
24914/// which can remove the bitwise instructions or sink them to colder places.
24915///
24916/// (store (or (zext (bitcast F to i32) to i64),
24917/// (shl (zext I to i64), 32)), addr) -->
24918/// (store F, addr) and (store I, addr+4)
24919///
24920/// Similarly, splitting for other merged store can also be beneficial, like:
24921/// For pair of {i32, i32}, i64 store --> two i32 stores.
24922/// For pair of {i32, i16}, i64 store --> two i32 stores.
24923/// For pair of {i16, i16}, i32 store --> two i16 stores.
24924/// For pair of {i16, i8}, i32 store --> two i16 stores.
24925/// For pair of {i8, i8}, i16 store --> two i8 stores.
24926///
24927/// We allow each target to determine specifically which kind of splitting is
24928/// supported.
24929///
24930/// The store patterns are commonly seen from the simple code snippet below
24931/// if only std::make_pair(...) is sroa transformed before inlined into hoo.
24932/// void goo(const std::pair<int, float> &);
24933/// hoo() {
24934/// ...
24935/// goo(std::make_pair(tmp, ftmp));
24936/// ...
24937/// }
24938///
24939SDValue DAGCombiner::splitMergedValStore(StoreSDNode *ST) {
24940 if (OptLevel == CodeGenOptLevel::None)
24941 return SDValue();
24942
24943 // Can't change the number of memory accesses for a volatile store or break
24944 // atomicity for an atomic one.
24945 if (!ST->isSimple())
24946 return SDValue();
24947
24948 SDValue Val = ST->getValue();
24949 SDLoc DL(ST);
24950
24951 // Match OR operand.
24952 if (!Val.getValueType().isScalarInteger() || Val.getOpcode() != ISD::OR)
24953 return SDValue();
24954
24955 // Match SHL operand and get Lower and Higher parts of Val.
24956 SDValue Op1 = Val.getOperand(i: 0);
24957 SDValue Op2 = Val.getOperand(i: 1);
24958 SDValue Lo, Hi;
24959 if (Op1.getOpcode() != ISD::SHL) {
24960 std::swap(a&: Op1, b&: Op2);
24961 if (Op1.getOpcode() != ISD::SHL)
24962 return SDValue();
24963 }
24964 Lo = Op2;
24965 Hi = Op1.getOperand(i: 0);
24966 if (!Op1.hasOneUse())
24967 return SDValue();
24968
24969 // Match shift amount to HalfValBitSize.
24970 unsigned HalfValBitSize = Val.getValueSizeInBits() / 2;
24971 ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(Val: Op1.getOperand(i: 1));
24972 if (!ShAmt || ShAmt->getAPIntValue() != HalfValBitSize)
24973 return SDValue();
24974
24975 // Lo and Hi are zero-extended from int with size less equal than 32
24976 // to i64.
24977 if (Lo.getOpcode() != ISD::ZERO_EXTEND || !Lo.hasOneUse() ||
24978 !Lo.getOperand(i: 0).getValueType().isScalarInteger() ||
24979 Lo.getOperand(i: 0).getValueSizeInBits() > HalfValBitSize ||
24980 Hi.getOpcode() != ISD::ZERO_EXTEND || !Hi.hasOneUse() ||
24981 !Hi.getOperand(i: 0).getValueType().isScalarInteger() ||
24982 Hi.getOperand(i: 0).getValueSizeInBits() > HalfValBitSize)
24983 return SDValue();
24984
24985 // Use the EVT of low and high parts before bitcast as the input
24986 // of target query.
24987 EVT LowTy = (Lo.getOperand(i: 0).getOpcode() == ISD::BITCAST)
24988 ? Lo.getOperand(i: 0).getValueType()
24989 : Lo.getValueType();
24990 EVT HighTy = (Hi.getOperand(i: 0).getOpcode() == ISD::BITCAST)
24991 ? Hi.getOperand(i: 0).getValueType()
24992 : Hi.getValueType();
24993 if (!TLI.isMultiStoresCheaperThanBitsMerge(LTy: LowTy, HTy: HighTy))
24994 return SDValue();
24995
24996 // Start to split store.
24997 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
24998 AAMDNodes AAInfo = ST->getAAInfo();
24999
25000 // Change the sizes of Lo and Hi's value types to HalfValBitSize.
25001 EVT VT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HalfValBitSize);
25002 Lo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Lo.getOperand(i: 0));
25003 Hi = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Hi.getOperand(i: 0));
25004
25005 SDValue Chain = ST->getChain();
25006 SDValue Ptr = ST->getBasePtr();
25007 // Lower value store.
25008 SDValue St0 = DAG.getStore(Chain, dl: DL, Val: Lo, Ptr, PtrInfo: ST->getPointerInfo(),
25009 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
25010 Ptr =
25011 DAG.getMemBasePlusOffset(Base: Ptr, Offset: TypeSize::getFixed(ExactSize: HalfValBitSize / 8), DL);
25012 // Higher value store.
25013 SDValue St1 = DAG.getStore(
25014 Chain: St0, dl: DL, Val: Hi, Ptr, PtrInfo: ST->getPointerInfo().getWithOffset(O: HalfValBitSize / 8),
25015 Alignment: ST->getBaseAlign(), MMOFlags, Metadata: AAInfo);
25016 return St1;
25017}
25018
25019// Merge an insertion into an existing shuffle:
25020// (insert_vector_elt (vector_shuffle X, Y, Mask),
25021// .(extract_vector_elt X, N), InsIndex)
25022// --> (vector_shuffle X, Y, NewMask)
25023// and variations where shuffle operands may be CONCAT_VECTORS.
25024static bool mergeEltWithShuffle(SDValue &X, SDValue &Y, ArrayRef<int> Mask,
25025 SmallVectorImpl<int> &NewMask, SDValue Elt,
25026 unsigned InsIndex) {
25027 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
25028 !isa<ConstantSDNode>(Val: Elt.getOperand(i: 1)))
25029 return false;
25030
25031 // Vec's operand 0 is using indices from 0 to N-1 and
25032 // operand 1 from N to 2N - 1, where N is the number of
25033 // elements in the vectors.
25034 SDValue InsertVal0 = Elt.getOperand(i: 0);
25035 int ElementOffset = -1;
25036
25037 // We explore the inputs of the shuffle in order to see if we find the
25038 // source of the extract_vector_elt. If so, we can use it to modify the
25039 // shuffle rather than perform an insert_vector_elt.
25040 SmallVector<std::pair<int, SDValue>, 8> ArgWorkList;
25041 ArgWorkList.emplace_back(Args: Mask.size(), Args&: Y);
25042 ArgWorkList.emplace_back(Args: 0, Args&: X);
25043
25044 while (!ArgWorkList.empty()) {
25045 int ArgOffset;
25046 SDValue ArgVal;
25047 std::tie(args&: ArgOffset, args&: ArgVal) = ArgWorkList.pop_back_val();
25048
25049 if (ArgVal == InsertVal0) {
25050 ElementOffset = ArgOffset;
25051 break;
25052 }
25053
25054 // Peek through concat_vector.
25055 if (ArgVal.getOpcode() == ISD::CONCAT_VECTORS) {
25056 int CurrentArgOffset =
25057 ArgOffset + ArgVal.getValueType().getVectorNumElements();
25058 int Step = ArgVal.getOperand(i: 0).getValueType().getVectorNumElements();
25059 for (SDValue Op : reverse(C: ArgVal->ops())) {
25060 CurrentArgOffset -= Step;
25061 ArgWorkList.emplace_back(Args&: CurrentArgOffset, Args&: Op);
25062 }
25063
25064 // Make sure we went through all the elements and did not screw up index
25065 // computation.
25066 assert(CurrentArgOffset == ArgOffset);
25067 }
25068 }
25069
25070 // If we failed to find a match, see if we can replace an UNDEF shuffle
25071 // operand.
25072 if (ElementOffset == -1) {
25073 if (!Y.isUndef() || InsertVal0.getValueType() != Y.getValueType())
25074 return false;
25075 ElementOffset = Mask.size();
25076 Y = InsertVal0;
25077 }
25078
25079 NewMask.assign(in_start: Mask.begin(), in_end: Mask.end());
25080 NewMask[InsIndex] = ElementOffset + Elt.getConstantOperandVal(i: 1);
25081 assert(NewMask[InsIndex] < (int)(2 * Mask.size()) && NewMask[InsIndex] >= 0 &&
25082 "NewMask[InsIndex] is out of bound");
25083 return true;
25084}
25085
25086// Merge an insertion into an existing shuffle:
25087// (insert_vector_elt (vector_shuffle X, Y), (extract_vector_elt X, N),
25088// InsIndex)
25089// --> (vector_shuffle X, Y) and variations where shuffle operands may be
25090// CONCAT_VECTORS.
25091SDValue DAGCombiner::mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex) {
25092 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT &&
25093 "Expected extract_vector_elt");
25094 SDValue InsertVal = N->getOperand(Num: 1);
25095 SDValue Vec = N->getOperand(Num: 0);
25096
25097 auto *SVN = dyn_cast<ShuffleVectorSDNode>(Val&: Vec);
25098 if (!SVN || !Vec.hasOneUse())
25099 return SDValue();
25100
25101 ArrayRef<int> Mask = SVN->getMask();
25102 SDValue X = Vec.getOperand(i: 0);
25103 SDValue Y = Vec.getOperand(i: 1);
25104
25105 SmallVector<int, 16> NewMask(Mask);
25106 if (mergeEltWithShuffle(X, Y, Mask, NewMask, Elt: InsertVal, InsIndex)) {
25107 SDValue LegalShuffle = TLI.buildLegalVectorShuffle(
25108 VT: Vec.getValueType(), DL: SDLoc(N), N0: X, N1: Y, Mask: NewMask, DAG);
25109 if (LegalShuffle)
25110 return LegalShuffle;
25111 }
25112
25113 return SDValue();
25114}
25115
25116// Convert a disguised subvector insertion into a shuffle:
25117// insert_vector_elt V, (bitcast X from vector type), IdxC -->
25118// bitcast(shuffle (bitcast V), (extended X), Mask)
25119// Note: We do not use an insert_subvector node because that requires a
25120// legal subvector type.
25121SDValue DAGCombiner::combineInsertEltToShuffle(SDNode *N, unsigned InsIndex) {
25122 assert(N->getOpcode() == ISD::INSERT_VECTOR_ELT &&
25123 "Expected extract_vector_elt");
25124 SDValue InsertVal = N->getOperand(Num: 1);
25125
25126 if (InsertVal.getOpcode() != ISD::BITCAST || !InsertVal.hasOneUse() ||
25127 !InsertVal.getOperand(i: 0).getValueType().isVector())
25128 return SDValue();
25129
25130 SDValue SubVec = InsertVal.getOperand(i: 0);
25131 SDValue DestVec = N->getOperand(Num: 0);
25132 EVT SubVecVT = SubVec.getValueType();
25133 EVT VT = DestVec.getValueType();
25134 unsigned NumSrcElts = SubVecVT.getVectorNumElements();
25135 // Bail out if the inserted value is larger than the vector element, as
25136 // insert_vector_elt performs an implicit truncation in this case.
25137 if (InsertVal.getValueType() != VT.getVectorElementType())
25138 return SDValue();
25139 // If the source only has a single vector element, the cost of creating adding
25140 // it to a vector is likely to exceed the cost of a insert_vector_elt.
25141 if (NumSrcElts == 1)
25142 return SDValue();
25143 unsigned ExtendRatio = VT.getSizeInBits() / SubVecVT.getSizeInBits();
25144 unsigned NumMaskVals = ExtendRatio * NumSrcElts;
25145
25146 // Step 1: Create a shuffle mask that implements this insert operation. The
25147 // vector that we are inserting into will be operand 0 of the shuffle, so
25148 // those elements are just 'i'. The inserted subvector is in the first
25149 // positions of operand 1 of the shuffle. Example:
25150 // insert v4i32 V, (v2i16 X), 2 --> shuffle v8i16 V', X', {0,1,2,3,8,9,6,7}
25151 SmallVector<int, 16> Mask(NumMaskVals);
25152 for (unsigned i = 0; i != NumMaskVals; ++i) {
25153 if (i / NumSrcElts == InsIndex)
25154 Mask[i] = (i % NumSrcElts) + NumMaskVals;
25155 else
25156 Mask[i] = i;
25157 }
25158
25159 // Bail out if the target can not handle the shuffle we want to create, or
25160 // would create an illegal-typed shuffle after type legalization.
25161 EVT SubVecEltVT = SubVecVT.getVectorElementType();
25162 EVT ShufVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SubVecEltVT, NumElements: NumMaskVals);
25163 if ((LegalTypes && !TLI.isTypeLegal(VT: ShufVT)) ||
25164 !TLI.isShuffleMaskLegal(Mask, ShufVT))
25165 return SDValue();
25166
25167 // Step 2: Create a wide vector from the inserted source vector by appending
25168 // poison elements. This is the same size as our destination vector.
25169 SDLoc DL(N);
25170 SmallVector<SDValue, 8> ConcatOps(ExtendRatio, DAG.getPOISON(VT: SubVecVT));
25171 ConcatOps[0] = SubVec;
25172 SDValue PaddedSubV = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ShufVT, Ops: ConcatOps);
25173
25174 // Step 3: Shuffle in the padded subvector.
25175 SDValue DestVecBC = DAG.getBitcast(VT: ShufVT, V: DestVec);
25176 SDValue Shuf = DAG.getVectorShuffle(VT: ShufVT, dl: DL, N1: DestVecBC, N2: PaddedSubV, Mask);
25177 AddToWorklist(N: PaddedSubV.getNode());
25178 AddToWorklist(N: DestVecBC.getNode());
25179 AddToWorklist(N: Shuf.getNode());
25180 return DAG.getBitcast(VT, V: Shuf);
25181}
25182
25183// Combine insert(shuffle(load, <u,0,1,2>), load, 0) into a single load if
25184// possible and the new load will be quick. We use more loads but less shuffles
25185// and inserts.
25186SDValue DAGCombiner::combineInsertEltToLoad(SDNode *N, unsigned InsIndex) {
25187 EVT VT = N->getValueType(ResNo: 0);
25188
25189 // InsIndex is expected to be the first of last lane.
25190 if (!VT.isFixedLengthVector() ||
25191 (InsIndex != 0 && InsIndex != VT.getVectorNumElements() - 1))
25192 return SDValue();
25193
25194 // Look for a shuffle with the mask u,0,1,2,3,4,5,6 or 1,2,3,4,5,6,7,u
25195 // depending on the InsIndex.
25196 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 0));
25197 SDValue Scalar = N->getOperand(Num: 1);
25198 if (!Shuffle || !all_of(Range: enumerate(First: Shuffle->getMask()), P: [&](auto P) {
25199 return InsIndex == P.index() || P.value() < 0 ||
25200 (InsIndex == 0 && P.value() == (int)P.index() - 1) ||
25201 (InsIndex == VT.getVectorNumElements() - 1 &&
25202 P.value() == (int)P.index() + 1);
25203 }))
25204 return SDValue();
25205
25206 // We optionally skip over an extend so long as both loads are extended in the
25207 // same way from the same type.
25208 unsigned Extend = 0;
25209 if (Scalar.getOpcode() == ISD::ZERO_EXTEND ||
25210 Scalar.getOpcode() == ISD::SIGN_EXTEND ||
25211 Scalar.getOpcode() == ISD::ANY_EXTEND) {
25212 Extend = Scalar.getOpcode();
25213 Scalar = Scalar.getOperand(i: 0);
25214 }
25215
25216 auto *ScalarLoad = dyn_cast<LoadSDNode>(Val&: Scalar);
25217 if (!ScalarLoad)
25218 return SDValue();
25219
25220 SDValue Vec = Shuffle->getOperand(Num: 0);
25221 if (Extend) {
25222 if (Vec.getOpcode() != Extend)
25223 return SDValue();
25224 Vec = Vec.getOperand(i: 0);
25225 }
25226 auto *VecLoad = dyn_cast<LoadSDNode>(Val&: Vec);
25227 if (!VecLoad || Vec.getValueType().getScalarType() != Scalar.getValueType())
25228 return SDValue();
25229
25230 int EltSize = ScalarLoad->getValueType(ResNo: 0).getScalarSizeInBits();
25231 if (EltSize == 0 || EltSize % 8 != 0 || !ScalarLoad->isSimple() ||
25232 !VecLoad->isSimple() || VecLoad->getExtensionType() != ISD::NON_EXTLOAD ||
25233 ScalarLoad->getExtensionType() != ISD::NON_EXTLOAD ||
25234 ScalarLoad->getAddressSpace() != VecLoad->getAddressSpace())
25235 return SDValue();
25236
25237 // Check that the offset between the pointers to produce a single continuous
25238 // load.
25239 if (InsIndex == 0) {
25240 if (!DAG.areNonVolatileConsecutiveLoads(LD: ScalarLoad, Base: VecLoad, Bytes: EltSize / 8,
25241 Dist: -1))
25242 return SDValue();
25243 } else {
25244 if (!DAG.areNonVolatileConsecutiveLoads(
25245 LD: VecLoad, Base: ScalarLoad, Bytes: VT.getVectorNumElements() * EltSize / 8, Dist: -1))
25246 return SDValue();
25247 }
25248
25249 // And that the new unaligned load will be fast.
25250 unsigned IsFast = 0;
25251 Align NewAlign = commonAlignment(A: VecLoad->getAlign(), Offset: EltSize / 8);
25252 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
25253 VT: Vec.getValueType(), AddrSpace: VecLoad->getAddressSpace(),
25254 Alignment: NewAlign, Flags: VecLoad->getMemOperand()->getFlags(),
25255 Fast: &IsFast) ||
25256 !IsFast)
25257 return SDValue();
25258
25259 // Calculate the new Ptr and create the new load.
25260 SDLoc DL(N);
25261 SDValue Ptr = ScalarLoad->getBasePtr();
25262 if (InsIndex != 0)
25263 Ptr = DAG.getNode(Opcode: ISD::ADD, DL, VT: Ptr.getValueType(), N1: VecLoad->getBasePtr(),
25264 N2: DAG.getConstant(Val: EltSize / 8, DL, VT: Ptr.getValueType()));
25265 MachinePointerInfo PtrInfo =
25266 InsIndex == 0 ? ScalarLoad->getPointerInfo()
25267 : VecLoad->getPointerInfo().getWithOffset(O: EltSize / 8);
25268
25269 SDValue Load = DAG.getLoad(VT: VecLoad->getValueType(ResNo: 0), dl: DL,
25270 Chain: ScalarLoad->getChain(), Ptr, PtrInfo, Alignment: NewAlign);
25271 DAG.makeEquivalentMemoryOrdering(OldLoad: ScalarLoad, NewMemOp: Load.getValue(R: 1));
25272 DAG.makeEquivalentMemoryOrdering(OldLoad: VecLoad, NewMemOp: Load.getValue(R: 1));
25273 return Extend ? DAG.getNode(Opcode: Extend, DL, VT, Operand: Load) : Load;
25274}
25275
25276SDValue DAGCombiner::visitINSERT_VECTOR_ELT(SDNode *N) {
25277 SDValue InVec = N->getOperand(Num: 0);
25278 SDValue InVal = N->getOperand(Num: 1);
25279 SDValue EltNo = N->getOperand(Num: 2);
25280 SDLoc DL(N);
25281
25282 EVT VT = InVec.getValueType();
25283 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
25284
25285 // Insert into out-of-bounds element is poison.
25286 if (IndexC && VT.isFixedLengthVector() &&
25287 IndexC->getZExtValue() >= VT.getVectorNumElements())
25288 return DAG.getPOISON(VT);
25289
25290 // Remove redundant insertions:
25291 // (insert_vector_elt x (extract_vector_elt x idx) idx) -> x
25292 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
25293 InVec == InVal.getOperand(i: 0) && EltNo == InVal.getOperand(i: 1))
25294 return InVec;
25295
25296 // Remove insert of UNDEF/POISON elements.
25297 if (InVal.isUndef()) {
25298 if (InVal.getOpcode() == ISD::POISON || InVec.getOpcode() == ISD::UNDEF)
25299 return InVec;
25300 return DAG.getFreeze(V: InVec);
25301 }
25302
25303 if (!IndexC) {
25304 // If this is variable insert to undef vector, it might be better to splat:
25305 // inselt undef, InVal, EltNo --> build_vector < InVal, InVal, ... >
25306 if (InVec.isUndef() && TLI.shouldSplatInsEltVarIndex(VT))
25307 return DAG.getSplat(VT, DL, Op: InVal);
25308
25309 // Extend this type to be byte-addressable
25310 EVT OldVT = VT;
25311 EVT EltVT = VT.getVectorElementType();
25312 bool IsByteSized = EltVT.isByteSized();
25313 if (!IsByteSized) {
25314 EltVT =
25315 EltVT.changeTypeToInteger().getRoundIntegerType(Context&: *DAG.getContext());
25316 VT = VT.changeElementType(Context&: *DAG.getContext(), EltVT);
25317 }
25318
25319 // Check if this operation will be handled the default way for its type.
25320 auto IsTypeDefaultHandled = [this](EVT VT) {
25321 return TLI.getTypeAction(Context&: *DAG.getContext(), VT) ==
25322 TargetLowering::TypeSplitVector ||
25323 TLI.isOperationExpand(Op: ISD::INSERT_VECTOR_ELT, VT);
25324 };
25325
25326 // Check if this operation is illegal and will be handled the default way,
25327 // even after extending the type to be byte-addressable.
25328 if (IsTypeDefaultHandled(OldVT) && IsTypeDefaultHandled(VT)) {
25329 // For each dynamic insertelt, the default way will save the vector to
25330 // the stack, store at an offset, and load the modified vector. This can
25331 // dramatically increase code size if we have a chain of insertelts on a
25332 // large vector: requiring O(V*C) stores/loads where V = length of
25333 // vector and C is length of chain. If each insertelt is only fed into the
25334 // next, the vector is write-only across this chain, and we can just
25335 // save once before the chain and load after in O(V + C) operations.
25336 SmallVector<SDNode *> Seq{N};
25337 unsigned NumDynamic = 1;
25338 while (true) {
25339 SDValue InVec = Seq.back()->getOperand(Num: 0);
25340 if (InVec.getOpcode() != ISD::INSERT_VECTOR_ELT)
25341 break;
25342 Seq.push_back(Elt: InVec.getNode());
25343 NumDynamic += !isa<ConstantSDNode>(Val: InVec.getOperand(i: 2));
25344 }
25345
25346 // It always and only makes sense to lower this sequence when we have more
25347 // than one dynamic insertelt, since we will not have more than V constant
25348 // insertelts, so we will be reducing the total number of stores+loads.
25349 if (NumDynamic > 1) {
25350 // In cases where the vector is illegal it will be broken down into
25351 // parts and stored in parts - we should use the alignment for the
25352 // smallest part.
25353 Align SmallestAlign = DAG.getReducedAlign(VT, /*UseABI=*/false);
25354 SDValue StackPtr =
25355 DAG.CreateStackTemporary(Bytes: VT.getStoreSize(), Alignment: SmallestAlign);
25356 auto &MF = DAG.getMachineFunction();
25357 int FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
25358 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
25359
25360 // Save the vector to the stack
25361 SDValue InVec = Seq.back()->getOperand(Num: 0);
25362 if (!IsByteSized)
25363 InVec = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT, Operand: InVec);
25364 SDValue Store = DAG.getStore(Chain: DAG.getEntryNode(), dl: DL, Val: InVec, Ptr: StackPtr,
25365 PtrInfo, Alignment: SmallestAlign);
25366
25367 // Lower each dynamic insertelt to a store
25368 for (SDNode *N : reverse(C&: Seq)) {
25369 SDValue Elmnt = N->getOperand(Num: 1);
25370 SDValue Index = N->getOperand(Num: 2);
25371
25372 // Check if we have to extend the element type
25373 if (!IsByteSized && Elmnt.getValueType().bitsLT(VT: EltVT))
25374 Elmnt = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: EltVT, Operand: Elmnt);
25375
25376 // Store the new element. This may be larger than the vector element
25377 // type, so use a truncating store.
25378 SDValue EltPtr =
25379 TLI.getVectorElementPointer(DAG, VecPtr: StackPtr, VecVT: VT, Index);
25380 EVT EltVT = Elmnt.getValueType();
25381 Store = DAG.getTruncStore(
25382 Chain: Store, dl: DL, Val: Elmnt, Ptr: EltPtr, PtrInfo: MachinePointerInfo::getUnknownStack(MF),
25383 SVT: EltVT,
25384 Alignment: commonAlignment(A: SmallestAlign, Offset: EltVT.getFixedSizeInBits() / 8));
25385 }
25386
25387 // Load the saved vector from the stack
25388 SDValue Load =
25389 DAG.getLoad(VT, dl: DL, Chain: Store, Ptr: StackPtr, PtrInfo, Alignment: SmallestAlign);
25390 SDValue LoadV = Load.getValue(R: 0);
25391 return IsByteSized ? LoadV : DAG.getAnyExtOrTrunc(Op: LoadV, DL, VT: OldVT);
25392 }
25393 }
25394
25395 return SDValue();
25396 }
25397
25398 if (VT.isScalableVector())
25399 return SDValue();
25400
25401 unsigned NumElts = VT.getVectorNumElements();
25402
25403 // We must know which element is being inserted for folds below here.
25404 unsigned Elt = IndexC->getZExtValue();
25405
25406 // Handle <1 x ???> vector insertion special cases.
25407 if (NumElts == 1) {
25408 // insert_vector_elt(x, extract_vector_elt(y, 0), 0) -> y
25409 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
25410 InVal.getOperand(i: 0).getValueType() == VT &&
25411 isNullConstant(V: InVal.getOperand(i: 1)))
25412 return InVal.getOperand(i: 0);
25413 }
25414
25415 // Canonicalize insert_vector_elt dag nodes.
25416 // Example:
25417 // (insert_vector_elt (insert_vector_elt A, Idx0), Idx1)
25418 // -> (insert_vector_elt (insert_vector_elt A, Idx1), Idx0)
25419 //
25420 // Do this only if the child insert_vector node has one use; also
25421 // do this only if indices are both constants and Idx1 < Idx0.
25422 if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse()
25423 && isa<ConstantSDNode>(Val: InVec.getOperand(i: 2))) {
25424 unsigned OtherElt = InVec.getConstantOperandVal(i: 2);
25425 if (Elt < OtherElt) {
25426 // Swap nodes.
25427 SDValue NewOp = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT,
25428 N1: InVec.getOperand(i: 0), N2: InVal, N3: EltNo);
25429 AddToWorklist(N: NewOp.getNode());
25430 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(InVec.getNode()),
25431 VT, N1: NewOp, N2: InVec.getOperand(i: 1), N3: InVec.getOperand(i: 2));
25432 }
25433 }
25434
25435 if (SDValue Shuf = mergeInsertEltWithShuffle(N, InsIndex: Elt))
25436 return Shuf;
25437
25438 if (SDValue Shuf = combineInsertEltToShuffle(N, InsIndex: Elt))
25439 return Shuf;
25440
25441 if (SDValue Shuf = combineInsertEltToLoad(N, InsIndex: Elt))
25442 return Shuf;
25443
25444 // Attempt to convert an insert_vector_elt chain into a legal build_vector.
25445 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::BUILD_VECTOR, VT)) {
25446 // vXi1 vector - we don't need to recurse.
25447 if (NumElts == 1)
25448 return DAG.getBuildVector(VT, DL, Ops: {InVal});
25449
25450 // If we haven't already collected the element, insert into the op list.
25451 EVT MaxEltVT = InVal.getValueType();
25452 auto AddBuildVectorOp = [&](SmallVectorImpl<SDValue> &Ops, SDValue Elt,
25453 unsigned Idx) {
25454 if (!Ops[Idx]) {
25455 Ops[Idx] = Elt;
25456 if (VT.isInteger()) {
25457 EVT EltVT = Elt.getValueType();
25458 MaxEltVT = MaxEltVT.bitsGE(VT: EltVT) ? MaxEltVT : EltVT;
25459 }
25460 }
25461 };
25462
25463 // Ensure all the operands are the same value type, fill any missing
25464 // operands with UNDEF and create the BUILD_VECTOR.
25465 auto CanonicalizeBuildVector = [&](SmallVectorImpl<SDValue> &Ops,
25466 bool FreezeUndef = false) {
25467 assert(Ops.size() == NumElts && "Unexpected vector size");
25468 SDValue UndefOp = FreezeUndef ? DAG.getFreeze(V: DAG.getUNDEF(VT: MaxEltVT))
25469 : DAG.getUNDEF(VT: MaxEltVT);
25470 for (SDValue &Op : Ops) {
25471 if (Op)
25472 Op = VT.isInteger() ? DAG.getAnyExtOrTrunc(Op, DL, VT: MaxEltVT) : Op;
25473 else
25474 Op = UndefOp;
25475 }
25476 return DAG.getBuildVector(VT, DL, Ops);
25477 };
25478
25479 SmallVector<SDValue, 8> Ops(NumElts, SDValue());
25480 Ops[Elt] = InVal;
25481
25482 // Recurse up a INSERT_VECTOR_ELT chain to build a BUILD_VECTOR.
25483 for (SDValue CurVec = InVec; CurVec;) {
25484 // UNDEF - build new BUILD_VECTOR from already inserted operands.
25485 if (CurVec.isUndef())
25486 return CanonicalizeBuildVector(Ops);
25487
25488 // FREEZE(UNDEF) - build new BUILD_VECTOR from already inserted operands.
25489 if (ISD::isFreezeUndef(N: CurVec.getNode()) && CurVec.hasOneUse())
25490 return CanonicalizeBuildVector(Ops, /*FreezeUndef=*/true);
25491
25492 // BUILD_VECTOR - insert unused operands and build new BUILD_VECTOR.
25493 // A multi-use base is allowed when the target prefers build_vector
25494 // sources: rebuilding only re-references the base's scalar operands, and
25495 // it un-shares the base so each vector is built independently.
25496 if (CurVec.getOpcode() == ISD::BUILD_VECTOR &&
25497 (CurVec.hasOneUse() ||
25498 TLI.aggressivelyPreferBuildVectorSources(VecVT: VT))) {
25499 for (unsigned I = 0; I != NumElts; ++I)
25500 AddBuildVectorOp(Ops, CurVec.getOperand(i: I), I);
25501 return CanonicalizeBuildVector(Ops);
25502 }
25503
25504 // SCALAR_TO_VECTOR - insert unused scalar and build new BUILD_VECTOR.
25505 if (CurVec.getOpcode() == ISD::SCALAR_TO_VECTOR && CurVec.hasOneUse()) {
25506 AddBuildVectorOp(Ops, CurVec.getOperand(i: 0), 0);
25507 return CanonicalizeBuildVector(Ops);
25508 }
25509
25510 // INSERT_VECTOR_ELT - insert operand and continue up the chain.
25511 if (CurVec.getOpcode() == ISD::INSERT_VECTOR_ELT && CurVec.hasOneUse())
25512 if (auto *CurIdx = dyn_cast<ConstantSDNode>(Val: CurVec.getOperand(i: 2)))
25513 if (CurIdx->getAPIntValue().ult(RHS: NumElts)) {
25514 unsigned Idx = CurIdx->getZExtValue();
25515 AddBuildVectorOp(Ops, CurVec.getOperand(i: 1), Idx);
25516
25517 // Found entire BUILD_VECTOR.
25518 if (all_of(Range&: Ops, P: [](SDValue Op) { return !!Op; }))
25519 return CanonicalizeBuildVector(Ops);
25520
25521 CurVec = CurVec->getOperand(Num: 0);
25522 continue;
25523 }
25524
25525 // VECTOR_SHUFFLE - if all the operands match the shuffle's sources,
25526 // update the shuffle mask (and second operand if we started with unary
25527 // shuffle) and create a new legal shuffle.
25528 if (CurVec.getOpcode() == ISD::VECTOR_SHUFFLE && CurVec.hasOneUse()) {
25529 auto *SVN = cast<ShuffleVectorSDNode>(Val&: CurVec);
25530 SDValue LHS = SVN->getOperand(Num: 0);
25531 SDValue RHS = SVN->getOperand(Num: 1);
25532 SmallVector<int, 16> Mask(SVN->getMask());
25533 bool Merged = true;
25534 for (auto I : enumerate(First&: Ops)) {
25535 SDValue &Op = I.value();
25536 if (Op) {
25537 SmallVector<int, 16> NewMask;
25538 if (!mergeEltWithShuffle(X&: LHS, Y&: RHS, Mask, NewMask, Elt: Op, InsIndex: I.index())) {
25539 Merged = false;
25540 break;
25541 }
25542 Mask = std::move(NewMask);
25543 }
25544 }
25545 if (Merged)
25546 if (SDValue NewShuffle =
25547 TLI.buildLegalVectorShuffle(VT, DL, N0: LHS, N1: RHS, Mask, DAG))
25548 return NewShuffle;
25549 }
25550
25551 if (!LegalOperations) {
25552 bool IsNull = llvm::isNullConstant(V: InVal);
25553 // We can convert to AND/OR mask if all insertions are zero or -1
25554 // respectively.
25555 if ((IsNull || llvm::isAllOnesConstant(V: InVal)) &&
25556 all_of(Range&: Ops, P: [InVal](SDValue Op) { return !Op || Op == InVal; }) &&
25557 count_if(Range&: Ops, P: [InVal](SDValue Op) { return Op == InVal; }) >= 2) {
25558 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MaxEltVT);
25559 SDValue AllOnes = DAG.getAllOnesConstant(DL, VT: MaxEltVT);
25560 SmallVector<SDValue, 8> Mask(NumElts);
25561
25562 // Build the mask and return the corresponding DAG node.
25563 auto BuildMaskAndNode = [&](SDValue TrueVal, SDValue FalseVal,
25564 unsigned MaskOpcode) {
25565 APInt InsertedEltMask = APInt::getZero(numBits: NumElts);
25566 for (unsigned I = 0; I != NumElts; ++I) {
25567 Mask[I] = Ops[I] ? TrueVal : FalseVal;
25568 if (Ops[I])
25569 InsertedEltMask.setBit(I);
25570 }
25571 // Make sure to freeze the source vector in case any of the elements
25572 // overwritten by the insert may be poison. Otherwise those elements
25573 // could end up being poison instead of 0/-1 after the AND/OR.
25574 CurVec = DAG.getFreeze(V: CurVec, DemandedElts: InsertedEltMask,
25575 Kind: UndefPoisonKind::PoisonOnly);
25576 return DAG.getNode(Opcode: MaskOpcode, DL, VT, N1: CurVec,
25577 N2: DAG.getBuildVector(VT, DL, Ops: Mask));
25578 };
25579
25580 // If all elements are zero, we can use AND with all ones.
25581 if (IsNull)
25582 return BuildMaskAndNode(Zero, AllOnes, ISD::AND);
25583
25584 // If all elements are -1, we can use OR with zero.
25585 return BuildMaskAndNode(AllOnes, Zero, ISD::OR);
25586 }
25587 }
25588
25589 // Failed to find a match in the chain - bail.
25590 break;
25591 }
25592
25593 // See if we can fill in the missing constant elements as zeros.
25594 // TODO: Should we do this for any constant?
25595 APInt DemandedZeroElts = APInt::getZero(numBits: NumElts);
25596 for (unsigned I = 0; I != NumElts; ++I)
25597 if (!Ops[I])
25598 DemandedZeroElts.setBit(I);
25599
25600 if (DAG.MaskedVectorIsZero(Op: InVec, DemandedElts: DemandedZeroElts)) {
25601 SDValue Zero = VT.isInteger() ? DAG.getConstant(Val: 0, DL, VT: MaxEltVT)
25602 : DAG.getConstantFP(Val: 0, DL, VT: MaxEltVT);
25603 for (unsigned I = 0; I != NumElts; ++I)
25604 if (!Ops[I])
25605 Ops[I] = Zero;
25606
25607 return CanonicalizeBuildVector(Ops);
25608 }
25609 }
25610
25611 return SDValue();
25612}
25613
25614/// Transform a vector binary operation into a scalar binary operation by moving
25615/// the math/logic after an extract element of a vector.
25616static SDValue scalarizeExtractedBinOp(SDNode *ExtElt, SelectionDAG &DAG,
25617 const SDLoc &DL, bool LegalTypes) {
25618 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25619 SDValue Vec = ExtElt->getOperand(Num: 0);
25620 SDValue Index = ExtElt->getOperand(Num: 1);
25621 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: Index);
25622 unsigned Opc = Vec.getOpcode();
25623 if (!IndexC || !Vec.hasOneUse() || (!TLI.isBinOp(Opcode: Opc) && Opc != ISD::SETCC) ||
25624 Vec->getNumValues() != 1)
25625 return SDValue();
25626
25627 // Targets may want to avoid this to prevent an expensive register transfer.
25628 if (!TLI.shouldScalarizeBinop(VecOp: Vec))
25629 return SDValue();
25630
25631 EVT ResVT = ExtElt->getValueType(ResNo: 0);
25632 if (Opc == ISD::SETCC &&
25633 (ResVT != Vec.getValueType().getVectorElementType() || LegalTypes))
25634 return SDValue();
25635
25636 // Extracting an element of a vector constant is constant-folded, so this
25637 // transform is just replacing a vector op with a scalar op while moving the
25638 // extract.
25639 auto IsExtractFree = [](SDValue Op) {
25640 APInt SplatVal;
25641 return isAnyConstantBuildVector(V: Op, NoOpaques: true) ||
25642 ISD::isConstantSplatVector(N: Op.getNode(), SplatValue&: SplatVal) ||
25643 (Op.getOpcode() == ISD::BUILD_VECTOR && Op.hasOneUse());
25644 };
25645 SDValue Op0 = Vec.getOperand(i: 0);
25646 SDValue Op1 = Vec.getOperand(i: 1);
25647 if (!IsExtractFree(Op0) && !IsExtractFree(Op1))
25648 return SDValue();
25649
25650 // extractelt (op X, C), IndexC --> op (extractelt X, IndexC), C'
25651 // extractelt (op C, X), IndexC --> op C', (extractelt X, IndexC)
25652 if (Opc == ISD::SETCC) {
25653 EVT OpVT = Op0.getValueType().getVectorElementType();
25654 Op0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Op0, N2: Index);
25655 Op1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: OpVT, N1: Op1, N2: Index);
25656 SDValue NewVal = DAG.getSetCC(
25657 DL, VT: ResVT, LHS: Op0, RHS: Op1, Cond: cast<CondCodeSDNode>(Val: Vec->getOperand(Num: 2))->get());
25658 // We may need to sign- or zero-extend the result to match the same
25659 // behaviour as the vector version of SETCC.
25660 unsigned VecBoolContents = TLI.getBooleanContents(Type: Vec.getValueType());
25661 if (ResVT != MVT::i1 &&
25662 VecBoolContents != TargetLowering::UndefinedBooleanContent &&
25663 VecBoolContents != TLI.getBooleanContents(Type: ResVT)) {
25664 if (VecBoolContents == TargetLowering::ZeroOrNegativeOneBooleanContent)
25665 NewVal = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: ResVT, N1: NewVal,
25666 N2: DAG.getValueType(MVT::i1));
25667 else
25668 NewVal = DAG.getZeroExtendInReg(Op: NewVal, DL, VT: MVT::i1);
25669 }
25670 return NewVal;
25671 }
25672 Op0 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op0, N2: Index);
25673 Op1 = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ResVT, N1: Op1, N2: Index);
25674 return DAG.getNode(Opcode: Opc, DL, VT: ResVT, N1: Op0, N2: Op1);
25675}
25676
25677// Given a ISD::EXTRACT_VECTOR_ELT, which is a glorified bit sequence extract,
25678// recursively analyse all of it's users. and try to model themselves as
25679// bit sequence extractions. If all of them agree on the new, narrower element
25680// type, and all of them can be modelled as ISD::EXTRACT_VECTOR_ELT's of that
25681// new element type, do so now.
25682// This is mainly useful to recover from legalization that scalarized
25683// the vector as wide elements, but tries to rebuild it with narrower elements.
25684//
25685// Some more nodes could be modelled if that helps cover interesting patterns.
25686bool DAGCombiner::refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
25687 SDNode *N) {
25688 // We perform this optimization post type-legalization because
25689 // the type-legalizer often scalarizes integer-promoted vectors.
25690 // Performing this optimization before may cause legalizaton cycles.
25691 if (Level != AfterLegalizeVectorOps && Level != AfterLegalizeTypes)
25692 return false;
25693
25694 // TODO: Add support for big-endian.
25695 if (DAG.getDataLayout().isBigEndian())
25696 return false;
25697
25698 SDValue VecOp = N->getOperand(Num: 0);
25699 EVT VecVT = VecOp.getValueType();
25700 assert(!VecVT.isScalableVector() && "Only for fixed vectors.");
25701
25702 // We must start with a constant extraction index.
25703 auto *IndexC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
25704 if (!IndexC)
25705 return false;
25706
25707 assert(IndexC->getZExtValue() < VecVT.getVectorNumElements() &&
25708 "Original ISD::EXTRACT_VECTOR_ELT is undefinend?");
25709
25710 // TODO: deal with the case of implicit anyext of the extraction.
25711 unsigned VecEltBitWidth = VecVT.getScalarSizeInBits();
25712 EVT ScalarVT = N->getValueType(ResNo: 0);
25713 if (VecVT.getScalarType() != ScalarVT)
25714 return false;
25715
25716 // TODO: deal with the cases other than everything being integer-typed.
25717 if (!ScalarVT.isScalarInteger())
25718 return false;
25719
25720 struct Entry {
25721 SDNode *Producer;
25722
25723 // Which bits of VecOp does it contain?
25724 unsigned BitPos;
25725 int NumBits;
25726 // NOTE: the actual width of \p Producer may be wider than NumBits!
25727
25728 Entry(Entry &&) = default;
25729 Entry(SDNode *Producer_, unsigned BitPos_, int NumBits_)
25730 : Producer(Producer_), BitPos(BitPos_), NumBits(NumBits_) {}
25731
25732 Entry() = delete;
25733 Entry(const Entry &) = delete;
25734 Entry &operator=(const Entry &) = delete;
25735 Entry &operator=(Entry &&) = delete;
25736 };
25737 SmallVector<Entry, 32> Worklist;
25738 SmallVector<Entry, 32> Leafs;
25739
25740 // We start at the "root" ISD::EXTRACT_VECTOR_ELT.
25741 Worklist.emplace_back(Args&: N, /*BitPos=*/Args: VecEltBitWidth * IndexC->getZExtValue(),
25742 /*NumBits=*/Args&: VecEltBitWidth);
25743
25744 while (!Worklist.empty()) {
25745 Entry E = Worklist.pop_back_val();
25746 // Does the node not even use any of the VecOp bits?
25747 if (!(E.NumBits > 0 && E.BitPos < VecVT.getSizeInBits() &&
25748 E.BitPos + E.NumBits <= VecVT.getSizeInBits()))
25749 return false; // Let's allow the other combines clean this up first.
25750 // Did we fail to model any of the users of the Producer?
25751 bool ProducerIsLeaf = false;
25752 // Look at each user of this Producer.
25753 for (SDNode *User : E.Producer->users()) {
25754 switch (User->getOpcode()) {
25755 // TODO: support ISD::BITCAST
25756 // TODO: support ISD::ANY_EXTEND
25757 // TODO: support ISD::ZERO_EXTEND
25758 // TODO: support ISD::SIGN_EXTEND
25759 case ISD::TRUNCATE:
25760 // Truncation simply means we keep position, but extract less bits.
25761 Worklist.emplace_back(Args&: User, Args&: E.BitPos,
25762 /*NumBits=*/Args: User->getValueSizeInBits(ResNo: 0));
25763 break;
25764 // TODO: support ISD::SRA
25765 // TODO: support ISD::SHL
25766 case ISD::SRL:
25767 // We should be shifting the Producer by a constant amount.
25768 if (auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: User->getOperand(Num: 1));
25769 User->getOperand(Num: 0).getNode() == E.Producer && ShAmtC) {
25770 // Logical right-shift means that we start extraction later,
25771 // but stop it at the same position we did previously.
25772 unsigned ShAmt = ShAmtC->getZExtValue();
25773 Worklist.emplace_back(Args&: User, Args: E.BitPos + ShAmt, Args: E.NumBits - ShAmt);
25774 break;
25775 }
25776 [[fallthrough]];
25777 default:
25778 // We can not model this user of the Producer.
25779 // Which means the current Producer will be a ISD::EXTRACT_VECTOR_ELT.
25780 ProducerIsLeaf = true;
25781 // Profitability check: all users that we can not model
25782 // must be ISD::BUILD_VECTOR's.
25783 if (User->getOpcode() != ISD::BUILD_VECTOR)
25784 return false;
25785 break;
25786 }
25787 }
25788 if (ProducerIsLeaf)
25789 Leafs.emplace_back(Args: std::move(E));
25790 }
25791
25792 unsigned NewVecEltBitWidth = Leafs.front().NumBits;
25793
25794 // If we are still at the same element granularity, give up,
25795 if (NewVecEltBitWidth == VecEltBitWidth)
25796 return false;
25797
25798 // The vector width must be a multiple of the new element width.
25799 if (VecVT.getSizeInBits() % NewVecEltBitWidth != 0)
25800 return false;
25801
25802 // All leafs must agree on the new element width.
25803 // All leafs must not expect any "padding" bits ontop of that width.
25804 // All leafs must start extraction from multiple of that width.
25805 if (!all_of(Range&: Leafs, P: [NewVecEltBitWidth](const Entry &E) {
25806 return (unsigned)E.NumBits == NewVecEltBitWidth &&
25807 E.Producer->getValueSizeInBits(ResNo: 0) == NewVecEltBitWidth &&
25808 E.BitPos % NewVecEltBitWidth == 0;
25809 }))
25810 return false;
25811
25812 EVT NewScalarVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewVecEltBitWidth);
25813 EVT NewVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewScalarVT,
25814 NumElements: VecVT.getSizeInBits() / NewVecEltBitWidth);
25815
25816 if (LegalTypes &&
25817 !(TLI.isTypeLegal(VT: NewScalarVT) && TLI.isTypeLegal(VT: NewVecVT)))
25818 return false;
25819
25820 if (LegalOperations &&
25821 !(TLI.isOperationLegalOrCustom(Op: ISD::BITCAST, VT: NewVecVT) &&
25822 TLI.isOperationLegalOrCustom(Op: ISD::EXTRACT_VECTOR_ELT, VT: NewVecVT)))
25823 return false;
25824
25825 SDValue NewVecOp = DAG.getBitcast(VT: NewVecVT, V: VecOp);
25826 for (const Entry &E : Leafs) {
25827 SDLoc DL(E.Producer);
25828 unsigned NewIndex = E.BitPos / NewVecEltBitWidth;
25829 assert(NewIndex < NewVecVT.getVectorNumElements() &&
25830 "Creating out-of-bounds ISD::EXTRACT_VECTOR_ELT?");
25831 SDValue V = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: NewScalarVT, N1: NewVecOp,
25832 N2: DAG.getVectorIdxConstant(Val: NewIndex, DL));
25833 CombineTo(N: E.Producer, Res: V);
25834 }
25835
25836 return true;
25837}
25838
25839SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
25840 SDValue VecOp = N->getOperand(Num: 0);
25841 SDValue Index = N->getOperand(Num: 1);
25842 EVT ScalarVT = N->getValueType(ResNo: 0);
25843 EVT VecVT = VecOp.getValueType();
25844 if (VecOp.isUndef())
25845 return DAG.getUNDEF(VT: ScalarVT);
25846
25847 // extract_vector_elt (insert_vector_elt vec, val, idx), idx) -> val
25848 //
25849 // This only really matters if the index is non-constant since other combines
25850 // on the constant elements already work.
25851 SDLoc DL(N);
25852 if (VecOp.getOpcode() == ISD::INSERT_VECTOR_ELT &&
25853 Index == VecOp.getOperand(i: 2)) {
25854 SDValue Elt = VecOp.getOperand(i: 1);
25855 AddUsersToWorklist(N: VecOp.getNode());
25856 return VecVT.isInteger() ? DAG.getAnyExtOrTrunc(Op: Elt, DL, VT: ScalarVT) : Elt;
25857 }
25858
25859 // (vextract (scalar_to_vector val, 0) -> val
25860 if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR) {
25861 // Only 0'th element of SCALAR_TO_VECTOR is defined.
25862 if (DAG.isKnownNeverZero(Op: Index))
25863 return DAG.getPOISON(VT: ScalarVT);
25864
25865 // Check if the result type doesn't match the inserted element type.
25866 // The inserted element and extracted element may have mismatched bitwidth.
25867 // As a result, EXTRACT_VECTOR_ELT may extend or truncate the extracted vector.
25868 SDValue InOp = VecOp.getOperand(i: 0);
25869 if (InOp.getValueType() != ScalarVT) {
25870 assert(InOp.getValueType().isInteger() && ScalarVT.isInteger());
25871 if (InOp.getValueType().bitsGT(VT: ScalarVT))
25872 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ScalarVT, Operand: InOp);
25873 return DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: ScalarVT, Operand: InOp);
25874 }
25875 return InOp;
25876 }
25877
25878 // extract_vector_elt of out-of-bounds element -> POISON
25879 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: Index);
25880 if (IndexC && VecVT.isFixedLengthVector() &&
25881 IndexC->getAPIntValue().uge(RHS: VecVT.getVectorNumElements()))
25882 return DAG.getPOISON(VT: ScalarVT);
25883
25884 // extract_vector_elt (build_vector x, y), 1 -> y
25885 if (((IndexC && VecOp.getOpcode() == ISD::BUILD_VECTOR) ||
25886 VecOp.getOpcode() == ISD::SPLAT_VECTOR) &&
25887 TLI.isTypeLegal(VT: VecVT)) {
25888 assert((VecOp.getOpcode() != ISD::BUILD_VECTOR ||
25889 VecVT.isFixedLengthVector()) &&
25890 "BUILD_VECTOR used for scalable vectors");
25891 unsigned IndexVal =
25892 VecOp.getOpcode() == ISD::BUILD_VECTOR ? IndexC->getZExtValue() : 0;
25893 SDValue Elt = VecOp.getOperand(i: IndexVal);
25894 EVT InEltVT = Elt.getValueType();
25895
25896 if (VecOp.hasOneUse() || TLI.aggressivelyPreferBuildVectorSources(VecVT) ||
25897 isNullConstant(V: Elt)) {
25898 // Sometimes build_vector's scalar input types do not match result type.
25899 if (ScalarVT == InEltVT)
25900 return Elt;
25901
25902 // TODO: It may be useful to truncate if free if the build_vector
25903 // implicitly converts.
25904 }
25905 }
25906
25907 if (SDValue BO = scalarizeExtractedBinOp(ExtElt: N, DAG, DL, LegalTypes))
25908 return BO;
25909
25910 if (VecVT.isScalableVector())
25911 return SDValue();
25912
25913 // All the code from this point onwards assumes fixed width vectors, but it's
25914 // possible that some of the combinations could be made to work for scalable
25915 // vectors too.
25916 unsigned NumElts = VecVT.getVectorNumElements();
25917 unsigned VecEltBitWidth = VecVT.getScalarSizeInBits();
25918
25919 // See if the extracted element is constant, in which case fold it if its
25920 // a legal fp immediate.
25921 if (IndexC && ScalarVT.isFloatingPoint()) {
25922 APInt EltMask = APInt::getOneBitSet(numBits: NumElts, BitNo: IndexC->getZExtValue());
25923 KnownBits KnownElt = DAG.computeKnownBits(Op: VecOp, DemandedElts: EltMask);
25924 if (KnownElt.isConstant()) {
25925 APFloat CstFP =
25926 APFloat(ScalarVT.getFltSemantics(), KnownElt.getConstant());
25927 if (TLI.isFPImmLegal(CstFP, ScalarVT))
25928 return DAG.getConstantFP(Val: CstFP, DL, VT: ScalarVT);
25929 }
25930 }
25931
25932 // TODO: These transforms should not require the 'hasOneUse' restriction, but
25933 // there are regressions on multiple targets without it. We can end up with a
25934 // mess of scalar and vector code if we reduce only part of the DAG to scalar.
25935 if (IndexC && VecOp.getOpcode() == ISD::BITCAST && VecVT.isInteger() &&
25936 VecOp.hasOneUse()) {
25937 // The vector index of the LSBs of the source depend on the endian-ness.
25938 bool IsLE = DAG.getDataLayout().isLittleEndian();
25939 unsigned ExtractIndex = IndexC->getZExtValue();
25940 // extract_elt (v2i32 (bitcast i64:x)), BCTruncElt -> i32 (trunc i64:x)
25941 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1;
25942 SDValue BCSrc = VecOp.getOperand(i: 0);
25943 if (ExtractIndex == BCTruncElt && BCSrc.getValueType().isScalarInteger())
25944 return DAG.getAnyExtOrTrunc(Op: BCSrc, DL, VT: ScalarVT);
25945
25946 // TODO: Add support for SCALAR_TO_VECTOR implicit truncation.
25947 if (LegalTypes && BCSrc.getValueType().isInteger() &&
25948 BCSrc.getOpcode() == ISD::SCALAR_TO_VECTOR &&
25949 BCSrc.getScalarValueSizeInBits() ==
25950 BCSrc.getOperand(i: 0).getScalarValueSizeInBits()) {
25951 // ext_elt (bitcast (scalar_to_vec i64 X to v2i64) to v4i32), TruncElt -->
25952 // trunc i64 X to i32
25953 SDValue X = BCSrc.getOperand(i: 0);
25954 EVT XVT = X.getValueType();
25955 assert(XVT.isScalarInteger() && ScalarVT.isScalarInteger() &&
25956 "Extract element and scalar to vector can't change element type "
25957 "from FP to integer.");
25958 unsigned XBitWidth = X.getValueSizeInBits();
25959 unsigned Scale = XBitWidth / VecEltBitWidth;
25960 BCTruncElt = IsLE ? 0 : Scale - 1;
25961
25962 // An extract element return value type can be wider than its vector
25963 // operand element type. In that case, the high bits are undefined, so
25964 // it's possible that we may need to extend rather than truncate.
25965 if (ExtractIndex < Scale && XBitWidth > VecEltBitWidth) {
25966 assert(XBitWidth % VecEltBitWidth == 0 &&
25967 "Scalar bitwidth must be a multiple of vector element bitwidth");
25968
25969 if (ExtractIndex != BCTruncElt) {
25970 unsigned ShiftIndex =
25971 IsLE ? ExtractIndex : (Scale - 1) - ExtractIndex;
25972 X = DAG.getNode(
25973 Opcode: ISD::SRL, DL, VT: XVT, N1: X,
25974 N2: DAG.getShiftAmountConstant(Val: ShiftIndex * VecEltBitWidth, VT: XVT, DL));
25975 }
25976
25977 return DAG.getAnyExtOrTrunc(Op: X, DL, VT: ScalarVT);
25978 }
25979 }
25980 }
25981
25982 // Transform: (EXTRACT_VECTOR_ELT( VECTOR_SHUFFLE )) -> EXTRACT_VECTOR_ELT.
25983 // We only perform this optimization before the op legalization phase because
25984 // we may introduce new vector instructions which are not backed by TD
25985 // patterns. For example on AVX, extracting elements from a wide vector
25986 // without using extract_subvector. However, if we can find an underlying
25987 // scalar value, then we can always use that.
25988 if (IndexC && VecOp.getOpcode() == ISD::VECTOR_SHUFFLE) {
25989 auto *Shuf = cast<ShuffleVectorSDNode>(Val&: VecOp);
25990 // Find the new index to extract from.
25991 int OrigElt = Shuf->getMaskElt(Idx: IndexC->getZExtValue());
25992
25993 // Extracting an undef index is undef.
25994 if (OrigElt == -1)
25995 return DAG.getUNDEF(VT: ScalarVT);
25996
25997 // Select the right vector half to extract from.
25998 SDValue SVInVec;
25999 if (OrigElt < (int)NumElts) {
26000 SVInVec = VecOp.getOperand(i: 0);
26001 } else {
26002 SVInVec = VecOp.getOperand(i: 1);
26003 OrigElt -= NumElts;
26004 }
26005
26006 if (SVInVec.getOpcode() == ISD::BUILD_VECTOR) {
26007 // TODO: Check if shuffle mask is legal?
26008 if (LegalOperations && TLI.isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: VecVT) &&
26009 !VecOp.hasOneUse())
26010 return SDValue();
26011
26012 SDValue InOp = SVInVec.getOperand(i: OrigElt);
26013 if (InOp.getValueType() != ScalarVT) {
26014 assert(InOp.getValueType().isInteger() && ScalarVT.isInteger());
26015 InOp = DAG.getSExtOrTrunc(Op: InOp, DL, VT: ScalarVT);
26016 }
26017
26018 return InOp;
26019 }
26020
26021 // FIXME: We should handle recursing on other vector shuffles and
26022 // scalar_to_vector here as well.
26023
26024 if (!LegalOperations ||
26025 // FIXME: Should really be just isOperationLegalOrCustom.
26026 TLI.isOperationLegal(Op: ISD::EXTRACT_VECTOR_ELT, VT: VecVT) ||
26027 TLI.isOperationExpand(Op: ISD::VECTOR_SHUFFLE, VT: VecVT)) {
26028 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ScalarVT, N1: SVInVec,
26029 N2: DAG.getVectorIdxConstant(Val: OrigElt, DL));
26030 }
26031 }
26032
26033 // If only EXTRACT_VECTOR_ELT nodes use the source vector we can
26034 // simplify it based on the (valid) extraction indices.
26035 if (llvm::all_of(Range: VecOp->users(), P: [&](SDNode *Use) {
26036 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
26037 Use->getOperand(Num: 0) == VecOp &&
26038 isa<ConstantSDNode>(Val: Use->getOperand(Num: 1));
26039 })) {
26040 APInt DemandedElts = APInt::getZero(numBits: NumElts);
26041 for (SDNode *User : VecOp->users()) {
26042 auto *CstElt = cast<ConstantSDNode>(Val: User->getOperand(Num: 1));
26043 if (CstElt->getAPIntValue().ult(RHS: NumElts))
26044 DemandedElts.setBit(CstElt->getZExtValue());
26045 }
26046 if (SimplifyDemandedVectorElts(Op: VecOp, DemandedElts, AssumeSingleUse: true)) {
26047 // We simplified the vector operand of this extract element. If this
26048 // extract is not dead, visit it again so it is folded properly.
26049 if (N->getOpcode() != ISD::DELETED_NODE)
26050 AddToWorklist(N);
26051 return SDValue(N, 0);
26052 }
26053 APInt DemandedBits = APInt::getAllOnes(numBits: VecEltBitWidth);
26054 if (SimplifyDemandedBits(Op: VecOp, DemandedBits, DemandedElts, AssumeSingleUse: true)) {
26055 // We simplified the vector operand of this extract element. If this
26056 // extract is not dead, visit it again so it is folded properly.
26057 if (N->getOpcode() != ISD::DELETED_NODE)
26058 AddToWorklist(N);
26059 return SDValue(N, 0);
26060 }
26061 }
26062
26063 if (refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(N))
26064 return SDValue(N, 0);
26065
26066 // Everything under here is trying to match an extract of a loaded value.
26067 // If the result of load has to be truncated, then it's not necessarily
26068 // profitable.
26069 bool BCNumEltsChanged = false;
26070 EVT ExtVT = VecVT.getVectorElementType();
26071 EVT LVT = ExtVT;
26072 if (ScalarVT.bitsLT(VT: LVT) && !TLI.isTruncateFree(FromVT: LVT, ToVT: ScalarVT))
26073 return SDValue();
26074
26075 if (VecOp.getOpcode() == ISD::BITCAST) {
26076 // Don't duplicate a load with other uses.
26077 if (!VecOp.hasOneUse())
26078 return SDValue();
26079
26080 EVT BCVT = VecOp.getOperand(i: 0).getValueType();
26081 if (!BCVT.isVector() || ExtVT.bitsGT(VT: BCVT.getVectorElementType()))
26082 return SDValue();
26083 if (NumElts != BCVT.getVectorNumElements())
26084 BCNumEltsChanged = true;
26085 VecOp = VecOp.getOperand(i: 0);
26086 ExtVT = BCVT.getVectorElementType();
26087 }
26088
26089 // extract (vector load $addr), i --> load $addr + i * size
26090 if (!LegalOperations && !IndexC && VecOp.hasOneUse() &&
26091 ISD::isNormalLoad(N: VecOp.getNode()) &&
26092 !Index->hasPredecessor(N: VecOp.getNode())) {
26093 auto *VecLoad = dyn_cast<LoadSDNode>(Val&: VecOp);
26094 if (VecLoad && VecLoad->isSimple()) {
26095 if (SDValue Scalarized = TLI.scalarizeExtractedVectorLoad(
26096 ResultVT: ScalarVT, DL: SDLoc(N), InVecVT: VecVT, EltNo: Index, OriginalLoad: VecLoad, DAG)) {
26097 ++OpsNarrowed;
26098 return Scalarized;
26099 }
26100 }
26101 }
26102
26103 // Perform only after legalization to ensure build_vector / vector_shuffle
26104 // optimizations have already been done.
26105 if (!LegalOperations || !IndexC)
26106 return SDValue();
26107
26108 bool IsFrozen = false;
26109 if (VecOp.getOpcode() == ISD::FREEZE && VecOp.hasOneUse()) {
26110 VecOp = VecOp.getOperand(i: 0);
26111 IsFrozen = true;
26112 }
26113
26114 // (vextract (v4f32 load $addr), c) -> (f32 load $addr+c*size)
26115 // (vextract (v4f32 s2v (f32 load $addr)), c) -> (f32 load $addr+c*size)
26116 // (vextract (v4f32 shuffle (load $addr), <1,u,u,u>), 0) -> (f32 load $addr)
26117 int Elt = IndexC->getZExtValue();
26118 LoadSDNode *LN0 = nullptr;
26119 if (ISD::isNormalLoad(N: VecOp.getNode())) {
26120 LN0 = cast<LoadSDNode>(Val&: VecOp);
26121 } else if (VecOp.getOpcode() == ISD::SCALAR_TO_VECTOR &&
26122 VecOp.getOperand(i: 0).getValueType() == ExtVT &&
26123 ISD::isNormalLoad(N: VecOp.getOperand(i: 0).getNode())) {
26124 // Don't duplicate a load with other uses.
26125 if (!VecOp.hasOneUse())
26126 return SDValue();
26127
26128 LN0 = cast<LoadSDNode>(Val: VecOp.getOperand(i: 0));
26129 }
26130 if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(Val&: VecOp)) {
26131 // (vextract (vector_shuffle (load $addr), v2, <1, u, u, u>), 1)
26132 // =>
26133 // (load $addr+1*size)
26134
26135 // Don't duplicate a load with other uses.
26136 if (!VecOp.hasOneUse())
26137 return SDValue();
26138
26139 // If the bit convert changed the number of elements, it is unsafe
26140 // to examine the mask.
26141 if (BCNumEltsChanged)
26142 return SDValue();
26143
26144 // Select the input vector, guarding against out of range extract vector.
26145 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Idx: Elt);
26146 VecOp = (Idx < (int)NumElts) ? VecOp.getOperand(i: 0) : VecOp.getOperand(i: 1);
26147
26148 if (VecOp.getOpcode() == ISD::BITCAST) {
26149 // Don't duplicate a load with other uses.
26150 if (!VecOp.hasOneUse())
26151 return SDValue();
26152
26153 VecOp = VecOp.getOperand(i: 0);
26154 }
26155 if (ISD::isNormalLoad(N: VecOp.getNode())) {
26156 LN0 = cast<LoadSDNode>(Val&: VecOp);
26157 Elt = (Idx < (int)NumElts) ? Idx : Idx - (int)NumElts;
26158 Index = DAG.getConstant(Val: Elt, DL, VT: Index.getValueType());
26159 }
26160 } else if (VecOp.getOpcode() == ISD::CONCAT_VECTORS && !BCNumEltsChanged &&
26161 VecVT.getVectorElementType() == ScalarVT &&
26162 (!LegalTypes ||
26163 TLI.isTypeLegal(
26164 VT: VecOp.getOperand(i: 0).getValueType().getVectorElementType()))) {
26165 // extract_vector_elt (concat_vectors v2i16:a, v2i16:b), 0
26166 // -> extract_vector_elt a, 0
26167 // extract_vector_elt (concat_vectors v2i16:a, v2i16:b), 1
26168 // -> extract_vector_elt a, 1
26169 // extract_vector_elt (concat_vectors v2i16:a, v2i16:b), 2
26170 // -> extract_vector_elt b, 0
26171 // extract_vector_elt (concat_vectors v2i16:a, v2i16:b), 3
26172 // -> extract_vector_elt b, 1
26173 EVT ConcatVT = VecOp.getOperand(i: 0).getValueType();
26174 unsigned ConcatNumElts = ConcatVT.getVectorNumElements();
26175 SDValue NewIdx = DAG.getConstant(Val: Elt % ConcatNumElts, DL,
26176 VT: Index.getValueType());
26177
26178 SDValue ConcatOp = VecOp.getOperand(i: Elt / ConcatNumElts);
26179 SDValue Elt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL,
26180 VT: ConcatVT.getVectorElementType(),
26181 N1: ConcatOp, N2: NewIdx);
26182 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ScalarVT, Operand: Elt);
26183 }
26184
26185 // Make sure we found a non-volatile load and the extractelement is
26186 // the only use.
26187 if (!LN0 || !LN0->hasNUsesOfValue(NUses: 1,Value: 0) || !LN0->isSimple())
26188 return SDValue();
26189
26190 // If Idx was -1 above, Elt is going to be -1, so just return undef.
26191 if (Elt == -1)
26192 return DAG.getUNDEF(VT: LVT);
26193
26194 if (SDValue Scalarized =
26195 TLI.scalarizeExtractedVectorLoad(ResultVT: LVT, DL, InVecVT: VecVT, EltNo: Index, OriginalLoad: LN0, DAG)) {
26196 ++OpsNarrowed;
26197 if (IsFrozen)
26198 return DAG.getFreeze(V: Scalarized);
26199 return Scalarized;
26200 }
26201
26202 return SDValue();
26203}
26204
26205// Simplify (build_vec (ext )) to (bitcast (build_vec ))
26206SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) {
26207 // We perform this optimization post type-legalization because
26208 // the type-legalizer often scalarizes integer-promoted vectors.
26209 // Performing this optimization before may create bit-casts which
26210 // will be type-legalized to complex code sequences.
26211 // We perform this optimization only before the operation legalizer because we
26212 // may introduce illegal operations.
26213 if (Level != AfterLegalizeVectorOps && Level != AfterLegalizeTypes)
26214 return SDValue();
26215
26216 unsigned NumInScalars = N->getNumOperands();
26217 SDLoc DL(N);
26218 EVT VT = N->getValueType(ResNo: 0);
26219
26220 // Check to see if this is a BUILD_VECTOR of a bunch of values
26221 // which come from any_extend or zero_extend nodes. If so, we can create
26222 // a new BUILD_VECTOR using bit-casts which may enable other BUILD_VECTOR
26223 // optimizations. We do not handle sign-extend because we can't fill the sign
26224 // using shuffles.
26225 EVT SourceType = MVT::Other;
26226 bool AllAnyExt = true;
26227
26228 for (unsigned i = 0; i != NumInScalars; ++i) {
26229 SDValue In = N->getOperand(Num: i);
26230 // Ignore undef inputs.
26231 if (In.isUndef()) continue;
26232
26233 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND;
26234 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND;
26235
26236 // Abort if the element is not an extension.
26237 if (!ZeroExt && !AnyExt) {
26238 SourceType = MVT::Other;
26239 break;
26240 }
26241
26242 // The input is a ZeroExt or AnyExt. Check the original type.
26243 EVT InTy = In.getOperand(i: 0).getValueType();
26244
26245 // Check that all of the widened source types are the same.
26246 if (SourceType == MVT::Other)
26247 // First time.
26248 SourceType = InTy;
26249 else if (InTy != SourceType) {
26250 // Multiple income types. Abort.
26251 SourceType = MVT::Other;
26252 break;
26253 }
26254
26255 // Check if all of the extends are ANY_EXTENDs.
26256 AllAnyExt &= AnyExt;
26257 }
26258
26259 // In order to have valid types, all of the inputs must be extended from the
26260 // same source type and all of the inputs must be any or zero extend.
26261 // Scalar sizes must be a power of two.
26262 EVT OutScalarTy = VT.getScalarType();
26263 bool ValidTypes =
26264 SourceType != MVT::Other &&
26265 llvm::has_single_bit<uint32_t>(Value: OutScalarTy.getSizeInBits()) &&
26266 llvm::has_single_bit<uint32_t>(Value: SourceType.getSizeInBits());
26267
26268 // Create a new simpler BUILD_VECTOR sequence which other optimizations can
26269 // turn into a single shuffle instruction.
26270 if (!ValidTypes)
26271 return SDValue();
26272
26273 // If we already have a splat buildvector, then don't fold it if it means
26274 // introducing zeros.
26275 if (!AllAnyExt && DAG.isSplatValue(V: SDValue(N, 0), /*AllowUndefs*/ true))
26276 return SDValue();
26277
26278 bool isLE = DAG.getDataLayout().isLittleEndian();
26279 unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
26280 assert(ElemRatio > 1 && "Invalid element size ratio");
26281 SDValue Filler = AllAnyExt ? DAG.getPOISON(VT: SourceType)
26282 : DAG.getConstant(Val: 0, DL, VT: SourceType);
26283
26284 unsigned NewBVElems = ElemRatio * VT.getVectorNumElements();
26285 SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
26286
26287 // Populate the new build_vector
26288 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
26289 SDValue Cast = N->getOperand(Num: i);
26290 assert((Cast.getOpcode() == ISD::ANY_EXTEND ||
26291 Cast.getOpcode() == ISD::ZERO_EXTEND ||
26292 Cast.isUndef()) && "Invalid cast opcode");
26293 SDValue In;
26294 if (Cast.isUndef())
26295 In = DAG.getUNDEF(VT: SourceType);
26296 else
26297 In = Cast->getOperand(Num: 0);
26298 unsigned Index = isLE ? (i * ElemRatio) :
26299 (i * ElemRatio + (ElemRatio - 1));
26300
26301 assert(Index < Ops.size() && "Invalid index");
26302 Ops[Index] = In;
26303 }
26304
26305 // The type of the new BUILD_VECTOR node.
26306 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SourceType, NumElements: NewBVElems);
26307 assert(VecVT.getSizeInBits() == VT.getSizeInBits() &&
26308 "Invalid vector size");
26309 // Check if the new vector type is legal.
26310 if (!isTypeLegal(VT: VecVT) ||
26311 (!TLI.isOperationLegal(Op: ISD::BUILD_VECTOR, VT: VecVT) &&
26312 TLI.isOperationLegal(Op: ISD::BUILD_VECTOR, VT)))
26313 return SDValue();
26314
26315 // Make the new BUILD_VECTOR.
26316 SDValue BV = DAG.getBuildVector(VT: VecVT, DL, Ops);
26317
26318 // The new BUILD_VECTOR node has the potential to be further optimized.
26319 AddToWorklist(N: BV.getNode());
26320 // Bitcast to the desired type.
26321 return DAG.getBitcast(VT, V: BV);
26322}
26323
26324// Simplify (build_vec (trunc $1)
26325// (trunc (srl $1 half-width))
26326// (trunc (srl $1 (2 * half-width))))
26327// to (bitcast $1)
26328SDValue DAGCombiner::reduceBuildVecTruncToBitCast(SDNode *N) {
26329 assert(N->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector");
26330
26331 EVT VT = N->getValueType(ResNo: 0);
26332
26333 // Don't run this before LegalizeTypes if VT is legal.
26334 // Targets may have other preferences.
26335 if (Level < AfterLegalizeTypes && TLI.isTypeLegal(VT))
26336 return SDValue();
26337
26338 // Only for little endian
26339 if (!DAG.getDataLayout().isLittleEndian())
26340 return SDValue();
26341
26342 EVT OutScalarTy = VT.getScalarType();
26343 uint64_t ScalarTypeBitsize = OutScalarTy.getSizeInBits();
26344
26345 // Only for power of two types to be sure that bitcast works well
26346 if (!isPowerOf2_64(Value: ScalarTypeBitsize))
26347 return SDValue();
26348
26349 unsigned NumInScalars = N->getNumOperands();
26350
26351 // Look through bitcasts
26352 auto PeekThroughBitcast = [](SDValue Op) {
26353 if (Op.getOpcode() == ISD::BITCAST)
26354 return Op.getOperand(i: 0);
26355 return Op;
26356 };
26357
26358 // The source value where all the parts are extracted.
26359 SDValue Src;
26360 for (unsigned i = 0; i != NumInScalars; ++i) {
26361 SDValue In = PeekThroughBitcast(N->getOperand(Num: i));
26362 // Ignore undef inputs.
26363 if (In.isUndef()) continue;
26364
26365 if (In.getOpcode() != ISD::TRUNCATE)
26366 return SDValue();
26367
26368 In = PeekThroughBitcast(In.getOperand(i: 0));
26369
26370 if (In.getOpcode() != ISD::SRL) {
26371 // For now only build_vec without shuffling, handle shifts here in the
26372 // future.
26373 if (i != 0)
26374 return SDValue();
26375
26376 Src = In;
26377 } else {
26378 // In is SRL
26379 SDValue part = PeekThroughBitcast(In.getOperand(i: 0));
26380
26381 if (!Src) {
26382 Src = part;
26383 } else if (Src != part) {
26384 // Vector parts do not stem from the same variable
26385 return SDValue();
26386 }
26387
26388 SDValue ShiftAmtVal = In.getOperand(i: 1);
26389 if (!isa<ConstantSDNode>(Val: ShiftAmtVal))
26390 return SDValue();
26391
26392 uint64_t ShiftAmt = In.getConstantOperandVal(i: 1);
26393
26394 // The extracted value is not extracted at the right position
26395 if (ShiftAmt != i * ScalarTypeBitsize)
26396 return SDValue();
26397 }
26398 }
26399
26400 // Only cast if the size is the same
26401 if (!Src || Src.getValueType().getSizeInBits() != VT.getSizeInBits())
26402 return SDValue();
26403
26404 return DAG.getBitcast(VT, V: Src);
26405}
26406
26407SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N,
26408 ArrayRef<int> VectorMask,
26409 SDValue VecIn1, SDValue VecIn2,
26410 unsigned LeftIdx, bool DidSplitVec) {
26411 EVT VT = N->getValueType(ResNo: 0);
26412 EVT InVT1 = VecIn1.getValueType();
26413 EVT InVT2 = VecIn2.getNode() ? VecIn2.getValueType() : InVT1;
26414
26415 unsigned NumElems = VT.getVectorNumElements();
26416 unsigned ShuffleNumElems = NumElems;
26417
26418 // If we artificially split a vector in two already, then the offsets in the
26419 // operands will all be based off of VecIn1, even those in VecIn2.
26420 unsigned Vec2Offset = DidSplitVec ? 0 : InVT1.getVectorNumElements();
26421
26422 uint64_t VTSize = VT.getFixedSizeInBits();
26423 uint64_t InVT1Size = InVT1.getFixedSizeInBits();
26424 uint64_t InVT2Size = InVT2.getFixedSizeInBits();
26425
26426 assert(InVT2Size <= InVT1Size &&
26427 "Inputs must be sorted to be in non-increasing vector size order.");
26428
26429 // We can't generate a shuffle node with mismatched input and output types.
26430 // Try to make the types match the type of the output.
26431 if (InVT1 != VT || InVT2 != VT) {
26432 if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
26433 // If the output vector length is a multiple of both input lengths,
26434 // we can concatenate them and pad the rest with poison.
26435 unsigned NumConcats = VTSize / InVT1Size;
26436 assert(NumConcats >= 2 && "Concat needs at least two inputs!");
26437 SmallVector<SDValue, 2> ConcatOps(NumConcats, DAG.getPOISON(VT: InVT1));
26438 ConcatOps[0] = VecIn1;
26439 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.getPOISON(VT: InVT1);
26440 VecIn1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps);
26441 VecIn2 = SDValue();
26442 } else if (InVT1Size == VTSize * 2) {
26443 if (TLI.getExtractSubvectorCost(ResVT: VT, SrcVT: InVT1, Index: NumElems) >
26444 TargetLowering::ExtractSubvectorCost::Cheap)
26445 return SDValue();
26446
26447 if (!VecIn2.getNode()) {
26448 // If we only have one input vector, and it's twice the size of the
26449 // output, split it in two.
26450 VecIn2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: VecIn1,
26451 N2: DAG.getVectorIdxConstant(Val: NumElems, DL));
26452 VecIn1 = DAG.getExtractSubvector(DL, VT, Vec: VecIn1, Idx: 0);
26453 // Since we now have shorter input vectors, adjust the offset of the
26454 // second vector's start.
26455 Vec2Offset = NumElems;
26456 } else {
26457 assert(InVT2Size <= InVT1Size &&
26458 "Second input is not going to be larger than the first one.");
26459
26460 // VecIn1 is wider than the output, and we have another, possibly
26461 // smaller input. Pad the smaller input with undefs, shuffle at the
26462 // input vector width, and extract the output.
26463 // The shuffle type is different than VT, so check legality again.
26464 if (LegalOperations &&
26465 !TLI.isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: InVT1))
26466 return SDValue();
26467
26468 // Legalizing INSERT_SUBVECTOR is tricky - you basically have to
26469 // lower it back into a BUILD_VECTOR. So if the inserted type is
26470 // illegal, don't even try.
26471 if (InVT1 != InVT2) {
26472 if (!TLI.isTypeLegal(VT: InVT2))
26473 return SDValue();
26474 VecIn2 = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: InVT1), SubVec: VecIn2, Idx: 0);
26475 }
26476 ShuffleNumElems = NumElems * 2;
26477 }
26478 } else if (InVT2Size * 2 == VTSize && InVT1Size == VTSize) {
26479 SmallVector<SDValue, 2> ConcatOps(2, DAG.getPOISON(VT: InVT2));
26480 ConcatOps[0] = VecIn2;
26481 VecIn2 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps);
26482 } else if (InVT1Size / VTSize > 1 && InVT1Size % VTSize == 0) {
26483 if (TLI.getExtractSubvectorCost(ResVT: VT, SrcVT: InVT1, Index: NumElems) >
26484 TargetLowering::ExtractSubvectorCost::Cheap ||
26485 !TLI.isTypeLegal(VT: InVT1) || !TLI.isTypeLegal(VT: InVT2))
26486 return SDValue();
26487 // If dest vector has less than two elements, then use shuffle and extract
26488 // from larger regs will cost even more.
26489 if (VT.getVectorNumElements() <= 2 || !VecIn2.getNode())
26490 return SDValue();
26491 assert(InVT2Size <= InVT1Size &&
26492 "Second input is not going to be larger than the first one.");
26493
26494 // VecIn1 is wider than the output, and we have another, possibly
26495 // smaller input. Pad the smaller input with undefs, shuffle at the
26496 // input vector width, and extract the output.
26497 // The shuffle type is different than VT, so check legality again.
26498 if (LegalOperations && !TLI.isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT: InVT1))
26499 return SDValue();
26500
26501 if (InVT1 != InVT2) {
26502 VecIn2 = DAG.getInsertSubvector(DL, Vec: DAG.getPOISON(VT: InVT1), SubVec: VecIn2, Idx: 0);
26503 }
26504 ShuffleNumElems = InVT1Size / VTSize * NumElems;
26505 } else {
26506 // TODO: Support cases where the length mismatch isn't exactly by a
26507 // factor of 2.
26508 // TODO: Move this check upwards, so that if we have bad type
26509 // mismatches, we don't create any DAG nodes.
26510 return SDValue();
26511 }
26512 }
26513
26514 // Initialize mask to undef.
26515 SmallVector<int, 8> Mask(ShuffleNumElems, -1);
26516
26517 // Only need to run up to the number of elements actually used, not the
26518 // total number of elements in the shuffle - if we are shuffling a wider
26519 // vector, the high lanes should be set to undef.
26520 for (unsigned i = 0; i != NumElems; ++i) {
26521 if (VectorMask[i] <= 0)
26522 continue;
26523
26524 unsigned ExtIndex = N->getOperand(Num: i).getConstantOperandVal(i: 1);
26525 if (VectorMask[i] == (int)LeftIdx) {
26526 Mask[i] = ExtIndex;
26527 } else if (VectorMask[i] == (int)LeftIdx + 1) {
26528 Mask[i] = Vec2Offset + ExtIndex;
26529 }
26530 }
26531
26532 // The type the input vectors may have changed above.
26533 InVT1 = VecIn1.getValueType();
26534
26535 // If we already have a VecIn2, it should have the same type as VecIn1.
26536 // If we don't, get an poison/zero vector of the appropriate type.
26537 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getPOISON(VT: InVT1);
26538 assert(InVT1 == VecIn2.getValueType() && "Unexpected second input type.");
26539
26540 SDValue Shuffle = DAG.getVectorShuffle(VT: InVT1, dl: DL, N1: VecIn1, N2: VecIn2, Mask);
26541 if (ShuffleNumElems > NumElems)
26542 Shuffle = DAG.getExtractSubvector(DL, VT, Vec: Shuffle, Idx: 0);
26543
26544 return Shuffle;
26545}
26546
26547static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) {
26548 assert(BV->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector");
26549
26550 // First, determine where the build vector is not undef.
26551 // TODO: We could extend this to handle zero elements as well as undefs.
26552 int NumBVOps = BV->getNumOperands();
26553 int ZextElt = -1;
26554 for (int i = 0; i != NumBVOps; ++i) {
26555 SDValue Op = BV->getOperand(Num: i);
26556 if (Op.isUndef())
26557 continue;
26558 if (ZextElt == -1)
26559 ZextElt = i;
26560 else
26561 return SDValue();
26562 }
26563 // Bail out if there's no non-undef element.
26564 if (ZextElt == -1)
26565 return SDValue();
26566
26567 // The build vector contains some number of undef elements and exactly
26568 // one other element. That other element must be a zero-extended scalar
26569 // extracted from a vector at a constant index to turn this into a shuffle.
26570 // Also, require that the build vector does not implicitly truncate/extend
26571 // its elements.
26572 // TODO: This could be enhanced to allow ANY_EXTEND as well as ZERO_EXTEND.
26573 EVT VT = BV->getValueType(ResNo: 0);
26574 SDValue Zext = BV->getOperand(Num: ZextElt);
26575 if (Zext.getOpcode() != ISD::ZERO_EXTEND || !Zext.hasOneUse() ||
26576 Zext.getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
26577 !isa<ConstantSDNode>(Val: Zext.getOperand(i: 0).getOperand(i: 1)) ||
26578 Zext.getValueSizeInBits() != VT.getScalarSizeInBits())
26579 return SDValue();
26580
26581 // The zero-extend must be a multiple of the source size, and we must be
26582 // building a vector of the same size as the source of the extract element.
26583 SDValue Extract = Zext.getOperand(i: 0);
26584 unsigned DestSize = Zext.getValueSizeInBits();
26585 unsigned SrcSize = Extract.getValueSizeInBits();
26586 if (DestSize % SrcSize != 0 ||
26587 Extract.getOperand(i: 0).getValueSizeInBits() != VT.getSizeInBits())
26588 return SDValue();
26589
26590 // Create a shuffle mask that will combine the extracted element with zeros
26591 // and undefs.
26592 int ZextRatio = DestSize / SrcSize;
26593 int NumMaskElts = NumBVOps * ZextRatio;
26594 SmallVector<int, 32> ShufMask(NumMaskElts, -1);
26595 for (int i = 0; i != NumMaskElts; ++i) {
26596 if (i / ZextRatio == ZextElt) {
26597 // The low bits of the (potentially translated) extracted element map to
26598 // the source vector. The high bits map to zero. We will use a zero vector
26599 // as the 2nd source operand of the shuffle, so use the 1st element of
26600 // that vector (mask value is number-of-elements) for the high bits.
26601 int Low = DAG.getDataLayout().isBigEndian() ? (ZextRatio - 1) : 0;
26602 ShufMask[i] = (i % ZextRatio == Low) ? Extract.getConstantOperandVal(i: 1)
26603 : NumMaskElts;
26604 }
26605
26606 // Undef elements of the build vector remain undef because we initialize
26607 // the shuffle mask with -1.
26608 }
26609
26610 // buildvec undef, ..., (zext (extractelt V, IndexC)), undef... -->
26611 // bitcast (shuffle V, ZeroVec, VectorMask)
26612 SDLoc DL(BV);
26613 EVT VecVT = Extract.getOperand(i: 0).getValueType();
26614 SDValue ZeroVec = DAG.getConstant(Val: 0, DL, VT: VecVT);
26615 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
26616 SDValue Shuf = TLI.buildLegalVectorShuffle(VT: VecVT, DL, N0: Extract.getOperand(i: 0),
26617 N1: ZeroVec, Mask: ShufMask, DAG);
26618 if (!Shuf)
26619 return SDValue();
26620 return DAG.getBitcast(VT, V: Shuf);
26621}
26622
26623// FIXME: promote to STLExtras.
26624template <typename R, typename T>
26625static auto getFirstIndexOf(R &&Range, const T &Val) {
26626 auto I = find(Range, Val);
26627 if (I == Range.end())
26628 return static_cast<decltype(std::distance(Range.begin(), I))>(-1);
26629 return std::distance(Range.begin(), I);
26630}
26631
26632// Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
26633// operations. If the types of the vectors we're extracting from allow it,
26634// turn this into a vector_shuffle node.
26635SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
26636 SDLoc DL(N);
26637 EVT VT = N->getValueType(ResNo: 0);
26638
26639 // Only type-legal BUILD_VECTOR nodes are converted to shuffle nodes.
26640 if (!isTypeLegal(VT))
26641 return SDValue();
26642
26643 if (SDValue V = reduceBuildVecToShuffleWithZero(BV: N, DAG))
26644 return V;
26645
26646 // May only combine to shuffle after legalize if shuffle is legal.
26647 if (LegalOperations && !TLI.isOperationLegal(Op: ISD::VECTOR_SHUFFLE, VT))
26648 return SDValue();
26649
26650 bool UsesZeroVector = false;
26651 unsigned NumElems = N->getNumOperands();
26652
26653 // Record, for each element of the newly built vector, which input vector
26654 // that element comes from. -1 stands for undef, 0 for the zero vector,
26655 // and positive values for the input vectors.
26656 // VectorMask maps each element to its vector number, and VecIn maps vector
26657 // numbers to their initial SDValues.
26658
26659 SmallVector<int, 8> VectorMask(NumElems, -1);
26660 SmallVector<SDValue, 8> VecIn;
26661 VecIn.push_back(Elt: SDValue());
26662
26663 // If we have a single extract_element with a constant index, track the index
26664 // value.
26665 unsigned OneConstExtractIndex = ~0u;
26666
26667 // Count the number of extract_vector_elt sources (i.e. non-constant or undef)
26668 unsigned NumExtracts = 0;
26669
26670 for (unsigned i = 0; i != NumElems; ++i) {
26671 SDValue Op = N->getOperand(Num: i);
26672
26673 if (Op.isUndef())
26674 continue;
26675
26676 // See if we can use a blend with a zero vector.
26677 // TODO: Should we generalize this to a blend with an arbitrary constant
26678 // vector?
26679 if (isNullConstant(V: Op) || isNullFPConstant(V: Op)) {
26680 UsesZeroVector = true;
26681 VectorMask[i] = 0;
26682 continue;
26683 }
26684
26685 // Not an undef or zero. If the input is something other than an
26686 // EXTRACT_VECTOR_ELT with an in-range constant index, bail out.
26687 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
26688 return SDValue();
26689
26690 SDValue ExtractedFromVec = Op.getOperand(i: 0);
26691 if (ExtractedFromVec.getValueType().isScalableVector())
26692 return SDValue();
26693 auto *ExtractIdx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
26694 if (!ExtractIdx)
26695 return SDValue();
26696
26697 if (ExtractIdx->getAsAPIntVal().uge(
26698 RHS: ExtractedFromVec.getValueType().getVectorNumElements()))
26699 return SDValue();
26700
26701 // All inputs must have the same element type as the output.
26702 if (VT.getVectorElementType() !=
26703 ExtractedFromVec.getValueType().getVectorElementType())
26704 return SDValue();
26705
26706 OneConstExtractIndex = ExtractIdx->getZExtValue();
26707 ++NumExtracts;
26708
26709 // Have we seen this input vector before?
26710 // The vectors are expected to be tiny (usually 1 or 2 elements), so using
26711 // a map back from SDValues to numbers isn't worth it.
26712 int Idx = getFirstIndexOf(Range&: VecIn, Val: ExtractedFromVec);
26713 if (Idx == -1) { // A new source vector?
26714 Idx = VecIn.size();
26715 VecIn.push_back(Elt: ExtractedFromVec);
26716 }
26717
26718 VectorMask[i] = Idx;
26719 }
26720
26721 // If we didn't find at least one input vector, bail out.
26722 if (VecIn.size() < 2)
26723 return SDValue();
26724
26725 // If all the Operands of BUILD_VECTOR extract from same
26726 // vector, then split the vector efficiently based on the maximum
26727 // vector access index and adjust the VectorMask and
26728 // VecIn accordingly.
26729 bool DidSplitVec = false;
26730 if (VecIn.size() == 2) {
26731 // If we only found a single constant indexed extract_vector_elt feeding the
26732 // build_vector, do not produce a more complicated shuffle if the extract is
26733 // cheap with other constant/undef elements. Skip broadcast patterns with
26734 // multiple uses in the build_vector.
26735
26736 // TODO: This should be more aggressive about skipping the shuffle
26737 // formation, particularly if VecIn[1].hasOneUse(), and regardless of the
26738 // index.
26739 if (NumExtracts == 1 &&
26740 TLI.isOperationLegalOrCustom(Op: ISD::EXTRACT_VECTOR_ELT, VT) &&
26741 TLI.isTypeLegal(VT: VT.getVectorElementType()) &&
26742 TLI.isExtractVecEltCheap(VT, Index: OneConstExtractIndex))
26743 return SDValue();
26744
26745 unsigned MaxIndex = 0;
26746 unsigned NearestPow2 = 0;
26747 SDValue Vec = VecIn.back();
26748 EVT InVT = Vec.getValueType();
26749 SmallVector<unsigned, 8> IndexVec(NumElems, 0);
26750
26751 for (unsigned i = 0; i < NumElems; i++) {
26752 if (VectorMask[i] <= 0)
26753 continue;
26754 unsigned Index = N->getOperand(Num: i).getConstantOperandVal(i: 1);
26755 IndexVec[i] = Index;
26756 MaxIndex = std::max(a: MaxIndex, b: Index);
26757 }
26758
26759 NearestPow2 = PowerOf2Ceil(A: MaxIndex);
26760 if (InVT.isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
26761 NumElems * 2 < NearestPow2) {
26762 unsigned SplitSize = NearestPow2 / 2;
26763 EVT SplitVT = EVT::getVectorVT(Context&: *DAG.getContext(),
26764 VT: InVT.getVectorElementType(), NumElements: SplitSize);
26765 if (TLI.isTypeLegal(VT: SplitVT) &&
26766 SplitSize + SplitVT.getVectorNumElements() <=
26767 InVT.getVectorNumElements()) {
26768 SDValue VecIn2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1: Vec,
26769 N2: DAG.getVectorIdxConstant(Val: SplitSize, DL));
26770 SDValue VecIn1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1: Vec,
26771 N2: DAG.getVectorIdxConstant(Val: 0, DL));
26772 VecIn.pop_back();
26773 VecIn.push_back(Elt: VecIn1);
26774 VecIn.push_back(Elt: VecIn2);
26775 DidSplitVec = true;
26776
26777 for (unsigned i = 0; i < NumElems; i++) {
26778 if (VectorMask[i] <= 0)
26779 continue;
26780 VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
26781 }
26782 }
26783 }
26784 }
26785
26786 // Sort input vectors by decreasing vector element count,
26787 // while preserving the relative order of equally-sized vectors.
26788 // Note that we keep the first "implicit zero vector as-is.
26789 SmallVector<SDValue, 8> SortedVecIn(VecIn);
26790 llvm::stable_sort(Range: MutableArrayRef<SDValue>(SortedVecIn).drop_front(),
26791 C: [](const SDValue &a, const SDValue &b) {
26792 return a.getValueType().getVectorNumElements() >
26793 b.getValueType().getVectorNumElements();
26794 });
26795
26796 // We now also need to rebuild the VectorMask, because it referenced element
26797 // order in VecIn, and we just sorted them.
26798 for (int &SourceVectorIndex : VectorMask) {
26799 if (SourceVectorIndex <= 0)
26800 continue;
26801 unsigned Idx = getFirstIndexOf(Range&: SortedVecIn, Val: VecIn[SourceVectorIndex]);
26802 assert(Idx > 0 && Idx < SortedVecIn.size() &&
26803 VecIn[SourceVectorIndex] == SortedVecIn[Idx] && "Remapping failure");
26804 SourceVectorIndex = Idx;
26805 }
26806
26807 VecIn = std::move(SortedVecIn);
26808
26809 // TODO: Should this fire if some of the input vectors has illegal type (like
26810 // it does now), or should we let legalization run its course first?
26811
26812 // Shuffle phase:
26813 // Take pairs of vectors, and shuffle them so that the result has elements
26814 // from these vectors in the correct places.
26815 // For example, given:
26816 // t10: i32 = extract_vector_elt t1, Constant:i64<0>
26817 // t11: i32 = extract_vector_elt t2, Constant:i64<0>
26818 // t12: i32 = extract_vector_elt t3, Constant:i64<0>
26819 // t13: i32 = extract_vector_elt t1, Constant:i64<1>
26820 // t14: v4i32 = BUILD_VECTOR t10, t11, t12, t13
26821 // We will generate:
26822 // t20: v4i32 = vector_shuffle<0,4,u,1> t1, t2
26823 // t21: v4i32 = vector_shuffle<u,u,0,u> t3, undef
26824 SmallVector<SDValue, 4> Shuffles;
26825 for (unsigned In = 0, Len = (VecIn.size() / 2); In < Len; ++In) {
26826 unsigned LeftIdx = 2 * In + 1;
26827 SDValue VecLeft = VecIn[LeftIdx];
26828 SDValue VecRight =
26829 (LeftIdx + 1) < VecIn.size() ? VecIn[LeftIdx + 1] : SDValue();
26830
26831 if (SDValue Shuffle = createBuildVecShuffle(DL, N, VectorMask, VecIn1: VecLeft,
26832 VecIn2: VecRight, LeftIdx, DidSplitVec))
26833 Shuffles.push_back(Elt: Shuffle);
26834 else
26835 return SDValue();
26836 }
26837
26838 // If we need the zero vector as an "ingredient" in the blend tree, add it
26839 // to the list of shuffles.
26840 if (UsesZeroVector)
26841 Shuffles.push_back(Elt: VT.isInteger() ? DAG.getConstant(Val: 0, DL, VT)
26842 : DAG.getConstantFP(Val: 0.0, DL, VT));
26843
26844 // If we only have one shuffle, we're done.
26845 if (Shuffles.size() == 1)
26846 return Shuffles[0];
26847
26848 // Update the vector mask to point to the post-shuffle vectors.
26849 for (int &Vec : VectorMask)
26850 if (Vec == 0)
26851 Vec = Shuffles.size() - 1;
26852 else
26853 Vec = (Vec - 1) / 2;
26854
26855 // More than one shuffle. Generate a binary tree of blends, e.g. if from
26856 // the previous step we got the set of shuffles t10, t11, t12, t13, we will
26857 // generate:
26858 // t10: v8i32 = vector_shuffle<0,8,u,u,u,u,u,u> t1, t2
26859 // t11: v8i32 = vector_shuffle<u,u,0,8,u,u,u,u> t3, t4
26860 // t12: v8i32 = vector_shuffle<u,u,u,u,0,8,u,u> t5, t6
26861 // t13: v8i32 = vector_shuffle<u,u,u,u,u,u,0,8> t7, t8
26862 // t20: v8i32 = vector_shuffle<0,1,10,11,u,u,u,u> t10, t11
26863 // t21: v8i32 = vector_shuffle<u,u,u,u,4,5,14,15> t12, t13
26864 // t30: v8i32 = vector_shuffle<0,1,2,3,12,13,14,15> t20, t21
26865
26866 // Make sure the initial size of the shuffle list is even.
26867 if (Shuffles.size() % 2)
26868 Shuffles.push_back(Elt: DAG.getPOISON(VT));
26869
26870 for (unsigned CurSize = Shuffles.size(); CurSize > 1; CurSize /= 2) {
26871 if (CurSize % 2) {
26872 Shuffles[CurSize] = DAG.getPOISON(VT);
26873 CurSize++;
26874 }
26875 for (unsigned In = 0, Len = CurSize / 2; In < Len; ++In) {
26876 int Left = 2 * In;
26877 int Right = 2 * In + 1;
26878 SmallVector<int, 8> Mask(NumElems, -1);
26879 SDValue L = Shuffles[Left];
26880 ArrayRef<int> LMask;
26881 bool IsLeftShuffle = L.getOpcode() == ISD::VECTOR_SHUFFLE &&
26882 L.use_empty() && L.getOperand(i: 1).isUndef() &&
26883 L.getOperand(i: 0).getValueType() == L.getValueType();
26884 if (IsLeftShuffle) {
26885 LMask = cast<ShuffleVectorSDNode>(Val: L.getNode())->getMask();
26886 L = L.getOperand(i: 0);
26887 }
26888 SDValue R = Shuffles[Right];
26889 ArrayRef<int> RMask;
26890 bool IsRightShuffle = R.getOpcode() == ISD::VECTOR_SHUFFLE &&
26891 R.use_empty() && R.getOperand(i: 1).isUndef() &&
26892 R.getOperand(i: 0).getValueType() == R.getValueType();
26893 if (IsRightShuffle) {
26894 RMask = cast<ShuffleVectorSDNode>(Val: R.getNode())->getMask();
26895 R = R.getOperand(i: 0);
26896 }
26897 for (unsigned I = 0; I != NumElems; ++I) {
26898 if (VectorMask[I] == Left) {
26899 Mask[I] = I;
26900 if (IsLeftShuffle)
26901 Mask[I] = LMask[I];
26902 VectorMask[I] = In;
26903 } else if (VectorMask[I] == Right) {
26904 Mask[I] = I + NumElems;
26905 if (IsRightShuffle)
26906 Mask[I] = RMask[I] + NumElems;
26907 VectorMask[I] = In;
26908 }
26909 }
26910
26911 Shuffles[In] = DAG.getVectorShuffle(VT, dl: DL, N1: L, N2: R, Mask);
26912 }
26913 }
26914 return Shuffles[0];
26915}
26916
26917// Try to turn a build vector of zero/sign extends of extract vector elts into
26918// a vector zero/sign extend and possibly an extract subvector.
26919// TODO: Allow undef elements?
26920SDValue DAGCombiner::convertBuildVecExtToExt(SDNode *N) {
26921 if (LegalOperations)
26922 return SDValue();
26923
26924 EVT VT = N->getValueType(ResNo: 0);
26925
26926 bool FoundZeroExtend = false;
26927 bool FoundSignExtend = false;
26928 SDValue Op0 = N->getOperand(Num: 0);
26929 auto checkElem = [&](SDValue Op) -> int64_t {
26930 unsigned Opc = Op.getOpcode();
26931 FoundZeroExtend |= (Opc == ISD::ZERO_EXTEND);
26932 FoundSignExtend |= (Opc == ISD::SIGN_EXTEND);
26933 if ((Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND ||
26934 Opc == ISD::ANY_EXTEND) &&
26935 Op.getOperand(i: 0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
26936 Op0.getOperand(i: 0).getOperand(i: 0) == Op.getOperand(i: 0).getOperand(i: 0))
26937 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 0).getOperand(i: 1)))
26938 return C->getZExtValue();
26939 return -1;
26940 };
26941
26942 // Make sure the first element matches
26943 // (zext (extract_vector_elt X, C))
26944 // Offset must be a constant multiple of the
26945 // known-minimum vector length of the result type.
26946 int64_t Offset = checkElem(Op0);
26947 if (Offset < 0 || (Offset % VT.getVectorNumElements()) != 0)
26948 return SDValue();
26949
26950 unsigned NumElems = N->getNumOperands();
26951 SDValue In = Op0.getOperand(i: 0).getOperand(i: 0);
26952 EVT InSVT = In.getValueType().getScalarType();
26953 EVT InVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: InSVT, NumElements: NumElems);
26954
26955 // Don't create an illegal input type after type legalization.
26956 if (LegalTypes && !TLI.isTypeLegal(VT: InVT))
26957 return SDValue();
26958
26959 // Ensure all the elements come from the same vector and are adjacent.
26960 for (unsigned i = 1; i != NumElems; ++i) {
26961 if ((Offset + i) != checkElem(N->getOperand(Num: i)))
26962 return SDValue();
26963 }
26964
26965 // Can't mix zero and sign extends in the same build_vector.
26966 if (FoundZeroExtend && FoundSignExtend)
26967 return SDValue();
26968
26969 unsigned ExtOpc = ISD::ANY_EXTEND;
26970 if (FoundSignExtend)
26971 ExtOpc = ISD::SIGN_EXTEND;
26972 else if (FoundZeroExtend)
26973 ExtOpc = ISD::ZERO_EXTEND;
26974
26975 SDLoc DL(N);
26976 In = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: InVT, N1: In,
26977 N2: Op0.getOperand(i: 0).getOperand(i: 1));
26978 return DAG.getNode(Opcode: ExtOpc, DL, VT, Operand: In);
26979}
26980
26981// If this is a very simple BUILD_VECTOR with first element being a ZERO_EXTEND,
26982// and all other elements being constant zero's, granularize the BUILD_VECTOR's
26983// element width, absorbing the ZERO_EXTEND, turning it into a constant zero op.
26984// This patten can appear during legalization.
26985//
26986// NOTE: This can be generalized to allow more than a single
26987// non-constant-zero op, UNDEF's, and to be KnownBits-based,
26988SDValue DAGCombiner::convertBuildVecZextToBuildVecWithZeros(SDNode *N) {
26989 // Don't run this after legalization. Targets may have other preferences.
26990 if (Level >= AfterLegalizeDAG)
26991 return SDValue();
26992
26993 // FIXME: support big-endian.
26994 if (DAG.getDataLayout().isBigEndian())
26995 return SDValue();
26996
26997 EVT VT = N->getValueType(ResNo: 0);
26998 EVT OpVT = N->getOperand(Num: 0).getValueType();
26999 assert(!VT.isScalableVector() && "Encountered scalable BUILD_VECTOR?");
27000
27001 EVT OpIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OpVT.getSizeInBits());
27002
27003 if (!TLI.isTypeLegal(VT: OpIntVT) ||
27004 (LegalOperations && !TLI.isOperationLegalOrCustom(Op: ISD::BITCAST, VT: OpIntVT)))
27005 return SDValue();
27006
27007 unsigned EltBitwidth = VT.getScalarSizeInBits();
27008 // NOTE: the actual width of operands may be wider than that!
27009
27010 // Analyze all operands of this BUILD_VECTOR. What is the largest number of
27011 // active bits they all have? We'll want to truncate them all to that width.
27012 unsigned ActiveBits = 0;
27013 APInt KnownZeroOps(VT.getVectorNumElements(), 0);
27014 for (auto I : enumerate(First: N->ops())) {
27015 SDValue Op = I.value();
27016 // FIXME: support UNDEF elements?
27017 if (auto *Cst = dyn_cast<ConstantSDNode>(Val&: Op)) {
27018 unsigned OpActiveBits =
27019 Cst->getAPIntValue().trunc(width: EltBitwidth).getActiveBits();
27020 if (OpActiveBits == 0) {
27021 KnownZeroOps.setBit(I.index());
27022 continue;
27023 }
27024 // Profitability check: don't allow non-zero constant operands.
27025 return SDValue();
27026 }
27027 // Profitability check: there must only be a single non-zero operand,
27028 // and it must be the first operand of the BUILD_VECTOR.
27029 if (I.index() != 0)
27030 return SDValue();
27031 // The operand must be a zero-extension itself.
27032 // FIXME: this could be generalized to known leading zeros check.
27033 if (Op.getOpcode() != ISD::ZERO_EXTEND)
27034 return SDValue();
27035 unsigned CurrActiveBits =
27036 Op.getOperand(i: 0).getValueSizeInBits().getFixedValue();
27037 assert(!ActiveBits && "Already encountered non-constant-zero operand?");
27038 ActiveBits = CurrActiveBits;
27039 // We want to at least halve the element size.
27040 if (2 * ActiveBits > EltBitwidth)
27041 return SDValue();
27042 }
27043
27044 // This BUILD_VECTOR must have at least one non-constant-zero operand.
27045 if (ActiveBits == 0)
27046 return SDValue();
27047
27048 // We have EltBitwidth bits, the *minimal* chunk size is ActiveBits,
27049 // into how many chunks can we split our element width?
27050 EVT NewScalarIntVT, NewIntVT;
27051 std::optional<unsigned> Factor;
27052 // We can split the element into at least two chunks, but not into more
27053 // than |_ EltBitwidth / ActiveBits _| chunks. Find a largest split factor
27054 // for which the element width is a multiple of it,
27055 // and the resulting types/operations on that chunk width are legal.
27056 assert(2 * ActiveBits <= EltBitwidth &&
27057 "We know that half or less bits of the element are active.");
27058 for (unsigned Scale = EltBitwidth / ActiveBits; Scale >= 2; --Scale) {
27059 if (EltBitwidth % Scale != 0)
27060 continue;
27061 unsigned ChunkBitwidth = EltBitwidth / Scale;
27062 assert(ChunkBitwidth >= ActiveBits && "As per starting point.");
27063 NewScalarIntVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ChunkBitwidth);
27064 NewIntVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: NewScalarIntVT,
27065 NumElements: Scale * N->getNumOperands());
27066 if (!TLI.isTypeLegal(VT: NewScalarIntVT) || !TLI.isTypeLegal(VT: NewIntVT) ||
27067 (LegalOperations &&
27068 !(TLI.isOperationLegalOrCustom(Op: ISD::TRUNCATE, VT: NewScalarIntVT) &&
27069 TLI.isOperationLegalOrCustom(Op: ISD::BUILD_VECTOR, VT: NewIntVT))))
27070 continue;
27071 Factor = Scale;
27072 break;
27073 }
27074 if (!Factor)
27075 return SDValue();
27076
27077 SDLoc DL(N);
27078 SDValue ZeroOp = DAG.getConstant(Val: 0, DL, VT: NewScalarIntVT);
27079
27080 // Recreate the BUILD_VECTOR, with elements now being Factor times smaller.
27081 SmallVector<SDValue, 16> NewOps;
27082 NewOps.reserve(N: NewIntVT.getVectorNumElements());
27083 for (auto I : enumerate(First: N->ops())) {
27084 SDValue Op = I.value();
27085 assert(!Op.isUndef() && "FIXME: after allowing UNDEF's, handle them here.");
27086 unsigned SrcOpIdx = I.index();
27087 if (KnownZeroOps[SrcOpIdx]) {
27088 NewOps.append(NumInputs: *Factor, Elt: ZeroOp);
27089 continue;
27090 }
27091 Op = DAG.getBitcast(VT: OpIntVT, V: Op);
27092 Op = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: NewScalarIntVT, Operand: Op);
27093 NewOps.emplace_back(Args&: Op);
27094 NewOps.append(NumInputs: *Factor - 1, Elt: ZeroOp);
27095 }
27096 assert(NewOps.size() == NewIntVT.getVectorNumElements());
27097 SDValue NewBV = DAG.getBuildVector(VT: NewIntVT, DL, Ops: NewOps);
27098 NewBV = DAG.getBitcast(VT, V: NewBV);
27099 return NewBV;
27100}
27101
27102SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
27103 EVT VT = N->getValueType(ResNo: 0);
27104
27105 // A vector built entirely of undefs is undef.
27106 if (ISD::allOperandsUndef(N))
27107 return DAG.getUNDEF(VT);
27108
27109 // If this is a splat of a bitcast from another vector, change to a
27110 // concat_vector.
27111 // For example:
27112 // (build_vector (i64 (bitcast (v2i32 X))), (i64 (bitcast (v2i32 X)))) ->
27113 // (v2i64 (bitcast (concat_vectors (v2i32 X), (v2i32 X))))
27114 //
27115 // If X is a build_vector itself, the concat can become a larger build_vector.
27116 // TODO: Maybe this is useful for non-splat too?
27117 if (!LegalOperations) {
27118 SDValue Splat = cast<BuildVectorSDNode>(Val: N)->getSplatValue();
27119 // Only change build_vector to a concat_vector if the splat value type is
27120 // same as the vector element type.
27121 if (Splat && Splat.getValueType() == VT.getVectorElementType()) {
27122 Splat = peekThroughBitcasts(V: Splat);
27123 EVT SrcVT = Splat.getValueType();
27124 if (SrcVT.isVector()) {
27125 unsigned NumElts = N->getNumOperands() * SrcVT.getVectorNumElements();
27126 EVT NewVT = EVT::getVectorVT(Context&: *DAG.getContext(),
27127 VT: SrcVT.getVectorElementType(), NumElements: NumElts);
27128 if (!LegalTypes || TLI.isTypeLegal(VT: NewVT)) {
27129 SmallVector<SDValue, 8> Ops(N->getNumOperands(), Splat);
27130 SDValue Concat =
27131 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT: NewVT, Ops);
27132 return DAG.getBitcast(VT, V: Concat);
27133 }
27134 }
27135 }
27136 }
27137
27138 // Check if we can express BUILD VECTOR via subvector extract.
27139 if (!LegalTypes && (N->getNumOperands() > 1)) {
27140 SDValue Op0 = N->getOperand(Num: 0);
27141 auto checkElem = [&](SDValue Op) -> uint64_t {
27142 if ((Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT) &&
27143 (Op0.getOperand(i: 0) == Op.getOperand(i: 0)))
27144 if (auto CNode = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1)))
27145 return CNode->getZExtValue();
27146 return -1;
27147 };
27148
27149 int Offset = checkElem(Op0);
27150 for (unsigned i = 0; i < N->getNumOperands(); ++i) {
27151 if (Offset + i != checkElem(N->getOperand(Num: i))) {
27152 Offset = -1;
27153 break;
27154 }
27155 }
27156
27157 if ((Offset == 0) &&
27158 (Op0.getOperand(i: 0).getValueType() == N->getValueType(ResNo: 0)))
27159 return Op0.getOperand(i: 0);
27160 if ((Offset != -1) &&
27161 ((Offset % N->getValueType(ResNo: 0).getVectorNumElements()) ==
27162 0)) // IDX must be multiple of output size.
27163 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
27164 N1: Op0.getOperand(i: 0), N2: Op0.getOperand(i: 1));
27165 }
27166
27167 if (SDValue V = convertBuildVecExtToExt(N))
27168 return V;
27169
27170 if (SDValue V = convertBuildVecZextToBuildVecWithZeros(N))
27171 return V;
27172
27173 if (SDValue V = reduceBuildVecExtToExtBuildVec(N))
27174 return V;
27175
27176 if (SDValue V = reduceBuildVecTruncToBitCast(N))
27177 return V;
27178
27179 if (SDValue V = reduceBuildVecToShuffle(N))
27180 return V;
27181
27182 // A splat of a single element is a SPLAT_VECTOR if supported on the target.
27183 // Do this late as some of the above may replace the splat.
27184 if (TLI.getOperationAction(Op: ISD::SPLAT_VECTOR, VT) != TargetLowering::Expand)
27185 if (SDValue V = cast<BuildVectorSDNode>(Val: N)->getSplatValue()) {
27186 assert(!V.isUndef() && "Splat of undef should have been handled earlier");
27187 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: SDLoc(N), VT, Operand: V);
27188 }
27189
27190 return SDValue();
27191}
27192
27193static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG) {
27194 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27195 EVT OpVT = N->getOperand(Num: 0).getValueType();
27196
27197 // If the operands are legal vectors, leave them alone.
27198 if (TLI.isTypeLegal(VT: OpVT) || OpVT.isScalableVector())
27199 return SDValue();
27200
27201 SDLoc DL(N);
27202 EVT VT = N->getValueType(ResNo: 0);
27203 SmallVector<SDValue, 8> Ops;
27204 EVT SVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: OpVT.getSizeInBits());
27205
27206 // Keep track of what we encounter.
27207 EVT AnyFPVT;
27208
27209 for (const SDValue &Op : N->ops()) {
27210 if (ISD::BITCAST == Op.getOpcode() &&
27211 !Op.getOperand(i: 0).getValueType().isVector())
27212 Ops.push_back(Elt: Op.getOperand(i: 0));
27213 else if (Op.isUndef())
27214 Ops.push_back(Elt: DAG.getNode(Opcode: Op.getOpcode(), DL, VT: SVT));
27215 else
27216 return SDValue();
27217
27218 // Note whether we encounter an integer or floating point scalar.
27219 // If it's neither, bail out, it could be something weird like x86mmx.
27220 EVT LastOpVT = Ops.back().getValueType();
27221 if (LastOpVT.isFloatingPoint())
27222 AnyFPVT = LastOpVT;
27223 else if (!LastOpVT.isInteger())
27224 return SDValue();
27225 }
27226
27227 // If any of the operands is a floating point scalar bitcast to a vector,
27228 // use floating point types throughout, and bitcast everything.
27229 // Replace UNDEFs by another scalar UNDEF node, of the final desired type.
27230 if (AnyFPVT != EVT()) {
27231 SVT = AnyFPVT;
27232 for (SDValue &Op : Ops) {
27233 if (Op.getValueType() == SVT)
27234 continue;
27235 if (Op.isUndef())
27236 Op = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: SVT);
27237 else
27238 Op = DAG.getBitcast(VT: SVT, V: Op);
27239 }
27240 }
27241
27242 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SVT,
27243 NumElements: VT.getSizeInBits() / SVT.getSizeInBits());
27244 return DAG.getBitcast(VT, V: DAG.getBuildVector(VT: VecVT, DL, Ops));
27245}
27246
27247// Attempt to merge nested concat_vectors/undefs.
27248// Fold concat_vectors(concat_vectors(x,y,z,w),u,u,concat_vectors(a,b,c,d))
27249// --> concat_vectors(x,y,z,w,u,u,u,u,u,u,u,u,a,b,c,d)
27250static SDValue combineConcatVectorOfConcatVectors(SDNode *N,
27251 SelectionDAG &DAG) {
27252 EVT VT = N->getValueType(ResNo: 0);
27253
27254 // Ensure we're concatenating UNDEF and CONCAT_VECTORS nodes of similar types.
27255 EVT SubVT;
27256 SDValue FirstConcat;
27257 for (const SDValue &Op : N->ops()) {
27258 if (Op.isUndef())
27259 continue;
27260 if (Op.getOpcode() != ISD::CONCAT_VECTORS)
27261 return SDValue();
27262 if (!FirstConcat) {
27263 SubVT = Op.getOperand(i: 0).getValueType();
27264 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT: SubVT))
27265 return SDValue();
27266 FirstConcat = Op;
27267 continue;
27268 }
27269 if (SubVT != Op.getOperand(i: 0).getValueType())
27270 return SDValue();
27271 }
27272 assert(FirstConcat && "Concat of all-undefs found");
27273
27274 SmallVector<SDValue> ConcatOps;
27275 for (const SDValue &Op : N->ops()) {
27276 if (Op.isUndef()) {
27277 ConcatOps.append(NumInputs: FirstConcat->getNumOperands(),
27278 Elt: DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(), VT: SubVT));
27279 continue;
27280 }
27281 ConcatOps.append(in_start: Op->op_begin(), in_end: Op->op_end());
27282 }
27283 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT, Ops: ConcatOps);
27284}
27285
27286// Check to see if this is a CONCAT_VECTORS of a bunch of EXTRACT_SUBVECTOR
27287// operations. If so, and if the EXTRACT_SUBVECTOR vector inputs come from at
27288// most two distinct vectors the same size as the result, attempt to turn this
27289// into a legal shuffle.
27290static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG) {
27291 EVT VT = N->getValueType(ResNo: 0);
27292 EVT OpVT = N->getOperand(Num: 0).getValueType();
27293
27294 // We currently can't generate an appropriate shuffle for a scalable vector.
27295 if (VT.isScalableVector())
27296 return SDValue();
27297
27298 int NumElts = VT.getVectorNumElements();
27299 int NumOpElts = OpVT.getVectorNumElements();
27300
27301 SDValue SV0 = DAG.getPOISON(VT), SV1 = DAG.getPOISON(VT);
27302 SmallVector<int, 8> Mask;
27303
27304 for (SDValue Op : N->ops()) {
27305 Op = peekThroughBitcasts(V: Op);
27306
27307 // UNDEF nodes convert to UNDEF shuffle mask values.
27308 if (Op.isUndef()) {
27309 Mask.append(NumInputs: (unsigned)NumOpElts, Elt: -1);
27310 continue;
27311 }
27312
27313 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR)
27314 return SDValue();
27315
27316 // What vector are we extracting the subvector from and at what index?
27317 SDValue ExtVec = Op.getOperand(i: 0);
27318 int ExtIdx = Op.getConstantOperandVal(i: 1);
27319
27320 // We want the EVT of the original extraction to correctly scale the
27321 // extraction index.
27322 EVT ExtVT = ExtVec.getValueType();
27323 ExtVec = peekThroughBitcasts(V: ExtVec);
27324
27325 // UNDEF nodes convert to UNDEF shuffle mask values.
27326 if (ExtVec.isUndef()) {
27327 Mask.append(NumInputs: (unsigned)NumOpElts, Elt: -1);
27328 continue;
27329 }
27330
27331 // Ensure that we are extracting a subvector from a vector the same
27332 // size as the result.
27333 if (ExtVT.getSizeInBits() != VT.getSizeInBits())
27334 return SDValue();
27335
27336 // Scale the subvector index to account for any bitcast.
27337 int NumExtElts = ExtVT.getVectorNumElements();
27338 if (0 == (NumExtElts % NumElts))
27339 ExtIdx /= (NumExtElts / NumElts);
27340 else if (0 == (NumElts % NumExtElts))
27341 ExtIdx *= (NumElts / NumExtElts);
27342 else
27343 return SDValue();
27344
27345 // At most we can reference 2 inputs in the final shuffle.
27346 if (SV0.isUndef() || SV0 == ExtVec) {
27347 SV0 = ExtVec;
27348 for (int i = 0; i != NumOpElts; ++i)
27349 Mask.push_back(Elt: i + ExtIdx);
27350 } else if (SV1.isUndef() || SV1 == ExtVec) {
27351 SV1 = ExtVec;
27352 for (int i = 0; i != NumOpElts; ++i)
27353 Mask.push_back(Elt: i + ExtIdx + NumElts);
27354 } else {
27355 return SDValue();
27356 }
27357 }
27358
27359 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27360 return TLI.buildLegalVectorShuffle(VT, DL: SDLoc(N), N0: DAG.getBitcast(VT, V: SV0),
27361 N1: DAG.getBitcast(VT, V: SV1), Mask, DAG);
27362}
27363
27364static SDValue combineConcatVectorOfCasts(SDNode *N, SelectionDAG &DAG) {
27365 unsigned CastOpcode = N->getOperand(Num: 0).getOpcode();
27366 switch (CastOpcode) {
27367 case ISD::SINT_TO_FP:
27368 case ISD::UINT_TO_FP:
27369 case ISD::FP_TO_SINT:
27370 case ISD::FP_TO_UINT:
27371 // TODO: Allow more opcodes?
27372 // case ISD::BITCAST:
27373 // case ISD::TRUNCATE:
27374 // case ISD::ZERO_EXTEND:
27375 // case ISD::SIGN_EXTEND:
27376 // case ISD::FP_EXTEND:
27377 break;
27378 default:
27379 return SDValue();
27380 }
27381
27382 EVT SrcVT = N->getOperand(Num: 0).getOperand(i: 0).getValueType();
27383 if (!SrcVT.isVector())
27384 return SDValue();
27385
27386 // All operands of the concat must be the same kind of cast from the same
27387 // source type.
27388 SmallVector<SDValue, 4> SrcOps;
27389 for (SDValue Op : N->ops()) {
27390 if (Op.getOpcode() != CastOpcode || !Op.hasOneUse() ||
27391 Op.getOperand(i: 0).getValueType() != SrcVT)
27392 return SDValue();
27393 SrcOps.push_back(Elt: Op.getOperand(i: 0));
27394 }
27395
27396 // The wider cast must be supported by the target. This is unusual because
27397 // the operation support type parameter depends on the opcode. In addition,
27398 // check the other type in the cast to make sure this is really legal.
27399 EVT VT = N->getValueType(ResNo: 0);
27400 ElementCount NumElts = SrcVT.getVectorElementCount() * N->getNumOperands();
27401 EVT ConcatSrcVT = SrcVT.changeVectorElementCount(Context&: *DAG.getContext(), EC: NumElts);
27402 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27403 switch (CastOpcode) {
27404 case ISD::SINT_TO_FP:
27405 case ISD::UINT_TO_FP:
27406 if (!TLI.isOperationLegalOrCustom(Op: CastOpcode, VT: ConcatSrcVT) ||
27407 !TLI.isTypeLegal(VT))
27408 return SDValue();
27409 break;
27410 case ISD::FP_TO_SINT:
27411 case ISD::FP_TO_UINT:
27412 if (!TLI.isOperationLegalOrCustom(Op: CastOpcode, VT) ||
27413 !TLI.isTypeLegal(VT: ConcatSrcVT))
27414 return SDValue();
27415 break;
27416 default:
27417 llvm_unreachable("Unexpected cast opcode");
27418 }
27419
27420 // concat (cast X), (cast Y)... -> cast (concat X, Y...)
27421 SDLoc DL(N);
27422 SDValue NewConcat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatSrcVT, Ops: SrcOps);
27423 return DAG.getNode(Opcode: CastOpcode, DL, VT, Operand: NewConcat);
27424}
27425
27426// See if this is a simple CONCAT_VECTORS with no UNDEF operands, and if one of
27427// the operands is a SHUFFLE_VECTOR, and all other operands are also operands
27428// to that SHUFFLE_VECTOR, create wider SHUFFLE_VECTOR.
27429static SDValue combineConcatVectorOfShuffleAndItsOperands(
27430 SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalTypes,
27431 bool LegalOperations) {
27432 EVT VT = N->getValueType(ResNo: 0);
27433 EVT OpVT = N->getOperand(Num: 0).getValueType();
27434 if (VT.isScalableVector())
27435 return SDValue();
27436
27437 // For now, only allow simple 2-operand concatenations.
27438 if (N->getNumOperands() != 2)
27439 return SDValue();
27440
27441 // Don't create illegal types/shuffles when not allowed to.
27442 if ((LegalTypes && !TLI.isTypeLegal(VT)) ||
27443 (LegalOperations &&
27444 !TLI.isOperationLegalOrCustom(Op: ISD::VECTOR_SHUFFLE, VT)))
27445 return SDValue();
27446
27447 // Analyze all of the operands of the CONCAT_VECTORS. Out of all of them,
27448 // we want to find one that is: (1) a SHUFFLE_VECTOR (2) only used by us,
27449 // and (3) all operands of CONCAT_VECTORS must be either that SHUFFLE_VECTOR,
27450 // or one of the operands of that SHUFFLE_VECTOR (but not UNDEF!).
27451 // (4) and for now, the SHUFFLE_VECTOR must be unary.
27452 ShuffleVectorSDNode *SVN = nullptr;
27453 for (SDValue Op : N->ops()) {
27454 if (auto *CurSVN = dyn_cast<ShuffleVectorSDNode>(Val&: Op);
27455 CurSVN && CurSVN->getOperand(Num: 1).isUndef() && N->isOnlyUserOf(N: CurSVN) &&
27456 all_of(Range: N->ops(), P: [CurSVN](SDValue Op) {
27457 // FIXME: can we allow UNDEF operands?
27458 return !Op.isUndef() &&
27459 (Op.getNode() == CurSVN || is_contained(Range: CurSVN->ops(), Element: Op));
27460 })) {
27461 SVN = CurSVN;
27462 break;
27463 }
27464 }
27465 if (!SVN)
27466 return SDValue();
27467
27468 // We are going to pad the shuffle operands, so any indice, that was picking
27469 // from the second operand, must be adjusted.
27470 SmallVector<int, 16> AdjustedMask(SVN->getMask());
27471 assert(SVN->getOperand(1).isUndef() && "Expected unary shuffle!");
27472
27473 // Identity masks for the operands of the (padded) shuffle.
27474 SmallVector<int, 32> IdentityMask(2 * OpVT.getVectorNumElements());
27475 MutableArrayRef<int> FirstShufOpIdentityMask =
27476 MutableArrayRef<int>(IdentityMask)
27477 .take_front(N: OpVT.getVectorNumElements());
27478 MutableArrayRef<int> SecondShufOpIdentityMask =
27479 MutableArrayRef<int>(IdentityMask).take_back(N: OpVT.getVectorNumElements());
27480 std::iota(first: FirstShufOpIdentityMask.begin(), last: FirstShufOpIdentityMask.end(), value: 0);
27481 std::iota(first: SecondShufOpIdentityMask.begin(), last: SecondShufOpIdentityMask.end(),
27482 value: VT.getVectorNumElements());
27483
27484 // New combined shuffle mask.
27485 SmallVector<int, 32> Mask;
27486 Mask.reserve(N: VT.getVectorNumElements());
27487 for (SDValue Op : N->ops()) {
27488 assert(!Op.isUndef() && "Not expecting to concatenate UNDEF.");
27489 if (Op.getNode() == SVN) {
27490 append_range(C&: Mask, R&: AdjustedMask);
27491 continue;
27492 }
27493 if (Op == SVN->getOperand(Num: 0)) {
27494 append_range(C&: Mask, R&: FirstShufOpIdentityMask);
27495 continue;
27496 }
27497 if (Op == SVN->getOperand(Num: 1)) {
27498 append_range(C&: Mask, R&: SecondShufOpIdentityMask);
27499 continue;
27500 }
27501 llvm_unreachable("Unexpected operand!");
27502 }
27503
27504 // Don't create illegal shuffle masks.
27505 if (!TLI.isShuffleMaskLegal(Mask, VT))
27506 return SDValue();
27507
27508 // Pad the shuffle operands with poison.
27509 SDLoc dl(N);
27510 std::array<SDValue, 2> ShufOps;
27511 for (auto I : zip(t: SVN->ops(), u&: ShufOps)) {
27512 SDValue ShufOp = std::get<0>(t&: I);
27513 SDValue &NewShufOp = std::get<1>(t&: I);
27514 if (ShufOp.isUndef())
27515 NewShufOp = DAG.getNode(Opcode: ShufOp.getOpcode(), DL: SDLoc(), VT);
27516 else {
27517 SmallVector<SDValue, 2> ShufOpParts(N->getNumOperands(),
27518 DAG.getPOISON(VT: OpVT));
27519 ShufOpParts[0] = ShufOp;
27520 NewShufOp = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT, Ops: ShufOpParts);
27521 }
27522 }
27523 // Finally, create the new wide shuffle.
27524 return DAG.getVectorShuffle(VT, dl, N1: ShufOps[0], N2: ShufOps[1], Mask);
27525}
27526
27527// concat(shuffle(loadA, loadB, mask0), shuffle(loadA, loadB, mask1))
27528// -> shuffle(loadAB, poison, concat(mask0, mask1))
27529// only if loadA and loadB can be proven consecutive.
27530static SDValue combineConcatVectorOfShuffles(SDNode *N, SelectionDAG &DAG,
27531 const TargetLowering &TLI,
27532 bool LegalOperations) {
27533 SDValue A, B;
27534 ArrayRef<int> M0, M1;
27535 if (!sd_match(N,
27536 P: m_Node(Opcode: ISD::CONCAT_VECTORS,
27537 preds: m_OneUse(P: m_Shuffle(v1: m_NUses<2>(P: m_Value(N&: A)),
27538 v2: m_NUses<2>(P: m_Value(N&: B)), mask: m_Mask(M0))),
27539 preds: m_OneUse(P: m_Shuffle(v1: m_Deferred(V&: A), v2: m_Deferred(V&: B),
27540 mask: m_Mask(M1))))))
27541 return SDValue();
27542 auto *LoadA = dyn_cast<LoadSDNode>(Val: A.getNode());
27543 auto *LoadB = dyn_cast<LoadSDNode>(Val: B.getNode());
27544 if (!LoadA || !LoadB || !ISD::isNON_EXTLoad(N: LoadA) ||
27545 !ISD::isNON_EXTLoad(N: LoadB))
27546 return SDValue();
27547
27548 // Check if the address spaces of both loads are the same.
27549 if (LoadA->getAddressSpace() != LoadB->getAddressSpace())
27550 return SDValue();
27551
27552 // Check if the loads are consecutive.
27553 LoadSDNode *Base = nullptr;
27554 if (DAG.areNonVolatileConsecutiveLoads(
27555 LD: LoadB, Base: LoadA, Bytes: LoadB->getMemoryVT().getStoreSize(), /*Dist=*/1)) {
27556 Base = LoadA;
27557 } else if (DAG.areNonVolatileConsecutiveLoads(
27558 LD: LoadA, Base: LoadB, Bytes: LoadA->getMemoryVT().getStoreSize(),
27559 /*Dist=*/1)) {
27560 Base = LoadB;
27561 } else {
27562 return SDValue(); // not adjacent
27563 }
27564
27565 unsigned Fast = 0;
27566 Align NewAlign = Base->getAlign();
27567 EVT WideVT =
27568 LoadA->getMemoryVT().getDoubleNumVectorElementsVT(Context&: *DAG.getContext());
27569 if (!TLI.allowsMemoryAccess(Context&: *DAG.getContext(), DL: DAG.getDataLayout(), VT: WideVT,
27570 AddrSpace: Base->getAddressSpace(), Alignment: NewAlign,
27571 Flags: Base->getMemOperand()->getFlags(), Fast: &Fast) ||
27572 !Fast)
27573 return SDValue();
27574
27575 // Create a shuffle of the wide load.
27576 SmallVector<int, 32> Mask;
27577 if (Base == LoadA) {
27578 llvm::append_range(C&: Mask, R&: M0);
27579 llvm::append_range(C&: Mask, R&: M1);
27580 } else {
27581 SmallVector<int, 16> C0(M0), C1(M1);
27582 ShuffleVectorSDNode::commuteMask(Mask: C0);
27583 ShuffleVectorSDNode::commuteMask(Mask: C1);
27584 llvm::append_range(C&: Mask, R&: C0);
27585 llvm::append_range(C&: Mask, R&: C1);
27586 }
27587
27588 // Check if the wide load, new shuffle and it's mask is legal.
27589 if (LegalOperations &&
27590 (!TLI.isOperationLegal(Op: ISD::LOAD, VT: WideVT) ||
27591 !TLI.isOperationLegalOrCustom(Op: ISD::VECTOR_SHUFFLE, VT: WideVT) ||
27592 !TLI.isShuffleMaskLegal(Mask, WideVT)))
27593 return SDValue();
27594
27595 // Create a wide load of twice the size of the original load.
27596 MachineFunction &MF = DAG.getMachineFunction();
27597 MachineMemOperand *WideMMO = MF.getMachineMemOperand(
27598 MMO: Base->getMemOperand(), /*Offset=*/0, Size: WideVT.getStoreSize());
27599 SDValue WideLoad = DAG.getLoad(VT: WideVT, dl: SDLoc(N), Chain: Base->getChain(),
27600 Ptr: Base->getBasePtr(), MMO: WideMMO);
27601 // Redirect old chain users to the new chain.
27602 DAG.makeEquivalentMemoryOrdering(OldLoad: LoadA, NewMemOp: WideLoad);
27603 DAG.makeEquivalentMemoryOrdering(OldLoad: LoadB, NewMemOp: WideLoad);
27604
27605 // Create a new shuffle with the new mask.
27606 return DAG.getVectorShuffle(VT: WideVT, dl: SDLoc(N), N1: WideLoad, N2: DAG.getPOISON(VT: WideVT),
27607 Mask);
27608}
27609
27610static SDValue combineConcatVectorOfSplats(SDNode *N, SelectionDAG &DAG,
27611 const TargetLowering &TLI,
27612 bool LegalTypes,
27613 bool LegalOperations) {
27614 EVT VT = N->getValueType(ResNo: 0);
27615
27616 // Post-legalization we can only create wider SPLAT_VECTOR operations if both
27617 // the type and operation is legal. The Hexagon target has custom
27618 // legalization for SPLAT_VECTOR that splits the operation into two parts and
27619 // concatenates them. Therefore, custom lowering must also be rejected in
27620 // order to avoid an infinite loop.
27621 if ((LegalTypes && !TLI.isTypeLegal(VT)) ||
27622 (LegalOperations && !TLI.isOperationLegal(Op: ISD::SPLAT_VECTOR, VT)))
27623 return SDValue();
27624
27625 SDValue Op0 = N->getOperand(Num: 0);
27626 if (!llvm::all_equal(Range: N->op_values()) || Op0.getOpcode() != ISD::SPLAT_VECTOR)
27627 return SDValue();
27628
27629 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: SDLoc(N), VT, Operand: Op0.getOperand(i: 0));
27630}
27631
27632SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) {
27633 // If we only have one input vector, we don't need to do any concatenation.
27634 if (N->getNumOperands() == 1)
27635 return N->getOperand(Num: 0);
27636
27637 // Check if all of the operands are undefs.
27638 EVT VT = N->getValueType(ResNo: 0);
27639 if (ISD::allOperandsUndef(N))
27640 return DAG.getUNDEF(VT);
27641
27642 // Optimize concat_vectors where all but the first of the vectors are undef.
27643 if (all_of(Range: drop_begin(RangeOrContainer: N->ops()),
27644 P: [](const SDValue &Op) { return Op.isUndef(); })) {
27645 SDValue In = N->getOperand(Num: 0);
27646 assert(In.getValueType().isVector() && "Must concat vectors");
27647
27648 // If the input is a concat_vectors, just make a larger concat by padding
27649 // with smaller undefs.
27650 //
27651 // Legalizing in AArch64TargetLowering::LowerCONCAT_VECTORS() and combining
27652 // here could cause an infinite loop. That legalizing happens when LegalDAG
27653 // is true and input of AArch64TargetLowering::LowerCONCAT_VECTORS() is
27654 // scalable.
27655 if (In.getOpcode() == ISD::CONCAT_VECTORS && In.hasOneUse() &&
27656 !(LegalDAG && In.getValueType().isScalableVector())) {
27657 unsigned NumOps = N->getNumOperands() * In.getNumOperands();
27658 SmallVector<SDValue, 4> Ops(In->ops());
27659 Ops.resize(N: NumOps, NV: DAG.getPOISON(VT: Ops[0].getValueType()));
27660 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT, Ops);
27661 }
27662
27663 SDValue Scalar = peekThroughOneUseBitcasts(V: In);
27664
27665 // concat_vectors(scalar_to_vector(scalar), undef) ->
27666 // scalar_to_vector(scalar)
27667 if (!LegalOperations && Scalar.getOpcode() == ISD::SCALAR_TO_VECTOR &&
27668 Scalar.hasOneUse()) {
27669 EVT SVT = Scalar.getValueType().getVectorElementType();
27670 if (SVT == Scalar.getOperand(i: 0).getValueType())
27671 Scalar = Scalar.getOperand(i: 0);
27672 }
27673
27674 // concat_vectors(scalar, undef) -> scalar_to_vector(scalar)
27675 if (!Scalar.getValueType().isVector() && In.hasOneUse()) {
27676 // If the bitcast type isn't legal, it might be a trunc of a legal type;
27677 // look through the trunc so we can still do the transform:
27678 // concat_vectors(trunc(scalar), undef) -> scalar_to_vector(scalar)
27679 // However, this is only equivalent on little-endian targets.
27680 if (Scalar->getOpcode() == ISD::TRUNCATE &&
27681 !TLI.isTypeLegal(VT: Scalar.getValueType()) &&
27682 TLI.isTypeLegal(VT: Scalar->getOperand(Num: 0).getValueType()) &&
27683 DAG.getDataLayout().isLittleEndian())
27684 Scalar = Scalar->getOperand(Num: 0);
27685
27686 EVT SclTy = Scalar.getValueType();
27687
27688 if (!SclTy.isFloatingPoint() && !SclTy.isInteger())
27689 return SDValue();
27690
27691 // Bail out if the vector size is not a multiple of the scalar size.
27692 if (VT.getSizeInBits() % SclTy.getSizeInBits())
27693 return SDValue();
27694
27695 unsigned VNTNumElms = VT.getSizeInBits() / SclTy.getSizeInBits();
27696 if (VNTNumElms < 2)
27697 return SDValue();
27698
27699 EVT NVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: SclTy, NumElements: VNTNumElms);
27700 if (!TLI.isTypeLegal(VT: NVT) || !TLI.isTypeLegal(VT: Scalar.getValueType()))
27701 return SDValue();
27702
27703 SDValue Res = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT: NVT, Operand: Scalar);
27704 return DAG.getBitcast(VT, V: Res);
27705 }
27706 }
27707
27708 // Fold any combination of BUILD_VECTOR or UNDEF nodes into one BUILD_VECTOR.
27709 // We have already tested above for an UNDEF only concatenation.
27710 // fold (concat_vectors (BUILD_VECTOR A, B, ...), (BUILD_VECTOR C, D, ...))
27711 // -> (BUILD_VECTOR A, B, ..., C, D, ...)
27712 auto IsBuildVectorOrUndef = [](const SDValue &Op) {
27713 return Op.isUndef() || ISD::BUILD_VECTOR == Op.getOpcode();
27714 };
27715 if (llvm::all_of(Range: N->ops(), P: IsBuildVectorOrUndef)) {
27716 SmallVector<SDValue, 8> Opnds;
27717 EVT SVT = VT.getScalarType();
27718
27719 EVT MinVT = SVT;
27720 if (!SVT.isFloatingPoint()) {
27721 // If BUILD_VECTOR are from built from integer, they may have different
27722 // operand types. Get the smallest type and truncate all operands to it.
27723 bool FoundMinVT = false;
27724 for (const SDValue &Op : N->ops())
27725 if (ISD::BUILD_VECTOR == Op.getOpcode()) {
27726 EVT OpSVT = Op.getOperand(i: 0).getValueType();
27727 MinVT = (!FoundMinVT || OpSVT.bitsLE(VT: MinVT)) ? OpSVT : MinVT;
27728 FoundMinVT = true;
27729 }
27730 assert(FoundMinVT && "Concat vector type mismatch");
27731 }
27732
27733 for (const SDValue &Op : N->ops()) {
27734 EVT OpVT = Op.getValueType();
27735 unsigned NumElts = OpVT.getVectorNumElements();
27736
27737 if (Op.isUndef())
27738 Opnds.append(NumInputs: NumElts, Elt: DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(), VT: MinVT));
27739
27740 if (ISD::BUILD_VECTOR == Op.getOpcode()) {
27741 if (SVT.isFloatingPoint()) {
27742 assert(SVT == OpVT.getScalarType() && "Concat vector type mismatch");
27743 Opnds.append(in_start: Op->op_begin(), in_end: Op->op_begin() + NumElts);
27744 } else {
27745 for (unsigned i = 0; i != NumElts; ++i)
27746 Opnds.push_back(
27747 Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT: MinVT, Operand: Op.getOperand(i)));
27748 }
27749 }
27750 }
27751
27752 assert(VT.getVectorNumElements() == Opnds.size() &&
27753 "Concat vector type mismatch");
27754 return DAG.getBuildVector(VT, DL: SDLoc(N), Ops: Opnds);
27755 }
27756
27757 if (SDValue V =
27758 combineConcatVectorOfSplats(N, DAG, TLI, LegalTypes, LegalOperations))
27759 return V;
27760
27761 // Fold CONCAT_VECTORS of only bitcast scalars (or undef) to BUILD_VECTOR.
27762 // FIXME: Add support for concat_vectors(bitcast(vec0),bitcast(vec1),...).
27763 if (SDValue V = combineConcatVectorOfScalars(N, DAG))
27764 return V;
27765
27766 if (Level <= AfterLegalizeVectorOps && TLI.isTypeLegal(VT)) {
27767 // Fold CONCAT_VECTORS of CONCAT_VECTORS (or undef) to VECTOR_SHUFFLE.
27768 if (SDValue V = combineConcatVectorOfConcatVectors(N, DAG))
27769 return V;
27770
27771 // Fold CONCAT_VECTORS of EXTRACT_SUBVECTOR (or undef) to VECTOR_SHUFFLE.
27772 if (SDValue V = combineConcatVectorOfExtracts(N, DAG))
27773 return V;
27774 }
27775
27776 if (SDValue V = combineConcatVectorOfCasts(N, DAG))
27777 return V;
27778
27779 if (SDValue V = combineConcatVectorOfShuffleAndItsOperands(
27780 N, DAG, TLI, LegalTypes, LegalOperations))
27781 return V;
27782
27783 if (SDValue V = combineConcatVectorOfShuffles(N, DAG, TLI, LegalOperations))
27784 return V;
27785
27786 // Type legalization of vectors and DAG canonicalization of SHUFFLE_VECTOR
27787 // nodes often generate nop CONCAT_VECTOR nodes. Scan the CONCAT_VECTOR
27788 // operands and look for a CONCAT operations that place the incoming vectors
27789 // at the exact same location.
27790 //
27791 // For scalable vectors, EXTRACT_SUBVECTOR indexes are implicitly scaled.
27792 SDValue SingleSource = SDValue();
27793 unsigned PartNumElem =
27794 N->getOperand(Num: 0).getValueType().getVectorMinNumElements();
27795
27796 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
27797 SDValue Op = N->getOperand(Num: i);
27798
27799 if (Op.isUndef())
27800 continue;
27801
27802 // Check if this is the identity extract:
27803 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR)
27804 return SDValue();
27805
27806 // Find the single incoming vector for the extract_subvector.
27807 if (SingleSource.getNode()) {
27808 if (Op.getOperand(i: 0) != SingleSource)
27809 return SDValue();
27810 } else {
27811 SingleSource = Op.getOperand(i: 0);
27812
27813 // Check the source type is the same as the type of the result.
27814 // If not, this concat may extend the vector, so we can not
27815 // optimize it away.
27816 if (SingleSource.getValueType() != N->getValueType(ResNo: 0))
27817 return SDValue();
27818 }
27819
27820 // Check that we are reading from the identity index.
27821 unsigned IdentityIndex = i * PartNumElem;
27822 if (Op.getConstantOperandAPInt(i: 1) != IdentityIndex)
27823 return SDValue();
27824 }
27825
27826 if (SingleSource.getNode())
27827 return SingleSource;
27828
27829 return SDValue();
27830}
27831
27832SDValue DAGCombiner::visitVECTOR_INTERLEAVE(SDNode *N) {
27833 EVT VT = N->getValueType(ResNo: 0);
27834 SDValue Op0 = N->getOperand(Num: 0);
27835
27836 // Fold an interleave of fixed-length BUILD_VECTORs by rearranging their
27837 // scalar operands directly.
27838 if (Op0.getOpcode() == ISD::BUILD_VECTOR) {
27839 EVT EltVT = Op0.getOperand(i: 0).getValueType();
27840 if (llvm::all_of(Range: N->op_values(), P: [&](SDValue Op) {
27841 return Op.getOpcode() == ISD::BUILD_VECTOR &&
27842 Op.getOperand(i: 0).getValueType() == EltVT;
27843 })) {
27844 unsigned Factor = N->getNumOperands();
27845 unsigned NumElts = VT.getVectorNumElements();
27846 SDLoc DL(N);
27847 SmallVector<SDValue, 4> Results;
27848 SmallVector<SDValue, 16> InterleavedElts;
27849 for (unsigned I = 0; I != NumElts; ++I) {
27850 for (SDValue op : N->op_values())
27851 InterleavedElts.push_back(Elt: op.getOperand(i: I));
27852 }
27853 for (unsigned I = 0; I < Factor; I++)
27854 Results.push_back(Elt: DAG.getBuildVector(
27855 VT, DL, Ops: ArrayRef(InterleavedElts).slice(N: I * NumElts, M: NumElts)));
27856 return CombineTo(N, To: &Results);
27857 }
27858 }
27859
27860 // Check to see if all operands are identical.
27861 if (!llvm::all_equal(Range: N->op_values()))
27862 return SDValue();
27863
27864 // Check to see if the identical operand is a splat.
27865 if (!DAG.isSplatValue(V: N->getOperand(Num: 0)))
27866 return SDValue();
27867
27868 // interleave splat(X), splat(X).... --> splat(X), splat(X)....
27869 SmallVector<SDValue, 4> Ops;
27870 Ops.append(in_start: N->op_values().begin(), in_end: N->op_values().end());
27871 return CombineTo(N, To: &Ops);
27872}
27873
27874// Helper that peeks through INSERT_SUBVECTOR/CONCAT_VECTORS to find
27875// if the subvector can be sourced for free.
27876static SDValue getSubVectorSrc(SDValue V, unsigned Index, EVT SubVT) {
27877 if (V.getOpcode() == ISD::INSERT_SUBVECTOR &&
27878 V.getOperand(i: 1).getValueType() == SubVT &&
27879 V.getConstantOperandAPInt(i: 2) == Index) {
27880 return V.getOperand(i: 1);
27881 }
27882 if (V.getOpcode() == ISD::CONCAT_VECTORS &&
27883 V.getOperand(i: 0).getValueType() == SubVT &&
27884 (Index % SubVT.getVectorMinNumElements()) == 0) {
27885 uint64_t SubIdx = Index / SubVT.getVectorMinNumElements();
27886 return V.getOperand(i: SubIdx);
27887 }
27888 return SDValue();
27889}
27890
27891static SDValue narrowInsertExtractVectorBinOp(EVT SubVT, SDValue BinOp,
27892 unsigned Index, const SDLoc &DL,
27893 SelectionDAG &DAG,
27894 bool LegalOperations) {
27895 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27896 unsigned BinOpcode = BinOp.getOpcode();
27897 if (!TLI.isBinOp(Opcode: BinOpcode) || BinOp->getNumValues() != 1)
27898 return SDValue();
27899
27900 EVT VecVT = BinOp.getValueType();
27901 SDValue Bop0 = BinOp.getOperand(i: 0), Bop1 = BinOp.getOperand(i: 1);
27902 if (VecVT != Bop0.getValueType() || VecVT != Bop1.getValueType())
27903 return SDValue();
27904 if (!TLI.isOperationLegalOrCustom(Op: BinOpcode, VT: SubVT, LegalOnly: LegalOperations))
27905 return SDValue();
27906
27907 SDValue Sub0 = getSubVectorSrc(V: Bop0, Index, SubVT);
27908 SDValue Sub1 = getSubVectorSrc(V: Bop1, Index, SubVT);
27909
27910 // TODO: We could handle the case where only 1 operand is being inserted by
27911 // creating an extract of the other operand, but that requires checking
27912 // number of uses and/or costs.
27913 if (!Sub0 || !Sub1)
27914 return SDValue();
27915
27916 // We are inserting both operands of the wide binop only to extract back
27917 // to the narrow vector size. Eliminate all of the insert/extract:
27918 // ext (binop (ins ?, X, Index), (ins ?, Y, Index)), Index --> binop X, Y
27919 return DAG.getNode(Opcode: BinOpcode, DL, VT: SubVT, N1: Sub0, N2: Sub1, Flags: BinOp->getFlags());
27920}
27921
27922/// If we are extracting a subvector produced by a wide binary operator try
27923/// to use a narrow binary operator and/or avoid concatenation and extraction.
27924static SDValue narrowExtractedVectorBinOp(EVT VT, SDValue Src, unsigned Index,
27925 const SDLoc &DL, SelectionDAG &DAG,
27926 bool LegalOperations) {
27927 // TODO: Refactor with the caller (visitEXTRACT_SUBVECTOR), so we can share
27928 // some of these bailouts with other transforms.
27929
27930 if (SDValue V = narrowInsertExtractVectorBinOp(SubVT: VT, BinOp: Src, Index, DL, DAG,
27931 LegalOperations))
27932 return V;
27933
27934 // We are looking for an optionally bitcasted wide vector binary operator
27935 // feeding an extract subvector.
27936 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
27937 SDValue BinOp = peekThroughBitcasts(V: Src);
27938 unsigned BOpcode = BinOp.getOpcode();
27939 if (!TLI.isBinOp(Opcode: BOpcode) || BinOp->getNumValues() != 1)
27940 return SDValue();
27941
27942 // Exclude the fake form of fneg (fsub -0.0, x) because that is likely to be
27943 // reduced to the unary fneg when it is visited, and we probably want to deal
27944 // with fneg in a target-specific way.
27945 if (BOpcode == ISD::FSUB) {
27946 auto *C = isConstOrConstSplatFP(N: BinOp.getOperand(i: 0), /*AllowUndefs*/ true);
27947 if (C && C->getValueAPF().isNegZero())
27948 return SDValue();
27949 }
27950
27951 // The binop must be a vector type, so we can extract some fraction of it.
27952 EVT WideBVT = BinOp.getValueType();
27953 // The optimisations below currently assume we are dealing with fixed length
27954 // vectors. It is possible to add support for scalable vectors, but at the
27955 // moment we've done no analysis to prove whether they are profitable or not.
27956 if (!WideBVT.isFixedLengthVector())
27957 return SDValue();
27958
27959 assert((Index % VT.getVectorNumElements()) == 0 &&
27960 "Extract index is not a multiple of the vector length.");
27961
27962 // Bail out if this is not a proper multiple width extraction.
27963 unsigned WideWidth = WideBVT.getSizeInBits();
27964 unsigned NarrowWidth = VT.getSizeInBits();
27965 if (WideWidth % NarrowWidth != 0)
27966 return SDValue();
27967
27968 // Bail out if we are extracting a fraction of a single operation. This can
27969 // occur because we potentially looked through a bitcast of the binop.
27970 unsigned NarrowingRatio = WideWidth / NarrowWidth;
27971 unsigned WideNumElts = WideBVT.getVectorNumElements();
27972 if (WideNumElts % NarrowingRatio != 0)
27973 return SDValue();
27974
27975 // Bail out if the target does not support a narrower version of the binop.
27976 EVT NarrowBVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideBVT.getScalarType(),
27977 NumElements: WideNumElts / NarrowingRatio);
27978 if (!TLI.isOperationLegalOrCustomOrPromote(Op: BOpcode, VT: NarrowBVT,
27979 LegalOnly: LegalOperations))
27980 return SDValue();
27981
27982 // If extraction is cheap, we don't need to look at the binop operands
27983 // for concat ops. The narrow binop alone makes this transform profitable.
27984 // We can't just reuse the original extract index operand because we may have
27985 // bitcasted.
27986 unsigned ConcatOpNum = Index / VT.getVectorNumElements();
27987 unsigned ExtBOIdx = ConcatOpNum * NarrowBVT.getVectorNumElements();
27988 if (TLI.getExtractSubvectorCost(ResVT: NarrowBVT, SrcVT: WideBVT, Index: ExtBOIdx) <=
27989 TargetLowering::ExtractSubvectorCost::Cheap &&
27990 BinOp.hasOneUse() && Src->hasOneUse()) {
27991 // extract (binop B0, B1), N --> binop (extract B0, N), (extract B1, N)
27992 SDValue NewExtIndex = DAG.getVectorIdxConstant(Val: ExtBOIdx, DL);
27993 SDValue X = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NarrowBVT,
27994 N1: BinOp.getOperand(i: 0), N2: NewExtIndex);
27995 SDValue Y = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NarrowBVT,
27996 N1: BinOp.getOperand(i: 1), N2: NewExtIndex);
27997 SDValue NarrowBinOp =
27998 DAG.getNode(Opcode: BOpcode, DL, VT: NarrowBVT, N1: X, N2: Y, Flags: BinOp->getFlags());
27999 return DAG.getBitcast(VT, V: NarrowBinOp);
28000 }
28001
28002 // Only handle the case where we are doubling and then halving. A larger ratio
28003 // may require more than two narrow binops to replace the wide binop.
28004 if (NarrowingRatio != 2)
28005 return SDValue();
28006
28007 // TODO: The motivating case for this transform is an x86 AVX1 target. That
28008 // target has temptingly almost legal versions of bitwise logic ops in 256-bit
28009 // flavors, but no other 256-bit integer support. This could be extended to
28010 // handle any binop, but that may require fixing/adding other folds to avoid
28011 // codegen regressions.
28012 if (BOpcode != ISD::AND && BOpcode != ISD::OR && BOpcode != ISD::XOR)
28013 return SDValue();
28014
28015 // We need at least one concatenation operation of a binop operand to make
28016 // this transform worthwhile. The concat must double the input vector sizes.
28017 auto GetSubVector = [ConcatOpNum](SDValue V) -> SDValue {
28018 if (V.getOpcode() == ISD::CONCAT_VECTORS && V.getNumOperands() == 2)
28019 return V.getOperand(i: ConcatOpNum);
28020 return SDValue();
28021 };
28022 SDValue SubVecL = GetSubVector(peekThroughBitcasts(V: BinOp.getOperand(i: 0)));
28023 SDValue SubVecR = GetSubVector(peekThroughBitcasts(V: BinOp.getOperand(i: 1)));
28024
28025 if (SubVecL || SubVecR) {
28026 // If a binop operand was not the result of a concat, we must extract a
28027 // half-sized operand for our new narrow binop:
28028 // extract (binop (concat X1, X2), (concat Y1, Y2)), N --> binop XN, YN
28029 // extract (binop (concat X1, X2), Y), N --> binop XN, (extract Y, IndexC)
28030 // extract (binop X, (concat Y1, Y2)), N --> binop (extract X, IndexC), YN
28031 SDValue IndexC = DAG.getVectorIdxConstant(Val: ExtBOIdx, DL);
28032 SDValue X = SubVecL ? DAG.getBitcast(VT: NarrowBVT, V: SubVecL)
28033 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NarrowBVT,
28034 N1: BinOp.getOperand(i: 0), N2: IndexC);
28035
28036 SDValue Y = SubVecR ? DAG.getBitcast(VT: NarrowBVT, V: SubVecR)
28037 : DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NarrowBVT,
28038 N1: BinOp.getOperand(i: 1), N2: IndexC);
28039
28040 SDValue NarrowBinOp =
28041 DAG.getNode(Opcode: BOpcode, DL, VT: NarrowBVT, N1: X, N2: Y, Flags: BinOp->getFlags());
28042 return DAG.getBitcast(VT, V: NarrowBinOp);
28043 }
28044
28045 return SDValue();
28046}
28047
28048/// If we are extracting a subvector from a wide vector load, convert to a
28049/// narrow load to eliminate the extraction:
28050/// (extract_subvector (load wide vector)) --> (load narrow vector)
28051static SDValue narrowExtractedVectorLoad(EVT VT, SDValue Src, unsigned Index,
28052 const SDLoc &DL, SelectionDAG &DAG) {
28053 // TODO: Add support for big-endian. The offset calculation must be adjusted.
28054 if (DAG.getDataLayout().isBigEndian())
28055 return SDValue();
28056
28057 auto *Ld = dyn_cast<LoadSDNode>(Val&: Src);
28058 if (!Ld || !ISD::isNormalLoad(N: Ld) || !Ld->isSimple())
28059 return SDValue();
28060
28061 // We can only create byte sized loads.
28062 if (!VT.isByteSized())
28063 return SDValue();
28064
28065 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28066 if (!TLI.isOperationLegalOrCustomOrPromote(Op: ISD::LOAD, VT))
28067 return SDValue();
28068
28069 unsigned NumElts = VT.getVectorMinNumElements();
28070 // A fixed length vector being extracted from a scalable vector
28071 // may not be any *smaller* than the scalable one.
28072 if (Index == 0 && NumElts >= Ld->getValueType(ResNo: 0).getVectorMinNumElements())
28073 return SDValue();
28074
28075 // The definition of EXTRACT_SUBVECTOR states that the index must be a
28076 // multiple of the minimum number of elements in the result type.
28077 assert(Index % NumElts == 0 && "The extract subvector index is not a "
28078 "multiple of the result's element count");
28079
28080 // It's fine to use TypeSize here as we know the offset will not be negative.
28081 TypeSize Offset = VT.getStoreSize() * (Index / NumElts);
28082 std::optional<unsigned> ByteOffset;
28083 if (Offset.isFixed())
28084 ByteOffset = Offset.getFixedValue();
28085
28086 if (!TLI.shouldReduceLoadWidth(Load: Ld, ExtTy: Ld->getExtensionType(), NewVT: VT, ByteOffset))
28087 return SDValue();
28088
28089 // The narrow load will be offset from the base address of the old load if
28090 // we are extracting from something besides index 0 (little-endian).
28091 // TODO: Use "BaseIndexOffset" to make this more effective.
28092 SDValue NewAddr = DAG.getMemBasePlusOffset(Base: Ld->getBasePtr(), Offset, DL);
28093
28094 MachineFunction &MF = DAG.getMachineFunction();
28095 MachineMemOperand *MMO;
28096 if (Offset.isScalable()) {
28097 MachinePointerInfo MPI =
28098 MachinePointerInfo(Ld->getPointerInfo().getAddrSpace());
28099 MMO = MF.getMachineMemOperand(MMO: Ld->getMemOperand(), PtrInfo: MPI, Size: VT.getStoreSize());
28100 } else
28101 MMO = MF.getMachineMemOperand(MMO: Ld->getMemOperand(), Offset: Offset.getFixedValue(),
28102 Size: VT.getStoreSize());
28103
28104 SDValue NewLd = DAG.getLoad(VT, dl: DL, Chain: Ld->getChain(), Ptr: NewAddr, MMO);
28105 DAG.makeEquivalentMemoryOrdering(OldLoad: Ld, NewMemOp: NewLd);
28106 return NewLd;
28107}
28108
28109/// Given EXTRACT_SUBVECTOR(VECTOR_SHUFFLE(Op0, Op1, Mask)),
28110/// try to produce VECTOR_SHUFFLE(EXTRACT_SUBVECTOR(Op?, ?),
28111/// EXTRACT_SUBVECTOR(Op?, ?),
28112/// Mask'))
28113/// iff it is legal and profitable to do so. Notably, the trimmed mask
28114/// (containing only the elements that are extracted)
28115/// must reference at most two subvectors.
28116static SDValue foldExtractSubvectorFromShuffleVector(EVT NarrowVT, SDValue Src,
28117 unsigned Index,
28118 const SDLoc &DL,
28119 SelectionDAG &DAG,
28120 bool LegalOperations) {
28121 // Only deal with non-scalable vectors.
28122 EVT WideVT = Src.getValueType();
28123 if (!NarrowVT.isFixedLengthVector() || !WideVT.isFixedLengthVector())
28124 return SDValue();
28125
28126 // The operand must be a shufflevector.
28127 auto *WideShuffleVector = dyn_cast<ShuffleVectorSDNode>(Val&: Src);
28128 if (!WideShuffleVector)
28129 return SDValue();
28130
28131 // The old shuffleneeds to go away.
28132 if (!WideShuffleVector->hasOneUse())
28133 return SDValue();
28134
28135 // And the narrow shufflevector that we'll form must be legal.
28136 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28137 if (LegalOperations &&
28138 !TLI.isOperationLegalOrCustom(Op: ISD::VECTOR_SHUFFLE, VT: NarrowVT))
28139 return SDValue();
28140
28141 int NumEltsExtracted = NarrowVT.getVectorNumElements();
28142 assert((Index % NumEltsExtracted) == 0 &&
28143 "Extract index is not a multiple of the output vector length.");
28144
28145 int WideNumElts = WideVT.getVectorNumElements();
28146
28147 SmallVector<int, 16> NewMask;
28148 NewMask.reserve(N: NumEltsExtracted);
28149 SmallSetVector<std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>, 2>
28150 DemandedSubvectors;
28151
28152 // Try to decode the wide mask into narrow mask from at most two subvectors.
28153 for (int M : WideShuffleVector->getMask().slice(N: Index, M: NumEltsExtracted)) {
28154 assert((M >= -1) && (M < (2 * WideNumElts)) &&
28155 "Out-of-bounds shuffle mask?");
28156
28157 if (M < 0) {
28158 // Does not depend on operands, does not require adjustment.
28159 NewMask.emplace_back(Args&: M);
28160 continue;
28161 }
28162
28163 // From which operand of the shuffle does this shuffle mask element pick?
28164 int WideShufOpIdx = M / WideNumElts;
28165 // Which element of that operand is picked?
28166 int OpEltIdx = M % WideNumElts;
28167
28168 assert((OpEltIdx + WideShufOpIdx * WideNumElts) == M &&
28169 "Shuffle mask vector decomposition failure.");
28170
28171 // And which NumEltsExtracted-sized subvector of that operand is that?
28172 int OpSubvecIdx = OpEltIdx / NumEltsExtracted;
28173 // And which element within that subvector of that operand is that?
28174 int OpEltIdxInSubvec = OpEltIdx % NumEltsExtracted;
28175
28176 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted) == OpEltIdx &&
28177 "Shuffle mask subvector decomposition failure.");
28178
28179 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted +
28180 WideShufOpIdx * WideNumElts) == M &&
28181 "Shuffle mask full decomposition failure.");
28182
28183 SDValue Op = WideShuffleVector->getOperand(Num: WideShufOpIdx);
28184
28185 if (Op.isUndef()) {
28186 // Picking from an undef operand. Let's adjust mask instead.
28187 NewMask.emplace_back(Args: -1);
28188 continue;
28189 }
28190
28191 const std::pair<SDValue, int> DemandedSubvector =
28192 std::make_pair(x&: Op, y&: OpSubvecIdx);
28193
28194 if (DemandedSubvectors.insert(X: DemandedSubvector)) {
28195 if (DemandedSubvectors.size() > 2)
28196 return SDValue(); // We can't handle more than two subvectors.
28197 // How many elements into the WideVT does this subvector start?
28198 int Index = NumEltsExtracted * OpSubvecIdx;
28199 // Bail out if the extraction isn't going to be cheap.
28200 if (TLI.getExtractSubvectorCost(ResVT: NarrowVT, SrcVT: WideVT, Index) >
28201 TargetLowering::ExtractSubvectorCost::Cheap)
28202 return SDValue();
28203 }
28204
28205 // Ok, but from which operand of the new shuffle will this element pick?
28206 int NewOpIdx =
28207 getFirstIndexOf(Range: DemandedSubvectors.getArrayRef(), Val: DemandedSubvector);
28208 assert((NewOpIdx == 0 || NewOpIdx == 1) && "Unexpected operand index.");
28209
28210 int AdjM = OpEltIdxInSubvec + NewOpIdx * NumEltsExtracted;
28211 NewMask.emplace_back(Args&: AdjM);
28212 }
28213 assert(NewMask.size() == (unsigned)NumEltsExtracted && "Produced bad mask.");
28214 assert(DemandedSubvectors.size() <= 2 &&
28215 "Should have ended up demanding at most two subvectors.");
28216
28217 // Did we discover that the shuffle does not actually depend on operands?
28218 if (DemandedSubvectors.empty())
28219 return DAG.getPOISON(VT: NarrowVT);
28220
28221 // Profitability check: only deal with extractions from the first subvector
28222 // unless the mask becomes an identity mask.
28223 if (!ShuffleVectorInst::isIdentityMask(Mask: NewMask, NumSrcElts: NewMask.size()) ||
28224 any_of(Range&: NewMask, P: [](int M) { return M < 0; }))
28225 for (auto &DemandedSubvector : DemandedSubvectors)
28226 if (DemandedSubvector.second != 0)
28227 return SDValue();
28228
28229 // We still perform the exact same EXTRACT_SUBVECTOR, just on different
28230 // operand[s]/index[es], so there is no point in checking for it's legality.
28231
28232 // Do not turn a legal shuffle into an illegal one.
28233 if (TLI.isShuffleMaskLegal(WideShuffleVector->getMask(), WideVT) &&
28234 !TLI.isShuffleMaskLegal(NewMask, NarrowVT))
28235 return SDValue();
28236
28237 SmallVector<SDValue, 2> NewOps;
28238 for (const std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>
28239 &DemandedSubvector : DemandedSubvectors) {
28240 // How many elements into the WideVT does this subvector start?
28241 int Index = NumEltsExtracted * DemandedSubvector.second;
28242 SDValue IndexC = DAG.getVectorIdxConstant(Val: Index, DL);
28243 NewOps.emplace_back(Args: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NarrowVT,
28244 N1: DemandedSubvector.first, N2: IndexC));
28245 }
28246 assert((NewOps.size() == 1 || NewOps.size() == 2) &&
28247 "Should end up with either one or two ops");
28248
28249 // If we ended up with only one operand, pad with poison.
28250 if (NewOps.size() == 1)
28251 NewOps.emplace_back(Args: DAG.getPOISON(VT: NarrowVT));
28252
28253 return DAG.getVectorShuffle(VT: NarrowVT, dl: DL, N1: NewOps[0], N2: NewOps[1], Mask: NewMask);
28254}
28255
28256SDValue DAGCombiner::foldExtractSubvectorFromConcatVectors(EVT VT, SDValue V,
28257 uint64_t ExtIdx,
28258 const SDLoc &DL) {
28259 assert(V.getOpcode() == ISD::CONCAT_VECTORS &&
28260 "Expected a CONCAT_VECTORS operand");
28261 ElementCount ExtNumElts = VT.getVectorElementCount();
28262 assert(ExtIdx % ExtNumElts.getKnownMinValue() == 0 &&
28263 "subvector extract is alligned");
28264 EVT ConcatSrcVT = V.getOperand(i: 0).getValueType();
28265
28266 ElementCount ConcatSrcNumElts = ConcatSrcVT.getVectorElementCount();
28267 unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts.getKnownMinValue();
28268 if (ConcatOpIdx >= V.getNumOperands())
28269 return SDValue();
28270
28271 // If the concatenated source types match this extract, it's a direct
28272 // simplification:
28273 // extract_subvector (concat V1, V2, ...), i --> Vi
28274 if (VT.getVectorElementCount() == ConcatSrcVT.getVectorElementCount())
28275 return V.getOperand(i: ConcatOpIdx);
28276
28277 // If the concatenated source vectors are a multiple length of this extract,
28278 // then extract a fraction of one of those source vectors directly from a
28279 // concat operand. Example:
28280 // v2i8 extract_subvector (v16i8 concat_subvector v8i8:X, v8i8:Y), 14 -->
28281 // v2i8 extract_subvector v8i8:Y, 6
28282 if (ConcatSrcNumElts.hasKnownScalarFactor(RHS: ExtNumElts)) {
28283 uint64_t NewExtIdx =
28284 ExtIdx - ConcatOpIdx * ConcatSrcNumElts.getKnownMinValue();
28285 return DAG.getExtractSubvector(DL, VT, Vec: V.getOperand(i: ConcatOpIdx),
28286 Idx: NewExtIdx);
28287 }
28288
28289 // If the extract covers multiple whole concat operands, rebuild that smaller
28290 // concat directly.
28291 if (ExtNumElts.hasKnownScalarFactor(RHS: ConcatSrcNumElts) &&
28292 ExtIdx % ConcatSrcNumElts.getKnownMinValue() == 0 &&
28293 (!LegalOperations || hasOperation(Opcode: ISD::CONCAT_VECTORS, VT))) {
28294 unsigned NumConcatOps = ExtNumElts.getKnownScalarFactor(RHS: ConcatSrcNumElts);
28295 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT,
28296 Ops: V->ops().slice(N: ConcatOpIdx, M: NumConcatOps));
28297 }
28298
28299 return SDValue();
28300}
28301
28302SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
28303 EVT NVT = N->getValueType(ResNo: 0);
28304 SDValue V = N->getOperand(Num: 0);
28305 uint64_t ExtIdx = N->getConstantOperandVal(Num: 1);
28306 SDLoc DL(N);
28307
28308 // Extract from UNDEF is UNDEF.
28309 if (V.isUndef())
28310 return DAG.getUNDEF(VT: NVT);
28311
28312 if (SDValue NarrowLoad = narrowExtractedVectorLoad(VT: NVT, Src: V, Index: ExtIdx, DL, DAG))
28313 return NarrowLoad;
28314
28315 // Peek through frozen loads, but ensure the load has a single use.
28316 if (V.getOpcode() == ISD::FREEZE && V.hasOneUse() &&
28317 V.getOperand(i: 0).hasOneUse())
28318 if (SDValue NarrowLoad =
28319 narrowExtractedVectorLoad(VT: NVT, Src: V.getOperand(i: 0), Index: ExtIdx, DL, DAG))
28320 return DAG.getFreeze(V: NarrowLoad);
28321
28322 // Combine an extract of an extract into a single extract_subvector.
28323 // ext (ext X, C1), C2 --> ext X, C1 + C2
28324 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR && V.hasOneUse()) {
28325 // Both indices must have the same scaling factor and C has to be a
28326 // multiple of the new result type's known minimum vector length.
28327 uint64_t InnerExtIdx = V.getConstantOperandVal(i: 1);
28328 uint64_t NewExtIdx = InnerExtIdx + ExtIdx;
28329 if (V.getValueType().isScalableVector() == NVT.isScalableVector() &&
28330 NewExtIdx % NVT.getVectorMinNumElements() == 0 &&
28331 TLI.getExtractSubvectorCost(ResVT: NVT, SrcVT: V.getOperand(i: 0).getValueType(),
28332 Index: NewExtIdx) <=
28333 TargetLowering::ExtractSubvectorCost::Cheap &&
28334 TLI.isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: NVT))
28335 return DAG.getExtractSubvector(DL, VT: NVT, Vec: V.getOperand(i: 0), Idx: NewExtIdx);
28336 }
28337
28338 // ty1 extract_vector(ty2 splat(V))) -> ty1 splat(V)
28339 if (V.getOpcode() == ISD::SPLAT_VECTOR)
28340 if ((DAG.isConstantValueOfAnyType(N: V.getOperand(i: 0)) &&
28341 !(NVT.isScalableVector() &&
28342 TLI.getExtractSubvectorCost(ResVT: NVT, SrcVT: V.getValueType(), Index: ExtIdx) <=
28343 TargetLowering::ExtractSubvectorCost::Cheap)) ||
28344 V.hasOneUse())
28345 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::SPLAT_VECTOR, VT: NVT))
28346 return DAG.getSplatVector(VT: NVT, DL, Op: V.getOperand(i: 0));
28347
28348 // ty1 extract_vector(ty2 get_active_lane_mask(X, Y), 0) --> ty1
28349 // get_active_lane_mask(X, Y)
28350 if (ExtIdx == 0 && V.getOpcode() == ISD::GET_ACTIVE_LANE_MASK &&
28351 V.hasOneUse() &&
28352 (!LegalOperations ||
28353 TLI.isOperationLegal(Op: ISD::GET_ACTIVE_LANE_MASK, VT: NVT)))
28354 return DAG.getNode(Opcode: ISD::GET_ACTIVE_LANE_MASK, DL, VT: NVT, N1: V.getOperand(i: 0),
28355 N2: V.getOperand(i: 1));
28356
28357 // extract_subvector(insert_subvector(x,y,c1),c2)
28358 // --> extract_subvector(y,c2-c1)
28359 // iff we're just extracting from the inserted subvector.
28360 if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
28361 SDValue InsSub = V.getOperand(i: 1);
28362 EVT InsSubVT = InsSub.getValueType();
28363 unsigned NumInsElts = InsSubVT.getVectorMinNumElements();
28364 unsigned InsIdx = V.getConstantOperandVal(i: 2);
28365 unsigned NumSubElts = NVT.getVectorMinNumElements();
28366 if (InsIdx <= ExtIdx && (ExtIdx + NumSubElts) <= (InsIdx + NumInsElts) &&
28367 TLI.getExtractSubvectorCost(ResVT: NVT, SrcVT: InsSubVT, Index: ExtIdx - InsIdx) <=
28368 TargetLowering::ExtractSubvectorCost::Cheap &&
28369 InsSubVT.isFixedLengthVector() && NVT.isFixedLengthVector() &&
28370 V.getValueType().isFixedLengthVector())
28371 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NVT, N1: InsSub,
28372 N2: DAG.getVectorIdxConstant(Val: ExtIdx - InsIdx, DL));
28373 }
28374
28375 // Try to move vector bitcast after extract_subv by scaling extraction index:
28376 // extract_subv (bitcast X), Index --> bitcast (extract_subv X, Index')
28377 if (V.getOpcode() == ISD::BITCAST &&
28378 V.getOperand(i: 0).getValueType().isVector() &&
28379 (!LegalOperations || TLI.isOperationLegal(Op: ISD::BITCAST, VT: NVT))) {
28380 SDValue SrcOp = V.getOperand(i: 0);
28381 EVT SrcVT = SrcOp.getValueType();
28382 unsigned SrcNumElts = SrcVT.getVectorMinNumElements();
28383 unsigned DestNumElts = V.getValueType().getVectorMinNumElements();
28384 if ((SrcNumElts % DestNumElts) == 0) {
28385 unsigned SrcDestRatio = SrcNumElts / DestNumElts;
28386 ElementCount NewExtEC = NVT.getVectorElementCount() * SrcDestRatio;
28387 EVT NewExtVT =
28388 EVT::getVectorVT(Context&: *DAG.getContext(), VT: SrcVT.getScalarType(), EC: NewExtEC);
28389 if (TLI.isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: NewExtVT)) {
28390 SDValue NewIndex = DAG.getVectorIdxConstant(Val: ExtIdx * SrcDestRatio, DL);
28391 SDValue NewExtract = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NewExtVT,
28392 N1: V.getOperand(i: 0), N2: NewIndex);
28393 return DAG.getBitcast(VT: NVT, V: NewExtract);
28394 }
28395 }
28396 if ((DestNumElts % SrcNumElts) == 0) {
28397 unsigned DestSrcRatio = DestNumElts / SrcNumElts;
28398 if (NVT.getVectorElementCount().isKnownMultipleOf(RHS: DestSrcRatio)) {
28399 ElementCount NewExtEC =
28400 NVT.getVectorElementCount().divideCoefficientBy(RHS: DestSrcRatio);
28401 EVT ScalarVT = SrcVT.getScalarType();
28402 if ((ExtIdx % DestSrcRatio) == 0) {
28403 unsigned IndexValScaled = ExtIdx / DestSrcRatio;
28404 EVT NewExtVT =
28405 EVT::getVectorVT(Context&: *DAG.getContext(), VT: ScalarVT, EC: NewExtEC);
28406 if (TLI.isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: NewExtVT)) {
28407 SDValue NewIndex = DAG.getVectorIdxConstant(Val: IndexValScaled, DL);
28408 SDValue NewExtract =
28409 DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NewExtVT,
28410 N1: V.getOperand(i: 0), N2: NewIndex);
28411 return DAG.getBitcast(VT: NVT, V: NewExtract);
28412 }
28413 if (NewExtEC.isScalar() &&
28414 TLI.isOperationLegalOrCustom(Op: ISD::EXTRACT_VECTOR_ELT, VT: ScalarVT)) {
28415 SDValue NewIndex = DAG.getVectorIdxConstant(Val: IndexValScaled, DL);
28416 SDValue NewExtract =
28417 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: ScalarVT,
28418 N1: V.getOperand(i: 0), N2: NewIndex);
28419 return DAG.getBitcast(VT: NVT, V: NewExtract);
28420 }
28421 }
28422 }
28423 }
28424 }
28425
28426 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
28427 if (SDValue Folded =
28428 foldExtractSubvectorFromConcatVectors(VT: NVT, V, ExtIdx, DL))
28429 return Folded;
28430 }
28431
28432 if (SDValue Shuffle = foldExtractSubvectorFromShuffleVector(
28433 NarrowVT: NVT, Src: V, Index: ExtIdx, DL, DAG, LegalOperations))
28434 return Shuffle;
28435
28436 if (SDValue NarrowBOp =
28437 narrowExtractedVectorBinOp(VT: NVT, Src: V, Index: ExtIdx, DL, DAG, LegalOperations))
28438 return NarrowBOp;
28439
28440 V = peekThroughBitcasts(V);
28441
28442 // If the input is a build vector. Try to make a smaller build vector.
28443 if (V.getOpcode() == ISD::BUILD_VECTOR) {
28444 EVT InVT = V.getValueType();
28445 unsigned ExtractSize = NVT.getSizeInBits();
28446 unsigned EltSize = InVT.getScalarSizeInBits();
28447 // Only do this if we won't split any elements.
28448 if (ExtractSize % EltSize == 0) {
28449 unsigned NumElems = ExtractSize / EltSize;
28450 EVT EltVT = InVT.getVectorElementType();
28451 EVT ExtractVT =
28452 NumElems == 1 ? EltVT
28453 : EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElems);
28454 if ((Level < AfterLegalizeDAG ||
28455 (NumElems == 1 ||
28456 TLI.isOperationLegal(Op: ISD::BUILD_VECTOR, VT: ExtractVT))) &&
28457 (!LegalTypes || TLI.isTypeLegal(VT: ExtractVT))) {
28458 unsigned IdxVal = (ExtIdx * NVT.getScalarSizeInBits()) / EltSize;
28459
28460 if (NumElems == 1) {
28461 SDValue Src = V->getOperand(Num: IdxVal);
28462 if (EltVT != Src.getValueType())
28463 Src = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Src);
28464 return DAG.getBitcast(VT: NVT, V: Src);
28465 }
28466
28467 // Extract the pieces from the original build_vector.
28468 SDValue BuildVec =
28469 DAG.getBuildVector(VT: ExtractVT, DL, Ops: V->ops().slice(N: IdxVal, M: NumElems));
28470 return DAG.getBitcast(VT: NVT, V: BuildVec);
28471 }
28472 }
28473 }
28474
28475 if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
28476 // Handle only simple case where vector being inserted and vector
28477 // being extracted are of same size.
28478 EVT SmallVT = V.getOperand(i: 1).getValueType();
28479 if (NVT.bitsEq(VT: SmallVT)) {
28480 // Combine:
28481 // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx)
28482 // Into:
28483 // indices are equal or bit offsets are equal => V1
28484 // otherwise => (extract_subvec V1, ExtIdx)
28485 uint64_t InsIdx = V.getConstantOperandVal(i: 2);
28486 if (InsIdx * SmallVT.getScalarSizeInBits() ==
28487 ExtIdx * NVT.getScalarSizeInBits()) {
28488 if (!LegalOperations || TLI.isOperationLegal(Op: ISD::BITCAST, VT: NVT))
28489 return DAG.getBitcast(VT: NVT, V: V.getOperand(i: 1));
28490 } else {
28491 return DAG.getNode(
28492 Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: NVT,
28493 N1: DAG.getBitcast(VT: N->getOperand(Num: 0).getValueType(), V: V.getOperand(i: 0)),
28494 N2: N->getOperand(Num: 1));
28495 }
28496 }
28497 }
28498
28499 // If only EXTRACT_SUBVECTOR nodes use the source vector we can
28500 // simplify it based on the (valid) extractions.
28501 if (!V.getValueType().isScalableVector() &&
28502 llvm::all_of(Range: V->users(), P: [&](SDNode *Use) {
28503 return Use->getOpcode() == ISD::EXTRACT_SUBVECTOR &&
28504 Use->getOperand(Num: 0) == V;
28505 })) {
28506 unsigned NumElts = V.getValueType().getVectorNumElements();
28507 APInt DemandedElts = APInt::getZero(numBits: NumElts);
28508 for (SDNode *User : V->users()) {
28509 unsigned ExtIdx = User->getConstantOperandVal(Num: 1);
28510 unsigned NumSubElts = User->getValueType(ResNo: 0).getVectorNumElements();
28511 DemandedElts.setBits(loBit: ExtIdx, hiBit: ExtIdx + NumSubElts);
28512 }
28513 if (SimplifyDemandedVectorElts(Op: V, DemandedElts, /*AssumeSingleUse=*/true)) {
28514 // We simplified the vector operand of this extract subvector. If this
28515 // extract is not dead, visit it again so it is folded properly.
28516 if (N->getOpcode() != ISD::DELETED_NODE)
28517 AddToWorklist(N);
28518 return SDValue(N, 0);
28519 }
28520 } else {
28521 if (SimplifyDemandedVectorElts(Op: SDValue(N, 0)))
28522 return SDValue(N, 0);
28523 }
28524
28525 return SDValue();
28526}
28527
28528/// Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles
28529/// followed by concatenation. Narrow vector ops may have better performance
28530/// than wide ops, and this can unlock further narrowing of other vector ops.
28531/// Targets can invert this transform later if it is not profitable.
28532static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf,
28533 SelectionDAG &DAG) {
28534 SDValue N0 = Shuf->getOperand(Num: 0), N1 = Shuf->getOperand(Num: 1);
28535 if (N0.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 ||
28536 N1.getOpcode() != ISD::CONCAT_VECTORS || N1.getNumOperands() != 2 ||
28537 !N0.getOperand(i: 1).isUndef() || !N1.getOperand(i: 1).isUndef())
28538 return SDValue();
28539
28540 // Split the wide shuffle mask into halves. Any mask element that is accessing
28541 // operand 1 is offset down to account for narrowing of the vectors.
28542 ArrayRef<int> Mask = Shuf->getMask();
28543 EVT VT = Shuf->getValueType(ResNo: 0);
28544 unsigned NumElts = VT.getVectorNumElements();
28545 unsigned HalfNumElts = NumElts / 2;
28546 SmallVector<int, 16> Mask0(HalfNumElts, -1);
28547 SmallVector<int, 16> Mask1(HalfNumElts, -1);
28548 for (unsigned i = 0; i != NumElts; ++i) {
28549 if (Mask[i] == -1)
28550 continue;
28551 // If we reference the upper (undef) subvector then the element is undef.
28552 if ((Mask[i] % NumElts) >= HalfNumElts)
28553 continue;
28554 int M = Mask[i] < (int)NumElts ? Mask[i] : Mask[i] - (int)HalfNumElts;
28555 if (i < HalfNumElts)
28556 Mask0[i] = M;
28557 else
28558 Mask1[i - HalfNumElts] = M;
28559 }
28560
28561 // Ask the target if this is a valid transform.
28562 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28563 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VT.getScalarType(),
28564 NumElements: HalfNumElts);
28565 if (!TLI.isShuffleMaskLegal(Mask0, HalfVT) ||
28566 !TLI.isShuffleMaskLegal(Mask1, HalfVT))
28567 return SDValue();
28568
28569 // shuffle (concat X, undef), (concat Y, undef), Mask -->
28570 // concat (shuffle X, Y, Mask0), (shuffle X, Y, Mask1)
28571 SDValue X = N0.getOperand(i: 0), Y = N1.getOperand(i: 0);
28572 SDLoc DL(Shuf);
28573 SDValue Shuf0 = DAG.getVectorShuffle(VT: HalfVT, dl: DL, N1: X, N2: Y, Mask: Mask0);
28574 SDValue Shuf1 = DAG.getVectorShuffle(VT: HalfVT, dl: DL, N1: X, N2: Y, Mask: Mask1);
28575 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Shuf0, N2: Shuf1);
28576}
28577
28578// Tries to turn a shuffle of two CONCAT_VECTORS into a single concat,
28579// or turn a shuffle of a single concat into simpler shuffle then concat.
28580static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) {
28581 EVT VT = N->getValueType(ResNo: 0);
28582 unsigned NumElts = VT.getVectorNumElements();
28583
28584 SDValue N0 = N->getOperand(Num: 0);
28585 SDValue N1 = N->getOperand(Num: 1);
28586 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
28587 ArrayRef<int> Mask = SVN->getMask();
28588
28589 SmallVector<SDValue, 4> Ops;
28590 EVT ConcatVT = N0.getOperand(i: 0).getValueType();
28591 unsigned NumElemsPerConcat = ConcatVT.getVectorNumElements();
28592 unsigned NumConcats = NumElts / NumElemsPerConcat;
28593
28594 auto IsUndefMaskElt = [](int i) { return i == -1; };
28595
28596 // Special case: shuffle(concat(A,B)) can be more efficiently represented
28597 // as concat(shuffle(A,B),UNDEF) if the shuffle doesn't set any of the high
28598 // half vector elements.
28599 if (NumElemsPerConcat * 2 == NumElts && N1.isUndef() &&
28600 llvm::all_of(Range: Mask.slice(N: NumElemsPerConcat, M: NumElemsPerConcat),
28601 P: IsUndefMaskElt)) {
28602 N0 = DAG.getVectorShuffle(VT: ConcatVT, dl: SDLoc(N), N1: N0.getOperand(i: 0),
28603 N2: N0.getOperand(i: 1),
28604 Mask: Mask.slice(N: 0, M: NumElemsPerConcat));
28605 N1 = DAG.getPOISON(VT: ConcatVT);
28606 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT, N1: N0, N2: N1);
28607 }
28608
28609 // Look at every vector that's inserted. We're looking for exact
28610 // subvector-sized copies from a concatenated vector
28611 for (unsigned I = 0; I != NumConcats; ++I) {
28612 unsigned Begin = I * NumElemsPerConcat;
28613 ArrayRef<int> SubMask = Mask.slice(N: Begin, M: NumElemsPerConcat);
28614
28615 // Make sure we're dealing with a copy.
28616 if (llvm::all_of(Range&: SubMask, P: IsUndefMaskElt)) {
28617 Ops.push_back(Elt: DAG.getUNDEF(VT: ConcatVT));
28618 continue;
28619 }
28620
28621 int OpIdx = -1;
28622 for (int i = 0; i != (int)NumElemsPerConcat; ++i) {
28623 if (IsUndefMaskElt(SubMask[i]))
28624 continue;
28625 if ((SubMask[i] % (int)NumElemsPerConcat) != i)
28626 return SDValue();
28627 int EltOpIdx = SubMask[i] / NumElemsPerConcat;
28628 if (0 <= OpIdx && EltOpIdx != OpIdx)
28629 return SDValue();
28630 OpIdx = EltOpIdx;
28631 }
28632 assert(0 <= OpIdx && "Unknown concat_vectors op");
28633
28634 if (OpIdx < (int)N0.getNumOperands())
28635 Ops.push_back(Elt: N0.getOperand(i: OpIdx));
28636 else
28637 Ops.push_back(Elt: N1.getOperand(i: OpIdx - N0.getNumOperands()));
28638 }
28639
28640 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT, Ops);
28641}
28642
28643// Attempt to combine a shuffle of 2 inputs of 'scalar sources' -
28644// BUILD_VECTOR or SCALAR_TO_VECTOR into a single BUILD_VECTOR.
28645//
28646// SHUFFLE(BUILD_VECTOR(), BUILD_VECTOR()) -> BUILD_VECTOR() is always
28647// a simplification in some sense, but it isn't appropriate in general: some
28648// BUILD_VECTORs are substantially cheaper than others. The general case
28649// of a BUILD_VECTOR requires inserting each element individually (or
28650// performing the equivalent in a temporary stack variable). A BUILD_VECTOR of
28651// all constants is a single constant pool load. A BUILD_VECTOR where each
28652// element is identical is a splat. A BUILD_VECTOR where most of the operands
28653// are undef lowers to a small number of element insertions.
28654//
28655// To deal with this, we currently use a bunch of mostly arbitrary heuristics.
28656// We don't fold shuffles where one side is a non-zero constant, and we don't
28657// fold shuffles if the resulting (non-splat) BUILD_VECTOR would have duplicate
28658// non-constant operands. This seems to work out reasonably well in practice.
28659static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN,
28660 SelectionDAG &DAG,
28661 const TargetLowering &TLI) {
28662 EVT VT = SVN->getValueType(ResNo: 0);
28663 unsigned NumElts = VT.getVectorNumElements();
28664 SDValue N0 = SVN->getOperand(Num: 0);
28665 SDValue N1 = SVN->getOperand(Num: 1);
28666
28667 if (!N0->hasOneUse())
28668 return SDValue();
28669
28670 // If only one of N1,N2 is constant, bail out if it is not ALL_ZEROS as
28671 // discussed above.
28672 if (!N1.isUndef()) {
28673 if (!N1->hasOneUse())
28674 return SDValue();
28675
28676 bool N0AnyConst = isAnyConstantBuildVector(V: N0);
28677 bool N1AnyConst = isAnyConstantBuildVector(V: N1);
28678 if (N0AnyConst && !N1AnyConst && !ISD::isBuildVectorAllZeros(N: N0.getNode()))
28679 return SDValue();
28680 if (!N0AnyConst && N1AnyConst && !ISD::isBuildVectorAllZeros(N: N1.getNode()))
28681 return SDValue();
28682 }
28683
28684 // If both inputs are splats of the same value then we can safely merge this
28685 // to a single BUILD_VECTOR with undef elements based on the shuffle mask.
28686 bool IsSplat = false;
28687 auto *BV0 = dyn_cast<BuildVectorSDNode>(Val&: N0);
28688 auto *BV1 = dyn_cast<BuildVectorSDNode>(Val&: N1);
28689 if (BV0 && BV1)
28690 if (SDValue Splat0 = BV0->getSplatValue())
28691 IsSplat = (Splat0 == BV1->getSplatValue());
28692
28693 SmallVector<SDValue, 8> Ops;
28694 SmallSet<SDValue, 16> DuplicateOps;
28695 for (int M : SVN->getMask()) {
28696 SDValue Op = DAG.getPOISON(VT: VT.getScalarType());
28697 if (M >= 0) {
28698 int Idx = M < (int)NumElts ? M : M - NumElts;
28699 SDValue &S = (M < (int)NumElts ? N0 : N1);
28700 if (S.getOpcode() == ISD::BUILD_VECTOR) {
28701 Op = S.getOperand(i: Idx);
28702 } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR) {
28703 SDValue Op0 = S.getOperand(i: 0);
28704 Op = Idx == 0 ? Op0 : DAG.getPOISON(VT: Op0.getValueType());
28705 } else {
28706 // Operand can't be combined - bail out.
28707 return SDValue();
28708 }
28709 }
28710
28711 // Don't duplicate a non-constant BUILD_VECTOR operand unless we're
28712 // generating a splat; semantically, this is fine, but it's likely to
28713 // generate low-quality code if the target can't reconstruct an appropriate
28714 // shuffle.
28715 if (!Op.isUndef() && !isIntOrFPConstant(V: Op))
28716 if (!IsSplat && !DuplicateOps.insert(V: Op).second)
28717 return SDValue();
28718
28719 Ops.push_back(Elt: Op);
28720 }
28721
28722 // BUILD_VECTOR requires all inputs to be of the same type, find the
28723 // maximum type and extend them all.
28724 EVT SVT = VT.getScalarType();
28725 if (SVT.isInteger())
28726 for (SDValue &Op : Ops)
28727 SVT = (SVT.bitsLT(VT: Op.getValueType()) ? Op.getValueType() : SVT);
28728 if (SVT != VT.getScalarType())
28729 for (SDValue &Op : Ops)
28730 Op = Op.isUndef() ? DAG.getUNDEF(VT: SVT)
28731 : (TLI.isZExtFree(FromTy: Op.getValueType(), ToTy: SVT)
28732 ? DAG.getZExtOrTrunc(Op, DL: SDLoc(SVN), VT: SVT)
28733 : DAG.getSExtOrTrunc(Op, DL: SDLoc(SVN), VT: SVT));
28734 return DAG.getBuildVector(VT, DL: SDLoc(SVN), Ops);
28735}
28736
28737// Match shuffles that can be converted to *_vector_extend_in_reg.
28738// This is often generated during legalization.
28739// e.g. v4i32 <0,u,1,u> -> (v2i64 any_vector_extend_in_reg(v4i32 src)),
28740// and returns the EVT to which the extension should be performed.
28741// NOTE: this assumes that the src is the first operand of the shuffle.
28742static std::optional<EVT> canCombineShuffleToExtendVectorInreg(
28743 unsigned Opcode, EVT VT, std::function<bool(unsigned)> Match,
28744 SelectionDAG &DAG, const TargetLowering &TLI, bool LegalTypes,
28745 bool LegalOperations) {
28746 bool IsBigEndian = DAG.getDataLayout().isBigEndian();
28747
28748 // TODO Add support for big-endian when we have a test case.
28749 if (!VT.isInteger() || IsBigEndian)
28750 return std::nullopt;
28751
28752 unsigned NumElts = VT.getVectorNumElements();
28753 unsigned EltSizeInBits = VT.getScalarSizeInBits();
28754
28755 // Attempt to match a '*_extend_vector_inreg' shuffle, we just search for
28756 // power-of-2 extensions as they are the most likely.
28757 // FIXME: should try Scale == NumElts case too,
28758 for (unsigned Scale = 2; Scale < NumElts; Scale *= 2) {
28759 // The vector width must be a multiple of Scale.
28760 if (NumElts % Scale != 0)
28761 continue;
28762
28763 EVT OutSVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EltSizeInBits * Scale);
28764 EVT OutVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: OutSVT, NumElements: NumElts / Scale);
28765
28766 if ((LegalTypes && !TLI.isTypeLegal(VT: OutVT)) ||
28767 (LegalOperations && !TLI.isOperationLegalOrCustom(Op: Opcode, VT: OutVT)))
28768 continue;
28769
28770 if (Match(Scale))
28771 return OutVT;
28772 }
28773
28774 return std::nullopt;
28775}
28776
28777// Match shuffles that can be converted to any_vector_extend_in_reg.
28778// This is often generated during legalization.
28779// e.g. v4i32 <0,u,1,u> -> (v2i64 any_vector_extend_in_reg(v4i32 src))
28780static SDValue combineShuffleToAnyExtendVectorInreg(ShuffleVectorSDNode *SVN,
28781 SelectionDAG &DAG,
28782 const TargetLowering &TLI,
28783 bool LegalOperations) {
28784 EVT VT = SVN->getValueType(ResNo: 0);
28785 bool IsBigEndian = DAG.getDataLayout().isBigEndian();
28786
28787 // TODO Add support for big-endian when we have a test case.
28788 if (!VT.isInteger() || IsBigEndian)
28789 return SDValue();
28790
28791 // shuffle<0,-1,1,-1> == (v2i64 anyextend_vector_inreg(v4i32))
28792 auto isAnyExtend = [NumElts = VT.getVectorNumElements(),
28793 Mask = SVN->getMask()](unsigned Scale) {
28794 for (unsigned i = 0; i != NumElts; ++i) {
28795 if (Mask[i] < 0)
28796 continue;
28797 if ((i % Scale) == 0 && Mask[i] == (int)(i / Scale))
28798 continue;
28799 return false;
28800 }
28801 return true;
28802 };
28803
28804 unsigned Opcode = ISD::ANY_EXTEND_VECTOR_INREG;
28805 SDValue N0 = SVN->getOperand(Num: 0);
28806 // Never create an illegal type. Only create unsupported operations if we
28807 // are pre-legalization.
28808 std::optional<EVT> OutVT = canCombineShuffleToExtendVectorInreg(
28809 Opcode, VT, Match: isAnyExtend, DAG, TLI, /*LegalTypes=*/true, LegalOperations);
28810 if (!OutVT)
28811 return SDValue();
28812 return DAG.getBitcast(VT, V: DAG.getNode(Opcode, DL: SDLoc(SVN), VT: *OutVT, Operand: N0));
28813}
28814
28815// Match shuffles that can be converted to zero_extend_vector_inreg.
28816// This is often generated during legalization.
28817// e.g. v4i32 <0,z,1,u> -> (v2i64 zero_extend_vector_inreg(v4i32 src))
28818static SDValue combineShuffleToZeroExtendVectorInReg(ShuffleVectorSDNode *SVN,
28819 SelectionDAG &DAG,
28820 const TargetLowering &TLI,
28821 bool LegalOperations) {
28822 bool LegalTypes = true;
28823 EVT VT = SVN->getValueType(ResNo: 0);
28824 assert(!VT.isScalableVector() && "Encountered scalable shuffle?");
28825 unsigned NumElts = VT.getVectorNumElements();
28826 unsigned EltSizeInBits = VT.getScalarSizeInBits();
28827
28828 // TODO: add support for big-endian when we have a test case.
28829 bool IsBigEndian = DAG.getDataLayout().isBigEndian();
28830 if (!VT.isInteger() || IsBigEndian)
28831 return SDValue();
28832
28833 SmallVector<int, 16> Mask(SVN->getMask());
28834 auto ForEachDecomposedIndice = [NumElts, &Mask](auto Fn) {
28835 for (int &Indice : Mask) {
28836 if (Indice < 0)
28837 continue;
28838 int OpIdx = (unsigned)Indice < NumElts ? 0 : 1;
28839 int OpEltIdx = (unsigned)Indice < NumElts ? Indice : Indice - NumElts;
28840 Fn(Indice, OpIdx, OpEltIdx);
28841 }
28842 };
28843
28844 // Which elements of which operand does this shuffle demand?
28845 std::array<APInt, 2> OpsDemandedElts;
28846 for (APInt &OpDemandedElts : OpsDemandedElts)
28847 OpDemandedElts = APInt::getZero(numBits: NumElts);
28848 ForEachDecomposedIndice(
28849 [&OpsDemandedElts](int &Indice, int OpIdx, int OpEltIdx) {
28850 OpsDemandedElts[OpIdx].setBit(OpEltIdx);
28851 });
28852
28853 // Element-wise(!), which of these demanded elements are know to be zero?
28854 std::array<APInt, 2> OpsKnownZeroElts;
28855 for (auto I : zip(t: SVN->ops(), u&: OpsDemandedElts, args&: OpsKnownZeroElts))
28856 std::get<2>(t&: I) =
28857 DAG.computeVectorKnownZeroElements(Op: std::get<0>(t&: I), DemandedElts: std::get<1>(t&: I));
28858
28859 // Manifest zeroable element knowledge in the shuffle mask.
28860 // NOTE: we don't have 'zeroable' sentinel value in generic DAG,
28861 // this is a local invention, but it won't leak into DAG.
28862 // FIXME: should we not manifest them, but just check when matching?
28863 bool HadZeroableElts = false;
28864 ForEachDecomposedIndice([&OpsKnownZeroElts, &HadZeroableElts](
28865 int &Indice, int OpIdx, int OpEltIdx) {
28866 if (OpsKnownZeroElts[OpIdx][OpEltIdx]) {
28867 Indice = -2; // Zeroable element.
28868 HadZeroableElts = true;
28869 }
28870 });
28871
28872 // Don't proceed unless we've refined at least one zeroable mask indice.
28873 // If we didn't, then we are still trying to match the same shuffle mask
28874 // we previously tried to match as ISD::ANY_EXTEND_VECTOR_INREG,
28875 // and evidently failed. Proceeding will lead to endless combine loops.
28876 if (!HadZeroableElts)
28877 return SDValue();
28878
28879 // The shuffle may be more fine-grained than we want. Widen elements first.
28880 // FIXME: should we do this before manifesting zeroable shuffle mask indices?
28881 SmallVector<int, 16> ScaledMask;
28882 getShuffleMaskWithWidestElts(Mask, ScaledMask);
28883 assert(Mask.size() >= ScaledMask.size() &&
28884 Mask.size() % ScaledMask.size() == 0 && "Unexpected mask widening.");
28885 int Prescale = Mask.size() / ScaledMask.size();
28886
28887 NumElts = ScaledMask.size();
28888 EltSizeInBits *= Prescale;
28889
28890 EVT PrescaledVT = EVT::getVectorVT(
28891 Context&: *DAG.getContext(), VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: EltSizeInBits),
28892 NumElements: NumElts);
28893
28894 if (LegalTypes && !TLI.isTypeLegal(VT: PrescaledVT) && TLI.isTypeLegal(VT))
28895 return SDValue();
28896
28897 // For example,
28898 // shuffle<0,z,1,-1> == (v2i64 zero_extend_vector_inreg(v4i32))
28899 // But not shuffle<z,z,1,-1> and not shuffle<0,z,z,-1> ! (for same types)
28900 auto isZeroExtend = [NumElts, &ScaledMask](unsigned Scale) {
28901 assert(Scale >= 2 && Scale <= NumElts && NumElts % Scale == 0 &&
28902 "Unexpected mask scaling factor.");
28903 ArrayRef<int> Mask = ScaledMask;
28904 for (unsigned SrcElt = 0, NumSrcElts = NumElts / Scale;
28905 SrcElt != NumSrcElts; ++SrcElt) {
28906 // Analyze the shuffle mask in Scale-sized chunks.
28907 ArrayRef<int> MaskChunk = Mask.take_front(N: Scale);
28908 assert(MaskChunk.size() == Scale && "Unexpected mask size.");
28909 Mask = Mask.drop_front(N: MaskChunk.size());
28910 // The first indice in this chunk must be SrcElt, but not zero!
28911 // FIXME: undef should be fine, but that results in more-defined result.
28912 if (int FirstIndice = MaskChunk[0]; (unsigned)FirstIndice != SrcElt)
28913 return false;
28914 // The rest of the indices in this chunk must be zeros.
28915 // FIXME: undef should be fine, but that results in more-defined result.
28916 if (!all_of(Range: MaskChunk.drop_front(N: 1),
28917 P: [](int Indice) { return Indice == -2; }))
28918 return false;
28919 }
28920 assert(Mask.empty() && "Did not process the whole mask?");
28921 return true;
28922 };
28923
28924 unsigned Opcode = ISD::ZERO_EXTEND_VECTOR_INREG;
28925 for (bool Commuted : {false, true}) {
28926 SDValue Op = SVN->getOperand(Num: !Commuted ? 0 : 1);
28927 if (Commuted)
28928 ShuffleVectorSDNode::commuteMask(Mask: ScaledMask);
28929 std::optional<EVT> OutVT = canCombineShuffleToExtendVectorInreg(
28930 Opcode, VT: PrescaledVT, Match: isZeroExtend, DAG, TLI, LegalTypes,
28931 LegalOperations);
28932 if (OutVT)
28933 return DAG.getBitcast(VT, V: DAG.getNode(Opcode, DL: SDLoc(SVN), VT: *OutVT,
28934 Operand: DAG.getBitcast(VT: PrescaledVT, V: Op)));
28935 }
28936 return SDValue();
28937}
28938
28939// Detect 'truncate_vector_inreg' style shuffles that pack the lower parts of
28940// each source element of a large type into the lowest elements of a smaller
28941// destination type. This is often generated during legalization.
28942// If the source node itself was a '*_extend_vector_inreg' node then we should
28943// then be able to remove it.
28944static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN,
28945 SelectionDAG &DAG) {
28946 EVT VT = SVN->getValueType(ResNo: 0);
28947 bool IsBigEndian = DAG.getDataLayout().isBigEndian();
28948
28949 // TODO Add support for big-endian when we have a test case.
28950 if (!VT.isInteger() || IsBigEndian)
28951 return SDValue();
28952
28953 SDValue N0 = peekThroughBitcasts(V: SVN->getOperand(Num: 0));
28954
28955 unsigned Opcode = N0.getOpcode();
28956 if (!ISD::isExtVecInRegOpcode(Opcode))
28957 return SDValue();
28958
28959 SDValue N00 = N0.getOperand(i: 0);
28960 ArrayRef<int> Mask = SVN->getMask();
28961 unsigned NumElts = VT.getVectorNumElements();
28962 unsigned EltSizeInBits = VT.getScalarSizeInBits();
28963 unsigned ExtSrcSizeInBits = N00.getScalarValueSizeInBits();
28964 unsigned ExtDstSizeInBits = N0.getScalarValueSizeInBits();
28965
28966 if (ExtDstSizeInBits % ExtSrcSizeInBits != 0)
28967 return SDValue();
28968 unsigned ExtScale = ExtDstSizeInBits / ExtSrcSizeInBits;
28969
28970 // (v4i32 truncate_vector_inreg(v2i64)) == shuffle<0,2-1,-1>
28971 // (v8i16 truncate_vector_inreg(v4i32)) == shuffle<0,2,4,6,-1,-1,-1,-1>
28972 // (v8i16 truncate_vector_inreg(v2i64)) == shuffle<0,4,-1,-1,-1,-1,-1,-1>
28973 auto isTruncate = [&Mask, &NumElts](unsigned Scale) {
28974 for (unsigned i = 0; i != NumElts; ++i) {
28975 if (Mask[i] < 0)
28976 continue;
28977 if ((i * Scale) < NumElts && Mask[i] == (int)(i * Scale))
28978 continue;
28979 return false;
28980 }
28981 return true;
28982 };
28983
28984 // At the moment we just handle the case where we've truncated back to the
28985 // same size as before the extension.
28986 // TODO: handle more extension/truncation cases as cases arise.
28987 if (EltSizeInBits != ExtSrcSizeInBits)
28988 return SDValue();
28989 if (VT.getSizeInBits() != N00.getValueSizeInBits())
28990 return SDValue();
28991
28992 // We can remove *extend_vector_inreg only if the truncation happens at
28993 // the same scale as the extension.
28994 if (isTruncate(ExtScale))
28995 return DAG.getBitcast(VT, V: N00);
28996
28997 return SDValue();
28998}
28999
29000// Combine shuffles of splat-shuffles of the form:
29001// shuffle (shuffle V, undef, splat-mask), undef, M
29002// If splat-mask contains undef elements, we need to be careful about
29003// introducing undef's in the folded mask which are not the result of composing
29004// the masks of the shuffles.
29005static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf,
29006 SelectionDAG &DAG) {
29007 EVT VT = Shuf->getValueType(ResNo: 0);
29008 unsigned NumElts = VT.getVectorNumElements();
29009
29010 if (!Shuf->getOperand(Num: 1).isUndef())
29011 return SDValue();
29012
29013 // See if this unary non-splat shuffle actually *is* a splat shuffle,
29014 // in disguise, with all demanded elements being identical.
29015 // FIXME: this can be done per-operand.
29016 if (!Shuf->isSplat()) {
29017 APInt DemandedElts(NumElts, 0);
29018 for (int Idx : Shuf->getMask()) {
29019 if (Idx < 0)
29020 continue; // Ignore sentinel indices.
29021 assert((unsigned)Idx < NumElts && "Out-of-bounds shuffle indice?");
29022 DemandedElts.setBit(Idx);
29023 }
29024 assert(DemandedElts.popcount() > 1 && "Is a splat shuffle already?");
29025 APInt UndefElts;
29026 if (DAG.isSplatValue(V: Shuf->getOperand(Num: 0), DemandedElts, UndefElts)) {
29027 // Even if all demanded elements are splat, some of them could be undef.
29028 // Which lowest demanded element is *not* known-undef?
29029 std::optional<unsigned> MinNonUndefIdx;
29030 for (int Idx : Shuf->getMask()) {
29031 if (Idx < 0 || UndefElts[Idx])
29032 continue; // Ignore sentinel indices, and undef elements.
29033 MinNonUndefIdx = std::min<unsigned>(a: Idx, b: MinNonUndefIdx.value_or(u: ~0U));
29034 }
29035 if (!MinNonUndefIdx)
29036 return DAG.getUNDEF(VT); // All undef - result is undef.
29037 assert(*MinNonUndefIdx < NumElts && "Expected valid element index.");
29038 SmallVector<int, 8> SplatMask(Shuf->getMask());
29039 for (int &Idx : SplatMask) {
29040 if (Idx < 0)
29041 continue; // Passthrough sentinel indices.
29042 // Otherwise, just pick the lowest demanded non-undef element.
29043 // Or sentinel undef, if we know we'd pick a known-undef element.
29044 Idx = UndefElts[Idx] ? -1 : *MinNonUndefIdx;
29045 }
29046 assert(SplatMask != Shuf->getMask() && "Expected mask to change!");
29047 return DAG.getVectorShuffle(VT, dl: SDLoc(Shuf), N1: Shuf->getOperand(Num: 0),
29048 N2: Shuf->getOperand(Num: 1), Mask: SplatMask);
29049 }
29050 }
29051
29052 // If the inner operand is a known splat with no undefs, just return that directly.
29053 // TODO: Create DemandedElts mask from Shuf's mask.
29054 // TODO: Allow undef elements and merge with the shuffle code below.
29055 if (DAG.isSplatValue(V: Shuf->getOperand(Num: 0), /*AllowUndefs*/ false))
29056 return Shuf->getOperand(Num: 0);
29057
29058 auto *Splat = dyn_cast<ShuffleVectorSDNode>(Val: Shuf->getOperand(Num: 0));
29059 if (!Splat || !Splat->isSplat())
29060 return SDValue();
29061
29062 ArrayRef<int> ShufMask = Shuf->getMask();
29063 ArrayRef<int> SplatMask = Splat->getMask();
29064 assert(ShufMask.size() == SplatMask.size() && "Mask length mismatch");
29065
29066 // Prefer simplifying to the splat-shuffle, if possible. This is legal if
29067 // every undef mask element in the splat-shuffle has a corresponding undef
29068 // element in the user-shuffle's mask or if the composition of mask elements
29069 // would result in undef.
29070 // Examples for (shuffle (shuffle v, undef, SplatMask), undef, UserMask):
29071 // * UserMask=[0,2,u,u], SplatMask=[2,u,2,u] -> [2,2,u,u]
29072 // In this case it is not legal to simplify to the splat-shuffle because we
29073 // may be exposing the users of the shuffle an undef element at index 1
29074 // which was not there before the combine.
29075 // * UserMask=[0,u,2,u], SplatMask=[2,u,2,u] -> [2,u,2,u]
29076 // In this case the composition of masks yields SplatMask, so it's ok to
29077 // simplify to the splat-shuffle.
29078 // * UserMask=[3,u,2,u], SplatMask=[2,u,2,u] -> [u,u,2,u]
29079 // In this case the composed mask includes all undef elements of SplatMask
29080 // and in addition sets element zero to undef. It is safe to simplify to
29081 // the splat-shuffle.
29082 auto CanSimplifyToExistingSplat = [](ArrayRef<int> UserMask,
29083 ArrayRef<int> SplatMask) {
29084 for (unsigned i = 0, e = UserMask.size(); i != e; ++i)
29085 if (UserMask[i] != -1 && SplatMask[i] == -1 &&
29086 SplatMask[UserMask[i]] != -1)
29087 return false;
29088 return true;
29089 };
29090 if (CanSimplifyToExistingSplat(ShufMask, SplatMask))
29091 return Shuf->getOperand(Num: 0);
29092
29093 // Create a new shuffle with a mask that is composed of the two shuffles'
29094 // masks.
29095 SmallVector<int, 32> NewMask;
29096 for (int Idx : ShufMask)
29097 NewMask.push_back(Elt: Idx == -1 ? -1 : SplatMask[Idx]);
29098
29099 return DAG.getVectorShuffle(VT: Splat->getValueType(ResNo: 0), dl: SDLoc(Splat),
29100 N1: Splat->getOperand(Num: 0), N2: Splat->getOperand(Num: 1),
29101 Mask: NewMask);
29102}
29103
29104// Combine shuffles of bitcasts into a shuffle of the bitcast type, providing
29105// the mask can be treated as a larger type.
29106static SDValue combineShuffleOfBitcast(ShuffleVectorSDNode *SVN,
29107 SelectionDAG &DAG,
29108 const TargetLowering &TLI,
29109 bool LegalOperations) {
29110 SDValue Op0 = SVN->getOperand(Num: 0);
29111 SDValue Op1 = SVN->getOperand(Num: 1);
29112 EVT VT = SVN->getValueType(ResNo: 0);
29113 if (Op0.getOpcode() != ISD::BITCAST)
29114 return SDValue();
29115 EVT InVT = Op0.getOperand(i: 0).getValueType();
29116 if (!InVT.isVector() ||
29117 (!Op1.isUndef() && (Op1.getOpcode() != ISD::BITCAST ||
29118 Op1.getOperand(i: 0).getValueType() != InVT)))
29119 return SDValue();
29120 if (isAnyConstantBuildVector(V: Op0.getOperand(i: 0)) &&
29121 (Op1.isUndef() || isAnyConstantBuildVector(V: Op1.getOperand(i: 0))))
29122 return SDValue();
29123
29124 int VTLanes = VT.getVectorNumElements();
29125 int InLanes = InVT.getVectorNumElements();
29126 if (VTLanes <= InLanes || VTLanes % InLanes != 0 ||
29127 (LegalOperations &&
29128 !TLI.isOperationLegalOrCustom(Op: ISD::VECTOR_SHUFFLE, VT: InVT)))
29129 return SDValue();
29130 int Factor = VTLanes / InLanes;
29131
29132 // Check that each group of lanes in the mask are either undef or make a valid
29133 // mask for the wider lane type.
29134 ArrayRef<int> Mask = SVN->getMask();
29135 SmallVector<int> NewMask;
29136 if (!widenShuffleMaskElts(Scale: Factor, Mask, ScaledMask&: NewMask))
29137 return SDValue();
29138
29139 if (!TLI.isShuffleMaskLegal(NewMask, InVT))
29140 return SDValue();
29141
29142 // Create the new shuffle with the new mask and bitcast it back to the
29143 // original type.
29144 SDLoc DL(SVN);
29145 Op0 = Op0.getOperand(i: 0);
29146 Op1 = Op1.isUndef() ? DAG.getUNDEF(VT: InVT) : Op1.getOperand(i: 0);
29147 SDValue NewShuf = DAG.getVectorShuffle(VT: InVT, dl: DL, N1: Op0, N2: Op1, Mask: NewMask);
29148 return DAG.getBitcast(VT, V: NewShuf);
29149}
29150
29151/// Combine shuffle of shuffle of the form:
29152/// shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X
29153static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf,
29154 SelectionDAG &DAG) {
29155 if (!OuterShuf->getOperand(Num: 1).isUndef())
29156 return SDValue();
29157 auto *InnerShuf = dyn_cast<ShuffleVectorSDNode>(Val: OuterShuf->getOperand(Num: 0));
29158 if (!InnerShuf || !InnerShuf->getOperand(Num: 1).isUndef())
29159 return SDValue();
29160
29161 ArrayRef<int> OuterMask = OuterShuf->getMask();
29162 ArrayRef<int> InnerMask = InnerShuf->getMask();
29163 unsigned NumElts = OuterMask.size();
29164 assert(NumElts == InnerMask.size() && "Mask length mismatch");
29165 SmallVector<int, 32> CombinedMask(NumElts, -1);
29166 int SplatIndex = -1;
29167 for (unsigned i = 0; i != NumElts; ++i) {
29168 // Undef lanes remain undef.
29169 int OuterMaskElt = OuterMask[i];
29170 if (OuterMaskElt == -1)
29171 continue;
29172
29173 // Peek through the shuffle masks to get the underlying source element.
29174 int InnerMaskElt = InnerMask[OuterMaskElt];
29175 if (InnerMaskElt == -1)
29176 continue;
29177
29178 // Initialize the splatted element.
29179 if (SplatIndex == -1)
29180 SplatIndex = InnerMaskElt;
29181
29182 // Non-matching index - this is not a splat.
29183 if (SplatIndex != InnerMaskElt)
29184 return SDValue();
29185
29186 CombinedMask[i] = InnerMaskElt;
29187 }
29188 assert((all_of(CombinedMask, equal_to(-1)) ||
29189 getSplatIndex(CombinedMask) != -1) &&
29190 "Expected a splat mask");
29191
29192 // TODO: The transform may be a win even if the mask is not legal.
29193 EVT VT = OuterShuf->getValueType(ResNo: 0);
29194 assert(VT == InnerShuf->getValueType(0) && "Expected matching shuffle types");
29195 if (!DAG.getTargetLoweringInfo().isShuffleMaskLegal(CombinedMask, VT))
29196 return SDValue();
29197
29198 return DAG.getVectorShuffle(VT, dl: SDLoc(OuterShuf), N1: InnerShuf->getOperand(Num: 0),
29199 N2: InnerShuf->getOperand(Num: 1), Mask: CombinedMask);
29200}
29201
29202/// If the shuffle mask is taking exactly one element from the first vector
29203/// operand and passing through all other elements from the second vector
29204/// operand, return the index of the mask element that is choosing an element
29205/// from the first operand. Otherwise, return -1.
29206static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef<int> Mask) {
29207 int MaskSize = Mask.size();
29208 int EltFromOp0 = -1;
29209 // TODO: This does not match if there are undef elements in the shuffle mask.
29210 // Should we ignore undefs in the shuffle mask instead? The trade-off is
29211 // removing an instruction (a shuffle), but losing the knowledge that some
29212 // vector lanes are not needed.
29213 for (int i = 0; i != MaskSize; ++i) {
29214 if (Mask[i] >= 0 && Mask[i] < MaskSize) {
29215 // We're looking for a shuffle of exactly one element from operand 0.
29216 if (EltFromOp0 != -1)
29217 return -1;
29218 EltFromOp0 = i;
29219 } else if (Mask[i] != i + MaskSize) {
29220 // Nothing from operand 1 can change lanes.
29221 return -1;
29222 }
29223 }
29224 return EltFromOp0;
29225}
29226
29227/// If a shuffle inserts exactly one element from a source vector operand into
29228/// another vector operand and we can access the specified element as a scalar,
29229/// then we can eliminate the shuffle.
29230SDValue DAGCombiner::replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf) {
29231 // First, check if we are taking one element of a vector and shuffling that
29232 // element into another vector.
29233 ArrayRef<int> Mask = Shuf->getMask();
29234 SmallVector<int, 16> CommutedMask(Mask);
29235 SDValue Op0 = Shuf->getOperand(Num: 0);
29236 SDValue Op1 = Shuf->getOperand(Num: 1);
29237 int ShufOp0Index = getShuffleMaskIndexOfOneElementFromOp0IntoOp1(Mask);
29238 if (ShufOp0Index == -1) {
29239 // Commute mask and check again.
29240 ShuffleVectorSDNode::commuteMask(Mask: CommutedMask);
29241 ShufOp0Index = getShuffleMaskIndexOfOneElementFromOp0IntoOp1(Mask: CommutedMask);
29242 if (ShufOp0Index == -1)
29243 return SDValue();
29244 // Commute operands to match the commuted shuffle mask.
29245 std::swap(a&: Op0, b&: Op1);
29246 Mask = CommutedMask;
29247 }
29248
29249 // The shuffle inserts exactly one element from operand 0 into operand 1.
29250 // Now see if we can access that element as a scalar via a real insert element
29251 // instruction.
29252 // TODO: We can try harder to locate the element as a scalar. Examples: it
29253 // could be an operand of BUILD_VECTOR, or a constant.
29254 assert(Mask[ShufOp0Index] >= 0 && Mask[ShufOp0Index] < (int)Mask.size() &&
29255 "Shuffle mask value must be from operand 0");
29256
29257 SDValue Elt;
29258 if (sd_match(N: Op0, P: m_InsertElt(Vec: m_Value(), Val: m_Value(N&: Elt),
29259 Idx: m_SpecificInt(V: Mask[ShufOp0Index])))) {
29260 // There's an existing insertelement with constant insertion index, so we
29261 // don't need to check the legality/profitability of a replacement operation
29262 // that differs at most in the constant value. The target should be able to
29263 // lower any of those in a similar way. If not, legalization will expand
29264 // this to a scalar-to-vector plus shuffle.
29265 //
29266 // Note that the shuffle may move the scalar from the position that the
29267 // insert element used. Therefore, our new insert element occurs at the
29268 // shuffle's mask index value, not the insert's index value.
29269 //
29270 // shuffle (insertelt v1, x, C), v2, mask --> insertelt v2, x, C'
29271 SDValue NewInsIndex = DAG.getVectorIdxConstant(Val: ShufOp0Index, DL: SDLoc(Shuf));
29272 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(Shuf), VT: Op0.getValueType(),
29273 N1: Op1, N2: Elt, N3: NewInsIndex);
29274 }
29275
29276 if (!hasOperation(Opcode: ISD::INSERT_VECTOR_ELT, VT: Op0.getValueType()))
29277 return SDValue();
29278
29279 if (sd_match(N: Op0, P: m_UnaryOp(Opc: ISD::SCALAR_TO_VECTOR, Op: m_Value(N&: Elt))) &&
29280 Mask[ShufOp0Index] == 0) {
29281 SDValue NewInsIndex = DAG.getVectorIdxConstant(Val: ShufOp0Index, DL: SDLoc(Shuf));
29282 return DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: SDLoc(Shuf), VT: Op0.getValueType(),
29283 N1: Op1, N2: Elt, N3: NewInsIndex);
29284 }
29285
29286 return SDValue();
29287}
29288
29289/// If we have a unary shuffle of a shuffle, see if it can be folded away
29290/// completely. This has the potential to lose undef knowledge because the first
29291/// shuffle may not have an undef mask element where the second one does. So
29292/// only call this after doing simplifications based on demanded elements.
29293static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf) {
29294 // shuf (shuf0 X, Y, Mask0), undef, Mask
29295 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Val: Shuf->getOperand(Num: 0));
29296 if (!Shuf0 || !Shuf->getOperand(Num: 1).isUndef())
29297 return SDValue();
29298
29299 ArrayRef<int> Mask = Shuf->getMask();
29300 ArrayRef<int> Mask0 = Shuf0->getMask();
29301 for (int i = 0, e = (int)Mask.size(); i != e; ++i) {
29302 // Ignore undef elements.
29303 if (Mask[i] == -1)
29304 continue;
29305 assert(Mask[i] >= 0 && Mask[i] < e && "Unexpected shuffle mask value");
29306
29307 // Is the element of the shuffle operand chosen by this shuffle the same as
29308 // the element chosen by the shuffle operand itself?
29309 if (Mask0[Mask[i]] != Mask0[i])
29310 return SDValue();
29311 }
29312 // Every element of this shuffle is identical to the result of the previous
29313 // shuffle, so we can replace this value.
29314 return Shuf->getOperand(Num: 0);
29315}
29316
29317SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
29318 EVT VT = N->getValueType(ResNo: 0);
29319 unsigned NumElts = VT.getVectorNumElements();
29320
29321 SDValue N0 = N->getOperand(Num: 0);
29322 SDValue N1 = N->getOperand(Num: 1);
29323
29324 assert(N0.getValueType() == VT && "Vector shuffle must be normalized in DAG");
29325
29326 // Canonicalize shuffle undef, undef -> undef
29327 if (N0.isUndef() && N1.isUndef())
29328 return DAG.getUNDEF(VT);
29329
29330 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
29331
29332 // Canonicalize shuffle v, v -> v, poison
29333 if (N0 == N1)
29334 return DAG.getVectorShuffle(VT, dl: SDLoc(N), N1: N0, N2: DAG.getPOISON(VT),
29335 Mask: createUnaryMask(Mask: SVN->getMask(), NumElts));
29336
29337 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
29338 if (N0.isUndef())
29339 return DAG.getCommutedVectorShuffle(SV: *SVN);
29340
29341 // Remove references to rhs if it is undef
29342 if (N1.isUndef()) {
29343 bool Changed = false;
29344 SmallVector<int, 8> NewMask;
29345 for (unsigned i = 0; i != NumElts; ++i) {
29346 int Idx = SVN->getMaskElt(Idx: i);
29347 if (Idx >= (int)NumElts) {
29348 Idx = -1;
29349 Changed = true;
29350 }
29351 NewMask.push_back(Elt: Idx);
29352 }
29353 if (Changed)
29354 return DAG.getVectorShuffle(VT, dl: SDLoc(N), N1: N0, N2: N1, Mask: NewMask);
29355 }
29356
29357 if (SDValue InsElt = replaceShuffleOfInsert(Shuf: SVN))
29358 return InsElt;
29359
29360 // A shuffle of a single vector that is a splatted value can always be folded.
29361 if (SDValue V = combineShuffleOfSplatVal(Shuf: SVN, DAG))
29362 return V;
29363
29364 if (SDValue V = formSplatFromShuffles(OuterShuf: SVN, DAG))
29365 return V;
29366
29367 // If it is a splat, check if the argument vector is another splat or a
29368 // build_vector.
29369 if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) {
29370 int SplatIndex = SVN->getSplatIndex();
29371 if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, Index: SplatIndex) &&
29372 TLI.isBinOp(Opcode: N0.getOpcode()) && N0->getNumValues() == 1) {
29373 // splat (vector_bo L, R), Index -->
29374 // splat (scalar_bo (extelt L, Index), (extelt R, Index))
29375 SDValue L = N0.getOperand(i: 0), R = N0.getOperand(i: 1);
29376 SDLoc DL(N);
29377 EVT EltVT = VT.getScalarType();
29378 SDValue Index = DAG.getVectorIdxConstant(Val: SplatIndex, DL);
29379 SDValue ExtL = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: L, N2: Index);
29380 SDValue ExtR = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: R, N2: Index);
29381 SDValue NewBO =
29382 DAG.getNode(Opcode: N0.getOpcode(), DL, VT: EltVT, N1: ExtL, N2: ExtR, Flags: N0->getFlags());
29383 SDValue Insert = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT, Operand: NewBO);
29384 SmallVector<int, 16> ZeroMask(VT.getVectorNumElements(), 0);
29385 return DAG.getVectorShuffle(VT, dl: DL, N1: Insert, N2: DAG.getPOISON(VT), Mask: ZeroMask);
29386 }
29387
29388 // splat(scalar_to_vector(x), 0) -> build_vector(x,...,x)
29389 // splat(insert_vector_elt(v, x, c), c) -> build_vector(x,...,x)
29390 if ((!LegalOperations || TLI.isOperationLegal(Op: ISD::BUILD_VECTOR, VT)) &&
29391 N0.hasOneUse()) {
29392 if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && SplatIndex == 0)
29393 return DAG.getSplatBuildVector(VT, DL: SDLoc(N), Op: N0.getOperand(i: 0));
29394
29395 if (N0.getOpcode() == ISD::INSERT_VECTOR_ELT)
29396 if (auto *Idx = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 2)))
29397 if (Idx->getAPIntValue() == SplatIndex)
29398 return DAG.getSplatBuildVector(VT, DL: SDLoc(N), Op: N0.getOperand(i: 1));
29399
29400 // Look through a bitcast if LE and splatting lane 0, through to a
29401 // scalar_to_vector or a build_vector.
29402 if (N0.getOpcode() == ISD::BITCAST && N0.getOperand(i: 0).hasOneUse() &&
29403 SplatIndex == 0 && DAG.getDataLayout().isLittleEndian() &&
29404 (N0.getOperand(i: 0).getOpcode() == ISD::SCALAR_TO_VECTOR ||
29405 N0.getOperand(i: 0).getOpcode() == ISD::BUILD_VECTOR)) {
29406 EVT N00VT = N0.getOperand(i: 0).getValueType();
29407 if (VT.getScalarSizeInBits() <= N00VT.getScalarSizeInBits() &&
29408 VT.isInteger() && N00VT.isInteger()) {
29409 EVT InVT =
29410 TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: VT.getScalarType());
29411 SDValue Op = DAG.getZExtOrTrunc(Op: N0.getOperand(i: 0).getOperand(i: 0),
29412 DL: SDLoc(N), VT: InVT);
29413 return DAG.getSplatBuildVector(VT, DL: SDLoc(N), Op);
29414 }
29415 }
29416 }
29417
29418 // If this is a bit convert that changes the element type of the vector but
29419 // not the number of vector elements, look through it. Be careful not to
29420 // look though conversions that change things like v4f32 to v2f64.
29421 SDNode *V = N0.getNode();
29422 if (V->getOpcode() == ISD::BITCAST) {
29423 SDValue ConvInput = V->getOperand(Num: 0);
29424 if (ConvInput.getValueType().isVector() &&
29425 ConvInput.getValueType().getVectorNumElements() == NumElts)
29426 V = ConvInput.getNode();
29427 }
29428
29429 if (V->getOpcode() == ISD::BUILD_VECTOR) {
29430 assert(V->getNumOperands() == NumElts &&
29431 "BUILD_VECTOR has wrong number of operands");
29432 SDValue Base;
29433 bool AllSame = true;
29434 for (unsigned i = 0; i != NumElts; ++i) {
29435 if (!V->getOperand(Num: i).isUndef()) {
29436 Base = V->getOperand(Num: i);
29437 break;
29438 }
29439 }
29440 // Splat of <u, u, u, u>, return <u, u, u, u>
29441 if (!Base.getNode())
29442 return N0;
29443 for (unsigned i = 0; i != NumElts; ++i) {
29444 if (V->getOperand(Num: i) != Base) {
29445 AllSame = false;
29446 break;
29447 }
29448 }
29449 // Splat of <x, x, x, x>, return <x, x, x, x>
29450 if (AllSame)
29451 return N0;
29452
29453 // Canonicalize any other splat as a build_vector, but avoid defining any
29454 // undefined elements in the mask.
29455 SDValue Splatted = V->getOperand(Num: SplatIndex);
29456 SmallVector<SDValue, 8> Ops(NumElts, Splatted);
29457 EVT EltVT = Splatted.getValueType();
29458
29459 for (unsigned i = 0; i != NumElts; ++i) {
29460 if (SVN->getMaskElt(Idx: i) < 0)
29461 Ops[i] = DAG.getPOISON(VT: EltVT);
29462 }
29463
29464 SDValue NewBV = DAG.getBuildVector(VT: V->getValueType(ResNo: 0), DL: SDLoc(N), Ops);
29465
29466 // We may have jumped through bitcasts, so the type of the
29467 // BUILD_VECTOR may not match the type of the shuffle.
29468 if (V->getValueType(ResNo: 0) != VT)
29469 NewBV = DAG.getBitcast(VT, V: NewBV);
29470 return NewBV;
29471 }
29472 }
29473
29474 // Simplify source operands based on shuffle mask.
29475 if (SimplifyDemandedVectorElts(Op: SDValue(N, 0)))
29476 return SDValue(N, 0);
29477
29478 // This is intentionally placed after demanded elements simplification because
29479 // it could eliminate knowledge of undef elements created by this shuffle.
29480 if (SDValue ShufOp = simplifyShuffleOfShuffle(Shuf: SVN))
29481 return ShufOp;
29482
29483 // Match shuffles that can be converted to any_vector_extend_in_reg.
29484 if (SDValue V =
29485 combineShuffleToAnyExtendVectorInreg(SVN, DAG, TLI, LegalOperations))
29486 return V;
29487
29488 // Combine "truncate_vector_in_reg" style shuffles.
29489 if (SDValue V = combineTruncationShuffle(SVN, DAG))
29490 return V;
29491
29492 if (N0.getOpcode() == ISD::CONCAT_VECTORS &&
29493 Level < AfterLegalizeVectorOps &&
29494 (N1.isUndef() ||
29495 (N1.getOpcode() == ISD::CONCAT_VECTORS &&
29496 N0.getOperand(i: 0).getValueType() == N1.getOperand(i: 0).getValueType()))) {
29497 if (SDValue V = partitionShuffleOfConcats(N, DAG))
29498 return V;
29499 }
29500
29501 // A shuffle of a concat of the same narrow vector can be reduced to use
29502 // only low-half elements of a concat with undef:
29503 // shuf (concat X, X), undef, Mask --> shuf (concat X, undef), undef, Mask'
29504 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N1.isUndef() &&
29505 N0.getNumOperands() == 2 &&
29506 N0.getOperand(i: 0) == N0.getOperand(i: 1)) {
29507 int HalfNumElts = (int)NumElts / 2;
29508 SmallVector<int, 8> NewMask;
29509 for (unsigned i = 0; i != NumElts; ++i) {
29510 int Idx = SVN->getMaskElt(Idx: i);
29511 if (Idx >= HalfNumElts) {
29512 assert(Idx < (int)NumElts && "Shuffle mask chooses undef op");
29513 Idx -= HalfNumElts;
29514 }
29515 NewMask.push_back(Elt: Idx);
29516 }
29517 if (TLI.isShuffleMaskLegal(NewMask, VT)) {
29518 SDValue UndefVec = DAG.getPOISON(VT: N0.getOperand(i: 0).getValueType());
29519 SDValue NewCat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT,
29520 N1: N0.getOperand(i: 0), N2: UndefVec);
29521 return DAG.getVectorShuffle(VT, dl: SDLoc(N), N1: NewCat, N2: N1, Mask: NewMask);
29522 }
29523 }
29524
29525 // See if we can replace a shuffle with an insert_subvector.
29526 // e.g. v2i32 into v8i32:
29527 // shuffle(lhs,concat(rhs0,rhs1,rhs2,rhs3),0,1,2,3,10,11,6,7).
29528 // --> insert_subvector(lhs,rhs1,4).
29529 if (Level < AfterLegalizeVectorOps && TLI.isTypeLegal(VT) &&
29530 TLI.isOperationLegalOrCustom(Op: ISD::INSERT_SUBVECTOR, VT)) {
29531 auto ShuffleToInsert = [&](SDValue LHS, SDValue RHS, ArrayRef<int> Mask) {
29532 // Ensure RHS subvectors are legal.
29533 assert(RHS.getOpcode() == ISD::CONCAT_VECTORS && "Can't find subvectors");
29534 EVT SubVT = RHS.getOperand(i: 0).getValueType();
29535 int NumSubVecs = RHS.getNumOperands();
29536 int NumSubElts = SubVT.getVectorNumElements();
29537 assert((NumElts % NumSubElts) == 0 && "Subvector mismatch");
29538 if (!TLI.isTypeLegal(VT: SubVT))
29539 return SDValue();
29540
29541 // Don't bother if we have an unary shuffle (matches undef + LHS elts).
29542 if (all_of(Range&: Mask, P: [NumElts](int M) { return M < (int)NumElts; }))
29543 return SDValue();
29544
29545 // Search [NumSubElts] spans for RHS sequence.
29546 // TODO: Can we avoid nested loops to increase performance?
29547 SmallVector<int> InsertionMask(NumElts);
29548 for (int SubVec = 0; SubVec != NumSubVecs; ++SubVec) {
29549 for (int SubIdx = 0; SubIdx != (int)NumElts; SubIdx += NumSubElts) {
29550 // Reset mask to identity.
29551 std::iota(first: InsertionMask.begin(), last: InsertionMask.end(), value: 0);
29552
29553 // Add subvector insertion.
29554 std::iota(first: InsertionMask.begin() + SubIdx,
29555 last: InsertionMask.begin() + SubIdx + NumSubElts,
29556 value: NumElts + (SubVec * NumSubElts));
29557
29558 // See if the shuffle mask matches the reference insertion mask.
29559 bool MatchingShuffle = true;
29560 for (int i = 0; i != (int)NumElts; ++i) {
29561 int ExpectIdx = InsertionMask[i];
29562 int ActualIdx = Mask[i];
29563 if (0 <= ActualIdx && ExpectIdx != ActualIdx) {
29564 MatchingShuffle = false;
29565 break;
29566 }
29567 }
29568
29569 if (MatchingShuffle)
29570 return DAG.getInsertSubvector(DL: SDLoc(N), Vec: LHS, SubVec: RHS.getOperand(i: SubVec),
29571 Idx: SubIdx);
29572 }
29573 }
29574 return SDValue();
29575 };
29576 ArrayRef<int> Mask = SVN->getMask();
29577 if (N1.getOpcode() == ISD::CONCAT_VECTORS)
29578 if (SDValue InsertN1 = ShuffleToInsert(N0, N1, Mask))
29579 return InsertN1;
29580 if (N0.getOpcode() == ISD::CONCAT_VECTORS) {
29581 SmallVector<int> CommuteMask(Mask);
29582 ShuffleVectorSDNode::commuteMask(Mask: CommuteMask);
29583 if (SDValue InsertN0 = ShuffleToInsert(N1, N0, CommuteMask))
29584 return InsertN0;
29585 }
29586 }
29587
29588 // If we're not performing a select/blend shuffle, see if we can convert the
29589 // shuffle into a AND node, with all the out-of-lane elements are known zero.
29590 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) {
29591 bool IsInLaneMask = true;
29592 ArrayRef<int> Mask = SVN->getMask();
29593 SmallVector<int, 16> ClearMask(NumElts, -1);
29594 APInt DemandedLHS = APInt::getZero(numBits: NumElts);
29595 APInt DemandedRHS = APInt::getZero(numBits: NumElts);
29596 for (int I = 0; I != (int)NumElts; ++I) {
29597 int M = Mask[I];
29598 if (M < 0)
29599 continue;
29600 ClearMask[I] = M == I ? I : (I + NumElts);
29601 IsInLaneMask &= (M == I) || (M == (int)(I + NumElts));
29602 if (M != I) {
29603 APInt &Demanded = M < (int)NumElts ? DemandedLHS : DemandedRHS;
29604 Demanded.setBit(M % NumElts);
29605 }
29606 }
29607 // TODO: Should we try to mask with N1 as well?
29608 if (!IsInLaneMask && (!DemandedLHS.isZero() || !DemandedRHS.isZero()) &&
29609 (DemandedLHS.isZero() || DAG.MaskedVectorIsZero(Op: N0, DemandedElts: DemandedLHS)) &&
29610 (DemandedRHS.isZero() || DAG.MaskedVectorIsZero(Op: N1, DemandedElts: DemandedRHS))) {
29611 SDLoc DL(N);
29612 EVT IntVT = VT.changeVectorElementTypeToInteger();
29613 EVT IntSVT = VT.getVectorElementType().changeTypeToInteger();
29614 // Transform the type to a legal type so that the buildvector constant
29615 // elements are not illegal. Make sure that the result is larger than the
29616 // original type, incase the value is split into two (eg i64->i32).
29617 if (!TLI.isTypeLegal(VT: IntSVT) && LegalTypes)
29618 IntSVT = TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: IntSVT);
29619 if (IntSVT.getSizeInBits() >= IntVT.getScalarSizeInBits()) {
29620 SDValue ZeroElt = DAG.getConstant(Val: 0, DL, VT: IntSVT);
29621 SDValue AllOnesElt = DAG.getAllOnesConstant(DL, VT: IntSVT);
29622 SmallVector<SDValue, 16> AndMask(NumElts, DAG.getPOISON(VT: IntSVT));
29623 for (int I = 0; I != (int)NumElts; ++I)
29624 if (0 <= Mask[I])
29625 AndMask[I] = Mask[I] == I ? AllOnesElt : ZeroElt;
29626
29627 // See if a clear mask is legal instead of going via
29628 // XformToShuffleWithZero which loses UNDEF mask elements.
29629 if (TLI.isVectorClearMaskLegal(ClearMask, IntVT))
29630 return DAG.getBitcast(
29631 VT, V: DAG.getVectorShuffle(VT: IntVT, dl: DL, N1: DAG.getBitcast(VT: IntVT, V: N0),
29632 N2: DAG.getConstant(Val: 0, DL, VT: IntVT), Mask: ClearMask));
29633
29634 if (TLI.isOperationLegalOrCustom(Op: ISD::AND, VT: IntVT))
29635 return DAG.getBitcast(
29636 VT, V: DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: DAG.getBitcast(VT: IntVT, V: N0),
29637 N2: DAG.getBuildVector(VT: IntVT, DL, Ops: AndMask)));
29638 }
29639 }
29640 }
29641
29642 // Attempt to combine a shuffle of 2 inputs of 'scalar sources' -
29643 // BUILD_VECTOR or SCALAR_TO_VECTOR into a single BUILD_VECTOR.
29644 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT))
29645 if (SDValue Res = combineShuffleOfScalars(SVN, DAG, TLI))
29646 return Res;
29647
29648 // If this shuffle only has a single input that is a bitcasted shuffle,
29649 // attempt to merge the 2 shuffles and suitably bitcast the inputs/output
29650 // back to their original types.
29651 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
29652 N1.isUndef() && Level < AfterLegalizeVectorOps &&
29653 TLI.isTypeLegal(VT)) {
29654
29655 SDValue BC0 = peekThroughOneUseBitcasts(V: N0);
29656 if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) {
29657 EVT SVT = VT.getScalarType();
29658 EVT InnerVT = BC0->getValueType(ResNo: 0);
29659 EVT InnerSVT = InnerVT.getScalarType();
29660
29661 // Determine which shuffle works with the smaller scalar type.
29662 EVT ScaleVT = SVT.bitsLT(VT: InnerSVT) ? VT : InnerVT;
29663 EVT ScaleSVT = ScaleVT.getScalarType();
29664
29665 if (TLI.isTypeLegal(VT: ScaleVT) &&
29666 0 == (InnerSVT.getSizeInBits() % ScaleSVT.getSizeInBits()) &&
29667 0 == (SVT.getSizeInBits() % ScaleSVT.getSizeInBits())) {
29668 int InnerScale = InnerSVT.getSizeInBits() / ScaleSVT.getSizeInBits();
29669 int OuterScale = SVT.getSizeInBits() / ScaleSVT.getSizeInBits();
29670
29671 // Scale the shuffle masks to the smaller scalar type.
29672 ShuffleVectorSDNode *InnerSVN = cast<ShuffleVectorSDNode>(Val&: BC0);
29673 SmallVector<int, 8> InnerMask;
29674 SmallVector<int, 8> OuterMask;
29675 narrowShuffleMaskElts(Scale: InnerScale, Mask: InnerSVN->getMask(), ScaledMask&: InnerMask);
29676 narrowShuffleMaskElts(Scale: OuterScale, Mask: SVN->getMask(), ScaledMask&: OuterMask);
29677
29678 // Merge the shuffle masks.
29679 SmallVector<int, 8> NewMask;
29680 for (int M : OuterMask)
29681 NewMask.push_back(Elt: M < 0 ? -1 : InnerMask[M]);
29682
29683 // Test for shuffle mask legality over both commutations.
29684 SDValue SV0 = BC0->getOperand(Num: 0);
29685 SDValue SV1 = BC0->getOperand(Num: 1);
29686 bool LegalMask = TLI.isShuffleMaskLegal(NewMask, ScaleVT);
29687 if (!LegalMask) {
29688 std::swap(a&: SV0, b&: SV1);
29689 ShuffleVectorSDNode::commuteMask(Mask: NewMask);
29690 LegalMask = TLI.isShuffleMaskLegal(NewMask, ScaleVT);
29691 }
29692
29693 if (LegalMask) {
29694 SV0 = DAG.getBitcast(VT: ScaleVT, V: SV0);
29695 SV1 = DAG.getBitcast(VT: ScaleVT, V: SV1);
29696 return DAG.getBitcast(
29697 VT, V: DAG.getVectorShuffle(VT: ScaleVT, dl: SDLoc(N), N1: SV0, N2: SV1, Mask: NewMask));
29698 }
29699 }
29700 }
29701 }
29702
29703 // Match shuffles of bitcasts, so long as the mask can be treated as the
29704 // larger type.
29705 if (SDValue V = combineShuffleOfBitcast(SVN, DAG, TLI, LegalOperations))
29706 return V;
29707
29708 // Compute the combined shuffle mask for a shuffle with SV0 as the first
29709 // operand, and SV1 as the second operand.
29710 // i.e. Merge SVN(OtherSVN, N1) -> shuffle(SV0, SV1, Mask) iff Commute = false
29711 // Merge SVN(N1, OtherSVN) -> shuffle(SV0, SV1, Mask') iff Commute = true
29712 auto MergeInnerShuffle =
29713 [NumElts, &VT](bool Commute, ShuffleVectorSDNode *SVN,
29714 ShuffleVectorSDNode *OtherSVN, SDValue N1,
29715 const TargetLowering &TLI, SDValue &SV0, SDValue &SV1,
29716 SmallVectorImpl<int> &Mask) -> bool {
29717 // Don't try to fold splats; they're likely to simplify somehow, or they
29718 // might be free.
29719 if (OtherSVN->isSplat())
29720 return false;
29721
29722 SV0 = SV1 = SDValue();
29723 Mask.clear();
29724
29725 for (unsigned i = 0; i != NumElts; ++i) {
29726 int Idx = SVN->getMaskElt(Idx: i);
29727 if (Idx < 0) {
29728 // Propagate Undef.
29729 Mask.push_back(Elt: Idx);
29730 continue;
29731 }
29732
29733 if (Commute)
29734 Idx = (Idx < (int)NumElts) ? (Idx + NumElts) : (Idx - NumElts);
29735
29736 SDValue CurrentVec;
29737 if (Idx < (int)NumElts) {
29738 // This shuffle index refers to the inner shuffle N0. Lookup the inner
29739 // shuffle mask to identify which vector is actually referenced.
29740 Idx = OtherSVN->getMaskElt(Idx);
29741 if (Idx < 0) {
29742 // Propagate Undef.
29743 Mask.push_back(Elt: Idx);
29744 continue;
29745 }
29746 CurrentVec = (Idx < (int)NumElts) ? OtherSVN->getOperand(Num: 0)
29747 : OtherSVN->getOperand(Num: 1);
29748 } else {
29749 // This shuffle index references an element within N1.
29750 CurrentVec = N1;
29751 }
29752
29753 // Simple case where 'CurrentVec' is UNDEF.
29754 if (CurrentVec.isUndef()) {
29755 Mask.push_back(Elt: -1);
29756 continue;
29757 }
29758
29759 // Canonicalize the shuffle index. We don't know yet if CurrentVec
29760 // will be the first or second operand of the combined shuffle.
29761 Idx = Idx % NumElts;
29762 if (!SV0.getNode() || SV0 == CurrentVec) {
29763 // Ok. CurrentVec is the left hand side.
29764 // Update the mask accordingly.
29765 SV0 = CurrentVec;
29766 Mask.push_back(Elt: Idx);
29767 continue;
29768 }
29769 if (!SV1.getNode() || SV1 == CurrentVec) {
29770 // Ok. CurrentVec is the right hand side.
29771 // Update the mask accordingly.
29772 SV1 = CurrentVec;
29773 Mask.push_back(Elt: Idx + NumElts);
29774 continue;
29775 }
29776
29777 // Last chance - see if the vector is another shuffle and if it
29778 // uses one of the existing candidate shuffle ops.
29779 if (auto *CurrentSVN = dyn_cast<ShuffleVectorSDNode>(Val&: CurrentVec)) {
29780 int InnerIdx = CurrentSVN->getMaskElt(Idx);
29781 if (InnerIdx < 0) {
29782 Mask.push_back(Elt: -1);
29783 continue;
29784 }
29785 SDValue InnerVec = (InnerIdx < (int)NumElts)
29786 ? CurrentSVN->getOperand(Num: 0)
29787 : CurrentSVN->getOperand(Num: 1);
29788 if (InnerVec.isUndef()) {
29789 Mask.push_back(Elt: -1);
29790 continue;
29791 }
29792 InnerIdx %= NumElts;
29793 if (InnerVec == SV0) {
29794 Mask.push_back(Elt: InnerIdx);
29795 continue;
29796 }
29797 if (InnerVec == SV1) {
29798 Mask.push_back(Elt: InnerIdx + NumElts);
29799 continue;
29800 }
29801 }
29802
29803 // Bail out if we cannot convert the shuffle pair into a single shuffle.
29804 return false;
29805 }
29806
29807 if (llvm::all_of(Range&: Mask, P: [](int M) { return M < 0; }))
29808 return true;
29809
29810 // Avoid introducing shuffles with illegal mask.
29811 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, B, M2)
29812 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, C, M2)
29813 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(B, C, M2)
29814 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(B, A, M2)
29815 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(C, A, M2)
29816 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(C, B, M2)
29817 if (TLI.isShuffleMaskLegal(Mask, VT))
29818 return true;
29819
29820 std::swap(a&: SV0, b&: SV1);
29821 ShuffleVectorSDNode::commuteMask(Mask);
29822 return TLI.isShuffleMaskLegal(Mask, VT);
29823 };
29824
29825 if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT)) {
29826 // Canonicalize shuffles according to rules:
29827 // shuffle(A, shuffle(A, B)) -> shuffle(shuffle(A,B), A)
29828 // shuffle(B, shuffle(A, B)) -> shuffle(shuffle(A,B), B)
29829 // shuffle(B, shuffle(A, Undef)) -> shuffle(shuffle(A, Undef), B)
29830 if (N1.getOpcode() == ISD::VECTOR_SHUFFLE &&
29831 N0.getOpcode() != ISD::VECTOR_SHUFFLE) {
29832 // The incoming shuffle must be of the same type as the result of the
29833 // current shuffle.
29834 assert(N1->getOperand(0).getValueType() == VT &&
29835 "Shuffle types don't match");
29836
29837 SDValue SV0 = N1->getOperand(Num: 0);
29838 SDValue SV1 = N1->getOperand(Num: 1);
29839 bool HasSameOp0 = N0 == SV0;
29840 bool IsSV1Undef = SV1.isUndef();
29841 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
29842 // Commute the operands of this shuffle so merging below will trigger.
29843 return DAG.getCommutedVectorShuffle(SV: *SVN);
29844 }
29845
29846 // Canonicalize splat shuffles to the RHS to improve merging below.
29847 // shuffle(splat(A,u), shuffle(C,D)) -> shuffle'(shuffle(C,D), splat(A,u))
29848 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE &&
29849 N1.getOpcode() == ISD::VECTOR_SHUFFLE &&
29850 cast<ShuffleVectorSDNode>(Val&: N0)->isSplat() &&
29851 !cast<ShuffleVectorSDNode>(Val&: N1)->isSplat()) {
29852 return DAG.getCommutedVectorShuffle(SV: *SVN);
29853 }
29854
29855 // Try to fold according to rules:
29856 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, B, M2)
29857 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(A, C, M2)
29858 // shuffle(shuffle(A, B, M0), C, M1) -> shuffle(B, C, M2)
29859 // Don't try to fold shuffles with illegal type.
29860 // Only fold if this shuffle is the only user of the other shuffle.
29861 // Try matching shuffle(C,shuffle(A,B)) commutted patterns as well.
29862 for (int i = 0; i != 2; ++i) {
29863 if (N->getOperand(Num: i).getOpcode() == ISD::VECTOR_SHUFFLE &&
29864 N->isOnlyUserOf(N: N->getOperand(Num: i).getNode())) {
29865 // The incoming shuffle must be of the same type as the result of the
29866 // current shuffle.
29867 auto *OtherSV = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: i));
29868 assert(OtherSV->getOperand(0).getValueType() == VT &&
29869 "Shuffle types don't match");
29870
29871 SDValue SV0, SV1;
29872 SmallVector<int, 4> Mask;
29873 if (MergeInnerShuffle(i != 0, SVN, OtherSV, N->getOperand(Num: 1 - i), TLI,
29874 SV0, SV1, Mask)) {
29875 // Check if all indices in Mask are poison. In case, propagate poison.
29876 if (llvm::all_of(Range&: Mask, P: [](int M) { return M < 0; }))
29877 return DAG.getPOISON(VT);
29878
29879 return DAG.getVectorShuffle(VT, dl: SDLoc(N),
29880 N1: SV0 ? SV0 : DAG.getPOISON(VT),
29881 N2: SV1 ? SV1 : DAG.getPOISON(VT), Mask);
29882 }
29883 }
29884 }
29885
29886 // Merge shuffles through binops if we are able to merge it with at least
29887 // one other shuffles.
29888 // shuffle(bop(shuffle(x,y),shuffle(z,w)),undef)
29889 // shuffle(bop(shuffle(x,y),shuffle(z,w)),bop(shuffle(a,b),shuffle(c,d)))
29890 unsigned SrcOpcode = N0.getOpcode();
29891 if (TLI.isBinOp(Opcode: SrcOpcode) && N->isOnlyUserOf(N: N0.getNode()) &&
29892 (N1.isUndef() ||
29893 (SrcOpcode == N1.getOpcode() && N->isOnlyUserOf(N: N1.getNode()) &&
29894 N0.getResNo() == N1.getResNo()))) {
29895 // Get binop source ops, or just pass on the undef.
29896 SDValue Op00 = N0.getOperand(i: 0);
29897 SDValue Op01 = N0.getOperand(i: 1);
29898 SDValue Op10 = N1.isUndef() ? N1 : N1.getOperand(i: 0);
29899 SDValue Op11 = N1.isUndef() ? N1 : N1.getOperand(i: 1);
29900 // TODO: We might be able to relax the VT check but we don't currently
29901 // have any isBinOp() that has different result/ops VTs so play safe until
29902 // we have test coverage.
29903 if (Op00.getValueType() == VT && Op10.getValueType() == VT &&
29904 Op01.getValueType() == VT && Op11.getValueType() == VT &&
29905 (Op00.getOpcode() == ISD::VECTOR_SHUFFLE ||
29906 Op10.getOpcode() == ISD::VECTOR_SHUFFLE ||
29907 Op01.getOpcode() == ISD::VECTOR_SHUFFLE ||
29908 Op11.getOpcode() == ISD::VECTOR_SHUFFLE)) {
29909 auto CanMergeInnerShuffle = [&](SDValue &SV0, SDValue &SV1,
29910 SmallVectorImpl<int> &Mask, bool LeftOp,
29911 bool Commute) {
29912 SDValue InnerN = Commute ? N1 : N0;
29913 SDValue Op0 = LeftOp ? Op00 : Op01;
29914 SDValue Op1 = LeftOp ? Op10 : Op11;
29915 if (Commute)
29916 std::swap(a&: Op0, b&: Op1);
29917 // Only accept the merged shuffle if we don't introduce undef elements,
29918 // or the inner shuffle already contained undef elements.
29919 auto *SVN0 = dyn_cast<ShuffleVectorSDNode>(Val&: Op0);
29920 return SVN0 && InnerN->isOnlyUserOf(N: SVN0) &&
29921 MergeInnerShuffle(Commute, SVN, SVN0, Op1, TLI, SV0, SV1,
29922 Mask) &&
29923 (llvm::any_of(Range: SVN0->getMask(), P: [](int M) { return M < 0; }) ||
29924 llvm::none_of(Range&: Mask, P: [](int M) { return M < 0; }));
29925 };
29926
29927 // Ensure we don't increase the number of shuffles - we must merge a
29928 // shuffle from at least one of the LHS and RHS ops.
29929 bool MergedLeft = false;
29930 SDValue LeftSV0, LeftSV1;
29931 SmallVector<int, 4> LeftMask;
29932 if (CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask, true, false) ||
29933 CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask, true, true)) {
29934 MergedLeft = true;
29935 } else {
29936 LeftMask.assign(in_start: SVN->getMask().begin(), in_end: SVN->getMask().end());
29937 LeftSV0 = Op00, LeftSV1 = Op10;
29938 }
29939
29940 bool MergedRight = false;
29941 SDValue RightSV0, RightSV1;
29942 SmallVector<int, 4> RightMask;
29943 if (CanMergeInnerShuffle(RightSV0, RightSV1, RightMask, false, false) ||
29944 CanMergeInnerShuffle(RightSV0, RightSV1, RightMask, false, true)) {
29945 MergedRight = true;
29946 } else {
29947 RightMask.assign(in_start: SVN->getMask().begin(), in_end: SVN->getMask().end());
29948 RightSV0 = Op01, RightSV1 = Op11;
29949 }
29950
29951 if (MergedLeft || MergedRight) {
29952 SDLoc DL(N);
29953 SDValue LHS = DAG.getVectorShuffle(
29954 VT, dl: DL, N1: LeftSV0 ? LeftSV0 : DAG.getPOISON(VT),
29955 N2: LeftSV1 ? LeftSV1 : DAG.getPOISON(VT), Mask: LeftMask);
29956 SDValue RHS = DAG.getVectorShuffle(
29957 VT, dl: DL, N1: RightSV0 ? RightSV0 : DAG.getPOISON(VT),
29958 N2: RightSV1 ? RightSV1 : DAG.getPOISON(VT), Mask: RightMask);
29959 return DAG.getNode(Opcode: SrcOpcode, DL, VTList: N0->getVTList(), N1: LHS, N2: RHS)
29960 .getValue(R: N0.getResNo());
29961 }
29962 }
29963 }
29964 }
29965
29966 if (SDValue V = foldShuffleOfConcatUndefs(Shuf: SVN, DAG))
29967 return V;
29968
29969 // Match shuffles that can be converted to ISD::ZERO_EXTEND_VECTOR_INREG.
29970 // Perform this really late, because it could eliminate knowledge
29971 // of undef elements created by this shuffle.
29972 if (Level < AfterLegalizeTypes)
29973 if (SDValue V = combineShuffleToZeroExtendVectorInReg(SVN, DAG, TLI,
29974 LegalOperations))
29975 return V;
29976
29977 return SDValue();
29978}
29979
29980SDValue DAGCombiner::visitSCALAR_TO_VECTOR(SDNode *N) {
29981 EVT VT = N->getValueType(ResNo: 0);
29982 if (!VT.isFixedLengthVector())
29983 return SDValue();
29984
29985 // Try to convert a scalar binop with an extracted vector element to a vector
29986 // binop. This is intended to reduce potentially expensive register moves.
29987 // TODO: Check if both operands are extracted.
29988 // TODO: How to prefer scalar/vector ops with multiple uses of the extact?
29989 // TODO: Generalize this, so it can be called from visitINSERT_VECTOR_ELT().
29990 SDValue Scalar = N->getOperand(Num: 0);
29991 unsigned Opcode = Scalar.getOpcode();
29992 EVT VecEltVT = VT.getScalarType();
29993 if (Scalar.hasOneUse() && Scalar->getNumValues() == 1 &&
29994 TLI.isBinOp(Opcode) && Scalar.getValueType() == VecEltVT &&
29995 Scalar.getOperand(i: 0).getValueType() == VecEltVT &&
29996 Scalar.getOperand(i: 1).getValueType() == VecEltVT &&
29997 Scalar->isOnlyUserOf(N: Scalar.getOperand(i: 0).getNode()) &&
29998 Scalar->isOnlyUserOf(N: Scalar.getOperand(i: 1).getNode()) &&
29999 DAG.isSafeToSpeculativelyExecute(Opcode) && hasOperation(Opcode, VT)) {
30000 // Match an extract element and get a shuffle mask equivalent.
30001 SmallVector<int, 8> ShufMask(VT.getVectorNumElements(), -1);
30002
30003 for (int i : {0, 1}) {
30004 // s2v (bo (extelt V, Idx), C) --> shuffle (bo V, C'), {Idx, -1, -1...}
30005 // s2v (bo C, (extelt V, Idx)) --> shuffle (bo C', V), {Idx, -1, -1...}
30006 SDValue EE = Scalar.getOperand(i);
30007 auto *C = dyn_cast<ConstantSDNode>(Val: Scalar.getOperand(i: i ? 0 : 1));
30008 if (C && EE.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
30009 EE.getOperand(i: 0).getValueType() == VT &&
30010 isa<ConstantSDNode>(Val: EE.getOperand(i: 1))) {
30011 // Mask = {ExtractIndex, undef, undef....}
30012 ShufMask[0] = EE.getConstantOperandVal(i: 1);
30013 // Make sure the shuffle is legal if we are crossing lanes.
30014 if (TLI.isShuffleMaskLegal(ShufMask, VT)) {
30015 SDLoc DL(N);
30016 SDValue V[] = {EE.getOperand(i: 0),
30017 DAG.getConstant(Val: C->getAPIntValue(), DL, VT)};
30018 SDValue VecBO = DAG.getNode(Opcode, DL, VT, N1: V[i], N2: V[1 - i]);
30019 return DAG.getVectorShuffle(VT, dl: DL, N1: VecBO, N2: DAG.getPOISON(VT),
30020 Mask: ShufMask);
30021 }
30022 }
30023 }
30024 }
30025
30026 // Replace a SCALAR_TO_VECTOR(EXTRACT_VECTOR_ELT(V,C0)) pattern
30027 // with a VECTOR_SHUFFLE and possible truncate.
30028 if (Opcode != ISD::EXTRACT_VECTOR_ELT ||
30029 !Scalar.getOperand(i: 0).getValueType().isFixedLengthVector())
30030 return SDValue();
30031
30032 // If we have an implicit truncate, truncate here if it is legal.
30033 if (VecEltVT != Scalar.getValueType() &&
30034 Scalar.getValueType().isScalarInteger() && isTypeLegal(VT: VecEltVT)) {
30035 SDValue Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(Scalar), VT: VecEltVT, Operand: Scalar);
30036 return DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: SDLoc(N), VT, Operand: Val);
30037 }
30038
30039 auto *ExtIndexC = dyn_cast<ConstantSDNode>(Val: Scalar.getOperand(i: 1));
30040 if (!ExtIndexC)
30041 return SDValue();
30042
30043 SDValue SrcVec = Scalar.getOperand(i: 0);
30044 EVT SrcVT = SrcVec.getValueType();
30045 unsigned SrcNumElts = SrcVT.getVectorNumElements();
30046 unsigned VTNumElts = VT.getVectorNumElements();
30047 if (VecEltVT == SrcVT.getScalarType() && VTNumElts <= SrcNumElts) {
30048 // Create a shuffle equivalent for scalar-to-vector: {ExtIndex, -1, -1, ...}
30049 SmallVector<int, 8> Mask(SrcNumElts, -1);
30050 Mask[0] = ExtIndexC->getZExtValue();
30051 SDValue LegalShuffle = TLI.buildLegalVectorShuffle(
30052 VT: SrcVT, DL: SDLoc(N), N0: SrcVec, N1: DAG.getPOISON(VT: SrcVT), Mask, DAG);
30053 if (!LegalShuffle)
30054 return SDValue();
30055
30056 // If the initial vector is the same size, the shuffle is the result.
30057 if (VT == SrcVT)
30058 return LegalShuffle;
30059
30060 // If not, shorten the shuffled vector.
30061 if (VTNumElts != SrcNumElts) {
30062 SDValue ZeroIdx = DAG.getVectorIdxConstant(Val: 0, DL: SDLoc(N));
30063 EVT SubVT = EVT::getVectorVT(Context&: *DAG.getContext(),
30064 VT: SrcVT.getVectorElementType(), NumElements: VTNumElts);
30065 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N), VT: SubVT, N1: LegalShuffle,
30066 N2: ZeroIdx);
30067 }
30068 }
30069
30070 return SDValue();
30071}
30072
30073SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
30074 EVT VT = N->getValueType(ResNo: 0);
30075 SDValue N0 = N->getOperand(Num: 0);
30076 SDValue N1 = N->getOperand(Num: 1);
30077 SDValue N2 = N->getOperand(Num: 2);
30078 uint64_t InsIdx = N->getConstantOperandVal(Num: 2);
30079
30080 // Remove insert of UNDEF/POISON.
30081 if (N1.isUndef()) {
30082 if (N1.getOpcode() == ISD::POISON || N0.getOpcode() == ISD::UNDEF)
30083 return N0;
30084 return DAG.getFreeze(V: N0);
30085 }
30086
30087 // If this is an insert of an extracted vector into an undef/poison vector, we
30088 // can just use the input to the extract if the types match, and can simplify
30089 // in some cases even if they don't.
30090 if (N0.isUndef() && N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
30091 N1.getOperand(i: 1) == N2) {
30092 EVT N1VT = N1.getValueType();
30093 EVT SrcVT = N1.getOperand(i: 0).getValueType();
30094 if (SrcVT == VT) {
30095 // Need to ensure that result isn't more poisonous if skipping both the
30096 // extract+insert.
30097 if (N0.getOpcode() == ISD::POISON)
30098 return N1.getOperand(i: 0);
30099 if (VT.isFixedLengthVector() && N1VT.isFixedLengthVector()) {
30100 unsigned SubVecNumElts = N1VT.getVectorNumElements();
30101 APInt EltMask = APInt::getBitsSet(numBits: VT.getVectorNumElements(), loBit: InsIdx,
30102 hiBit: InsIdx + SubVecNumElts);
30103 if (DAG.isGuaranteedNotToBePoison(Op: N1.getOperand(i: 0), DemandedElts: ~EltMask))
30104 return N1.getOperand(i: 0);
30105 } else if (DAG.isGuaranteedNotToBePoison(Op: N1.getOperand(i: 0)))
30106 return N1.getOperand(i: 0);
30107 }
30108 // TODO: To remove the zero check, need to adjust the offset to
30109 // a multiple of the new src type.
30110 if (isNullConstant(V: N2)) {
30111 if (VT.knownBitsGE(VT: SrcVT) &&
30112 !(VT.isFixedLengthVector() && SrcVT.isScalableVector()))
30113 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N),
30114 VT, N1: N0, N2: N1.getOperand(i: 0), N3: N2);
30115 else if (VT.knownBitsLE(VT: SrcVT) &&
30116 !(VT.isScalableVector() && SrcVT.isFixedLengthVector()))
30117 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL: SDLoc(N),
30118 VT, N1: N1.getOperand(i: 0), N2);
30119 }
30120 }
30121
30122 // Handle case where we've ended up inserting back into the source vector
30123 // we extracted the subvector from.
30124 // insert_subvector(N0, extract_subvector(N0, N2), N2) --> N0
30125 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR && N1.getOperand(i: 0) == N0 &&
30126 N1.getOperand(i: 1) == N2)
30127 return N0;
30128
30129 // Simplify scalar inserts into an undef vector:
30130 // insert_subvector undef, (splat X), N2 -> splat X
30131 if (N0.isUndef() && N1.getOpcode() == ISD::SPLAT_VECTOR)
30132 if (DAG.isConstantValueOfAnyType(N: N1.getOperand(i: 0)) || N1.hasOneUse())
30133 return DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: SDLoc(N), VT, Operand: N1.getOperand(i: 0));
30134
30135 // insert_subvector (splat X), (splat X), N2 -> splat X
30136 if (N0.getOpcode() == ISD::SPLAT_VECTOR && N0.getOpcode() == N1.getOpcode() &&
30137 N0.getOperand(i: 0) == N1.getOperand(i: 0))
30138 return N0;
30139
30140 // If we are inserting a bitcast value into an undef, with the same
30141 // number of elements, just use the bitcast input of the extract.
30142 // i.e. INSERT_SUBVECTOR UNDEF (BITCAST N1) N2 ->
30143 // BITCAST (INSERT_SUBVECTOR UNDEF N1 N2)
30144 if (N0.isUndef() && N1.getOpcode() == ISD::BITCAST &&
30145 N1.getOperand(i: 0).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
30146 N1.getOperand(i: 0).getOperand(i: 1) == N2 &&
30147 N1.getOperand(i: 0).getOperand(i: 0).getValueType().getVectorElementCount() ==
30148 VT.getVectorElementCount() &&
30149 N1.getOperand(i: 0).getOperand(i: 0).getValueType().getSizeInBits() ==
30150 VT.getSizeInBits()) {
30151 return DAG.getBitcast(VT, V: N1.getOperand(i: 0).getOperand(i: 0));
30152 }
30153
30154 // If both N1 and N2 are bitcast values on which insert_subvector
30155 // would makes sense, pull the bitcast through.
30156 // i.e. INSERT_SUBVECTOR (BITCAST N0) (BITCAST N1) N2 ->
30157 // BITCAST (INSERT_SUBVECTOR N0 N1 N2)
30158 if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST) {
30159 SDValue CN0 = N0.getOperand(i: 0);
30160 SDValue CN1 = N1.getOperand(i: 0);
30161 EVT CN0VT = CN0.getValueType();
30162 EVT CN1VT = CN1.getValueType();
30163 if (CN0VT.isVector() && CN1VT.isVector() &&
30164 CN0VT.getVectorElementType() == CN1VT.getVectorElementType() &&
30165 CN0VT.getVectorElementCount() == VT.getVectorElementCount()) {
30166 SDValue NewINSERT = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N),
30167 VT: CN0.getValueType(), N1: CN0, N2: CN1, N3: N2);
30168 return DAG.getBitcast(VT, V: NewINSERT);
30169 }
30170 }
30171
30172 // Combine INSERT_SUBVECTORs where we are inserting to the same index.
30173 // INSERT_SUBVECTOR( INSERT_SUBVECTOR( Vec, SubOld, Idx ), SubNew, Idx )
30174 // --> INSERT_SUBVECTOR( Vec, SubNew, Idx )
30175 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR &&
30176 N0.getOperand(i: 1).getValueType() == N1.getValueType() &&
30177 N0.getOperand(i: 2) == N2)
30178 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT, N1: N0.getOperand(i: 0),
30179 N2: N1, N3: N2);
30180
30181 // Eliminate an intermediate insert into an undef vector:
30182 // insert_subvector undef, (insert_subvector undef, X, 0), 0 -->
30183 // insert_subvector undef, X, 0
30184 if (N0.isUndef() && N1.getOpcode() == ISD::INSERT_SUBVECTOR &&
30185 N1.getOperand(i: 0).isUndef() && isNullConstant(V: N1.getOperand(i: 2)) &&
30186 isNullConstant(V: N2))
30187 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT, N1: N0,
30188 N2: N1.getOperand(i: 1), N3: N2);
30189
30190 // Push subvector bitcasts to the output, adjusting the index as we go.
30191 // insert_subvector(bitcast(v), bitcast(s), c1)
30192 // -> bitcast(insert_subvector(v, s, c2))
30193 if ((N0.isUndef() || N0.getOpcode() == ISD::BITCAST) &&
30194 N1.getOpcode() == ISD::BITCAST) {
30195 SDValue N0Src = peekThroughBitcasts(V: N0);
30196 SDValue N1Src = peekThroughBitcasts(V: N1);
30197 EVT N0SrcSVT = N0Src.getValueType().getScalarType();
30198 EVT N1SrcSVT = N1Src.getValueType().getScalarType();
30199 if ((N0.isUndef() || N0SrcSVT == N1SrcSVT) &&
30200 N0Src.getValueType().isVector() && N1Src.getValueType().isVector()) {
30201 EVT NewVT;
30202 SDLoc DL(N);
30203 SDValue NewIdx;
30204 LLVMContext &Ctx = *DAG.getContext();
30205 ElementCount NumElts = VT.getVectorElementCount();
30206 unsigned EltSizeInBits = VT.getScalarSizeInBits();
30207 if ((EltSizeInBits % N1SrcSVT.getSizeInBits()) == 0) {
30208 unsigned Scale = EltSizeInBits / N1SrcSVT.getSizeInBits();
30209 NewVT = EVT::getVectorVT(Context&: Ctx, VT: N1SrcSVT, EC: NumElts * Scale);
30210 NewIdx = DAG.getVectorIdxConstant(Val: InsIdx * Scale, DL);
30211 } else if ((N1SrcSVT.getSizeInBits() % EltSizeInBits) == 0) {
30212 unsigned Scale = N1SrcSVT.getSizeInBits() / EltSizeInBits;
30213 if (NumElts.isKnownMultipleOf(RHS: Scale) && (InsIdx % Scale) == 0) {
30214 NewVT = EVT::getVectorVT(Context&: Ctx, VT: N1SrcSVT,
30215 EC: NumElts.divideCoefficientBy(RHS: Scale));
30216 NewIdx = DAG.getVectorIdxConstant(Val: InsIdx / Scale, DL);
30217 }
30218 }
30219 if (NewIdx && hasOperation(Opcode: ISD::INSERT_SUBVECTOR, VT: NewVT)) {
30220 SDValue Res = DAG.getBitcast(VT: NewVT, V: N0Src);
30221 Res = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: NewVT, N1: Res, N2: N1Src, N3: NewIdx);
30222 return DAG.getBitcast(VT, V: Res);
30223 }
30224 }
30225 }
30226
30227 // Canonicalize insert_subvector dag nodes.
30228 // Example:
30229 // (insert_subvector (insert_subvector A, Idx0), Idx1)
30230 // -> (insert_subvector (insert_subvector A, Idx1), Idx0)
30231 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR && N0.hasOneUse() &&
30232 N1.getValueType() == N0.getOperand(i: 1).getValueType()) {
30233 unsigned OtherIdx = N0.getConstantOperandVal(i: 2);
30234 if (InsIdx < OtherIdx) {
30235 // Swap nodes.
30236 SDValue NewOp = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N), VT,
30237 N1: N0.getOperand(i: 0), N2: N1, N3: N2);
30238 AddToWorklist(N: NewOp.getNode());
30239 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: SDLoc(N0.getNode()),
30240 VT, N1: NewOp, N2: N0.getOperand(i: 1), N3: N0.getOperand(i: 2));
30241 }
30242 }
30243
30244 // If the input vector is a concatenation and the insert is wholly contained
30245 // in one of its operands, push the insertion into that operand.
30246 if (N0.getOpcode() == ISD::CONCAT_VECTORS && N0.hasOneUse()) {
30247 EVT ConcatOpVT = N0.getOperand(i: 0).getValueType();
30248 EVT InsVT = N1.getValueType();
30249 unsigned Factor = ConcatOpVT.getVectorMinNumElements();
30250 unsigned ConcatOpIdx = InsIdx / Factor;
30251 unsigned RelativeIdx = InsIdx - ConcatOpIdx * Factor;
30252 assert(ConcatOpIdx < N0.getNumOperands() && "subvector index mismatch");
30253
30254 // If the insert replaces a whole concat operand, optimize into a single
30255 // concat_vectors.
30256 if (RelativeIdx == 0 && ConcatOpVT == InsVT) {
30257 SmallVector<SDValue, 8> Ops(N0->ops());
30258 Ops[ConcatOpIdx] = N1;
30259 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT, Ops);
30260 }
30261
30262 if (VT.isFixedLengthVector() && ConcatOpVT.isFixedLengthVector() &&
30263 InsVT.isFixedLengthVector() &&
30264 hasOperation(Opcode: ISD::INSERT_SUBVECTOR, VT: ConcatOpVT)) {
30265 unsigned NumConcatOpElts = ConcatOpVT.getVectorNumElements();
30266 unsigned NumInsElts = InsVT.getVectorNumElements();
30267 if (RelativeIdx % NumInsElts == 0 &&
30268 RelativeIdx + NumInsElts <= NumConcatOpElts) {
30269 SmallVector<SDValue, 8> Ops(N0->ops());
30270 Ops[ConcatOpIdx] =
30271 DAG.getInsertSubvector(DL: SDLoc(N), Vec: Ops[ConcatOpIdx], SubVec: N1, Idx: RelativeIdx);
30272 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: SDLoc(N), VT, Ops);
30273 }
30274 }
30275 }
30276
30277 // Simplify source operands based on insertion.
30278 if (SimplifyDemandedVectorElts(Op: SDValue(N, 0)))
30279 return SDValue(N, 0);
30280
30281 return SDValue();
30282}
30283
30284SDValue DAGCombiner::visitFP_TO_FP16(SDNode *N) {
30285 SDValue N0 = N->getOperand(Num: 0);
30286
30287 // fold (fp_to_fp16 (fp16_to_fp op)) -> op
30288 if (N0->getOpcode() == ISD::FP16_TO_FP)
30289 return N0->getOperand(Num: 0);
30290
30291 return SDValue();
30292}
30293
30294SDValue DAGCombiner::visitFP16_TO_FP(SDNode *N) {
30295 SelectionDAG::FlagInserter FlagsInserter(DAG, N);
30296 auto Op = N->getOpcode();
30297 assert((Op == ISD::FP16_TO_FP || Op == ISD::BF16_TO_FP) &&
30298 "opcode should be FP16_TO_FP or BF16_TO_FP.");
30299 SDValue N0 = N->getOperand(Num: 0);
30300
30301 // fold fp16_to_fp(op & 0xffff) -> fp16_to_fp(op) or
30302 // fold bf16_to_fp(op & 0xffff) -> bf16_to_fp(op)
30303 if (!TLI.shouldKeepZExtForFP16Conv() && N0->getOpcode() == ISD::AND) {
30304 ConstantSDNode *AndConst = getAsNonOpaqueConstant(N: N0.getOperand(i: 1));
30305 if (AndConst && AndConst->getAPIntValue() == 0xffff) {
30306 return DAG.getNode(Opcode: Op, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: N0.getOperand(i: 0));
30307 }
30308 }
30309
30310 if (SDValue CastEliminated = eliminateFPCastPair(N))
30311 return CastEliminated;
30312
30313 // Sometimes constants manage to survive very late in the pipeline, e.g.,
30314 // because they are wrapped inside the <1 x f16> type. Try one last time to
30315 // get rid of them.
30316 SDValue Folded = DAG.FoldConstantArithmetic(Opcode: N->getOpcode(), DL: SDLoc(N),
30317 VT: N->getValueType(ResNo: 0), Ops: {N0});
30318 return Folded;
30319}
30320
30321SDValue DAGCombiner::visitFP_TO_BF16(SDNode *N) {
30322 SDValue N0 = N->getOperand(Num: 0);
30323
30324 // fold (fp_to_bf16 (bf16_to_fp op)) -> op
30325 if (N0->getOpcode() == ISD::BF16_TO_FP)
30326 return N0->getOperand(Num: 0);
30327
30328 return SDValue();
30329}
30330
30331SDValue DAGCombiner::visitBF16_TO_FP(SDNode *N) {
30332 // fold bf16_to_fp(op & 0xffff) -> bf16_to_fp(op)
30333 return visitFP16_TO_FP(N);
30334}
30335
30336SDValue DAGCombiner::visitVECREDUCE(SDNode *N) {
30337 SDValue N0 = N->getOperand(Num: 0);
30338 EVT VT = N0.getValueType();
30339 unsigned Opcode = N->getOpcode();
30340
30341 // VECREDUCE over 1-element vector is just an extract.
30342 if (VT.getVectorElementCount().isScalar()) {
30343 SDLoc dl(N);
30344 SDValue Res =
30345 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: VT.getVectorElementType(), N1: N0,
30346 N2: DAG.getVectorIdxConstant(Val: 0, DL: dl));
30347 if (Res.getValueType() != N->getValueType(ResNo: 0))
30348 Res = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: N->getValueType(ResNo: 0), Operand: Res);
30349 return Res;
30350 }
30351
30352 // On an boolean vector an and/or reduction is the same as a umin/umax
30353 // reduction. Convert them if the latter is legal while the former isn't.
30354 if (Opcode == ISD::VECREDUCE_AND || Opcode == ISD::VECREDUCE_OR) {
30355 unsigned NewOpcode = Opcode == ISD::VECREDUCE_AND
30356 ? ISD::VECREDUCE_UMIN : ISD::VECREDUCE_UMAX;
30357 if (!TLI.isOperationLegalOrCustom(Op: Opcode, VT) &&
30358 TLI.isOperationLegalOrCustom(Op: NewOpcode, VT) &&
30359 DAG.ComputeNumSignBits(Op: N0) == VT.getScalarSizeInBits())
30360 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: N0);
30361 }
30362
30363 // vecreduce_or(insert_subvector(zero or undef, val)) -> vecreduce_or(val)
30364 // vecreduce_and(insert_subvector(ones or undef, val)) -> vecreduce_and(val)
30365 if (N0.getOpcode() == ISD::INSERT_SUBVECTOR &&
30366 TLI.isTypeLegal(VT: N0.getOperand(i: 1).getValueType())) {
30367 SDValue Vec = N0.getOperand(i: 0);
30368 SDValue Subvec = N0.getOperand(i: 1);
30369 if ((Opcode == ISD::VECREDUCE_OR &&
30370 (N0.getOperand(i: 0).isUndef() || isNullOrNullSplat(V: Vec))) ||
30371 (Opcode == ISD::VECREDUCE_AND &&
30372 (N0.getOperand(i: 0).isUndef() || isAllOnesOrAllOnesSplat(V: Vec))))
30373 return DAG.getNode(Opcode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Subvec);
30374 }
30375
30376 // vecreduce_or(sext(x)) -> sext(vecreduce_or(x))
30377 // Same for zext and anyext, and for and/or/xor reductions.
30378 if ((Opcode == ISD::VECREDUCE_OR || Opcode == ISD::VECREDUCE_AND ||
30379 Opcode == ISD::VECREDUCE_XOR) &&
30380 (N0.getOpcode() == ISD::SIGN_EXTEND ||
30381 N0.getOpcode() == ISD::ZERO_EXTEND ||
30382 N0.getOpcode() == ISD::ANY_EXTEND) &&
30383 TLI.isOperationLegalOrCustom(Op: Opcode, VT: N0.getOperand(i: 0).getValueType())) {
30384 SDValue Red = DAG.getNode(Opcode, DL: SDLoc(N),
30385 VT: N0.getOperand(i: 0).getValueType().getScalarType(),
30386 Operand: N0.getOperand(i: 0));
30387 return DAG.getNode(Opcode: N0.getOpcode(), DL: SDLoc(N), VT: N->getValueType(ResNo: 0), Operand: Red);
30388 }
30389 return SDValue();
30390}
30391
30392SDValue DAGCombiner::visitVPOp(SDNode *N) {
30393
30394 if (N->getOpcode() == ISD::VP_GATHER)
30395 if (SDValue SD = visitVPGATHER(N))
30396 return SD;
30397
30398 if (N->getOpcode() == ISD::VP_SCATTER)
30399 if (SDValue SD = visitVPSCATTER(N))
30400 return SD;
30401
30402 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD)
30403 if (SDValue SD = visitVP_STRIDED_LOAD(N))
30404 return SD;
30405
30406 if (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE)
30407 if (SDValue SD = visitVP_STRIDED_STORE(N))
30408 return SD;
30409
30410 // VP operations in which all vector elements are disabled - either by
30411 // determining that the mask is all false or that the EVL is 0 - can be
30412 // eliminated.
30413 bool AreAllEltsDisabled = false;
30414 if (auto EVLIdx = ISD::getVPExplicitVectorLengthIdx(Opcode: N->getOpcode()))
30415 AreAllEltsDisabled |= isNullConstant(V: N->getOperand(Num: *EVLIdx));
30416 if (auto MaskIdx = ISD::getVPMaskIdx(Opcode: N->getOpcode()))
30417 AreAllEltsDisabled |=
30418 ISD::isConstantSplatVectorAllZeros(N: N->getOperand(Num: *MaskIdx).getNode());
30419
30420 // This is the only generic VP combine we support for now.
30421 if (!AreAllEltsDisabled)
30422 return SDValue();
30423
30424 // Binary operations can be replaced by UNDEF.
30425 if (ISD::isVPBinaryOp(Opcode: N->getOpcode()))
30426 return DAG.getUNDEF(VT: N->getValueType(ResNo: 0));
30427
30428 // VP Memory operations can be replaced by either the chain (stores) or the
30429 // chain + undef (loads).
30430 if (const auto *MemSD = dyn_cast<MemSDNode>(Val: N)) {
30431 if (MemSD->writeMem())
30432 return MemSD->getChain();
30433 return CombineTo(N, Res0: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)), Res1: MemSD->getChain());
30434 }
30435
30436 // Reduction operations return the start operand when no elements are active.
30437 if (ISD::isVPReduction(Opcode: N->getOpcode()))
30438 return N->getOperand(Num: 0);
30439
30440 return SDValue();
30441}
30442
30443SDValue DAGCombiner::visitGET_FPENV_MEM(SDNode *N) {
30444 SDValue Chain = N->getOperand(Num: 0);
30445 SDValue Ptr = N->getOperand(Num: 1);
30446 EVT MemVT = cast<FPStateAccessSDNode>(Val: N)->getMemoryVT();
30447
30448 // Check if the memory, where FP state is written to, is used only in a single
30449 // load operation.
30450 LoadSDNode *LdNode = nullptr;
30451 for (auto *U : Ptr->users()) {
30452 if (U == N)
30453 continue;
30454 if (auto *Ld = dyn_cast<LoadSDNode>(Val: U)) {
30455 if (LdNode && LdNode != Ld)
30456 return SDValue();
30457 LdNode = Ld;
30458 continue;
30459 }
30460 return SDValue();
30461 }
30462 if (!LdNode || !LdNode->isSimple() || LdNode->isIndexed() ||
30463 !LdNode->getOffset().isUndef() || LdNode->getMemoryVT() != MemVT ||
30464 !LdNode->getChain().reachesChainWithoutSideEffects(Dest: SDValue(N, 0)))
30465 return SDValue();
30466
30467 // Check if the loaded value is used only in a store operation.
30468 StoreSDNode *StNode = nullptr;
30469 for (SDUse &U : LdNode->uses()) {
30470 if (U.getResNo() == 0) {
30471 if (auto *St = dyn_cast<StoreSDNode>(Val: U.getUser())) {
30472 if (StNode)
30473 return SDValue();
30474 StNode = St;
30475 } else {
30476 return SDValue();
30477 }
30478 }
30479 }
30480 if (!StNode || !StNode->isSimple() || StNode->isIndexed() ||
30481 !StNode->getOffset().isUndef() || StNode->getMemoryVT() != MemVT ||
30482 !StNode->getChain().reachesChainWithoutSideEffects(Dest: SDValue(LdNode, 1)))
30483 return SDValue();
30484
30485 // Create new node GET_FPENV_MEM, which uses the store address to write FP
30486 // environment.
30487 SDValue Res = DAG.getGetFPEnv(Chain, dl: SDLoc(N), Ptr: StNode->getBasePtr(), MemVT,
30488 MMO: StNode->getMemOperand());
30489 CombineTo(N: StNode, Res, AddTo: false);
30490 return Res;
30491}
30492
30493SDValue DAGCombiner::visitSET_FPENV_MEM(SDNode *N) {
30494 SDValue Chain = N->getOperand(Num: 0);
30495 SDValue Ptr = N->getOperand(Num: 1);
30496 EVT MemVT = cast<FPStateAccessSDNode>(Val: N)->getMemoryVT();
30497
30498 // Check if the address of FP state is used also in a store operation only.
30499 StoreSDNode *StNode = nullptr;
30500 for (auto *U : Ptr->users()) {
30501 if (U == N)
30502 continue;
30503 if (auto *St = dyn_cast<StoreSDNode>(Val: U)) {
30504 if (StNode && StNode != St)
30505 return SDValue();
30506 StNode = St;
30507 continue;
30508 }
30509 return SDValue();
30510 }
30511 if (!StNode || !StNode->isSimple() || StNode->isIndexed() ||
30512 !StNode->getOffset().isUndef() || StNode->getMemoryVT() != MemVT ||
30513 !Chain.reachesChainWithoutSideEffects(Dest: SDValue(StNode, 0)))
30514 return SDValue();
30515
30516 // Check if the stored value is loaded from some location and the loaded
30517 // value is used only in the store operation.
30518 SDValue StValue = StNode->getValue();
30519 auto *LdNode = dyn_cast<LoadSDNode>(Val&: StValue);
30520 if (!LdNode || !LdNode->isSimple() || LdNode->isIndexed() ||
30521 !LdNode->getOffset().isUndef() || LdNode->getMemoryVT() != MemVT ||
30522 !StNode->getChain().reachesChainWithoutSideEffects(Dest: SDValue(LdNode, 1)))
30523 return SDValue();
30524
30525 // Create new node SET_FPENV_MEM, which uses the load address to read FP
30526 // environment.
30527 SDValue Res =
30528 DAG.getSetFPEnv(Chain: LdNode->getChain(), dl: SDLoc(N), Ptr: LdNode->getBasePtr(), MemVT,
30529 MMO: LdNode->getMemOperand());
30530 return Res;
30531}
30532
30533/// Returns a vector_shuffle if it able to transform an AND to a vector_shuffle
30534/// with the destination vector and a zero vector.
30535/// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==>
30536/// vector_shuffle V, Zero, <0, 4, 2, 4>
30537SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) {
30538 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!");
30539
30540 EVT VT = N->getValueType(ResNo: 0);
30541 SDValue LHS = N->getOperand(Num: 0);
30542 SDValue RHS = peekThroughBitcasts(V: N->getOperand(Num: 1));
30543 SDLoc DL(N);
30544
30545 // Make sure we're not running after operation legalization where it
30546 // may have custom lowered the vector shuffles.
30547 if (LegalOperations)
30548 return SDValue();
30549
30550 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
30551 return SDValue();
30552
30553 EVT RVT = RHS.getValueType();
30554 unsigned NumElts = RHS.getNumOperands();
30555
30556 // Attempt to create a valid clear mask, splitting the mask into
30557 // sub elements and checking to see if each is
30558 // all zeros or all ones - suitable for shuffle masking.
30559 auto BuildClearMask = [&](int Split) {
30560 int NumSubElts = NumElts * Split;
30561 int NumSubBits = RVT.getScalarSizeInBits() / Split;
30562
30563 SmallVector<int, 8> Indices;
30564 for (int i = 0; i != NumSubElts; ++i) {
30565 int EltIdx = i / Split;
30566 int SubIdx = i % Split;
30567 SDValue Elt = RHS.getOperand(i: EltIdx);
30568 // X & undef --> 0 (not undef). So this lane must be converted to choose
30569 // from the zero constant vector (same as if the element had all 0-bits).
30570 if (Elt.isUndef()) {
30571 Indices.push_back(Elt: i + NumSubElts);
30572 continue;
30573 }
30574
30575 std::optional<APInt> Bits = Elt->bitcastToAPInt();
30576 if (!Bits)
30577 return SDValue();
30578
30579 // Extract the sub element from the constant bit mask.
30580 if (DAG.getDataLayout().isBigEndian())
30581 *Bits =
30582 Bits->extractBits(numBits: NumSubBits, bitPosition: (Split - SubIdx - 1) * NumSubBits);
30583 else
30584 *Bits = Bits->extractBits(numBits: NumSubBits, bitPosition: SubIdx * NumSubBits);
30585
30586 if (Bits->isAllOnes())
30587 Indices.push_back(Elt: i);
30588 else if (*Bits == 0)
30589 Indices.push_back(Elt: i + NumSubElts);
30590 else
30591 return SDValue();
30592 }
30593
30594 // Let's see if the target supports this vector_shuffle.
30595 EVT ClearSVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NumSubBits);
30596 EVT ClearVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ClearSVT, NumElements: NumSubElts);
30597 if (!TLI.isVectorClearMaskLegal(Indices, ClearVT))
30598 return SDValue();
30599
30600 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: ClearVT);
30601 return DAG.getBitcast(VT, V: DAG.getVectorShuffle(VT: ClearVT, dl: DL,
30602 N1: DAG.getBitcast(VT: ClearVT, V: LHS),
30603 N2: Zero, Mask: Indices));
30604 };
30605
30606 // Determine maximum split level (byte level masking).
30607 int MaxSplit = 1;
30608 if (RVT.getScalarSizeInBits() % 8 == 0)
30609 MaxSplit = RVT.getScalarSizeInBits() / 8;
30610
30611 for (int Split = 1; Split <= MaxSplit; ++Split)
30612 if (RVT.getScalarSizeInBits() % Split == 0)
30613 if (SDValue S = BuildClearMask(Split))
30614 return S;
30615
30616 return SDValue();
30617}
30618
30619/// If a vector binop is performed on splat values, it may be profitable to
30620/// extract, scalarize, and insert/splat.
30621static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG,
30622 const SDLoc &DL, bool LegalTypes) {
30623 SDValue N0 = N->getOperand(Num: 0);
30624 SDValue N1 = N->getOperand(Num: 1);
30625 unsigned Opcode = N->getOpcode();
30626 EVT VT = N->getValueType(ResNo: 0);
30627 EVT EltVT = VT.getVectorElementType();
30628 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
30629
30630 // TODO: Remove/replace the extract cost check? If the elements are available
30631 // as scalars, then there may be no extract cost. Should we ask if
30632 // inserting a scalar back into a vector is cheap instead?
30633 int Index0, Index1;
30634 SDValue Src0 = DAG.getSplatSourceVector(V: N0, SplatIndex&: Index0);
30635 SDValue Src1 = DAG.getSplatSourceVector(V: N1, SplatIndex&: Index1);
30636 // Extract element from splat_vector should be free.
30637 // TODO: use DAG.isSplatValue instead?
30638 bool IsBothSplatVector = N0.getOpcode() == ISD::SPLAT_VECTOR &&
30639 N1.getOpcode() == ISD::SPLAT_VECTOR;
30640 if (!Src0 || !Src1 || Index0 != Index1 ||
30641 Src0.getValueType().getVectorElementType() != EltVT ||
30642 Src1.getValueType().getVectorElementType() != EltVT ||
30643 !(IsBothSplatVector || TLI.isExtractVecEltCheap(VT, Index: Index0)) ||
30644 // If before type legalization, allow scalar types that will eventually be
30645 // made legal.
30646 !TLI.isOperationLegalOrCustom(
30647 Op: Opcode, VT: LegalTypes
30648 ? EltVT
30649 : TLI.getTypeToTransformTo(Context&: *DAG.getContext(), VT: EltVT)))
30650 return SDValue();
30651
30652 // FIXME: Type legalization can't handle illegal MULHS/MULHU.
30653 if ((Opcode == ISD::MULHS || Opcode == ISD::MULHU) && !TLI.isTypeLegal(VT: EltVT))
30654 return SDValue();
30655
30656 if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getOpcode() == N1.getOpcode()) {
30657 // All but one element should have an undef input, which will fold to a
30658 // constant or undef. Avoid splatting which would over-define potentially
30659 // undefined elements.
30660
30661 // bo (build_vec ..undef, X, undef...), (build_vec ..undef, Y, undef...) -->
30662 // build_vec ..undef, (bo X, Y), undef...
30663 SmallVector<SDValue, 16> EltsX, EltsY, EltsResult;
30664 DAG.ExtractVectorElements(Op: Src0, Args&: EltsX);
30665 DAG.ExtractVectorElements(Op: Src1, Args&: EltsY);
30666
30667 for (auto [X, Y] : zip(t&: EltsX, u&: EltsY))
30668 EltsResult.push_back(Elt: DAG.getNode(Opcode, DL, VT: EltVT, N1: X, N2: Y, Flags: N->getFlags()));
30669 return DAG.getBuildVector(VT, DL, Ops: EltsResult);
30670 }
30671
30672 SDValue IndexC = DAG.getVectorIdxConstant(Val: Index0, DL);
30673 SDValue X = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Src0, N2: IndexC);
30674 SDValue Y = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Src1, N2: IndexC);
30675 SDValue ScalarBO = DAG.getNode(Opcode, DL, VT: EltVT, N1: X, N2: Y, Flags: N->getFlags());
30676
30677 // bo (splat X, Index), (splat Y, Index) --> splat (bo X, Y), Index
30678 return DAG.getSplat(VT, DL, Op: ScalarBO);
30679}
30680
30681/// Visit a vector cast operation, like FP_EXTEND.
30682SDValue DAGCombiner::SimplifyVCastOp(SDNode *N, const SDLoc &DL) {
30683 EVT VT = N->getValueType(ResNo: 0);
30684 assert(VT.isVector() && "SimplifyVCastOp only works on vectors!");
30685 EVT EltVT = VT.getVectorElementType();
30686 unsigned Opcode = N->getOpcode();
30687
30688 SDValue N0 = N->getOperand(Num: 0);
30689 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
30690
30691 // TODO: promote operation might be also good here?
30692 int Index0;
30693 SDValue Src0 = DAG.getSplatSourceVector(V: N0, SplatIndex&: Index0);
30694 if (Src0 &&
30695 (N0.getOpcode() == ISD::SPLAT_VECTOR ||
30696 TLI.isExtractVecEltCheap(VT, Index: Index0)) &&
30697 TLI.isOperationLegalOrCustom(Op: Opcode, VT: EltVT) &&
30698 TLI.preferScalarizeSplat(N)) {
30699 EVT SrcVT = N0.getValueType();
30700 EVT SrcEltVT = SrcVT.getVectorElementType();
30701 if (!LegalTypes || TLI.isTypeLegal(VT: SrcEltVT)) {
30702 SDValue IndexC = DAG.getVectorIdxConstant(Val: Index0, DL);
30703 SDValue Elt =
30704 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: SrcEltVT, N1: Src0, N2: IndexC);
30705 SDValue ScalarBO = DAG.getNode(Opcode, DL, VT: EltVT, Operand: Elt, Flags: N->getFlags());
30706 if (VT.isScalableVector())
30707 return DAG.getSplatVector(VT, DL, Op: ScalarBO);
30708 SmallVector<SDValue, 8> Ops(VT.getVectorNumElements(), ScalarBO);
30709 return DAG.getBuildVector(VT, DL, Ops);
30710 }
30711 }
30712
30713 return SDValue();
30714}
30715
30716/// Visit a binary vector operation, like ADD.
30717SDValue DAGCombiner::SimplifyVBinOp(SDNode *N, const SDLoc &DL) {
30718 EVT VT = N->getValueType(ResNo: 0);
30719 assert(VT.isVector() && "SimplifyVBinOp only works on vectors!");
30720
30721 SDValue LHS = N->getOperand(Num: 0);
30722 SDValue RHS = N->getOperand(Num: 1);
30723 unsigned Opcode = N->getOpcode();
30724 SDNodeFlags Flags = N->getFlags();
30725
30726 // Move unary shuffles with identical masks after a vector binop:
30727 // VBinOp (shuffle A, Undef, Mask), (shuffle B, Undef, Mask))
30728 // --> shuffle (VBinOp A, B), Undef, Mask
30729 // This does not require type legality checks because we are creating the
30730 // same types of operations that are in the original sequence. We do have to
30731 // restrict ops like integer div that have immediate UB (eg, div-by-zero)
30732 // though. This code is adapted from the identical transform in instcombine.
30733 if (DAG.isSafeToSpeculativelyExecute(Opcode)) {
30734 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Val&: LHS);
30735 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(Val&: RHS);
30736 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(RHS: Shuf1->getMask()) &&
30737 LHS.getOperand(i: 1).isUndef() && RHS.getOperand(i: 1).isUndef() &&
30738 (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) {
30739 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, N1: LHS.getOperand(i: 0),
30740 N2: RHS.getOperand(i: 0), Flags);
30741 SDValue UndefV = LHS.getOperand(i: 1);
30742 return DAG.getVectorShuffle(VT, dl: DL, N1: NewBinOp, N2: UndefV, Mask: Shuf0->getMask());
30743 }
30744
30745 // Try to sink a splat shuffle after a binop with a uniform constant.
30746 // This is limited to cases where neither the shuffle nor the constant have
30747 // undefined elements because that could be poison-unsafe or inhibit
30748 // demanded elements analysis. It is further limited to not change a splat
30749 // of an inserted scalar because that may be optimized better by
30750 // load-folding or other target-specific behaviors.
30751 if (isConstOrConstSplat(N: RHS) && Shuf0 && all_equal(Range: Shuf0->getMask()) &&
30752 Shuf0->hasOneUse() && Shuf0->getOperand(Num: 1).isUndef() &&
30753 Shuf0->getOperand(Num: 0).getOpcode() != ISD::INSERT_VECTOR_ELT) {
30754 // binop (splat X), (splat C) --> splat (binop X, C)
30755 SDValue X = Shuf0->getOperand(Num: 0);
30756 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, N1: X, N2: RHS, Flags);
30757 return DAG.getVectorShuffle(VT, dl: DL, N1: NewBinOp, N2: DAG.getPOISON(VT),
30758 Mask: Shuf0->getMask());
30759 }
30760 if (isConstOrConstSplat(N: LHS) && Shuf1 && all_equal(Range: Shuf1->getMask()) &&
30761 Shuf1->hasOneUse() && Shuf1->getOperand(Num: 1).isUndef() &&
30762 Shuf1->getOperand(Num: 0).getOpcode() != ISD::INSERT_VECTOR_ELT) {
30763 // binop (splat C), (splat X) --> splat (binop C, X)
30764 SDValue X = Shuf1->getOperand(Num: 0);
30765 SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, N1: LHS, N2: X, Flags);
30766 return DAG.getVectorShuffle(VT, dl: DL, N1: NewBinOp, N2: DAG.getPOISON(VT),
30767 Mask: Shuf1->getMask());
30768 }
30769 }
30770
30771 // The following pattern is likely to emerge with vector reduction ops. Moving
30772 // the binary operation ahead of insertion may allow using a narrower vector
30773 // instruction that has better performance than the wide version of the op:
30774 // VBinOp (ins undef, X, Z), (ins undef, Y, Z) --> ins VecC, (VBinOp X, Y), Z
30775 if (LHS.getOpcode() == ISD::INSERT_SUBVECTOR && LHS.getOperand(i: 0).isUndef() &&
30776 RHS.getOpcode() == ISD::INSERT_SUBVECTOR && RHS.getOperand(i: 0).isUndef() &&
30777 LHS.getOperand(i: 2) == RHS.getOperand(i: 2) &&
30778 (LHS.hasOneUse() || RHS.hasOneUse())) {
30779 SDValue X = LHS.getOperand(i: 1);
30780 SDValue Y = RHS.getOperand(i: 1);
30781 SDValue Z = LHS.getOperand(i: 2);
30782 EVT NarrowVT = X.getValueType();
30783 if (NarrowVT == Y.getValueType() &&
30784 TLI.isOperationLegalOrCustomOrPromote(Op: Opcode, VT: NarrowVT,
30785 LegalOnly: LegalOperations)) {
30786 // (binop undef, undef) may not return undef, so compute that result.
30787 SDValue VecC =
30788 DAG.getNode(Opcode, DL, VT, N1: DAG.getUNDEF(VT), N2: DAG.getUNDEF(VT));
30789 SDValue NarrowBO = DAG.getNode(Opcode, DL, VT: NarrowVT, N1: X, N2: Y);
30790 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: VecC, N2: NarrowBO, N3: Z);
30791 }
30792 }
30793
30794 // Make sure all but the first op are undef or constant.
30795 auto ConcatWithConstantOrUndef = [](SDValue Concat) {
30796 return Concat.getOpcode() == ISD::CONCAT_VECTORS &&
30797 all_of(Range: drop_begin(RangeOrContainer: Concat->ops()), P: [](const SDValue &Op) {
30798 return Op.isUndef() ||
30799 ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode());
30800 });
30801 };
30802
30803 // The following pattern is likely to emerge with vector reduction ops. Moving
30804 // the binary operation ahead of the concat may allow using a narrower vector
30805 // instruction that has better performance than the wide version of the op:
30806 // VBinOp (concat X, undef/constant), (concat Y, undef/constant) -->
30807 // concat (VBinOp X, Y), VecC
30808 if (ConcatWithConstantOrUndef(LHS) && ConcatWithConstantOrUndef(RHS) &&
30809 (LHS.hasOneUse() || RHS.hasOneUse())) {
30810 EVT NarrowVT = LHS.getOperand(i: 0).getValueType();
30811 if (NarrowVT == RHS.getOperand(i: 0).getValueType() &&
30812 TLI.isOperationLegalOrCustomOrPromote(Op: Opcode, VT: NarrowVT)) {
30813 unsigned NumOperands = LHS.getNumOperands();
30814 SmallVector<SDValue, 4> ConcatOps;
30815 for (unsigned i = 0; i != NumOperands; ++i) {
30816 // This constant fold for operands 1 and up.
30817 ConcatOps.push_back(Elt: DAG.getNode(Opcode, DL, VT: NarrowVT, N1: LHS.getOperand(i),
30818 N2: RHS.getOperand(i)));
30819 }
30820
30821 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps);
30822 }
30823 }
30824
30825 if (SDValue V = scalarizeBinOpOfSplats(N, DAG, DL, LegalTypes))
30826 return V;
30827
30828 return SDValue();
30829}
30830
30831SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1,
30832 SDValue N2) {
30833 assert(N0.getOpcode() == ISD::SETCC &&
30834 "First argument must be a SetCC node!");
30835
30836 SDValue SCC = SimplifySelectCC(DL, N0: N0.getOperand(i: 0), N1: N0.getOperand(i: 1), N2: N1, N3: N2,
30837 CC: cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get());
30838
30839 // If we got a simplified select_cc node back from SimplifySelectCC, then
30840 // break it down into a new SETCC node, and a new SELECT node, and then return
30841 // the SELECT node, since we were called with a SELECT node.
30842 if (SCC.getNode()) {
30843 // Check to see if we got a select_cc back (to turn into setcc/select).
30844 // Otherwise, just return whatever node we got back, like fabs.
30845 if (SCC.getOpcode() == ISD::SELECT_CC) {
30846 const SDNodeFlags Flags = N0->getFlags();
30847 SDValue SETCC = DAG.getNode(Opcode: ISD::SETCC, DL: SDLoc(N0),
30848 VT: N0.getValueType(),
30849 N1: SCC.getOperand(i: 0), N2: SCC.getOperand(i: 1),
30850 N3: SCC.getOperand(i: 4), Flags);
30851 AddToWorklist(N: SETCC.getNode());
30852 return DAG.getSelect(DL: SDLoc(SCC), VT: SCC.getValueType(), Cond: SETCC,
30853 LHS: SCC.getOperand(i: 2), RHS: SCC.getOperand(i: 3), Flags);
30854 }
30855
30856 return SCC;
30857 }
30858 return SDValue();
30859}
30860
30861/// Given a SELECT or a SELECT_CC node, where LHS and RHS are the two values
30862/// being selected between, see if we can simplify the select. Callers of this
30863/// should assume that TheSelect is deleted if this returns true. As such, they
30864/// should return the appropriate thing (e.g. the node) back to the top-level of
30865/// the DAG combiner loop to avoid it being looked at.
30866bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
30867 SDValue RHS) {
30868 // fold (select (setcc x, [+-]0.0, *lt), NaN, (fsqrt x))
30869 // The select + setcc is redundant, because fsqrt returns NaN for X < 0.
30870 if (const ConstantFPSDNode *NaN = isConstOrConstSplatFP(N: LHS)) {
30871 if (NaN->isNaN() && RHS.getOpcode() == ISD::FSQRT) {
30872 // We have: (select (setcc ?, ?, ?), NaN, (fsqrt ?))
30873 SDValue Sqrt = RHS;
30874 ISD::CondCode CC;
30875 SDValue CmpLHS;
30876 const ConstantFPSDNode *Zero = nullptr;
30877
30878 if (TheSelect->getOpcode() == ISD::SELECT_CC) {
30879 CC = cast<CondCodeSDNode>(Val: TheSelect->getOperand(Num: 4))->get();
30880 CmpLHS = TheSelect->getOperand(Num: 0);
30881 Zero = isConstOrConstSplatFP(N: TheSelect->getOperand(Num: 1));
30882 } else {
30883 // SELECT or VSELECT
30884 SDValue Cmp = TheSelect->getOperand(Num: 0);
30885 if (Cmp.getOpcode() == ISD::SETCC) {
30886 CC = cast<CondCodeSDNode>(Val: Cmp.getOperand(i: 2))->get();
30887 CmpLHS = Cmp.getOperand(i: 0);
30888 Zero = isConstOrConstSplatFP(N: Cmp.getOperand(i: 1));
30889 }
30890 }
30891 if (Zero && Zero->isZero() &&
30892 Sqrt.getOperand(i: 0) == CmpLHS && (CC == ISD::SETOLT ||
30893 CC == ISD::SETULT || CC == ISD::SETLT)) {
30894 // We have: (select (setcc x, [+-]0.0, *lt), NaN, (fsqrt x))
30895 CombineTo(N: TheSelect, Res: Sqrt);
30896 return true;
30897 }
30898 }
30899 }
30900 // Cannot simplify select with vector condition
30901 if (TheSelect->getOperand(Num: 0).getValueType().isVector()) return false;
30902
30903 // If this is a select from two identical things, try to pull the operation
30904 // through the select.
30905 if (LHS.getOpcode() != RHS.getOpcode() ||
30906 !LHS.hasOneUse() || !RHS.hasOneUse())
30907 return false;
30908
30909 // If this is a load and the token chain is identical, replace the select
30910 // of two loads with a load through a select of the address to load from.
30911 // This triggers in things like "select bool X, 10.0, 123.0" after the FP
30912 // constants have been dropped into the constant pool.
30913 if (LHS.getOpcode() == ISD::LOAD) {
30914 LoadSDNode *LLD = cast<LoadSDNode>(Val&: LHS);
30915 LoadSDNode *RLD = cast<LoadSDNode>(Val&: RHS);
30916
30917 // Token chains must be identical.
30918 if (LHS.getOperand(i: 0) != RHS.getOperand(i: 0) ||
30919 // Do not let this transformation reduce the number of volatile loads.
30920 // Be conservative for atomics for the moment
30921 // TODO: This does appear to be legal for unordered atomics (see D66309)
30922 !LLD->isSimple() || !RLD->isSimple() ||
30923 // FIXME: If either is a pre/post inc/dec load,
30924 // we'd need to split out the address adjustment.
30925 LLD->isIndexed() || RLD->isIndexed() ||
30926 // If this is an EXTLOAD, the VT's must match.
30927 LLD->getMemoryVT() != RLD->getMemoryVT() ||
30928 // If this is an EXTLOAD, the kind of extension must match.
30929 (LLD->getExtensionType() != RLD->getExtensionType() &&
30930 // The only exception is if one of the extensions is anyext.
30931 LLD->getExtensionType() != ISD::EXTLOAD &&
30932 RLD->getExtensionType() != ISD::EXTLOAD) ||
30933 // FIXME: this discards src value information. This is
30934 // over-conservative. It would be beneficial to be able to remember
30935 // both potential memory locations. Since we are discarding
30936 // src value info, don't do the transformation if the memory
30937 // locations are not in the same address space.
30938 LLD->getPointerInfo().getAddrSpace() !=
30939 RLD->getPointerInfo().getAddrSpace() ||
30940 // We can't produce a CMOV of a TargetFrameIndex since we won't
30941 // generate the address generation required.
30942 LLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex ||
30943 RLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex ||
30944 !TLI.isOperationLegalOrCustom(Op: TheSelect->getOpcode(),
30945 VT: LLD->getBasePtr().getValueType()))
30946 return false;
30947
30948 // The loads must not depend on one another.
30949 if (LLD->isPredecessorOf(N: RLD) || RLD->isPredecessorOf(N: LLD))
30950 return false;
30951
30952 // Check that the select condition doesn't reach either load. If so,
30953 // folding this will induce a cycle into the DAG. If not, this is safe to
30954 // xform, so create a select of the addresses.
30955
30956 SmallPtrSet<const SDNode *, 32> Visited;
30957 SmallVector<const SDNode *, 16> Worklist;
30958
30959 // Always fail if LLD and RLD are not independent. TheSelect is a
30960 // predecessor to all Nodes in question so we need not search past it.
30961
30962 Visited.insert(Ptr: TheSelect);
30963 Worklist.push_back(Elt: LLD);
30964 Worklist.push_back(Elt: RLD);
30965
30966 if (SDNode::hasPredecessorHelper(N: LLD, Visited, Worklist) ||
30967 SDNode::hasPredecessorHelper(N: RLD, Visited, Worklist))
30968 return false;
30969
30970 SDValue Addr;
30971 if (TheSelect->getOpcode() == ISD::SELECT) {
30972 // We cannot do this optimization if any pair of {RLD, LLD} is a
30973 // predecessor to {RLD, LLD, CondNode}. As we've already compared the
30974 // Loads, we only need to check if CondNode is a successor to one of the
30975 // loads. We can further avoid this if there's no use of their chain
30976 // value.
30977 SDNode *CondNode = TheSelect->getOperand(Num: 0).getNode();
30978 Worklist.push_back(Elt: CondNode);
30979
30980 if ((LLD->hasAnyUseOfValue(Value: 1) &&
30981 SDNode::hasPredecessorHelper(N: LLD, Visited, Worklist)) ||
30982 (RLD->hasAnyUseOfValue(Value: 1) &&
30983 SDNode::hasPredecessorHelper(N: RLD, Visited, Worklist)))
30984 return false;
30985
30986 // If the condition is poison, originally this would result in a poison
30987 // result. After the transform, this would result in a load of poison,
30988 // which is UB. Freeze the condition to prevent this.
30989 Addr = DAG.getSelect(DL: SDLoc(TheSelect), VT: LLD->getBasePtr().getValueType(),
30990 Cond: DAG.getFreeze(V: TheSelect->getOperand(Num: 0)),
30991 LHS: LLD->getBasePtr(), RHS: RLD->getBasePtr());
30992 } else { // Otherwise SELECT_CC
30993 // We cannot do this optimization if any pair of {RLD, LLD} is a
30994 // predecessor to {RLD, LLD, CondLHS, CondRHS}. As we've already compared
30995 // the Loads, we only need to check if CondLHS/CondRHS is a successor to
30996 // one of the loads. We can further avoid this if there's no use of their
30997 // chain value.
30998
30999 SDNode *CondLHS = TheSelect->getOperand(Num: 0).getNode();
31000 SDNode *CondRHS = TheSelect->getOperand(Num: 1).getNode();
31001 Worklist.push_back(Elt: CondLHS);
31002 Worklist.push_back(Elt: CondRHS);
31003
31004 if ((LLD->hasAnyUseOfValue(Value: 1) &&
31005 SDNode::hasPredecessorHelper(N: LLD, Visited, Worklist)) ||
31006 (RLD->hasAnyUseOfValue(Value: 1) &&
31007 SDNode::hasPredecessorHelper(N: RLD, Visited, Worklist)))
31008 return false;
31009
31010 SDValue FrozenOp0 = DAG.getFreeze(V: TheSelect->getOperand(Num: 0));
31011 SDValue FrozenOp1 = DAG.getFreeze(V: TheSelect->getOperand(Num: 1));
31012 Addr = DAG.getNode(Opcode: ISD::SELECT_CC, DL: SDLoc(TheSelect),
31013 VT: LLD->getBasePtr().getValueType(), N1: FrozenOp0, N2: FrozenOp1,
31014 N3: LLD->getBasePtr(), N4: RLD->getBasePtr(),
31015 N5: TheSelect->getOperand(Num: 4));
31016 }
31017
31018 SDValue Load;
31019 // It is safe to replace the two loads if they have different alignments,
31020 // but the new load must be the minimum (most restrictive) alignment of the
31021 // inputs.
31022 Align Alignment = std::min(a: LLD->getAlign(), b: RLD->getAlign());
31023 unsigned AddrSpace = LLD->getAddressSpace();
31024 assert(AddrSpace == RLD->getAddressSpace());
31025
31026 MachineMemOperand::Flags MMOFlags = LLD->getMemOperand()->getFlags();
31027 if (!RLD->isInvariant())
31028 MMOFlags &= ~MachineMemOperand::MOInvariant;
31029 if (!RLD->isDereferenceable())
31030 MMOFlags &= ~MachineMemOperand::MODereferenceable;
31031 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) {
31032 // FIXME: Discards pointer and AA info.
31033 Load = DAG.getLoad(VT: TheSelect->getValueType(ResNo: 0), dl: SDLoc(TheSelect),
31034 Chain: LLD->getChain(), Ptr: Addr, PtrInfo: MachinePointerInfo(AddrSpace),
31035 Alignment, MMOFlags);
31036 } else {
31037 // FIXME: Discards pointer and AA info.
31038 Load = DAG.getExtLoad(
31039 ExtType: LLD->getExtensionType() == ISD::EXTLOAD ? RLD->getExtensionType()
31040 : LLD->getExtensionType(),
31041 dl: SDLoc(TheSelect), VT: TheSelect->getValueType(ResNo: 0), Chain: LLD->getChain(), Ptr: Addr,
31042 PtrInfo: MachinePointerInfo(AddrSpace), MemVT: LLD->getMemoryVT(), Alignment,
31043 MMOFlags);
31044 }
31045
31046 // Users of the select now use the result of the load.
31047 CombineTo(N: TheSelect, Res: Load);
31048
31049 // Users of the old loads now use the new load's chain. We know the
31050 // old-load value is dead now.
31051 CombineTo(N: LHS.getNode(), Res0: Load.getValue(R: 0), Res1: Load.getValue(R: 1));
31052 CombineTo(N: RHS.getNode(), Res0: Load.getValue(R: 0), Res1: Load.getValue(R: 1));
31053 return true;
31054 }
31055
31056 return false;
31057}
31058
31059/// Try to fold an expression of the form (N0 cond N1) ? N2 : N3 to a shift and
31060/// bitwise 'and'.
31061SDValue DAGCombiner::foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0,
31062 SDValue N1, SDValue N2, SDValue N3,
31063 ISD::CondCode CC) {
31064 // If this is a select where the false operand is zero and the compare is a
31065 // check of the sign bit, see if we can perform the "gzip trick":
31066 // select_cc setlt X, 0, A, 0 -> and (sra X, size(X)-1), A
31067 // select_cc setgt X, 0, A, 0 -> and (not (sra X, size(X)-1)), A
31068 EVT XType = N0.getValueType();
31069 EVT AType = N2.getValueType();
31070 if (!isNullConstant(V: N3) || !XType.bitsGE(VT: AType))
31071 return SDValue();
31072
31073 // If the comparison is testing for a positive value, we have to invert
31074 // the sign bit mask, so only do that transform if the target has a bitwise
31075 // 'and not' instruction (the invert is free).
31076 if (CC == ISD::SETGT && TLI.hasAndNot(X: N2)) {
31077 // (X > -1) ? A : 0
31078 // (X > 0) ? X : 0 <-- This is canonical signed max.
31079 if (!(isAllOnesConstant(V: N1) || (isNullConstant(V: N1) && N0 == N2)))
31080 return SDValue();
31081 } else if (CC == ISD::SETLT) {
31082 // (X < 0) ? A : 0
31083 // (X < 1) ? X : 0 <-- This is un-canonicalized signed min.
31084 if (!(isNullConstant(V: N1) || (isOneConstant(V: N1) && N0 == N2)))
31085 return SDValue();
31086 } else {
31087 return SDValue();
31088 }
31089
31090 // and (sra X, size(X)-1), A -> "and (srl X, C2), A" iff A is a single-bit
31091 // constant.
31092 auto *N2C = dyn_cast<ConstantSDNode>(Val: N2.getNode());
31093 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
31094 unsigned ShCt = XType.getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
31095 if (!TLI.shouldAvoidTransformToShift(VT: XType, Amount: ShCt)) {
31096 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: ShCt, VT: XType, DL);
31097 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: XType, N1: N0, N2: ShiftAmt);
31098 AddToWorklist(N: Shift.getNode());
31099
31100 if (XType.bitsGT(VT: AType)) {
31101 Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: AType, Operand: Shift);
31102 AddToWorklist(N: Shift.getNode());
31103 }
31104
31105 if (CC == ISD::SETGT)
31106 Shift = DAG.getNOT(DL, Val: Shift, VT: AType);
31107
31108 return DAG.getNode(Opcode: ISD::AND, DL, VT: AType, N1: Shift, N2);
31109 }
31110 }
31111
31112 unsigned ShCt = XType.getSizeInBits() - 1;
31113 if (TLI.shouldAvoidTransformToShift(VT: XType, Amount: ShCt))
31114 return SDValue();
31115
31116 SDValue ShiftAmt = DAG.getShiftAmountConstant(Val: ShCt, VT: XType, DL);
31117 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL, VT: XType, N1: N0, N2: ShiftAmt);
31118 AddToWorklist(N: Shift.getNode());
31119
31120 if (XType.bitsGT(VT: AType)) {
31121 Shift = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: AType, Operand: Shift);
31122 AddToWorklist(N: Shift.getNode());
31123 }
31124
31125 if (CC == ISD::SETGT)
31126 Shift = DAG.getNOT(DL, Val: Shift, VT: AType);
31127
31128 return DAG.getNode(Opcode: ISD::AND, DL, VT: AType, N1: Shift, N2);
31129}
31130
31131// Fold select(cc, binop(), binop()) -> binop(select(), select()) etc.
31132SDValue DAGCombiner::foldSelectOfBinops(SDNode *N) {
31133 SDValue N0 = N->getOperand(Num: 0);
31134 SDValue N1 = N->getOperand(Num: 1);
31135 SDValue N2 = N->getOperand(Num: 2);
31136 SDLoc DL(N);
31137
31138 unsigned BinOpc = N1.getOpcode();
31139 if (!TLI.isBinOp(Opcode: BinOpc) || (N2.getOpcode() != BinOpc) ||
31140 (N1.getResNo() != N2.getResNo()))
31141 return SDValue();
31142
31143 // The use checks are intentionally on SDNode because we may be dealing
31144 // with opcodes that produce more than one SDValue.
31145 if (!N1->hasOneUse() || !N2->hasOneUse())
31146 return SDValue();
31147
31148 // Binops may include opcodes that return multiple values, so all values
31149 // must be created/propagated from the newly created binops below.
31150 SDVTList OpVTs = N1->getVTList();
31151
31152 // Fold select(cond, binop(x, y), binop(z, y))
31153 // --> binop(select(cond, x, z), y)
31154 if (N1.getOperand(i: 1) == N2.getOperand(i: 1)) {
31155 SDValue N10 = N1.getOperand(i: 0);
31156 SDValue N20 = N2.getOperand(i: 0);
31157 SDValue NewSel = DAG.getSelect(DL, VT: N10.getValueType(), Cond: N0, LHS: N10, RHS: N20);
31158 SDNodeFlags Flags = N1->getFlags() & N2->getFlags();
31159 SDValue NewBinOp =
31160 DAG.getNode(Opcode: BinOpc, DL, VTList: OpVTs, Ops: {NewSel, N1.getOperand(i: 1)}, Flags);
31161 return SDValue(NewBinOp.getNode(), N1.getResNo());
31162 }
31163
31164 // Fold select(cond, binop(x, y), binop(x, z))
31165 // --> binop(x, select(cond, y, z))
31166 if (N1.getOperand(i: 0) == N2.getOperand(i: 0)) {
31167 SDValue N11 = N1.getOperand(i: 1);
31168 SDValue N21 = N2.getOperand(i: 1);
31169 // Second op VT might be different (e.g. shift amount type)
31170 if (N11.getValueType() == N21.getValueType()) {
31171 SDValue NewSel = DAG.getSelect(DL, VT: N11.getValueType(), Cond: N0, LHS: N11, RHS: N21);
31172 SDNodeFlags Flags = N1->getFlags() & N2->getFlags();
31173 SDValue NewBinOp =
31174 DAG.getNode(Opcode: BinOpc, DL, VTList: OpVTs, Ops: {N1.getOperand(i: 0), NewSel}, Flags);
31175 return SDValue(NewBinOp.getNode(), N1.getResNo());
31176 }
31177 }
31178
31179 // TODO: Handle isCommutativeBinOp patterns as well?
31180 return SDValue();
31181}
31182
31183// Transform (fneg/fabs (bitconvert x)) to avoid loading constant pool values.
31184SDValue DAGCombiner::foldSignChangeInBitcast(SDNode *N) {
31185 SDValue N0 = N->getOperand(Num: 0);
31186 EVT VT = N->getValueType(ResNo: 0);
31187 bool IsFabs = N->getOpcode() == ISD::FABS;
31188 bool IsFree = IsFabs ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT);
31189
31190 if (IsFree || N0.getOpcode() != ISD::BITCAST || !N0.hasOneUse())
31191 return SDValue();
31192
31193 SDValue Int = N0.getOperand(i: 0);
31194 EVT IntVT = Int.getValueType();
31195
31196 // The operand to cast should be integer.
31197 if (!IntVT.isInteger() || IntVT.isVector())
31198 return SDValue();
31199
31200 // (fneg (bitconvert x)) -> (bitconvert (xor x sign))
31201 // (fabs (bitconvert x)) -> (bitconvert (and x ~sign))
31202 APInt SignMask;
31203 if (N0.getValueType().isVector()) {
31204 // For vector, create a sign mask (0x80...) or its inverse (for fabs,
31205 // 0x7f...) per element and splat it.
31206 SignMask = APInt::getSignMask(BitWidth: N0.getScalarValueSizeInBits());
31207 if (IsFabs)
31208 SignMask = ~SignMask;
31209 SignMask = APInt::getSplat(NewLen: IntVT.getSizeInBits(), V: SignMask);
31210 } else {
31211 // For scalar, just use the sign mask (0x80... or the inverse, 0x7f...)
31212 SignMask = APInt::getSignMask(BitWidth: IntVT.getSizeInBits());
31213 if (IsFabs)
31214 SignMask = ~SignMask;
31215 }
31216 SDLoc DL(N0);
31217 Int = DAG.getNode(Opcode: IsFabs ? ISD::AND : ISD::XOR, DL, VT: IntVT, N1: Int,
31218 N2: DAG.getConstant(Val: SignMask, DL, VT: IntVT));
31219 AddToWorklist(N: Int.getNode());
31220 return DAG.getBitcast(VT, V: Int);
31221}
31222
31223/// Turn "(a cond b) ? 1.0f : 2.0f" into "load (tmp + ((a cond b) ? 0 : 4)"
31224/// where "tmp" is a constant pool entry containing an array with 1.0 and 2.0
31225/// in it. This may be a win when the constant is not otherwise available
31226/// because it replaces two constant pool loads with one.
31227SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
31228 const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
31229 ISD::CondCode CC) {
31230 if (!TLI.reduceSelectOfFPConstantLoads(CmpOpVT: N0.getValueType()))
31231 return SDValue();
31232
31233 // If we are before legalize types, we want the other legalization to happen
31234 // first (for example, to avoid messing with soft float).
31235 auto *TV = dyn_cast<ConstantFPSDNode>(Val&: N2);
31236 auto *FV = dyn_cast<ConstantFPSDNode>(Val&: N3);
31237 EVT VT = N2.getValueType();
31238 if (!TV || !FV || !TLI.isTypeLegal(VT))
31239 return SDValue();
31240
31241 // If a constant can be materialized without loads, this does not make sense.
31242 if (TLI.getOperationAction(Op: ISD::ConstantFP, VT) == TargetLowering::Legal ||
31243 TLI.isFPImmLegal(TV->getValueAPF(), TV->getValueType(ResNo: 0), ForCodeSize) ||
31244 TLI.isFPImmLegal(FV->getValueAPF(), FV->getValueType(ResNo: 0), ForCodeSize))
31245 return SDValue();
31246
31247 // If both constants have multiple uses, then we won't need to do an extra
31248 // load. The values are likely around in registers for other users.
31249 if (!TV->hasOneUse() && !FV->hasOneUse())
31250 return SDValue();
31251
31252 Constant *Elts[] = { const_cast<ConstantFP*>(FV->getConstantFPValue()),
31253 const_cast<ConstantFP*>(TV->getConstantFPValue()) };
31254 Type *FPTy = Elts[0]->getType();
31255 const DataLayout &TD = DAG.getDataLayout();
31256
31257 // Create a ConstantArray of the two constants.
31258 Constant *CA = ConstantArray::get(T: ArrayType::get(ElementType: FPTy, NumElements: 2), V: Elts);
31259 SDValue CPIdx = DAG.getConstantPool(C: CA, VT: TLI.getPointerTy(DL: DAG.getDataLayout()),
31260 Align: TD.getPrefTypeAlign(Ty: FPTy));
31261 Align Alignment = cast<ConstantPoolSDNode>(Val&: CPIdx)->getAlign();
31262
31263 // Get offsets to the 0 and 1 elements of the array, so we can select between
31264 // them.
31265 SDValue Zero = DAG.getIntPtrConstant(Val: 0, DL);
31266 unsigned EltSize = (unsigned)TD.getTypeAllocSize(Ty: Elts[0]->getType());
31267 SDValue One = DAG.getIntPtrConstant(Val: EltSize, DL: SDLoc(FV));
31268 SDValue Cond =
31269 DAG.getSetCC(DL, VT: getSetCCResultType(VT: N0.getValueType()), LHS: N0, RHS: N1, Cond: CC);
31270 AddToWorklist(N: Cond.getNode());
31271 SDValue CstOffset = DAG.getSelect(DL, VT: Zero.getValueType(), Cond, LHS: One, RHS: Zero);
31272 AddToWorklist(N: CstOffset.getNode());
31273 CPIdx = DAG.getNode(Opcode: ISD::ADD, DL, VT: CPIdx.getValueType(), N1: CPIdx, N2: CstOffset);
31274 AddToWorklist(N: CPIdx.getNode());
31275 return DAG.getLoad(VT: TV->getValueType(ResNo: 0), dl: DL, Chain: DAG.getEntryNode(), Ptr: CPIdx,
31276 PtrInfo: MachinePointerInfo::getConstantPool(
31277 MF&: DAG.getMachineFunction()), Alignment);
31278}
31279
31280/// Simplify an expression of the form (N0 cond N1) ? N2 : N3
31281/// where 'cond' is the comparison specified by CC.
31282SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
31283 SDValue N2, SDValue N3, ISD::CondCode CC,
31284 bool NotExtCompare) {
31285 // (x ? y : y) -> y.
31286 if (N2 == N3) return N2;
31287
31288 EVT CmpOpVT = N0.getValueType();
31289 EVT CmpResVT = getSetCCResultType(VT: CmpOpVT);
31290 EVT VT = N2.getValueType();
31291 auto *N1C = dyn_cast<ConstantSDNode>(Val: N1.getNode());
31292 auto *N2C = dyn_cast<ConstantSDNode>(Val: N2.getNode());
31293 auto *N3C = dyn_cast<ConstantSDNode>(Val: N3.getNode());
31294
31295 // Determine if the condition we're dealing with is constant.
31296 if (SDValue SCC = DAG.FoldSetCC(VT: CmpResVT, N1: N0, N2: N1, Cond: CC, dl: DL)) {
31297 AddToWorklist(N: SCC.getNode());
31298 if (auto *SCCC = dyn_cast<ConstantSDNode>(Val&: SCC)) {
31299 // fold select_cc true, x, y -> x
31300 // fold select_cc false, x, y -> y
31301 return !(SCCC->isZero()) ? N2 : N3;
31302 }
31303 }
31304
31305 if (SDValue V =
31306 convertSelectOfFPConstantsToLoadOffset(DL, N0, N1, N2, N3, CC))
31307 return V;
31308
31309 if (SDValue V = foldSelectCCToShiftAnd(DL, N0, N1, N2, N3, CC))
31310 return V;
31311
31312 // fold (select_cc seteq (and x, y), 0, 0, A) -> (and (sra (shl x)) A)
31313 // where y is has a single bit set.
31314 // A plaintext description would be, we can turn the SELECT_CC into an AND
31315 // when the condition can be materialized as an all-ones register. Any
31316 // single bit-test can be materialized as an all-ones register with
31317 // shift-left and shift-right-arith.
31318 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND &&
31319 N0->getValueType(ResNo: 0) == VT && isNullConstant(V: N1) && isNullConstant(V: N2)) {
31320 SDValue AndLHS = N0->getOperand(Num: 0);
31321 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
31322 if (ConstAndRHS && ConstAndRHS->getAPIntValue().isPowerOf2()) {
31323 // Shift the tested bit over the sign bit.
31324 const APInt &AndMask = ConstAndRHS->getAPIntValue();
31325 if (TLI.shouldFoldSelectWithSingleBitTest(VT, AndMask)) {
31326 unsigned ShCt = AndMask.getBitWidth() - 1;
31327 SDValue ShlAmt = DAG.getShiftAmountConstant(Val: AndMask.countl_zero(), VT,
31328 DL: SDLoc(AndLHS));
31329 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL: SDLoc(N0), VT, N1: AndLHS, N2: ShlAmt);
31330
31331 // Now arithmetic right shift it all the way over, so the result is
31332 // either all-ones, or zero.
31333 SDValue ShrAmt = DAG.getShiftAmountConstant(Val: ShCt, VT, DL: SDLoc(Shl));
31334 SDValue Shr = DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N0), VT, N1: Shl, N2: ShrAmt);
31335
31336 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shr, N2: N3);
31337 }
31338 }
31339 }
31340
31341 // fold select C, 16, 0 -> shl C, 4
31342 bool Fold = N2C && isNullConstant(V: N3) && N2C->getAPIntValue().isPowerOf2();
31343 bool Swap = N3C && isNullConstant(V: N2) && N3C->getAPIntValue().isPowerOf2();
31344
31345 if ((Fold || Swap) &&
31346 TLI.getBooleanContents(Type: CmpOpVT) ==
31347 TargetLowering::ZeroOrOneBooleanContent &&
31348 (!LegalOperations || TLI.isOperationLegal(Op: ISD::SETCC, VT: CmpOpVT)) &&
31349 TLI.convertSelectOfConstantsToMath(VT)) {
31350
31351 if (Swap) {
31352 CC = ISD::getSetCCInverse(Operation: CC, Type: CmpOpVT);
31353 std::swap(a&: N2C, b&: N3C);
31354 }
31355
31356 // If the caller doesn't want us to simplify this into a zext of a compare,
31357 // don't do it.
31358 if (NotExtCompare && N2C->isOne())
31359 return SDValue();
31360
31361 SDValue Temp, SCC;
31362 // zext (setcc n0, n1)
31363 if (LegalTypes) {
31364 SCC = DAG.getSetCC(DL, VT: CmpResVT, LHS: N0, RHS: N1, Cond: CC);
31365 Temp = DAG.getZExtOrTrunc(Op: SCC, DL: SDLoc(N2), VT);
31366 } else {
31367 SCC = DAG.getSetCC(DL: SDLoc(N0), VT: MVT::i1, LHS: N0, RHS: N1, Cond: CC);
31368 Temp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(N2), VT, Operand: SCC);
31369 }
31370
31371 AddToWorklist(N: SCC.getNode());
31372 AddToWorklist(N: Temp.getNode());
31373
31374 if (N2C->isOne())
31375 return Temp;
31376
31377 unsigned ShCt = N2C->getAPIntValue().logBase2();
31378 if (TLI.shouldAvoidTransformToShift(VT, Amount: ShCt))
31379 return SDValue();
31380
31381 // shl setcc result by log2 n2c
31382 return DAG.getNode(
31383 Opcode: ISD::SHL, DL, VT: N2.getValueType(), N1: Temp,
31384 N2: DAG.getShiftAmountConstant(Val: ShCt, VT: N2.getValueType(), DL: SDLoc(Temp)));
31385 }
31386
31387 // select_cc seteq X, 0, sizeof(X), ctlz(X) -> ctlz(X)
31388 // select_cc seteq X, 0, sizeof(X), ctlz_zero_poison(X) -> ctlz(X)
31389 // select_cc seteq X, 0, sizeof(X), cttz(X) -> cttz(X)
31390 // select_cc seteq X, 0, sizeof(X), cttz_zero_poison(X) -> cttz(X)
31391 // select_cc setne X, 0, ctlz(X), sizeof(X) -> ctlz(X)
31392 // select_cc setne X, 0, ctlz_zero_poison(X), sizeof(X) -> ctlz(X)
31393 // select_cc setne X, 0, cttz(X), sizeof(X) -> cttz(X)
31394 // select_cc setne X, 0, cttz_zero_poison(X), sizeof(X) -> cttz(X)
31395 if (N1C && N1C->isZero() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
31396 SDValue ValueOnZero = N2;
31397 SDValue Count = N3;
31398 // If the condition is NE instead of E, swap the operands.
31399 if (CC == ISD::SETNE)
31400 std::swap(a&: ValueOnZero, b&: Count);
31401 // Check if the value on zero is a constant equal to the bits in the type.
31402 if (auto *ValueOnZeroC = dyn_cast<ConstantSDNode>(Val&: ValueOnZero)) {
31403 if (ValueOnZeroC->getAPIntValue() == VT.getSizeInBits()) {
31404 // If the other operand is cttz/cttz_zero_poison of N0, and cttz is
31405 // legal, combine to just cttz.
31406 if ((Count.getOpcode() == ISD::CTTZ ||
31407 Count.getOpcode() == ISD::CTTZ_ZERO_POISON) &&
31408 N0 == Count.getOperand(i: 0) &&
31409 (!LegalOperations || TLI.isOperationLegal(Op: ISD::CTTZ, VT)))
31410 return DAG.getNode(Opcode: ISD::CTTZ, DL, VT, Operand: N0);
31411 // If the other operand is ctlz/ctlz_zero_poison of N0, and ctlz is
31412 // legal, combine to just ctlz.
31413 if ((Count.getOpcode() == ISD::CTLZ ||
31414 Count.getOpcode() == ISD::CTLZ_ZERO_POISON) &&
31415 N0 == Count.getOperand(i: 0) &&
31416 (!LegalOperations || TLI.isOperationLegal(Op: ISD::CTLZ, VT)))
31417 return DAG.getNode(Opcode: ISD::CTLZ, DL, VT, Operand: N0);
31418 }
31419 }
31420 }
31421
31422 // Fold select_cc setgt X, -1, C, ~C -> xor (ashr X, BW-1), C
31423 // Fold select_cc setlt X, 0, C, ~C -> xor (ashr X, BW-1), ~C
31424 if (!NotExtCompare && N1C && N2C && N3C &&
31425 N2C->getAPIntValue() == ~N3C->getAPIntValue() &&
31426 ((N1C->isAllOnes() && CC == ISD::SETGT) ||
31427 (N1C->isZero() && CC == ISD::SETLT)) &&
31428 !TLI.shouldAvoidTransformToShift(VT, Amount: CmpOpVT.getScalarSizeInBits() - 1)) {
31429 SDValue ASHR =
31430 DAG.getNode(Opcode: ISD::SRA, DL, VT: CmpOpVT, N1: N0,
31431 N2: DAG.getShiftAmountConstant(
31432 Val: CmpOpVT.getScalarSizeInBits() - 1, VT: CmpOpVT, DL));
31433 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: DAG.getSExtOrTrunc(Op: ASHR, DL, VT),
31434 N2: DAG.getSExtOrTrunc(Op: CC == ISD::SETLT ? N3 : N2, DL, VT));
31435 }
31436
31437 // Fold sign pattern select_cc setgt X, -1, 1, -1 -> or (ashr X, BW-1), 1
31438 if (CC == ISD::SETGT && N1C && N2C && N3C && N1C->isAllOnes() &&
31439 N2C->isOne() && N3C->isAllOnes() &&
31440 !TLI.shouldAvoidTransformToShift(VT: CmpOpVT,
31441 Amount: CmpOpVT.getScalarSizeInBits() - 1)) {
31442 SDValue ASHR =
31443 DAG.getNode(Opcode: ISD::SRA, DL, VT: CmpOpVT, N1: N0,
31444 N2: DAG.getShiftAmountConstant(
31445 Val: CmpOpVT.getScalarSizeInBits() - 1, VT: CmpOpVT, DL));
31446 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: DAG.getSExtOrTrunc(Op: ASHR, DL, VT),
31447 N2: DAG.getConstant(Val: 1, DL, VT));
31448 }
31449
31450 if (SDValue S = PerformMinMaxFpToSatCombine(N0, N1, N2, N3, CC, DAG))
31451 return S;
31452 if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N2, N3, CC, DAG))
31453 return S;
31454 if (SDValue ABD = foldSelectToABD(LHS: N0, RHS: N1, True: N2, False: N3, CC, DL))
31455 return ABD;
31456
31457 return SDValue();
31458}
31459
31460static SDValue matchMergedBFX(SDValue Root, SelectionDAG &DAG,
31461 const TargetLowering &TLI) {
31462 // Match a pattern such as:
31463 // (X | (X >> C0) | (X >> C1) | ...) & Mask
31464 // This extracts contiguous parts of X and ORs them together before comparing.
31465 // We can optimize this so that we directly check (X & SomeMask) instead,
31466 // eliminating the shifts.
31467
31468 EVT VT = Root.getValueType();
31469
31470 // TODO: Support vectors?
31471 if (!VT.isScalarInteger() || Root.getOpcode() != ISD::AND)
31472 return SDValue();
31473
31474 SDValue N0 = Root.getOperand(i: 0);
31475 SDValue N1 = Root.getOperand(i: 1);
31476
31477 if (N0.getOpcode() != ISD::OR || !isa<ConstantSDNode>(Val: N1))
31478 return SDValue();
31479
31480 APInt RootMask = cast<ConstantSDNode>(Val&: N1)->getAsAPIntVal();
31481
31482 SDValue Src;
31483 const auto IsSrc = [&](SDValue V) {
31484 if (!Src) {
31485 Src = V;
31486 return true;
31487 }
31488
31489 return Src == V;
31490 };
31491
31492 SmallVector<SDValue> Worklist = {N0};
31493 APInt PartsMask(VT.getSizeInBits(), 0);
31494 while (!Worklist.empty()) {
31495 SDValue V = Worklist.pop_back_val();
31496 if (!V.hasOneUse() && (Src && Src != V))
31497 return SDValue();
31498
31499 if (V.getOpcode() == ISD::OR) {
31500 Worklist.push_back(Elt: V.getOperand(i: 0));
31501 Worklist.push_back(Elt: V.getOperand(i: 1));
31502 continue;
31503 }
31504
31505 if (V.getOpcode() == ISD::SRL) {
31506 SDValue ShiftSrc = V.getOperand(i: 0);
31507 SDValue ShiftAmt = V.getOperand(i: 1);
31508
31509 if (!IsSrc(ShiftSrc) || !isa<ConstantSDNode>(Val: ShiftAmt))
31510 return SDValue();
31511
31512 auto ShiftAmtVal = cast<ConstantSDNode>(Val&: ShiftAmt)->getAsZExtVal();
31513 if (ShiftAmtVal > RootMask.getBitWidth())
31514 return SDValue();
31515
31516 PartsMask |= (RootMask << ShiftAmtVal);
31517 continue;
31518 }
31519
31520 if (IsSrc(V)) {
31521 PartsMask |= RootMask;
31522 continue;
31523 }
31524
31525 return SDValue();
31526 }
31527
31528 if (!Src)
31529 return SDValue();
31530
31531 SDLoc DL(Root);
31532 return DAG.getNode(Opcode: ISD::AND, DL, VT,
31533 Ops: {Src, DAG.getConstant(Val: PartsMask, DL, VT)});
31534}
31535
31536/// This is a stub for TargetLowering::SimplifySetCC.
31537SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
31538 ISD::CondCode Cond, const SDLoc &DL,
31539 bool foldBooleans) {
31540 TargetLowering::DAGCombinerInfo
31541 DagCombineInfo(DAG, Level, false, this);
31542 if (SDValue C =
31543 TLI.SimplifySetCC(VT, N0, N1, Cond, foldBooleans, DCI&: DagCombineInfo, dl: DL))
31544 return C;
31545
31546 if (ISD::isIntEqualitySetCC(Code: Cond) && N0.getOpcode() == ISD::AND &&
31547 isNullConstant(V: N1)) {
31548
31549 if (SDValue Res = matchMergedBFX(Root: N0, DAG, TLI))
31550 return DAG.getSetCC(DL, VT, LHS: Res, RHS: N1, Cond);
31551 }
31552
31553 return SDValue();
31554}
31555
31556/// Given an ISD::SDIV node expressing a divide by constant, return
31557/// a DAG expression to select that will generate the same value by multiplying
31558/// by a magic number.
31559/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
31560SDValue DAGCombiner::BuildSDIV(SDNode *N) {
31561 // when optimising for minimum size, we don't want to expand a div to a mul
31562 // and a shift.
31563 if (DAG.getMachineFunction().getFunction().hasMinSize())
31564 return SDValue();
31565
31566 SmallVector<SDNode *, 8> Built;
31567 if (SDValue S = TLI.BuildSDIV(N, DAG, IsAfterLegalization: LegalOperations, IsAfterLegalTypes: LegalTypes, Created&: Built)) {
31568 for (SDNode *N : Built)
31569 AddToWorklist(N);
31570 return S;
31571 }
31572
31573 return SDValue();
31574}
31575
31576/// Given an ISD::SDIV node expressing a divide by constant power of 2, return a
31577/// DAG expression that will generate the same value by right shifting.
31578SDValue DAGCombiner::BuildSDIVPow2(SDNode *N) {
31579 ConstantSDNode *C = isConstOrConstSplat(N: N->getOperand(Num: 1));
31580 if (!C)
31581 return SDValue();
31582
31583 // Avoid division by zero.
31584 if (C->isZero())
31585 return SDValue();
31586
31587 SmallVector<SDNode *, 8> Built;
31588 if (SDValue S = TLI.BuildSDIVPow2(N, Divisor: C->getAPIntValue(), DAG, Created&: Built)) {
31589 for (SDNode *N : Built)
31590 AddToWorklist(N);
31591 return S;
31592 }
31593
31594 return SDValue();
31595}
31596
31597/// Given an ISD::UDIV node expressing a divide by constant, return a DAG
31598/// expression that will generate the same value by multiplying by a magic
31599/// number.
31600/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
31601SDValue DAGCombiner::BuildUDIV(SDNode *N) {
31602 // when optimising for minimum size, we don't want to expand a div to a mul
31603 // and a shift.
31604 if (DAG.getMachineFunction().getFunction().hasMinSize())
31605 return SDValue();
31606
31607 SmallVector<SDNode *, 8> Built;
31608 if (SDValue S = TLI.BuildUDIV(N, DAG, IsAfterLegalization: LegalOperations, IsAfterLegalTypes: LegalTypes, Created&: Built)) {
31609 for (SDNode *N : Built)
31610 AddToWorklist(N);
31611 return S;
31612 }
31613
31614 return SDValue();
31615}
31616
31617/// Given an ISD::SREM node expressing a remainder by constant power of 2,
31618/// return a DAG expression that will generate the same value.
31619SDValue DAGCombiner::BuildSREMPow2(SDNode *N) {
31620 ConstantSDNode *C = isConstOrConstSplat(N: N->getOperand(Num: 1));
31621 if (!C)
31622 return SDValue();
31623
31624 // Avoid division by zero.
31625 if (C->isZero())
31626 return SDValue();
31627
31628 SmallVector<SDNode *, 8> Built;
31629 if (SDValue S = TLI.BuildSREMPow2(N, Divisor: C->getAPIntValue(), DAG, Created&: Built)) {
31630 for (SDNode *N : Built)
31631 AddToWorklist(N);
31632 return S;
31633 }
31634
31635 return SDValue();
31636}
31637
31638// This is basically just a port of takeLog2 from InstCombineMulDivRem.cpp
31639//
31640// Returns the node that represents `Log2(Op)`. This may create a new node. If
31641// we are unable to compute `Log2(Op)` its return `SDValue()`.
31642//
31643// All nodes will be created at `DL` and the output will be of type `VT`.
31644//
31645// This will only return `Log2(Op)` if we can prove `Op` is non-zero. Set
31646// `AssumeNonZero` if this function should simply assume (not require proving
31647// `Op` is non-zero).
31648static SDValue takeInexpensiveLog2(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
31649 SDValue Op, unsigned Depth,
31650 bool AssumeNonZero) {
31651 assert(VT.isInteger() && "Only integer types are supported!");
31652
31653 auto PeekThroughCastsAndTrunc = [](SDValue V) {
31654 while (true) {
31655 switch (V.getOpcode()) {
31656 case ISD::TRUNCATE:
31657 case ISD::ZERO_EXTEND:
31658 V = V.getOperand(i: 0);
31659 break;
31660 default:
31661 return V;
31662 }
31663 }
31664 };
31665
31666 if (VT.isScalableVector())
31667 return SDValue();
31668
31669 Op = PeekThroughCastsAndTrunc(Op);
31670
31671 // Helper for determining whether a value is a power-2 constant scalar or a
31672 // vector of such elements.
31673 SmallVector<APInt> Pow2Constants;
31674 auto IsPowerOfTwo = [&Pow2Constants](ConstantSDNode *C) {
31675 if (C->isZero() || C->isOpaque())
31676 return false;
31677 // TODO: We may also be able to support negative powers of 2 here.
31678 if (C->getAPIntValue().isPowerOf2()) {
31679 Pow2Constants.emplace_back(Args: C->getAPIntValue());
31680 return true;
31681 }
31682 return false;
31683 };
31684
31685 if (ISD::matchUnaryPredicate(Op, Match: IsPowerOfTwo, /*AllowUndefs=*/false,
31686 /*AllowTruncation=*/true)) {
31687 if (!VT.isVector())
31688 return DAG.getConstant(Val: Pow2Constants.back().logBase2(), DL, VT);
31689 // We need to create a build vector
31690 if (Op.getOpcode() == ISD::SPLAT_VECTOR)
31691 return DAG.getSplat(VT, DL,
31692 Op: DAG.getConstant(Val: Pow2Constants.back().logBase2(), DL,
31693 VT: VT.getScalarType()));
31694 SmallVector<SDValue> Log2Ops;
31695 for (const APInt &Pow2 : Pow2Constants)
31696 Log2Ops.emplace_back(
31697 Args: DAG.getConstant(Val: Pow2.logBase2(), DL, VT: VT.getScalarType()));
31698 return DAG.getBuildVector(VT, DL, Ops: Log2Ops);
31699 }
31700
31701 if (Depth >= DAG.MaxRecursionDepth)
31702 return SDValue();
31703
31704 auto CastToVT = [&](EVT NewVT, SDValue ToCast) {
31705 // Peek through zero extend. We can't peek through truncates since this
31706 // function is called on a shift amount. We must ensure that all of the bits
31707 // above the original shift amount are zeroed by this function.
31708 while (ToCast.getOpcode() == ISD::ZERO_EXTEND)
31709 ToCast = ToCast.getOperand(i: 0);
31710 EVT CurVT = ToCast.getValueType();
31711 if (NewVT == CurVT)
31712 return ToCast;
31713
31714 if (NewVT.getSizeInBits() == CurVT.getSizeInBits())
31715 return DAG.getBitcast(VT: NewVT, V: ToCast);
31716
31717 return DAG.getZExtOrTrunc(Op: ToCast, DL, VT: NewVT);
31718 };
31719
31720 // log2(X << Y) -> log2(X) + Y
31721 if (Op.getOpcode() == ISD::SHL) {
31722 // 1 << Y and X nuw/nsw << Y are all non-zero.
31723 if (AssumeNonZero || Op->getFlags().hasNoUnsignedWrap() ||
31724 Op->getFlags().hasNoSignedWrap() || isOneConstant(V: Op.getOperand(i: 0)))
31725 if (SDValue LogX = takeInexpensiveLog2(DAG, DL, VT, Op: Op.getOperand(i: 0),
31726 Depth: Depth + 1, AssumeNonZero))
31727 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LogX,
31728 N2: CastToVT(VT, Op.getOperand(i: 1)));
31729 }
31730
31731 // c ? X : Y -> c ? Log2(X) : Log2(Y)
31732 SDValue Cond, TVal, FVal;
31733 if (sd_match(N: Op, P: m_OneUse(P: m_SelectLike(Cond: m_Value(N&: Cond), T: m_Value(N&: TVal),
31734 F: m_Value(N&: FVal))))) {
31735 if (SDValue LogX =
31736 takeInexpensiveLog2(DAG, DL, VT, Op: TVal, Depth: Depth + 1, AssumeNonZero))
31737 if (SDValue LogY =
31738 takeInexpensiveLog2(DAG, DL, VT, Op: FVal, Depth: Depth + 1, AssumeNonZero))
31739 return DAG.getSelect(DL, VT, Cond, LHS: LogX, RHS: LogY);
31740 }
31741
31742 // log2(umin(X, Y)) -> umin(log2(X), log2(Y))
31743 // log2(umax(X, Y)) -> umax(log2(X), log2(Y))
31744 if ((Op.getOpcode() == ISD::UMIN || Op.getOpcode() == ISD::UMAX) &&
31745 Op.hasOneUse()) {
31746 // Use AssumeNonZero as false here. Otherwise we can hit case where
31747 // log2(umax(X, Y)) != umax(log2(X), log2(Y)) (because overflow).
31748 if (SDValue LogX =
31749 takeInexpensiveLog2(DAG, DL, VT, Op: Op.getOperand(i: 0), Depth: Depth + 1,
31750 /*AssumeNonZero*/ false))
31751 if (SDValue LogY =
31752 takeInexpensiveLog2(DAG, DL, VT, Op: Op.getOperand(i: 1), Depth: Depth + 1,
31753 /*AssumeNonZero*/ false))
31754 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT, N1: LogX, N2: LogY);
31755 }
31756
31757 return SDValue();
31758}
31759
31760/// Determines the LogBase2 value for a non-null input value using the
31761/// transform: LogBase2(V) = (EltBits - 1) - ctlz(V).
31762SDValue DAGCombiner::BuildLogBase2(SDValue V, const SDLoc &DL,
31763 bool KnownNonZero, bool InexpensiveOnly,
31764 std::optional<EVT> OutVT) {
31765 EVT VT = OutVT ? *OutVT : V.getValueType();
31766 SDValue InexpensiveLogBase2 =
31767 takeInexpensiveLog2(DAG, DL, VT, Op: V, /*Depth*/ 0, AssumeNonZero: KnownNonZero);
31768 if (InexpensiveLogBase2 || InexpensiveOnly || !DAG.isKnownToBeAPowerOfTwo(Val: V))
31769 return InexpensiveLogBase2;
31770
31771 SDValue Ctlz = DAG.getNode(Opcode: ISD::CTLZ, DL, VT, Operand: V);
31772 SDValue Base = DAG.getConstant(Val: VT.getScalarSizeInBits() - 1, DL, VT);
31773 SDValue LogBase2 = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Base, N2: Ctlz);
31774 return LogBase2;
31775}
31776
31777/// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
31778/// For the reciprocal, we need to find the zero of the function:
31779/// F(X) = 1/X - A [which has a zero at X = 1/A]
31780/// =>
31781/// X_{i+1} = X_i (2 - A X_i) = X_i + X_i (1 - A X_i) [this second form
31782/// does not require additional intermediate precision]
31783/// For the last iteration, put numerator N into it to gain more precision:
31784/// Result = N X_i + X_i (N - N A X_i)
31785SDValue DAGCombiner::BuildDivEstimate(SDValue N, SDValue Op,
31786 SDNodeFlags Flags) {
31787 if (LegalDAG)
31788 return SDValue();
31789
31790 // TODO: Handle extended types?
31791 EVT VT = Op.getValueType();
31792 if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 &&
31793 VT.getScalarType() != MVT::f64)
31794 return SDValue();
31795
31796 // If estimates are explicitly disabled for this function, we're done.
31797 MachineFunction &MF = DAG.getMachineFunction();
31798 int Enabled = TLI.getRecipEstimateDivEnabled(VT, MF);
31799 if (Enabled == TLI.ReciprocalEstimate::Disabled)
31800 return SDValue();
31801
31802 // Estimates may be explicitly enabled for this type with a custom number of
31803 // refinement steps.
31804 int Iterations = TLI.getDivRefinementSteps(VT, MF);
31805 if (SDValue Est = TLI.getRecipEstimate(Operand: Op, DAG, Enabled, RefinementSteps&: Iterations)) {
31806 AddToWorklist(N: Est.getNode());
31807
31808 SDLoc DL(Op);
31809 if (Iterations) {
31810 SDValue FPOne = DAG.getConstantFP(Val: 1.0, DL, VT);
31811
31812 // Newton iterations: Est = Est + Est (N - Arg * Est)
31813 // If this is the last iteration, also multiply by the numerator.
31814 for (int i = 0; i < Iterations; ++i) {
31815 SDValue MulEst = Est;
31816
31817 if (i == Iterations - 1) {
31818 MulEst = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: N, N2: Est, Flags);
31819 AddToWorklist(N: MulEst.getNode());
31820 }
31821
31822 SDValue NewEst = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Op, N2: MulEst, Flags);
31823 AddToWorklist(N: NewEst.getNode());
31824
31825 NewEst = DAG.getNode(Opcode: ISD::FSUB, DL, VT,
31826 N1: (i == Iterations - 1 ? N : FPOne), N2: NewEst, Flags);
31827 AddToWorklist(N: NewEst.getNode());
31828
31829 NewEst = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Est, N2: NewEst, Flags);
31830 AddToWorklist(N: NewEst.getNode());
31831
31832 Est = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: MulEst, N2: NewEst, Flags);
31833 AddToWorklist(N: Est.getNode());
31834 }
31835 } else {
31836 // If no iterations are available, multiply with N.
31837 Est = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Est, N2: N, Flags);
31838 AddToWorklist(N: Est.getNode());
31839 }
31840
31841 return Est;
31842 }
31843
31844 return SDValue();
31845}
31846
31847/// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
31848/// For the reciprocal sqrt, we need to find the zero of the function:
31849/// F(X) = 1/X^2 - A [which has a zero at X = 1/sqrt(A)]
31850/// =>
31851/// X_{i+1} = X_i (1.5 - A X_i^2 / 2)
31852/// As a result, we precompute A/2 prior to the iteration loop.
31853SDValue DAGCombiner::buildSqrtNROneConst(SDValue Arg, SDValue Est,
31854 unsigned Iterations, bool Reciprocal) {
31855 EVT VT = Arg.getValueType();
31856 SDLoc DL(Arg);
31857 SDValue ThreeHalves = DAG.getConstantFP(Val: 1.5, DL, VT);
31858
31859 // We now need 0.5 * Arg which we can write as (1.5 * Arg - Arg) so that
31860 // this entire sequence requires only one FP constant.
31861 SDValue HalfArg = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: ThreeHalves, N2: Arg);
31862 HalfArg = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: HalfArg, N2: Arg);
31863
31864 // Newton iterations: Est = Est * (1.5 - HalfArg * Est * Est)
31865 for (unsigned i = 0; i < Iterations; ++i) {
31866 SDValue NewEst = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Est, N2: Est);
31867 NewEst = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: HalfArg, N2: NewEst);
31868 NewEst = DAG.getNode(Opcode: ISD::FSUB, DL, VT, N1: ThreeHalves, N2: NewEst);
31869 Est = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Est, N2: NewEst);
31870 }
31871
31872 // If non-reciprocal square root is requested, multiply the result by Arg.
31873 if (!Reciprocal)
31874 Est = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Est, N2: Arg);
31875
31876 return Est;
31877}
31878
31879/// Newton iteration for a function: F(X) is X_{i+1} = X_i - F(X_i)/F'(X_i)
31880/// For the reciprocal sqrt, we need to find the zero of the function:
31881/// F(X) = 1/X^2 - A [which has a zero at X = 1/sqrt(A)]
31882/// =>
31883/// X_{i+1} = (-0.5 * X_i) * (A * X_i * X_i + (-3.0))
31884SDValue DAGCombiner::buildSqrtNRTwoConst(SDValue Arg, SDValue Est,
31885 unsigned Iterations, bool Reciprocal) {
31886 EVT VT = Arg.getValueType();
31887 SDLoc DL(Arg);
31888 SDValue MinusThree = DAG.getConstantFP(Val: -3.0, DL, VT);
31889 SDValue MinusHalf = DAG.getConstantFP(Val: -0.5, DL, VT);
31890
31891 // This routine must enter the loop below to work correctly
31892 // when (Reciprocal == false).
31893 assert(Iterations > 0);
31894
31895 // Newton iterations for reciprocal square root:
31896 // E = (E * -0.5) * ((A * E) * E + -3.0)
31897 for (unsigned i = 0; i < Iterations; ++i) {
31898 SDValue AE = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Arg, N2: Est);
31899 SDValue AEE = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: AE, N2: Est);
31900 SDValue RHS = DAG.getNode(Opcode: ISD::FADD, DL, VT, N1: AEE, N2: MinusThree);
31901
31902 // When calculating a square root at the last iteration build:
31903 // S = ((A * E) * -0.5) * ((A * E) * E + -3.0)
31904 // (notice a common subexpression)
31905 SDValue LHS;
31906 if (Reciprocal || (i + 1) < Iterations) {
31907 // RSQRT: LHS = (E * -0.5)
31908 LHS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: Est, N2: MinusHalf);
31909 } else {
31910 // SQRT: LHS = (A * E) * -0.5
31911 LHS = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: AE, N2: MinusHalf);
31912 }
31913
31914 Est = DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: LHS, N2: RHS);
31915 }
31916
31917 return Est;
31918}
31919
31920/// Build code to calculate either rsqrt(Op) or sqrt(Op). In the latter case
31921/// Op*rsqrt(Op) is actually computed, so additional postprocessing is needed if
31922/// Op can be zero.
31923SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, bool Reciprocal,
31924 SDNodeFlags Flags) {
31925 if (LegalDAG)
31926 return SDValue();
31927
31928 // TODO: Handle extended types?
31929 EVT VT = Op.getValueType();
31930 if (VT.getScalarType() != MVT::f16 && VT.getScalarType() != MVT::f32 &&
31931 VT.getScalarType() != MVT::f64)
31932 return SDValue();
31933
31934 // If estimates are explicitly disabled for this function, we're done.
31935 MachineFunction &MF = DAG.getMachineFunction();
31936 int Enabled = TLI.getRecipEstimateSqrtEnabled(VT, MF);
31937 if (Enabled == TLI.ReciprocalEstimate::Disabled)
31938 return SDValue();
31939
31940 // Estimates may be explicitly enabled for this type with a custom number of
31941 // refinement steps.
31942 int Iterations = TLI.getSqrtRefinementSteps(VT, MF);
31943
31944 bool UseOneConstNR = false;
31945 if (SDValue Est =
31946 TLI.getSqrtEstimate(Operand: Op, DAG, Enabled, RefinementSteps&: Iterations, UseOneConstNR,
31947 Reciprocal)) {
31948 AddToWorklist(N: Est.getNode());
31949
31950 if (Iterations > 0)
31951 Est = UseOneConstNR
31952 ? buildSqrtNROneConst(Arg: Op, Est, Iterations, Reciprocal)
31953 : buildSqrtNRTwoConst(Arg: Op, Est, Iterations, Reciprocal);
31954 if (!Reciprocal) {
31955 SDLoc DL(Op);
31956 // Try the target specific test first.
31957 SDValue Test =
31958 TLI.getSqrtInputTest(Operand: Op, DAG, Mode: DAG.getDenormalMode(VT), Flags);
31959
31960 // The estimate is now completely wrong if the input was exactly 0.0 or
31961 // possibly a denormal. Force the answer to 0.0 or value provided by
31962 // target for those cases.
31963 Est = DAG.getSelect(DL, VT, Cond: Test,
31964 LHS: TLI.getSqrtResultForDenormInput(Operand: Op, DAG), RHS: Est);
31965 }
31966 return Est;
31967 }
31968
31969 return SDValue();
31970}
31971
31972SDValue DAGCombiner::buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags) {
31973 return buildSqrtEstimateImpl(Op, Reciprocal: true, Flags);
31974}
31975
31976SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) {
31977 return buildSqrtEstimateImpl(Op, Reciprocal: false, Flags);
31978}
31979
31980/// Return true if there is any possibility that the two addresses overlap.
31981bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const {
31982
31983 struct MemUseCharacteristics {
31984 bool IsVolatile;
31985 bool IsAtomic;
31986 SDValue BasePtr;
31987 int64_t Offset;
31988 LocationSize NumBytes;
31989 MachineMemOperand *MMO;
31990 };
31991
31992 auto getCharacteristics = [this](SDNode *N) -> MemUseCharacteristics {
31993 if (const auto *LSN = dyn_cast<LSBaseSDNode>(Val: N)) {
31994 int64_t Offset = 0;
31995 if (auto *C = dyn_cast<ConstantSDNode>(Val: LSN->getOffset()))
31996 Offset = (LSN->getAddressingMode() == ISD::PRE_INC) ? C->getSExtValue()
31997 : (LSN->getAddressingMode() == ISD::PRE_DEC)
31998 ? -1 * C->getSExtValue()
31999 : 0;
32000 TypeSize Size = LSN->getMemoryVT().getStoreSize();
32001 return {.IsVolatile: LSN->isVolatile(), .IsAtomic: LSN->isAtomic(),
32002 .BasePtr: LSN->getBasePtr(), .Offset: Offset /*base offset*/,
32003 .NumBytes: LocationSize::precise(Value: Size), .MMO: LSN->getMemOperand()};
32004 }
32005 if (const auto *LN = cast<LifetimeSDNode>(Val: N)) {
32006 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
32007 return {.IsVolatile: false /*isVolatile*/,
32008 /*isAtomic*/ .IsAtomic: false,
32009 .BasePtr: LN->getOperand(Num: 1),
32010 .Offset: 0,
32011 .NumBytes: LocationSize::precise(Value: MFI.getObjectSize(ObjectIdx: LN->getFrameIndex())),
32012 .MMO: (MachineMemOperand *)nullptr};
32013 }
32014 // Default.
32015 return {.IsVolatile: false /*isvolatile*/,
32016 /*isAtomic*/ .IsAtomic: false,
32017 .BasePtr: SDValue(),
32018 .Offset: (int64_t)0 /*offset*/,
32019 .NumBytes: LocationSize::beforeOrAfterPointer() /*size*/,
32020 .MMO: (MachineMemOperand *)nullptr};
32021 };
32022
32023 MemUseCharacteristics MUC0 = getCharacteristics(Op0),
32024 MUC1 = getCharacteristics(Op1);
32025
32026 // If they are to the same address, then they must be aliases.
32027 if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr &&
32028 MUC0.Offset == MUC1.Offset)
32029 return true;
32030
32031 // If they are both volatile then they cannot be reordered.
32032 if (MUC0.IsVolatile && MUC1.IsVolatile)
32033 return true;
32034
32035 // Be conservative about atomics for the moment
32036 // TODO: This is way overconservative for unordered atomics (see D66309)
32037 if (MUC0.IsAtomic && MUC1.IsAtomic)
32038 return true;
32039
32040 if (MUC0.MMO && MUC1.MMO) {
32041 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
32042 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
32043 return false;
32044 }
32045
32046 // If NumBytes is scalable and offset is not 0, conservatively return may
32047 // alias
32048 if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() &&
32049 MUC0.Offset != 0) ||
32050 (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() &&
32051 MUC1.Offset != 0))
32052 return true;
32053 // Try to prove that there is aliasing, or that there is no aliasing. Either
32054 // way, we can return now. If nothing can be proved, proceed with more tests.
32055 bool IsAlias;
32056 if (BaseIndexOffset::computeAliasing(Op0, NumBytes0: MUC0.NumBytes, Op1, NumBytes1: MUC1.NumBytes,
32057 DAG, IsAlias))
32058 return IsAlias;
32059
32060 // The following all rely on MMO0 and MMO1 being valid. Fail conservatively if
32061 // either are not known.
32062 if (!MUC0.MMO || !MUC1.MMO)
32063 return true;
32064
32065 // If one operation reads from invariant memory, and the other may store, they
32066 // cannot alias. These should really be checking the equivalent of mayWrite,
32067 // but it only matters for memory nodes other than load /store.
32068 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
32069 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
32070 return false;
32071
32072 // If we know required SrcValue1 and SrcValue2 have relatively large
32073 // alignment compared to the size and offset of the access, we may be able
32074 // to prove they do not alias. This check is conservative for now to catch
32075 // cases created by splitting vector types, it only works when the offsets are
32076 // multiples of the size of the data.
32077 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
32078 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
32079 Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
32080 Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
32081 LocationSize Size0 = MUC0.NumBytes;
32082 LocationSize Size1 = MUC1.NumBytes;
32083
32084 if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
32085 Size0.hasValue() && Size1.hasValue() && !Size0.isScalable() &&
32086 !Size1.isScalable() && Size0 == Size1 &&
32087 OrigAlignment0 > Size0.getValue().getKnownMinValue() &&
32088 SrcValOffset0 % Size0.getValue().getKnownMinValue() == 0 &&
32089 SrcValOffset1 % Size1.getValue().getKnownMinValue() == 0) {
32090 int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.value();
32091 int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.value();
32092
32093 // There is no overlap between these relatively aligned accesses of
32094 // similar size. Return no alias.
32095 if ((OffAlign0 + static_cast<int64_t>(
32096 Size0.getValue().getKnownMinValue())) <= OffAlign1 ||
32097 (OffAlign1 + static_cast<int64_t>(
32098 Size1.getValue().getKnownMinValue())) <= OffAlign0)
32099 return false;
32100 }
32101
32102 bool UseAA = CombinerGlobalAA.getNumOccurrences() > 0
32103 ? CombinerGlobalAA
32104 : DAG.getSubtarget().useAA();
32105#ifndef NDEBUG
32106 if (CombinerAAOnlyFunc.getNumOccurrences() &&
32107 CombinerAAOnlyFunc != DAG.getMachineFunction().getName())
32108 UseAA = false;
32109#endif
32110
32111 if (UseAA && BatchAA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
32112 Size0.hasValue() && Size1.hasValue() &&
32113 // Can't represent a scalable size + fixed offset in LocationSize
32114 (!Size0.isScalable() || SrcValOffset0 == 0) &&
32115 (!Size1.isScalable() || SrcValOffset1 == 0)) {
32116 // Use alias analysis information.
32117 int64_t MinOffset = std::min(a: SrcValOffset0, b: SrcValOffset1);
32118 int64_t Overlap0 =
32119 Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
32120 int64_t Overlap1 =
32121 Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
32122 LocationSize Loc0 =
32123 Size0.isScalable() ? Size0 : LocationSize::precise(Value: Overlap0);
32124 LocationSize Loc1 =
32125 Size1.isScalable() ? Size1 : LocationSize::precise(Value: Overlap1);
32126 if (BatchAA->isNoAlias(
32127 LocA: MemoryLocation(MUC0.MMO->getValue(), Loc0,
32128 UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()),
32129 LocB: MemoryLocation(MUC1.MMO->getValue(), Loc1,
32130 UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes())))
32131 return false;
32132 }
32133
32134 // Otherwise we have to assume they alias.
32135 return true;
32136}
32137
32138/// Walk up chain skipping non-aliasing memory nodes,
32139/// looking for aliasing nodes and adding them to the Aliases vector.
32140void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
32141 SmallVectorImpl<SDValue> &Aliases) {
32142 SmallVector<SDValue, 8> Chains; // List of chains to visit.
32143 SmallPtrSet<SDNode *, 16> Visited; // Visited node set.
32144
32145 // Get alias information for node.
32146 // TODO: relax aliasing for unordered atomics (see D66309)
32147 const bool IsLoad = isa<LoadSDNode>(Val: N) && cast<LoadSDNode>(Val: N)->isSimple();
32148
32149 // Starting off.
32150 Chains.push_back(Elt: OriginalChain);
32151 unsigned Depth = 0;
32152
32153 // Attempt to improve chain by a single step
32154 auto ImproveChain = [&](SDValue &C) -> bool {
32155 switch (C.getOpcode()) {
32156 case ISD::EntryToken:
32157 // No need to mark EntryToken.
32158 C = SDValue();
32159 return true;
32160 case ISD::LOAD:
32161 case ISD::STORE: {
32162 // Get alias information for C.
32163 // TODO: Relax aliasing for unordered atomics (see D66309)
32164 bool IsOpLoad = isa<LoadSDNode>(Val: C.getNode()) &&
32165 cast<LSBaseSDNode>(Val: C.getNode())->isSimple();
32166 if ((IsLoad && IsOpLoad) || !mayAlias(Op0: N, Op1: C.getNode())) {
32167 // Look further up the chain.
32168 C = C.getOperand(i: 0);
32169 return true;
32170 }
32171 // Alias, so stop here.
32172 return false;
32173 }
32174
32175 case ISD::CopyFromReg:
32176 // Always forward past CopyFromReg.
32177 C = C.getOperand(i: 0);
32178 return true;
32179
32180 case ISD::LIFETIME_START:
32181 case ISD::LIFETIME_END: {
32182 // We can forward past any lifetime start/end that can be proven not to
32183 // alias the memory access.
32184 if (!mayAlias(Op0: N, Op1: C.getNode())) {
32185 // Look further up the chain.
32186 C = C.getOperand(i: 0);
32187 return true;
32188 }
32189 return false;
32190 }
32191 default:
32192 return false;
32193 }
32194 };
32195
32196 // Look at each chain and determine if it is an alias. If so, add it to the
32197 // aliases list. If not, then continue up the chain looking for the next
32198 // candidate.
32199 while (!Chains.empty()) {
32200 SDValue Chain = Chains.pop_back_val();
32201
32202 // Don't bother if we've seen Chain before.
32203 if (!Visited.insert(Ptr: Chain.getNode()).second)
32204 continue;
32205
32206 // For TokenFactor nodes, look at each operand and only continue up the
32207 // chain until we reach the depth limit.
32208 //
32209 // FIXME: The depth check could be made to return the last non-aliasing
32210 // chain we found before we hit a tokenfactor rather than the original
32211 // chain.
32212 if (Depth > TLI.getGatherAllAliasesMaxDepth()) {
32213 Aliases.clear();
32214 Aliases.push_back(Elt: OriginalChain);
32215 return;
32216 }
32217
32218 if (Chain.getOpcode() == ISD::TokenFactor) {
32219 // We have to check each of the operands of the token factor for "small"
32220 // token factors, so we queue them up. Adding the operands to the queue
32221 // (stack) in reverse order maintains the original order and increases the
32222 // likelihood that getNode will find a matching token factor (CSE.)
32223 if (Chain.getNumOperands() > 16) {
32224 Aliases.push_back(Elt: Chain);
32225 continue;
32226 }
32227 for (unsigned n = Chain.getNumOperands(); n;)
32228 Chains.push_back(Elt: Chain.getOperand(i: --n));
32229 ++Depth;
32230 continue;
32231 }
32232 // Everything else
32233 if (ImproveChain(Chain)) {
32234 // Updated Chain Found, Consider new chain if one exists.
32235 if (Chain.getNode())
32236 Chains.push_back(Elt: Chain);
32237 ++Depth;
32238 continue;
32239 }
32240 // No Improved Chain Possible, treat as Alias.
32241 Aliases.push_back(Elt: Chain);
32242 }
32243}
32244
32245/// Walk up chain skipping non-aliasing memory nodes, looking for a better chain
32246/// (aliasing node.)
32247SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
32248 if (OptLevel == CodeGenOptLevel::None)
32249 return OldChain;
32250
32251 // Ops for replacing token factor.
32252 SmallVector<SDValue, 8> Aliases;
32253
32254 // Accumulate all the aliases to this node.
32255 GatherAllAliases(N, OriginalChain: OldChain, Aliases);
32256
32257 // If no operands then chain to entry token.
32258 if (Aliases.empty())
32259 return DAG.getEntryNode();
32260
32261 // If a single operand then chain to it. We don't need to revisit it.
32262 if (Aliases.size() == 1)
32263 return Aliases[0];
32264
32265 // Construct a custom tailored token factor.
32266 return DAG.getTokenFactor(DL: SDLoc(N), Vals&: Aliases);
32267}
32268
32269// This function tries to collect a bunch of potentially interesting
32270// nodes to improve the chains of, all at once. This might seem
32271// redundant, as this function gets called when visiting every store
32272// node, so why not let the work be done on each store as it's visited?
32273//
32274// I believe this is mainly important because mergeConsecutiveStores
32275// is unable to deal with merging stores of different sizes, so unless
32276// we improve the chains of all the potential candidates up-front
32277// before running mergeConsecutiveStores, it might only see some of
32278// the nodes that will eventually be candidates, and then not be able
32279// to go from a partially-merged state to the desired final
32280// fully-merged state.
32281
32282bool DAGCombiner::parallelizeChainedStores(StoreSDNode *St) {
32283 SmallVector<StoreSDNode *, 8> ChainedStores;
32284 StoreSDNode *STChain = St;
32285 // Intervals records which offsets from BaseIndex have been covered. In
32286 // the common case, every store writes to the immediately previous address
32287 // space and thus merged with the previous interval at insertion time.
32288
32289 using IMap = llvm::IntervalMap<int64_t, std::monostate, 8,
32290 IntervalMapHalfOpenInfo<int64_t>>;
32291 IMap::Allocator A;
32292 IMap Intervals(A);
32293
32294 // This holds the base pointer, index, and the offset in bytes from the base
32295 // pointer.
32296 const BaseIndexOffset BasePtr = BaseIndexOffset::match(N: St, DAG);
32297
32298 // We must have a base and an offset.
32299 if (!BasePtr.getBase().getNode())
32300 return false;
32301
32302 // Do not handle stores to undef base pointers.
32303 if (BasePtr.getBase().isUndef())
32304 return false;
32305
32306 // Do not handle stores to opaque types
32307 if (St->getMemoryVT().isZeroSized())
32308 return false;
32309
32310 // BaseIndexOffset assumes that offsets are fixed-size, which
32311 // is not valid for scalable vectors where the offsets are
32312 // scaled by `vscale`, so bail out early.
32313 if (St->getMemoryVT().isScalableVT())
32314 return false;
32315
32316 // Add ST's interval.
32317 Intervals.insert(a: 0, b: (St->getMemoryVT().getSizeInBits() + 7) / 8,
32318 y: std::monostate{});
32319
32320 while (StoreSDNode *Chain = dyn_cast<StoreSDNode>(Val: STChain->getChain())) {
32321 if (Chain->getMemoryVT().isScalableVector())
32322 return false;
32323
32324 // If the chain has more than one use, then we can't reorder the mem ops.
32325 if (!SDValue(Chain, 0)->hasOneUse())
32326 break;
32327 // TODO: Relax for unordered atomics (see D66309)
32328 if (!Chain->isSimple() || Chain->isIndexed())
32329 break;
32330
32331 // Find the base pointer and offset for this memory node.
32332 const BaseIndexOffset Ptr = BaseIndexOffset::match(N: Chain, DAG);
32333 // Check that the base pointer is the same as the original one.
32334 int64_t Offset;
32335 if (!BasePtr.equalBaseIndex(Other: Ptr, DAG, Off&: Offset))
32336 break;
32337 int64_t Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8;
32338 // Make sure we don't overlap with other intervals by checking the ones to
32339 // the left or right before inserting.
32340 auto I = Intervals.find(x: Offset);
32341 // If there's a next interval, we should end before it.
32342 if (I != Intervals.end() && I.start() < (Offset + Length))
32343 break;
32344 // If there's a previous interval, we should start after it.
32345 if (I != Intervals.begin() && (--I).stop() <= Offset)
32346 break;
32347 Intervals.insert(a: Offset, b: Offset + Length, y: std::monostate{});
32348
32349 ChainedStores.push_back(Elt: Chain);
32350 STChain = Chain;
32351 }
32352
32353 // If we didn't find a chained store, exit.
32354 if (ChainedStores.empty())
32355 return false;
32356
32357 // Improve all chained stores (St and ChainedStores members) starting from
32358 // where the store chain ended and return single TokenFactor.
32359 SDValue NewChain = STChain->getChain();
32360 SmallVector<SDValue, 8> TFOps;
32361 for (unsigned I = ChainedStores.size(); I;) {
32362 StoreSDNode *S = ChainedStores[--I];
32363 SDValue BetterChain = FindBetterChain(N: S, OldChain: NewChain);
32364 S = cast<StoreSDNode>(Val: DAG.UpdateNodeOperands(
32365 N: S, Op1: BetterChain, Op2: S->getOperand(Num: 1), Op3: S->getOperand(Num: 2), Op4: S->getOperand(Num: 3)));
32366 TFOps.push_back(Elt: SDValue(S, 0));
32367 ChainedStores[I] = S;
32368 }
32369
32370 // Improve St's chain. Use a new node to avoid creating a loop from CombineTo.
32371 SDValue BetterChain = FindBetterChain(N: St, OldChain: NewChain);
32372 SDValue NewST;
32373 if (St->isTruncatingStore())
32374 NewST = DAG.getTruncStore(Chain: BetterChain, dl: SDLoc(St), Val: St->getValue(),
32375 Ptr: St->getBasePtr(), SVT: St->getMemoryVT(),
32376 MMO: St->getMemOperand());
32377 else
32378 NewST = DAG.getStore(Chain: BetterChain, dl: SDLoc(St), Val: St->getValue(),
32379 Ptr: St->getBasePtr(), MMO: St->getMemOperand());
32380
32381 TFOps.push_back(Elt: NewST);
32382
32383 // If we improved every element of TFOps, then we've lost the dependence on
32384 // NewChain to successors of St and we need to add it back to TFOps. Do so at
32385 // the beginning to keep relative order consistent with FindBetterChains.
32386 auto hasImprovedChain = [&](SDValue ST) -> bool {
32387 return ST->getOperand(Num: 0) != NewChain;
32388 };
32389 bool AddNewChain = llvm::all_of(Range&: TFOps, P: hasImprovedChain);
32390 if (AddNewChain)
32391 TFOps.insert(I: TFOps.begin(), Elt: NewChain);
32392
32393 SDValue TF = DAG.getTokenFactor(DL: SDLoc(STChain), Vals&: TFOps);
32394 CombineTo(N: St, Res: TF);
32395
32396 // Add TF and its operands to the worklist.
32397 AddToWorklist(N: TF.getNode());
32398 for (const SDValue &Op : TF->ops())
32399 AddToWorklist(N: Op.getNode());
32400 AddToWorklist(N: STChain);
32401 return true;
32402}
32403
32404bool DAGCombiner::findBetterNeighborChains(StoreSDNode *St) {
32405 if (OptLevel == CodeGenOptLevel::None)
32406 return false;
32407
32408 const BaseIndexOffset BasePtr = BaseIndexOffset::match(N: St, DAG);
32409
32410 // We must have a base and an offset.
32411 if (!BasePtr.getBase().getNode())
32412 return false;
32413
32414 // Do not handle stores to undef base pointers.
32415 if (BasePtr.getBase().isUndef())
32416 return false;
32417
32418 // Directly improve a chain of disjoint stores starting at St.
32419 if (parallelizeChainedStores(St))
32420 return true;
32421
32422 // Improve St's Chain..
32423 SDValue BetterChain = FindBetterChain(N: St, OldChain: St->getChain());
32424 if (St->getChain() != BetterChain) {
32425 replaceStoreChain(ST: St, BetterChain);
32426 return true;
32427 }
32428 return false;
32429}
32430
32431/// This is the entry point for the file.
32432void SelectionDAG::Combine(CombineLevel Level, BatchAAResults *BatchAA,
32433 CodeGenOptLevel OptLevel) {
32434 /// This is the main entry point to this class.
32435 DAGCombiner(*this, BatchAA, OptLevel).Run(AtLevel: Level);
32436}
32437