1//===------- VectorCombine.cpp - Optimize partial vector operations -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass optimizes scalar/vector interactions using target cost models. The
10// transforms implemented here may not fit in traditional loop-based or SLP
11// vectorization passes.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Vectorize/VectorCombine.h"
16#include "llvm/ADT/DenseMap.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/ScopeExit.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/Analysis/AssumptionCache.h"
22#include "llvm/Analysis/BasicAliasAnalysis.h"
23#include "llvm/Analysis/GlobalsModRef.h"
24#include "llvm/Analysis/InstSimplifyFolder.h"
25#include "llvm/Analysis/Loads.h"
26#include "llvm/Analysis/TargetFolder.h"
27#include "llvm/Analysis/TargetTransformInfo.h"
28#include "llvm/Analysis/ValueTracking.h"
29#include "llvm/Analysis/VectorUtils.h"
30#include "llvm/IR/Dominators.h"
31#include "llvm/IR/Function.h"
32#include "llvm/IR/IRBuilder.h"
33#include "llvm/IR/Instructions.h"
34#include "llvm/IR/PatternMatch.h"
35#include "llvm/Support/CommandLine.h"
36#include "llvm/Support/KnownBits.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Transforms/Utils/Local.h"
39#include "llvm/Transforms/Utils/LoopUtils.h"
40#include <numeric>
41#include <optional>
42#include <queue>
43#include <set>
44
45#define DEBUG_TYPE "vector-combine"
46#include "llvm/Transforms/Utils/InstructionWorklist.h"
47
48using namespace llvm;
49using namespace llvm::PatternMatch;
50
51STATISTIC(NumVecLoad, "Number of vector loads formed");
52STATISTIC(NumVecCmp, "Number of vector compares formed");
53STATISTIC(NumVecBO, "Number of vector binops formed");
54STATISTIC(NumVecCmpBO, "Number of vector compare + binop formed");
55STATISTIC(NumShufOfBitcast, "Number of shuffles moved after bitcast");
56STATISTIC(NumScalarOps, "Number of scalar unary + binary ops formed");
57STATISTIC(NumScalarCmp, "Number of scalar compares formed");
58STATISTIC(NumScalarIntrinsic, "Number of scalar intrinsic calls formed");
59
60static cl::opt<bool> DisableVectorCombine(
61 "disable-vector-combine", cl::init(Val: false), cl::Hidden,
62 cl::desc("Disable all vector combine transforms"));
63
64static cl::opt<bool> DisableBinopExtractShuffle(
65 "disable-binop-extract-shuffle", cl::init(Val: false), cl::Hidden,
66 cl::desc("Disable binop extract to shuffle transforms"));
67
68static cl::opt<unsigned> MaxInstrsToScan(
69 "vector-combine-max-scan-instrs", cl::init(Val: 30), cl::Hidden,
70 cl::desc("Max number of instructions to scan for vector combining."));
71
72static const unsigned InvalidIndex = std::numeric_limits<unsigned>::max();
73
74namespace {
75class VectorCombine {
76public:
77 VectorCombine(Function &F, const TargetTransformInfo &TTI,
78 const DominatorTree &DT, AAResults &AA, AssumptionCache &AC,
79 const DataLayout *DL, TTI::TargetCostKind CostKind,
80 bool TryEarlyFoldsOnly)
81 : F(F), Builder(F.getContext(), InstSimplifyFolder(*DL)), TTI(TTI),
82 DT(DT), AA(AA), DL(DL), CostKind(CostKind),
83 SQ(*DL, /*TLI=*/nullptr, &DT, &AC),
84 TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
85
86 bool run();
87
88private:
89 Function &F;
90 IRBuilder<InstSimplifyFolder> Builder;
91 const TargetTransformInfo &TTI;
92 const DominatorTree &DT;
93 AAResults &AA;
94 const DataLayout *DL;
95 TTI::TargetCostKind CostKind;
96 const SimplifyQuery SQ;
97
98 /// If true, only perform beneficial early IR transforms. Do not introduce new
99 /// vector operations.
100 bool TryEarlyFoldsOnly;
101
102 InstructionWorklist Worklist;
103
104 /// Next instruction to iterate. It will be updated when it is erased by
105 /// RecursivelyDeleteTriviallyDeadInstructions.
106 Instruction *NextInst;
107
108 // TODO: Direct calls from the top-level "run" loop use a plain "Instruction"
109 // parameter. That should be updated to specific sub-classes because the
110 // run loop was changed to dispatch on opcode.
111 bool vectorizeLoadInsert(Instruction &I);
112 bool widenSubvectorLoad(Instruction &I);
113 ExtractElementInst *getShuffleExtract(ExtractElementInst *Ext0,
114 ExtractElementInst *Ext1,
115 unsigned PreferredExtractIndex) const;
116 bool isExtractExtractCheap(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
117 const Instruction &I,
118 ExtractElementInst *&ConvertToShuffle,
119 unsigned PreferredExtractIndex);
120 Value *foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
121 Value *foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
122 bool foldExtractExtract(Instruction &I);
123 bool foldInsExtFNeg(Instruction &I);
124 bool foldInsExtBinop(Instruction &I);
125 bool foldInsExtVectorToShuffle(Instruction &I);
126 bool foldBitOpOfCastops(Instruction &I);
127 bool foldBitOpOfCastConstant(Instruction &I);
128 bool foldBitcastShuffle(Instruction &I);
129 bool scalarizeOpOrCmp(Instruction &I);
130 bool scalarizeVPIntrinsic(Instruction &I);
131 bool foldExtractedCmps(Instruction &I);
132 bool foldSelectsFromBitcast(Instruction &I);
133 bool foldBinopOfReductions(Instruction &I);
134 bool foldSingleElementStore(Instruction &I);
135 bool scalarizeLoad(Instruction &I);
136 bool scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy, Value *Ptr);
137 bool scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy, Value *Ptr);
138 bool scalarizeExtExtract(Instruction &I);
139 bool foldConcatOfBoolMasks(Instruction &I);
140 bool foldPermuteOfBinops(Instruction &I);
141 bool foldShuffleOfBinops(Instruction &I);
142 bool foldShuffleOfSelects(Instruction &I);
143 bool foldShuffleOfCastops(Instruction &I);
144 bool foldShuffleOfShuffles(Instruction &I);
145 bool foldPermuteOfIntrinsic(Instruction &I);
146 bool foldShufflesOfLengthChangingShuffles(Instruction &I);
147 bool foldShuffleOfIntrinsics(Instruction &I);
148 bool foldShuffleToIdentity(Instruction &I);
149 bool foldShuffleFromReductions(Instruction &I);
150 bool foldShuffleChainsToReduce(Instruction &I);
151 bool foldCastFromReductions(Instruction &I);
152 bool foldSignBitReductionCmp(Instruction &I);
153 bool foldReductionZeroTest(Instruction &I);
154 bool foldICmpEqZeroVectorReduce(Instruction &I);
155 bool foldEquivalentReductionCmp(Instruction &I);
156 bool foldReduceAddCmpZero(Instruction &I);
157 bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
158 bool foldInterleaveIntrinsics(Instruction &I);
159 bool foldDeinterleaveIntrinsics(Instruction &I);
160 bool foldBitcastOfVPLoad(Instruction &I);
161 bool foldBitOrderReverseAndSwap(Instruction &I);
162 bool shrinkType(Instruction &I);
163 bool shrinkLoadForShuffles(Instruction &I);
164 bool shrinkPhiOfShuffles(Instruction &I);
165
166 void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
167 LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
168 LLVM_DEBUG(dbgs() << " With: " << New << '\n');
169 Old.replaceAllUsesWith(V: &New);
170 if (auto *NewI = dyn_cast<Instruction>(Val: &New)) {
171 New.takeName(V: &Old);
172 Worklist.pushUsersToWorkList(I&: *NewI);
173 Worklist.pushValue(V: NewI);
174 }
175 if (Erase && isInstructionTriviallyDead(I: &Old)) {
176 eraseInstruction(I&: Old);
177 } else {
178 Worklist.push(I: &Old);
179 }
180 }
181
182 void eraseInstruction(Instruction &I) {
183 LLVM_DEBUG(dbgs() << "VC: Erasing: " << I << '\n');
184 SmallVector<Value *> Ops(I.operands());
185 Worklist.remove(I: &I);
186 I.eraseFromParent();
187
188 // Push remaining users of the operands and then the operand itself - allows
189 // further folds that were hindered by OneUse limits.
190 SmallPtrSet<Value *, 4> Visited;
191 for (Value *Op : Ops) {
192 if (!Visited.contains(Ptr: Op)) {
193 if (auto *OpI = dyn_cast<Instruction>(Val: Op)) {
194 if (RecursivelyDeleteTriviallyDeadInstructions(
195 V: OpI, TLI: nullptr, MSSAU: nullptr, AboutToDeleteCallback: [&](Value *V) {
196 if (auto *I = dyn_cast<Instruction>(Val: V)) {
197 LLVM_DEBUG(dbgs() << "VC: Erased: " << *I << '\n');
198 Worklist.remove(I);
199 if (I == NextInst)
200 NextInst = NextInst->getNextNode();
201 Visited.insert(Ptr: I);
202 }
203 }))
204 continue;
205 Worklist.pushUsersToWorkList(I&: *OpI);
206 Worklist.pushValue(V: OpI);
207 }
208 }
209 }
210 }
211};
212} // namespace
213
214/// Return the source operand of a potentially bitcasted value. If there is no
215/// bitcast, return the input value itself.
216static Value *peekThroughBitcasts(Value *V) {
217 while (auto *BitCast = dyn_cast<BitCastInst>(Val: V))
218 V = BitCast->getOperand(i_nocapture: 0);
219 return V;
220}
221
222static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI) {
223 // Do not widen load if atomic/volatile or under asan/hwasan/memtag/tsan.
224 // The widened load may load data from dirty regions or create data races
225 // non-existent in the source.
226 if (!Load || !Load->isSimple() || !Load->hasOneUse() ||
227 Load->getFunction()->hasFnAttribute(Kind: Attribute::SanitizeMemTag) ||
228 mustSuppressSpeculation(LI: *Load))
229 return false;
230
231 // We are potentially transforming byte-sized (8-bit) memory accesses, so make
232 // sure we have all of our type-based constraints in place for this target.
233 Type *ScalarTy = Load->getType()->getScalarType();
234 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
235 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
236 if (!ScalarSize || !MinVectorSize || MinVectorSize % ScalarSize != 0 ||
237 ScalarSize % 8 != 0)
238 return false;
239
240 return true;
241}
242
243bool VectorCombine::vectorizeLoadInsert(Instruction &I) {
244 // Match insert into fixed vector of scalar value.
245 // TODO: Handle non-zero insert index.
246 Value *Scalar;
247 if (!match(V: &I,
248 P: m_InsertElt(Val: m_Poison(), Elt: m_OneUse(SubPattern: m_Value(V&: Scalar)), Idx: m_ZeroInt())))
249 return false;
250
251 // Optionally match an extract from another vector.
252 Value *X;
253 bool HasExtract = match(V: Scalar, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ZeroInt()));
254 if (!HasExtract)
255 X = Scalar;
256
257 auto *Load = dyn_cast<LoadInst>(Val: X);
258 if (!canWidenLoad(Load, TTI))
259 return false;
260
261 Type *ScalarTy = Scalar->getType();
262 uint64_t ScalarSize = ScalarTy->getPrimitiveSizeInBits();
263 unsigned MinVectorSize = TTI.getMinVectorRegisterBitWidth();
264
265 // Check safety of replacing the scalar load with a larger vector load.
266 // We use minimal alignment (maximum flexibility) because we only care about
267 // the dereferenceable region. When calculating cost and creating a new op,
268 // we may use a larger value based on alignment attributes.
269 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
270 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
271
272 unsigned MinVecNumElts = MinVectorSize / ScalarSize;
273 auto *MinVecTy = VectorType::get(ElementType: ScalarTy, NumElements: MinVecNumElts, Scalable: false);
274 unsigned OffsetEltIndex = 0;
275 Align Alignment = Load->getAlign();
276 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1), DL: *DL, ScanFrom: Load, AC: SQ.AC,
277 DT: SQ.DT)) {
278 // It is not safe to load directly from the pointer, but we can still peek
279 // through gep offsets and check if it safe to load from a base address with
280 // updated alignment. If it is, we can shuffle the element(s) into place
281 // after loading.
282 unsigned OffsetBitWidth = DL->getIndexTypeSizeInBits(Ty: SrcPtr->getType());
283 APInt Offset(OffsetBitWidth, 0);
284 SrcPtr = SrcPtr->stripAndAccumulateInBoundsConstantOffsets(DL: *DL, Offset);
285
286 // We want to shuffle the result down from a high element of a vector, so
287 // the offset must be positive.
288 if (Offset.isNegative())
289 return false;
290
291 // The offset must be a multiple of the scalar element to shuffle cleanly
292 // in the element's size.
293 uint64_t ScalarSizeInBytes = ScalarSize / 8;
294 if (Offset.urem(RHS: ScalarSizeInBytes) != 0)
295 return false;
296
297 // If we load MinVecNumElts, will our target element still be loaded?
298 OffsetEltIndex = Offset.udiv(RHS: ScalarSizeInBytes).getZExtValue();
299 if (OffsetEltIndex >= MinVecNumElts)
300 return false;
301
302 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty: MinVecTy, Alignment: Align(1), DL: *DL, ScanFrom: Load,
303 AC: SQ.AC, DT: SQ.DT))
304 return false;
305
306 // Update alignment with offset value. Note that the offset could be negated
307 // to more accurately represent "(new) SrcPtr - Offset = (old) SrcPtr", but
308 // negation does not change the result of the alignment calculation.
309 Alignment = commonAlignment(A: Alignment, Offset: Offset.getZExtValue());
310 }
311
312 // Original pattern: insertelt undef, load [free casts of] PtrOp, 0
313 // Use the greater of the alignment on the load or its source pointer.
314 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
315 Type *LoadTy = Load->getType();
316 unsigned AS = Load->getPointerAddressSpace();
317 InstructionCost OldCost =
318 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
319 APInt DemandedElts = APInt::getOneBitSet(numBits: MinVecNumElts, BitNo: 0);
320 OldCost +=
321 TTI.getScalarizationOverhead(Ty: MinVecTy, DemandedElts,
322 /* Insert */ true, Extract: HasExtract, CostKind);
323
324 // New pattern: load VecPtr
325 InstructionCost NewCost =
326 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: MinVecTy, Alignment, AddressSpace: AS, CostKind);
327 // Optionally, we are shuffling the loaded vector element(s) into place.
328 // For the mask set everything but element 0 to undef to prevent poison from
329 // propagating from the extra loaded memory. This will also optionally
330 // shrink/grow the vector from the loaded size to the output size.
331 // We assume this operation has no cost in codegen if there was no offset.
332 // Note that we could use freeze to avoid poison problems, but then we might
333 // still need a shuffle to change the vector size.
334 auto *Ty = cast<FixedVectorType>(Val: I.getType());
335 unsigned OutputNumElts = Ty->getNumElements();
336 SmallVector<int, 16> Mask(OutputNumElts, PoisonMaskElem);
337 assert(OffsetEltIndex < MinVecNumElts && "Address offset too big");
338 Mask[0] = OffsetEltIndex;
339 if (OffsetEltIndex)
340 NewCost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Ty, SrcTy: MinVecTy, Mask,
341 CostKind);
342
343 // We can aggressively convert to the vector form because the backend can
344 // invert this transform if it does not result in a performance win.
345 if (OldCost < NewCost || !NewCost.isValid())
346 return false;
347
348 // It is safe and potentially profitable to load a vector directly:
349 // inselt undef, load Scalar, 0 --> load VecPtr
350 IRBuilder<> Builder(Load);
351 Value *CastedPtr =
352 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
353 Value *VecLd = Builder.CreateAlignedLoad(Ty: MinVecTy, Ptr: CastedPtr, Align: Alignment);
354 VecLd = Builder.CreateShuffleVector(V: VecLd, Mask);
355
356 replaceValue(Old&: I, New&: *VecLd);
357 ++NumVecLoad;
358 return true;
359}
360
361/// If we are loading a vector and then inserting it into a larger vector with
362/// undefined elements, try to load the larger vector and eliminate the insert.
363/// This removes a shuffle in IR and may allow combining of other loaded values.
364bool VectorCombine::widenSubvectorLoad(Instruction &I) {
365 // Match subvector insert of fixed vector.
366 auto *Shuf = cast<ShuffleVectorInst>(Val: &I);
367 if (!Shuf->isIdentityWithPadding())
368 return false;
369
370 // Allow a non-canonical shuffle mask that is choosing elements from op1.
371 unsigned NumOpElts =
372 cast<FixedVectorType>(Val: Shuf->getOperand(i_nocapture: 0)->getType())->getNumElements();
373 unsigned OpIndex = any_of(Range: Shuf->getShuffleMask(), P: [&NumOpElts](int M) {
374 return M >= (int)(NumOpElts);
375 });
376
377 auto *Load = dyn_cast<LoadInst>(Val: Shuf->getOperand(i_nocapture: OpIndex));
378 if (!canWidenLoad(Load, TTI))
379 return false;
380
381 // We use minimal alignment (maximum flexibility) because we only care about
382 // the dereferenceable region. When calculating cost and creating a new op,
383 // we may use a larger value based on alignment attributes.
384 auto *Ty = cast<FixedVectorType>(Val: I.getType());
385 Value *SrcPtr = Load->getPointerOperand()->stripPointerCasts();
386 assert(isa<PointerType>(SrcPtr->getType()) && "Expected a pointer type");
387 Align Alignment = Load->getAlign();
388 if (!isSafeToLoadUnconditionally(V: SrcPtr, Ty, Alignment: Align(1), DL: *DL, ScanFrom: Load, AC: SQ.AC,
389 DT: SQ.DT))
390 return false;
391
392 Alignment = std::max(a: SrcPtr->getPointerAlignment(DL: *DL), b: Alignment);
393 Type *LoadTy = Load->getType();
394 unsigned AS = Load->getPointerAddressSpace();
395
396 // Original pattern: insert_subvector (load PtrOp)
397 // This conservatively assumes that the cost of a subvector insert into an
398 // undef value is 0. We could add that cost if the cost model accurately
399 // reflects the real cost of that operation.
400 InstructionCost OldCost =
401 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: LoadTy, Alignment, AddressSpace: AS, CostKind);
402
403 // New pattern: load PtrOp
404 InstructionCost NewCost =
405 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment, AddressSpace: AS, CostKind);
406
407 // We can aggressively convert to the vector form because the backend can
408 // invert this transform if it does not result in a performance win.
409 if (OldCost < NewCost || !NewCost.isValid())
410 return false;
411
412 IRBuilder<> Builder(Load);
413 Value *CastedPtr =
414 Builder.CreatePointerBitCastOrAddrSpaceCast(V: SrcPtr, DestTy: Builder.getPtrTy(AddrSpace: AS));
415 Value *VecLd = Builder.CreateAlignedLoad(Ty, Ptr: CastedPtr, Align: Alignment);
416 replaceValue(Old&: I, New&: *VecLd);
417 ++NumVecLoad;
418 return true;
419}
420
421/// Determine which, if any, of the inputs should be replaced by a shuffle
422/// followed by extract from a different index.
423ExtractElementInst *VectorCombine::getShuffleExtract(
424 ExtractElementInst *Ext0, ExtractElementInst *Ext1,
425 unsigned PreferredExtractIndex = InvalidIndex) const {
426 auto *Index0C = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
427 auto *Index1C = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
428 assert(Index0C && Index1C && "Expected constant extract indexes");
429
430 unsigned Index0 = Index0C->getZExtValue();
431 unsigned Index1 = Index1C->getZExtValue();
432
433 // If the extract indexes are identical, no shuffle is needed.
434 if (Index0 == Index1)
435 return nullptr;
436
437 Type *VecTy = Ext0->getVectorOperand()->getType();
438 assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types");
439 InstructionCost Cost0 =
440 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
441 InstructionCost Cost1 =
442 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
443
444 // If both costs are invalid no shuffle is needed
445 if (!Cost0.isValid() && !Cost1.isValid())
446 return nullptr;
447
448 // We are extracting from 2 different indexes, so one operand must be shuffled
449 // before performing a vector operation and/or extract. The more expensive
450 // extract will be replaced by a shuffle.
451 if (Cost0 > Cost1)
452 return Ext0;
453 if (Cost1 > Cost0)
454 return Ext1;
455
456 // If the costs are equal and there is a preferred extract index, shuffle the
457 // opposite operand.
458 if (PreferredExtractIndex == Index0)
459 return Ext1;
460 if (PreferredExtractIndex == Index1)
461 return Ext0;
462
463 // Otherwise, replace the extract with the higher index.
464 return Index0 > Index1 ? Ext0 : Ext1;
465}
466
467/// Compare the relative costs of 2 extracts followed by scalar operation vs.
468/// vector operation(s) followed by extract. Return true if the existing
469/// instructions are cheaper than a vector alternative. Otherwise, return false
470/// and if one of the extracts should be transformed to a shufflevector, set
471/// \p ConvertToShuffle to that extract instruction.
472bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0,
473 ExtractElementInst *Ext1,
474 const Instruction &I,
475 ExtractElementInst *&ConvertToShuffle,
476 unsigned PreferredExtractIndex) {
477 auto *Ext0IndexC = dyn_cast<ConstantInt>(Val: Ext0->getIndexOperand());
478 auto *Ext1IndexC = dyn_cast<ConstantInt>(Val: Ext1->getIndexOperand());
479 assert(Ext0IndexC && Ext1IndexC && "Expected constant extract indexes");
480
481 unsigned Opcode = I.getOpcode();
482 Value *Ext0Src = Ext0->getVectorOperand();
483 Value *Ext1Src = Ext1->getVectorOperand();
484 Type *ScalarTy = Ext0->getType();
485 auto *VecTy = cast<VectorType>(Val: Ext0Src->getType());
486 InstructionCost ScalarOpCost, VectorOpCost;
487
488 // Get cost estimates for scalar and vector versions of the operation.
489 bool IsBinOp = Instruction::isBinaryOp(Opcode);
490 if (IsBinOp) {
491 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
492 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
493 } else {
494 assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
495 "Expected a compare");
496 CmpInst::Predicate Pred = cast<CmpInst>(Val: I).getPredicate();
497 ScalarOpCost = TTI.getCmpSelInstrCost(
498 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
499 VectorOpCost = TTI.getCmpSelInstrCost(
500 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
501 }
502
503 // Get cost estimates for the extract elements. These costs will factor into
504 // both sequences.
505 unsigned Ext0Index = Ext0IndexC->getZExtValue();
506 unsigned Ext1Index = Ext1IndexC->getZExtValue();
507
508 InstructionCost Extract0Cost =
509 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Ext0Index);
510 InstructionCost Extract1Cost =
511 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Ext1Index);
512
513 // A more expensive extract will always be replaced by a splat shuffle.
514 // For example, if Ext0 is more expensive:
515 // opcode (extelt V0, Ext0), (ext V1, Ext1) -->
516 // extelt (opcode (splat V0, Ext0), V1), Ext1
517 // TODO: Evaluate whether that always results in lowest cost. Alternatively,
518 // check the cost of creating a broadcast shuffle and shuffling both
519 // operands to element 0.
520 unsigned BestExtIndex = Extract0Cost > Extract1Cost ? Ext0Index : Ext1Index;
521 unsigned BestInsIndex = Extract0Cost > Extract1Cost ? Ext1Index : Ext0Index;
522 InstructionCost CheapExtractCost = std::min(a: Extract0Cost, b: Extract1Cost);
523
524 // Extra uses of the extracts mean that we include those costs in the
525 // vector total because those instructions will not be eliminated.
526 InstructionCost OldCost, NewCost;
527 if (Ext0Src == Ext1Src && Ext0Index == Ext1Index) {
528 // Handle a special case. If the 2 extracts are identical, adjust the
529 // formulas to account for that. The extra use charge allows for either the
530 // CSE'd pattern or an unoptimized form with identical values:
531 // opcode (extelt V, C), (extelt V, C) --> extelt (opcode V, V), C
532 bool HasUseTax = Ext0 == Ext1 ? !Ext0->hasNUses(N: 2)
533 : !Ext0->hasOneUse() || !Ext1->hasOneUse();
534 OldCost = CheapExtractCost + ScalarOpCost;
535 NewCost = VectorOpCost + CheapExtractCost + HasUseTax * CheapExtractCost;
536 } else {
537 // Handle the general case. Each extract is actually a different value:
538 // opcode (extelt V0, C0), (extelt V1, C1) --> extelt (opcode V0, V1), C
539 OldCost = Extract0Cost + Extract1Cost + ScalarOpCost;
540 NewCost = VectorOpCost + CheapExtractCost +
541 !Ext0->hasOneUse() * Extract0Cost +
542 !Ext1->hasOneUse() * Extract1Cost;
543 }
544
545 ConvertToShuffle = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex);
546 if (ConvertToShuffle) {
547 if (IsBinOp && DisableBinopExtractShuffle)
548 return true;
549
550 // If we are extracting from 2 different indexes, then one operand must be
551 // shuffled before performing the vector operation. The shuffle mask is
552 // poison except for 1 lane that is being translated to the remaining
553 // extraction lane. Therefore, it is a splat shuffle. Ex:
554 // ShufMask = { poison, poison, 0, poison }
555 // TODO: The cost model has an option for a "broadcast" shuffle
556 // (splat-from-element-0), but no option for a more general splat.
557 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: VecTy)) {
558 SmallVector<int> ShuffleMask(FixedVecTy->getNumElements(),
559 PoisonMaskElem);
560 ShuffleMask[BestInsIndex] = BestExtIndex;
561 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
562 DstTy: VecTy, SrcTy: VecTy, Mask: ShuffleMask, CostKind, Index: 0,
563 SubTp: nullptr, Args: {ConvertToShuffle});
564 } else {
565 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
566 DstTy: VecTy, SrcTy: VecTy, Mask: {}, CostKind, Index: 0, SubTp: nullptr,
567 Args: {ConvertToShuffle});
568 }
569 }
570
571 LLVM_DEBUG(dbgs() << "Found a binop of extractions: " << I << "\n OldCost: "
572 << OldCost << " vs NewCost: " << NewCost << "\n");
573
574 // Aggressively form a vector op if the cost is equal because the transform
575 // may enable further optimization.
576 // Codegen can reverse this transform (scalarize) if it was not profitable.
577 return OldCost < NewCost;
578}
579
580/// Create a shuffle that translates (shifts) 1 element from the input vector
581/// to a new element location.
582static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
583 unsigned NewIndex, IRBuilderBase &Builder) {
584 // The shuffle mask is poison except for 1 lane that is being translated
585 // to the new element index. Example for OldIndex == 2 and NewIndex == 0:
586 // ShufMask = { 2, poison, poison, poison }
587 auto *VecTy = cast<FixedVectorType>(Val: Vec->getType());
588 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
589 ShufMask[NewIndex] = OldIndex;
590 return Builder.CreateShuffleVector(V: Vec, Mask: ShufMask, Name: "shift");
591}
592
593/// Given an extract element instruction with constant index operand, shuffle
594/// the source vector (shift the scalar element) to a NewIndex for extraction.
595/// Return null if the input can be constant folded, so that we are not creating
596/// unnecessary instructions.
597static Value *translateExtract(ExtractElementInst *ExtElt, unsigned NewIndex,
598 IRBuilderBase &Builder) {
599 // Shufflevectors can only be created for fixed-width vectors.
600 Value *X = ExtElt->getVectorOperand();
601 if (!isa<FixedVectorType>(Val: X->getType()))
602 return nullptr;
603
604 // If the extract can be constant-folded, this code is unsimplified. Defer
605 // to other passes to handle that.
606 Value *C = ExtElt->getIndexOperand();
607 assert(isa<ConstantInt>(C) && "Expected a constant index operand");
608 if (isa<Constant>(Val: X))
609 return nullptr;
610
611 Value *Shuf = createShiftShuffle(Vec: X, OldIndex: cast<ConstantInt>(Val: C)->getZExtValue(),
612 NewIndex, Builder);
613 return Shuf;
614}
615
616/// Try to reduce extract element costs by converting scalar compares to vector
617/// compares followed by extract.
618/// cmp (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
619Value *VectorCombine::foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex,
620 Instruction &I) {
621 assert(isa<CmpInst>(&I) && "Expected a compare");
622
623 // cmp Pred (extelt V0, ExtIndex), (extelt V1, ExtIndex)
624 // --> extelt (cmp Pred V0, V1), ExtIndex
625 ++NumVecCmp;
626 CmpInst::Predicate Pred = cast<CmpInst>(Val: &I)->getPredicate();
627 Value *VecCmp = Builder.CreateCmp(Pred, LHS: V0, RHS: V1);
628 return Builder.CreateExtractElement(Vec: VecCmp, Idx: ExtIndex, Name: "foldExtExtCmp");
629}
630
631/// Try to reduce extract element costs by converting scalar binops to vector
632/// binops followed by extract.
633/// bo (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
634Value *VectorCombine::foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex,
635 Instruction &I) {
636 assert(isa<BinaryOperator>(&I) && "Expected a binary operator");
637
638 // bo (extelt V0, ExtIndex), (extelt V1, ExtIndex)
639 // --> extelt (bo V0, V1), ExtIndex
640 ++NumVecBO;
641 Value *VecBO = Builder.CreateBinOp(Opc: cast<BinaryOperator>(Val: &I)->getOpcode(), LHS: V0,
642 RHS: V1, Name: "foldExtExtBinop");
643
644 // All IR flags are safe to back-propagate because any potential poison
645 // created in unused vector elements is discarded by the extract.
646 if (auto *VecBOInst = dyn_cast<Instruction>(Val: VecBO))
647 VecBOInst->copyIRFlags(V: &I);
648
649 return Builder.CreateExtractElement(Vec: VecBO, Idx: ExtIndex, Name: "foldExtExtBinop");
650}
651
652/// Match an instruction with extracted vector operands.
653bool VectorCombine::foldExtractExtract(Instruction &I) {
654 // It is not safe to transform things like div, urem, etc. because we may
655 // create undefined behavior when executing those on unknown vector elements.
656 if (!isSafeToSpeculativelyExecute(I: &I))
657 return false;
658
659 Instruction *I0, *I1;
660 CmpPredicate Pred = CmpInst::BAD_ICMP_PREDICATE;
661 if (!match(V: &I, P: m_Cmp(Pred, L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))) &&
662 !match(V: &I, P: m_BinOp(L: m_Instruction(I&: I0), R: m_Instruction(I&: I1))))
663 return false;
664
665 Value *V0, *V1;
666 uint64_t C0, C1;
667 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: V0), Idx: m_ConstantInt(V&: C0))) ||
668 !match(V: I1, P: m_ExtractElt(Val: m_Value(V&: V1), Idx: m_ConstantInt(V&: C1))) ||
669 V0->getType() != V1->getType())
670 return false;
671
672 // For fixed-width vectors, reject out-of-bounds extract indexes
673 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Val: V0->getType())) {
674 unsigned NumElts = FixedVecTy->getNumElements();
675 if (C0 >= NumElts || C1 >= NumElts)
676 return false;
677 }
678
679 // If the scalar value 'I' is going to be re-inserted into a vector, then try
680 // to create an extract to that same element. The extract/insert can be
681 // reduced to a "select shuffle".
682 // TODO: If we add a larger pattern match that starts from an insert, this
683 // probably becomes unnecessary.
684 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
685 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
686 uint64_t InsertIndex = InvalidIndex;
687 if (I.hasOneUse())
688 match(V: I.user_back(),
689 P: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ConstantInt(V&: InsertIndex)));
690
691 ExtractElementInst *ExtractToChange;
692 if (isExtractExtractCheap(Ext0, Ext1, I, ConvertToShuffle&: ExtractToChange, PreferredExtractIndex: InsertIndex))
693 return false;
694
695 Value *ExtOp0 = Ext0->getVectorOperand();
696 Value *ExtOp1 = Ext1->getVectorOperand();
697
698 if (ExtractToChange) {
699 unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
700 Value *NewExtOp =
701 translateExtract(ExtElt: ExtractToChange, NewIndex: CheapExtractIdx, Builder);
702 if (!NewExtOp)
703 return false;
704 if (ExtractToChange == Ext0)
705 ExtOp0 = NewExtOp;
706 else
707 ExtOp1 = NewExtOp;
708 }
709
710 Value *ExtIndex = ExtractToChange == Ext0 ? Ext1->getIndexOperand()
711 : Ext0->getIndexOperand();
712 Value *NewExt = Pred != CmpInst::BAD_ICMP_PREDICATE
713 ? foldExtExtCmp(V0: ExtOp0, V1: ExtOp1, ExtIndex, I)
714 : foldExtExtBinop(V0: ExtOp0, V1: ExtOp1, ExtIndex, I);
715 Worklist.push(I: Ext0);
716 Worklist.push(I: Ext1);
717 replaceValue(Old&: I, New&: *NewExt);
718 return true;
719}
720
721/// Try to replace an extract + scalar fneg + insert with a vector fneg +
722/// shuffle.
723bool VectorCombine::foldInsExtFNeg(Instruction &I) {
724 // Match an insert (op (extract)) pattern.
725 Value *DstVec;
726 uint64_t ExtIdx, InsIdx;
727 Instruction *FNeg;
728 if (!match(V: &I, P: m_InsertElt(Val: m_Value(V&: DstVec), Elt: m_OneUse(SubPattern: m_Instruction(I&: FNeg)),
729 Idx: m_ConstantInt(V&: InsIdx))))
730 return false;
731
732 // Note: This handles the canonical fneg instruction and "fsub -0.0, X".
733 Value *SrcVec;
734 Instruction *Extract;
735 if (!match(V: FNeg, P: m_FNeg(X: m_CombineAnd(
736 Ps: m_Instruction(I&: Extract),
737 Ps: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx))))))
738 return false;
739
740 auto *DstVecTy = cast<FixedVectorType>(Val: DstVec->getType());
741 auto *DstVecScalarTy = DstVecTy->getScalarType();
742 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
743 if (!SrcVecTy || DstVecScalarTy != SrcVecTy->getScalarType())
744 return false;
745
746 // Ignore if insert/extract index is out of bounds or destination vector has
747 // one element
748 unsigned NumDstElts = DstVecTy->getNumElements();
749 unsigned NumSrcElts = SrcVecTy->getNumElements();
750 if (ExtIdx > NumSrcElts || InsIdx >= NumDstElts || NumDstElts == 1)
751 return false;
752
753 // We are inserting the negated element into the same lane that we extracted
754 // from. This is equivalent to a select-shuffle that chooses all but the
755 // negated element from the destination vector.
756 SmallVector<int> Mask(NumDstElts);
757 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
758 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
759 InstructionCost OldCost =
760 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: DstVecScalarTy, CostKind) +
761 TTI.getVectorInstrCost(I, Val: DstVecTy, CostKind, Index: InsIdx);
762
763 // If the extract has one use, it will be eliminated, so count it in the
764 // original cost. If it has more than one use, ignore the cost because it will
765 // be the same before/after.
766 if (Extract->hasOneUse())
767 OldCost += TTI.getVectorInstrCost(I: *Extract, Val: SrcVecTy, CostKind, Index: ExtIdx);
768
769 InstructionCost NewCost =
770 TTI.getArithmeticInstrCost(Opcode: Instruction::FNeg, Ty: SrcVecTy, CostKind) +
771 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: DstVecTy,
772 SrcTy: DstVecTy, Mask, CostKind);
773
774 bool NeedLenChg = SrcVecTy->getNumElements() != NumDstElts;
775 // If the lengths of the two vectors are not equal,
776 // we need to add a length-change vector. Add this cost.
777 SmallVector<int> SrcMask;
778 if (NeedLenChg) {
779 SrcMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
780 SrcMask[ExtIdx % NumDstElts] = ExtIdx;
781 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
782 DstTy: DstVecTy, SrcTy: SrcVecTy, Mask: SrcMask, CostKind);
783 }
784
785 LLVM_DEBUG(dbgs() << "Found an insertion of (extract)fneg : " << I
786 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
787 << "\n");
788 if (NewCost > OldCost)
789 return false;
790
791 Value *NewShuf, *LenChgShuf = nullptr;
792 // insertelt DstVec, (fneg (extractelt SrcVec, Index)), Index
793 Value *VecFNeg = Builder.CreateFNegFMF(V: SrcVec, FMFSource: FNeg);
794 if (NeedLenChg) {
795 // shuffle DstVec, (shuffle (fneg SrcVec), poison, SrcMask), Mask
796 LenChgShuf = Builder.CreateShuffleVector(V: VecFNeg, Mask: SrcMask);
797 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: LenChgShuf, Mask);
798 Worklist.pushValue(V: LenChgShuf);
799 } else {
800 // shuffle DstVec, (fneg SrcVec), Mask
801 NewShuf = Builder.CreateShuffleVector(V1: DstVec, V2: VecFNeg, Mask);
802 }
803
804 Worklist.pushValue(V: VecFNeg);
805 replaceValue(Old&: I, New&: *NewShuf);
806 return true;
807}
808
809/// Try to fold insert(binop(x,y),binop(a,b),idx)
810/// --> binop(insert(x,a,idx),insert(y,b,idx))
811bool VectorCombine::foldInsExtBinop(Instruction &I) {
812 BinaryOperator *VecBinOp, *SclBinOp;
813 uint64_t Index;
814 if (!match(V: &I,
815 P: m_InsertElt(Val: m_OneUse(SubPattern: m_BinOp(I&: VecBinOp)),
816 Elt: m_OneUse(SubPattern: m_BinOp(I&: SclBinOp)), Idx: m_ConstantInt(V&: Index))))
817 return false;
818
819 // TODO: Add support for addlike etc.
820 Instruction::BinaryOps BinOpcode = VecBinOp->getOpcode();
821 if (BinOpcode != SclBinOp->getOpcode())
822 return false;
823
824 auto *ResultTy = dyn_cast<FixedVectorType>(Val: I.getType());
825 if (!ResultTy)
826 return false;
827
828 // TODO: Attempt to detect m_ExtractElt for scalar operands and convert to
829 // shuffle?
830
831 InstructionCost OldCost = TTI.getInstructionCost(U: &I, CostKind) +
832 TTI.getInstructionCost(U: VecBinOp, CostKind) +
833 TTI.getInstructionCost(U: SclBinOp, CostKind);
834 InstructionCost NewCost =
835 TTI.getArithmeticInstrCost(Opcode: BinOpcode, Ty: ResultTy, CostKind) +
836 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
837 Index, Op0: VecBinOp->getOperand(i_nocapture: 0),
838 Op1: SclBinOp->getOperand(i_nocapture: 0)) +
839 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: ResultTy, CostKind,
840 Index, Op0: VecBinOp->getOperand(i_nocapture: 1),
841 Op1: SclBinOp->getOperand(i_nocapture: 1));
842
843 LLVM_DEBUG(dbgs() << "Found an insertion of two binops: " << I
844 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
845 << "\n");
846 if (NewCost > OldCost)
847 return false;
848
849 Value *NewIns0 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 0),
850 NewElt: SclBinOp->getOperand(i_nocapture: 0), Idx: Index);
851 Value *NewIns1 = Builder.CreateInsertElement(Vec: VecBinOp->getOperand(i_nocapture: 1),
852 NewElt: SclBinOp->getOperand(i_nocapture: 1), Idx: Index);
853 Value *NewBO = Builder.CreateBinOp(Opc: BinOpcode, LHS: NewIns0, RHS: NewIns1);
854
855 // Intersect flags from the old binops.
856 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
857 NewInst->copyIRFlags(V: VecBinOp);
858 NewInst->andIRFlags(V: SclBinOp);
859 }
860
861 Worklist.pushValue(V: NewIns0);
862 Worklist.pushValue(V: NewIns1);
863 replaceValue(Old&: I, New&: *NewBO);
864 return true;
865}
866
867/// Match: bitop(castop(x), castop(y)) -> castop(bitop(x, y))
868/// Supports: bitcast, trunc, sext, zext
869bool VectorCombine::foldBitOpOfCastops(Instruction &I) {
870 // Check if this is a bitwise logic operation
871 auto *BinOp = dyn_cast<BinaryOperator>(Val: &I);
872 if (!BinOp || !BinOp->isBitwiseLogicOp())
873 return false;
874
875 // Get the cast instructions
876 auto *LHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 0));
877 auto *RHSCast = dyn_cast<CastInst>(Val: BinOp->getOperand(i_nocapture: 1));
878 if (!LHSCast || !RHSCast) {
879 LLVM_DEBUG(dbgs() << " One or both operands are not cast instructions\n");
880 return false;
881 }
882
883 // Both casts must be the same type
884 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
885 if (CastOpcode != RHSCast->getOpcode())
886 return false;
887
888 // Only handle supported cast operations
889 switch (CastOpcode) {
890 case Instruction::BitCast:
891 case Instruction::Trunc:
892 case Instruction::SExt:
893 case Instruction::ZExt:
894 break;
895 default:
896 return false;
897 }
898
899 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
900 Value *RHSSrc = RHSCast->getOperand(i_nocapture: 0);
901
902 // Source types must match
903 if (LHSSrc->getType() != RHSSrc->getType())
904 return false;
905
906 auto *SrcTy = LHSSrc->getType();
907 auto *DstTy = I.getType();
908 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
909 // Other casts only handle vector types with integer elements.
910 if (CastOpcode != Instruction::BitCast &&
911 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
912 return false;
913
914 // Only integer scalar/vector values are legal for bitwise logic operations.
915 if (!SrcTy->getScalarType()->isIntegerTy() ||
916 !DstTy->getScalarType()->isIntegerTy())
917 return false;
918
919 // Cost Check :
920 // OldCost = bitlogic + 2*casts
921 // NewCost = bitlogic + cast
922
923 // Calculate specific costs for each cast with instruction context
924 InstructionCost LHSCastCost = TTI.getCastInstrCost(
925 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
926 InstructionCost RHSCastCost = TTI.getCastInstrCost(
927 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: RHSCast);
928
929 InstructionCost OldCost =
930 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: DstTy, CostKind) +
931 LHSCastCost + RHSCastCost;
932
933 // For new cost, we can't provide an instruction (it doesn't exist yet)
934 InstructionCost GenericCastCost = TTI.getCastInstrCost(
935 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
936
937 InstructionCost NewCost =
938 TTI.getArithmeticInstrCost(Opcode: BinOp->getOpcode(), Ty: SrcTy, CostKind) +
939 GenericCastCost;
940
941 // Account for multi-use casts using specific costs
942 if (!LHSCast->hasOneUse())
943 NewCost += LHSCastCost;
944 if (!RHSCast->hasOneUse())
945 NewCost += RHSCastCost;
946
947 LLVM_DEBUG(dbgs() << "foldBitOpOfCastops: OldCost=" << OldCost
948 << " NewCost=" << NewCost << "\n");
949
950 if (NewCost > OldCost)
951 return false;
952
953 // Create the operation on the source type
954 Value *NewOp = Builder.CreateBinOp(Opc: BinOp->getOpcode(), LHS: LHSSrc, RHS: RHSSrc,
955 Name: BinOp->getName() + ".inner");
956 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
957 NewBinOp->copyIRFlags(V: BinOp);
958
959 Worklist.pushValue(V: NewOp);
960
961 // Create the cast operation directly to ensure we get a new instruction
962 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
963
964 // Preserve cast instruction flags
965 NewCast->copyIRFlags(V: LHSCast);
966 NewCast->andIRFlags(V: RHSCast);
967
968 // Insert the new instruction
969 Value *Result = Builder.Insert(I: NewCast);
970
971 replaceValue(Old&: I, New&: *Result);
972 return true;
973}
974
975/// Match:
976// bitop(castop(x), C) ->
977// bitop(castop(x), castop(InvC)) ->
978// castop(bitop(x, InvC))
979// Supports: bitcast
980bool VectorCombine::foldBitOpOfCastConstant(Instruction &I) {
981 Instruction *LHS;
982 Constant *C;
983
984 // Check if this is a bitwise logic operation
985 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_Instruction(I&: LHS), R: m_Constant(C))))
986 return false;
987
988 // Get the cast instructions
989 auto *LHSCast = dyn_cast<CastInst>(Val: LHS);
990 if (!LHSCast)
991 return false;
992
993 Instruction::CastOps CastOpcode = LHSCast->getOpcode();
994
995 // Only handle supported cast operations
996 switch (CastOpcode) {
997 case Instruction::BitCast:
998 case Instruction::ZExt:
999 case Instruction::SExt:
1000 case Instruction::Trunc:
1001 break;
1002 default:
1003 return false;
1004 }
1005
1006 Value *LHSSrc = LHSCast->getOperand(i_nocapture: 0);
1007
1008 auto *SrcTy = LHSSrc->getType();
1009 auto *DstTy = I.getType();
1010 // Bitcasts can handle scalar/vector mixes, such as i16 -> <16 x i1>.
1011 // Other casts only handle vector types with integer elements.
1012 if (CastOpcode != Instruction::BitCast &&
1013 (!isa<FixedVectorType>(Val: SrcTy) || !isa<FixedVectorType>(Val: DstTy)))
1014 return false;
1015
1016 // Only integer scalar/vector values are legal for bitwise logic operations.
1017 if (!SrcTy->getScalarType()->isIntegerTy() ||
1018 !DstTy->getScalarType()->isIntegerTy())
1019 return false;
1020
1021 // Find the constant InvC, such that castop(InvC) equals to C.
1022 PreservedCastFlags RHSFlags;
1023 Constant *InvC = getLosslessInvCast(C, InvCastTo: SrcTy, CastOp: CastOpcode, DL: *DL, Flags: &RHSFlags);
1024 if (!InvC)
1025 return false;
1026
1027 // Cost Check :
1028 // OldCost = bitlogic + cast
1029 // NewCost = bitlogic + cast
1030
1031 // Calculate specific costs for each cast with instruction context
1032 InstructionCost LHSCastCost = TTI.getCastInstrCost(
1033 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind, I: LHSCast);
1034
1035 InstructionCost OldCost =
1036 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: DstTy, CostKind) + LHSCastCost;
1037
1038 // For new cost, we can't provide an instruction (it doesn't exist yet)
1039 InstructionCost GenericCastCost = TTI.getCastInstrCost(
1040 Opcode: CastOpcode, Dst: DstTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
1041
1042 InstructionCost NewCost =
1043 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: SrcTy, CostKind) +
1044 GenericCastCost;
1045
1046 // Account for multi-use casts using specific costs
1047 if (!LHSCast->hasOneUse())
1048 NewCost += LHSCastCost;
1049
1050 LLVM_DEBUG(dbgs() << "foldBitOpOfCastConstant: OldCost=" << OldCost
1051 << " NewCost=" << NewCost << "\n");
1052
1053 if (NewCost > OldCost)
1054 return false;
1055
1056 // Create the operation on the source type
1057 Value *NewOp = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(),
1058 LHS: LHSSrc, RHS: InvC, Name: I.getName() + ".inner");
1059 if (auto *NewBinOp = dyn_cast<BinaryOperator>(Val: NewOp))
1060 NewBinOp->copyIRFlags(V: &I);
1061
1062 Worklist.pushValue(V: NewOp);
1063
1064 // Create the cast operation directly to ensure we get a new instruction
1065 Instruction *NewCast = CastInst::Create(CastOpcode, S: NewOp, Ty: I.getType());
1066
1067 // Preserve cast instruction flags
1068 if (RHSFlags.NNeg)
1069 NewCast->setNonNeg();
1070 if (RHSFlags.NUW)
1071 NewCast->setHasNoUnsignedWrap();
1072 if (RHSFlags.NSW)
1073 NewCast->setHasNoSignedWrap();
1074
1075 NewCast->andIRFlags(V: LHSCast);
1076
1077 // Insert the new instruction
1078 Value *Result = Builder.Insert(I: NewCast);
1079
1080 replaceValue(Old&: I, New&: *Result);
1081 return true;
1082}
1083
1084/// If this is a bitcast of a shuffle, try to bitcast the source vector to the
1085/// destination type followed by shuffle. This can enable further transforms by
1086/// moving bitcasts or shuffles together.
1087bool VectorCombine::foldBitcastShuffle(Instruction &I) {
1088 Value *V0, *V1;
1089 ArrayRef<int> Mask;
1090 if (!match(V: &I, P: m_BitCast(Op: m_OneUse(
1091 SubPattern: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(Mask))))))
1092 return false;
1093
1094 // 1) Do not fold bitcast shuffle for scalable type. First, shuffle cost for
1095 // scalable type is unknown; Second, we cannot reason if the narrowed shuffle
1096 // mask for scalable type is a splat or not.
1097 // 2) Disallow non-vector casts.
1098 // TODO: We could allow any shuffle.
1099 auto *DestTy = dyn_cast<FixedVectorType>(Val: I.getType());
1100 auto *SrcTy = dyn_cast<FixedVectorType>(Val: V0->getType());
1101 if (!DestTy || !SrcTy)
1102 return false;
1103
1104 unsigned DestEltSize = DestTy->getScalarSizeInBits();
1105 unsigned SrcEltSize = SrcTy->getScalarSizeInBits();
1106 if (SrcTy->getPrimitiveSizeInBits() % DestEltSize != 0)
1107 return false;
1108
1109 bool IsUnary = isa<UndefValue>(Val: V1);
1110
1111 // For binary shuffles, only fold bitcast(shuffle(X,Y))
1112 // if it won't increase the number of bitcasts.
1113 if (!IsUnary) {
1114 auto *BCTy0 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V0)->getType());
1115 auto *BCTy1 = dyn_cast<FixedVectorType>(Val: peekThroughBitcasts(V: V1)->getType());
1116 if (!(BCTy0 && BCTy0->getElementType() == DestTy->getElementType()) &&
1117 !(BCTy1 && BCTy1->getElementType() == DestTy->getElementType()))
1118 return false;
1119 }
1120
1121 SmallVector<int, 16> NewMask;
1122 if (DestEltSize <= SrcEltSize) {
1123 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
1124 // always be expanded to the equivalent form choosing narrower elements.
1125 if (SrcEltSize % DestEltSize != 0)
1126 return false;
1127 unsigned ScaleFactor = SrcEltSize / DestEltSize;
1128 narrowShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask);
1129 } else {
1130 // The bitcast is from narrow elements to wide elements. The shuffle mask
1131 // must choose consecutive elements to allow casting first.
1132 if (DestEltSize % SrcEltSize != 0)
1133 return false;
1134 unsigned ScaleFactor = DestEltSize / SrcEltSize;
1135 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask, ScaledMask&: NewMask))
1136 return false;
1137 }
1138
1139 // Bitcast the shuffle src - keep its original width but using the destination
1140 // scalar type.
1141 unsigned NumSrcElts = SrcTy->getPrimitiveSizeInBits() / DestEltSize;
1142 auto *NewShuffleTy =
1143 FixedVectorType::get(ElementType: DestTy->getScalarType(), NumElts: NumSrcElts);
1144 auto *OldShuffleTy =
1145 FixedVectorType::get(ElementType: SrcTy->getScalarType(), NumElts: Mask.size());
1146 unsigned NumOps = IsUnary ? 1 : 2;
1147
1148 // The new shuffle must not cost more than the old shuffle.
1149 TargetTransformInfo::ShuffleKind SK =
1150 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
1151 : TargetTransformInfo::SK_PermuteTwoSrc;
1152
1153 InstructionCost NewCost =
1154 TTI.getShuffleCost(Kind: SK, DstTy: DestTy, SrcTy: NewShuffleTy, Mask: NewMask, CostKind) +
1155 (NumOps * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewShuffleTy, Src: SrcTy,
1156 CCH: TargetTransformInfo::CastContextHint::None,
1157 CostKind));
1158 InstructionCost OldCost =
1159 TTI.getShuffleCost(Kind: SK, DstTy: OldShuffleTy, SrcTy, Mask, CostKind) +
1160 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: DestTy, Src: OldShuffleTy,
1161 CCH: TargetTransformInfo::CastContextHint::None,
1162 CostKind);
1163
1164 LLVM_DEBUG(dbgs() << "Found a bitcasted shuffle: " << I << "\n OldCost: "
1165 << OldCost << " vs NewCost: " << NewCost << "\n");
1166
1167 if (NewCost > OldCost || !NewCost.isValid())
1168 return false;
1169
1170 // bitcast (shuf V0, V1, MaskC) --> shuf (bitcast V0), (bitcast V1), MaskC'
1171 ++NumShufOfBitcast;
1172 Value *CastV0 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V0), DestTy: NewShuffleTy);
1173 Value *CastV1 = Builder.CreateBitCast(V: peekThroughBitcasts(V: V1), DestTy: NewShuffleTy);
1174 Value *Shuf = Builder.CreateShuffleVector(V1: CastV0, V2: CastV1, Mask: NewMask);
1175 replaceValue(Old&: I, New&: *Shuf);
1176 return true;
1177}
1178
1179/// VP Intrinsics whose vector operands are both splat values may be simplified
1180/// into the scalar version of the operation and the result splatted. This
1181/// can lead to scalarization down the line.
1182bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) {
1183 if (!isa<VPIntrinsic>(Val: I))
1184 return false;
1185 VPIntrinsic &VPI = cast<VPIntrinsic>(Val&: I);
1186 Value *Op0 = VPI.getArgOperand(i: 0);
1187 Value *Op1 = VPI.getArgOperand(i: 1);
1188
1189 if (!isSplatValue(V: Op0) || !isSplatValue(V: Op1))
1190 return false;
1191
1192 // Check getSplatValue early in this function, to avoid doing unnecessary
1193 // work.
1194 Value *ScalarOp0 = getSplatValue(V: Op0);
1195 Value *ScalarOp1 = getSplatValue(V: Op1);
1196 if (!ScalarOp0 || !ScalarOp1)
1197 return false;
1198
1199 // For the binary VP intrinsics supported here, the result on disabled lanes
1200 // is a poison value. For now, only do this simplification if all lanes
1201 // are active.
1202 // TODO: Relax the condition that all lanes are active by using insertelement
1203 // on inactive lanes.
1204 auto IsAllTrueMask = [](Value *MaskVal) {
1205 if (Value *SplattedVal = getSplatValue(V: MaskVal))
1206 if (auto *ConstValue = dyn_cast<Constant>(Val: SplattedVal))
1207 return ConstValue->isAllOnesValue();
1208 return false;
1209 };
1210 if (!IsAllTrueMask(VPI.getArgOperand(i: 2)))
1211 return false;
1212
1213 // Check to make sure we support scalarization of the intrinsic
1214 Intrinsic::ID IntrID = VPI.getIntrinsicID();
1215 if (!VPBinOpIntrinsic::isVPBinOp(ID: IntrID))
1216 return false;
1217
1218 // Calculate cost of splatting both operands into vectors and the vector
1219 // intrinsic
1220 VectorType *VecTy = cast<VectorType>(Val: VPI.getType());
1221 SmallVector<int> Mask;
1222 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: VecTy))
1223 Mask.resize(N: FVTy->getNumElements(), NV: 0);
1224 InstructionCost SplatCost =
1225 TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy, CostKind, Index: 0) +
1226 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Broadcast, DstTy: VecTy, SrcTy: VecTy, Mask,
1227 CostKind);
1228
1229 // Calculate the cost of the VP Intrinsic
1230 SmallVector<Type *, 4> Args;
1231 for (Value *V : VPI.args())
1232 Args.push_back(Elt: V->getType());
1233 IntrinsicCostAttributes Attrs(IntrID, VecTy, Args);
1234 InstructionCost VectorOpCost = TTI.getIntrinsicInstrCost(ICA: Attrs, CostKind);
1235 InstructionCost OldCost = 2 * SplatCost + VectorOpCost;
1236
1237 // Determine scalar opcode
1238 std::optional<unsigned> FunctionalOpcode =
1239 VPI.getFunctionalOpcode();
1240 std::optional<Intrinsic::ID> ScalarIntrID = std::nullopt;
1241 if (!FunctionalOpcode) {
1242 ScalarIntrID = VPI.getFunctionalIntrinsicID();
1243 if (!ScalarIntrID)
1244 return false;
1245 }
1246
1247 // Calculate cost of scalarizing
1248 InstructionCost ScalarOpCost = 0;
1249 if (ScalarIntrID) {
1250 IntrinsicCostAttributes Attrs(*ScalarIntrID, VecTy->getScalarType(), Args);
1251 ScalarOpCost = TTI.getIntrinsicInstrCost(ICA: Attrs, CostKind);
1252 } else {
1253 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode: *FunctionalOpcode,
1254 Ty: VecTy->getScalarType(), CostKind);
1255 }
1256
1257 // The existing splats may be kept around if other instructions use them.
1258 InstructionCost CostToKeepSplats =
1259 (SplatCost * !Op0->hasOneUse()) + (SplatCost * !Op1->hasOneUse());
1260 InstructionCost NewCost = ScalarOpCost + SplatCost + CostToKeepSplats;
1261
1262 LLVM_DEBUG(dbgs() << "Found a VP Intrinsic to scalarize: " << VPI
1263 << "\n");
1264 LLVM_DEBUG(dbgs() << "Cost of Intrinsic: " << OldCost
1265 << ", Cost of scalarizing:" << NewCost << "\n");
1266
1267 // We want to scalarize unless the vector variant actually has lower cost.
1268 if (OldCost < NewCost || !NewCost.isValid())
1269 return false;
1270
1271 // Scalarize the intrinsic
1272 ElementCount EC = cast<VectorType>(Val: Op0->getType())->getElementCount();
1273 Value *EVL = VPI.getArgOperand(i: 3);
1274
1275 // If the VP op might introduce UB or poison, we can scalarize it provided
1276 // that we know the EVL > 0: If the EVL is zero, then the original VP op
1277 // becomes a no-op and thus won't be UB, so make sure we don't introduce UB by
1278 // scalarizing it.
1279 bool SafeToSpeculate;
1280 if (ScalarIntrID)
1281 SafeToSpeculate = Intrinsic::getFnAttributes(C&: I.getContext(), id: *ScalarIntrID)
1282 .hasAttribute(Kind: Attribute::AttrKind::Speculatable);
1283 else
1284 SafeToSpeculate = isSafeToSpeculativelyExecuteWithOpcode(
1285 Opcode: *FunctionalOpcode, Inst: &VPI, CtxI: nullptr, AC: SQ.AC, DT: SQ.DT);
1286 if (!SafeToSpeculate &&
1287 !isKnownNonZero(V: EVL, Q: SimplifyQuery(*DL, SQ.DT, SQ.AC, &VPI)))
1288 return false;
1289
1290 Value *ScalarVal =
1291 ScalarIntrID
1292 ? Builder.CreateIntrinsic(RetTy: VecTy->getScalarType(), ID: *ScalarIntrID,
1293 Args: {ScalarOp0, ScalarOp1})
1294 : Builder.CreateBinOp(Opc: (Instruction::BinaryOps)(*FunctionalOpcode),
1295 LHS: ScalarOp0, RHS: ScalarOp1);
1296
1297 replaceValue(Old&: VPI, New&: *Builder.CreateVectorSplat(EC, V: ScalarVal));
1298 return true;
1299}
1300
1301/// Match a vector op/compare/intrinsic with at least one
1302/// inserted scalar operand and convert to scalar op/cmp/intrinsic followed
1303/// by insertelement.
1304bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
1305 auto *UO = dyn_cast<UnaryOperator>(Val: &I);
1306 auto *BO = dyn_cast<BinaryOperator>(Val: &I);
1307 auto *CI = dyn_cast<CmpInst>(Val: &I);
1308 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
1309 if (!UO && !BO && !CI && !II)
1310 return false;
1311
1312 // TODO: Allow intrinsics with different argument types
1313 if (II) {
1314 if (!isTriviallyVectorizable(ID: II->getIntrinsicID()))
1315 return false;
1316 for (auto [Idx, Arg] : enumerate(First: II->args()))
1317 if (Arg->getType() != II->getType() &&
1318 !isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI))
1319 return false;
1320 }
1321
1322 // Do not convert the vector condition of a vector select into a scalar
1323 // condition. That may cause problems for codegen because of differences in
1324 // boolean formats and register-file transfers.
1325 // TODO: Can we account for that in the cost model?
1326 if (CI)
1327 for (User *U : I.users())
1328 if (match(V: U, P: m_Select(C: m_Specific(V: &I), L: m_Value(), R: m_Value())))
1329 return false;
1330
1331 // Match constant vectors or scalars being inserted into constant vectors:
1332 // vec_op [VecC0 | (inselt VecC0, V0, Index)], ...
1333 SmallVector<Value *> VecCs, ScalarOps;
1334 std::optional<uint64_t> Index;
1335
1336 auto Ops = II ? II->args() : I.operands();
1337 for (auto [OpNum, Op] : enumerate(First&: Ops)) {
1338 Constant *VecC;
1339 Value *V;
1340 uint64_t InsIdx = 0;
1341 if (match(V: Op.get(), P: m_InsertElt(Val: m_Constant(C&: VecC), Elt: m_Value(V),
1342 Idx: m_ConstantInt(V&: InsIdx)))) {
1343 // Bail if any inserts are out of bounds.
1344 VectorType *OpTy = cast<VectorType>(Val: Op->getType());
1345 if (OpTy->getElementCount().getKnownMinValue() <= InsIdx)
1346 return false;
1347 // All inserts must have the same index.
1348 // TODO: Deal with mismatched index constants and variable indexes?
1349 if (!Index)
1350 Index = InsIdx;
1351 else if (InsIdx != *Index)
1352 return false;
1353 VecCs.push_back(Elt: VecC);
1354 ScalarOps.push_back(Elt: V);
1355 } else if (II && isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(),
1356 ScalarOpdIdx: OpNum, TTI: &TTI)) {
1357 VecCs.push_back(Elt: Op.get());
1358 ScalarOps.push_back(Elt: Op.get());
1359 } else if (match(V: Op.get(), P: m_Constant(C&: VecC))) {
1360 VecCs.push_back(Elt: VecC);
1361 ScalarOps.push_back(Elt: nullptr);
1362 } else {
1363 return false;
1364 }
1365 }
1366
1367 // Bail if all operands are constant.
1368 if (!Index.has_value())
1369 return false;
1370
1371 VectorType *VecTy = cast<VectorType>(Val: I.getType());
1372 Type *ScalarTy = VecTy->getScalarType();
1373 assert(VecTy->isVectorTy() &&
1374 (ScalarTy->isIntegerTy() || ScalarTy->isFloatingPointTy() ||
1375 ScalarTy->isPointerTy()) &&
1376 "Unexpected types for insert element into binop or cmp");
1377
1378 unsigned Opcode = I.getOpcode();
1379 InstructionCost ScalarOpCost, VectorOpCost;
1380 if (CI) {
1381 CmpInst::Predicate Pred = CI->getPredicate();
1382 ScalarOpCost = TTI.getCmpSelInstrCost(
1383 Opcode, ValTy: ScalarTy, CondTy: CmpInst::makeCmpResultType(opnd_type: ScalarTy), VecPred: Pred, CostKind);
1384 VectorOpCost = TTI.getCmpSelInstrCost(
1385 Opcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1386 } else if (UO || BO) {
1387 ScalarOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: ScalarTy, CostKind);
1388 VectorOpCost = TTI.getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1389 } else {
1390 IntrinsicCostAttributes ScalarICA(
1391 II->getIntrinsicID(), ScalarTy,
1392 SmallVector<Type *>(II->arg_size(), ScalarTy));
1393 ScalarOpCost = TTI.getIntrinsicInstrCost(ICA: ScalarICA, CostKind);
1394 IntrinsicCostAttributes VectorICA(
1395 II->getIntrinsicID(), VecTy,
1396 SmallVector<Type *>(II->arg_size(), VecTy));
1397 VectorOpCost = TTI.getIntrinsicInstrCost(ICA: VectorICA, CostKind);
1398 }
1399
1400 // Fold the vector constants in the original vectors into a new base vector to
1401 // get more accurate cost modelling.
1402 Value *NewVecC = nullptr;
1403 if (CI)
1404 NewVecC = simplifyCmpInst(Predicate: CI->getPredicate(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1405 else if (UO)
1406 NewVecC =
1407 simplifyUnOp(Opcode: UO->getOpcode(), Op: VecCs[0], FMF: UO->getFastMathFlags(), Q: SQ);
1408 else if (BO)
1409 NewVecC = simplifyBinOp(Opcode: BO->getOpcode(), LHS: VecCs[0], RHS: VecCs[1], Q: SQ);
1410 else if (II)
1411 NewVecC = simplifyCall(Call: II, Callee: II->getCalledOperand(), Args: VecCs, Q: SQ);
1412
1413 if (!NewVecC)
1414 return false;
1415
1416 // Get cost estimate for the insert element. This cost will factor into
1417 // both sequences.
1418 InstructionCost OldCost = VectorOpCost;
1419 InstructionCost NewCost =
1420 ScalarOpCost + TTI.getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy,
1421 CostKind, Index: *Index, Op0: NewVecC);
1422
1423 for (auto [Idx, Op, VecC, Scalar] : enumerate(First&: Ops, Rest&: VecCs, Rest&: ScalarOps)) {
1424 if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg(
1425 ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI: &TTI)))
1426 continue;
1427 InstructionCost InsertCost = TTI.getVectorInstrCost(
1428 Opcode: Instruction::InsertElement, Val: VecTy, CostKind, Index: *Index, Op0: VecC, Op1: Scalar);
1429 OldCost += InsertCost;
1430 NewCost += !Op->hasOneUse() * InsertCost;
1431 }
1432
1433 // We want to scalarize unless the vector variant actually has lower cost.
1434 if (OldCost < NewCost || !NewCost.isValid())
1435 return false;
1436
1437 // vec_op (inselt VecC0, V0, Index), (inselt VecC1, V1, Index) -->
1438 // inselt NewVecC, (scalar_op V0, V1), Index
1439 if (CI)
1440 ++NumScalarCmp;
1441 else if (UO || BO)
1442 ++NumScalarOps;
1443 else
1444 ++NumScalarIntrinsic;
1445
1446 // For constant cases, extract the scalar element, this should constant fold.
1447 for (auto [OpIdx, Scalar, VecC] : enumerate(First&: ScalarOps, Rest&: VecCs))
1448 if (!Scalar)
1449 ScalarOps[OpIdx] = ConstantExpr::getExtractElement(
1450 Vec: cast<Constant>(Val: VecC), Idx: Builder.getInt64(C: *Index));
1451
1452 Value *Scalar;
1453 if (CI)
1454 Scalar = Builder.CreateCmp(Pred: CI->getPredicate(), LHS: ScalarOps[0], RHS: ScalarOps[1]);
1455 else if (UO || BO)
1456 Scalar = Builder.CreateNAryOp(Opc: Opcode, Ops: ScalarOps);
1457 else
1458 Scalar = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: II->getIntrinsicID(), Args: ScalarOps);
1459
1460 Scalar->setName(I.getName() + ".scalar");
1461
1462 // All IR flags are safe to back-propagate. There is no potential for extra
1463 // poison to be created by the scalar instruction.
1464 if (auto *ScalarInst = dyn_cast<Instruction>(Val: Scalar))
1465 ScalarInst->copyIRFlags(V: &I);
1466
1467 Value *Insert = Builder.CreateInsertElement(Vec: NewVecC, NewElt: Scalar, Idx: *Index);
1468 replaceValue(Old&: I, New&: *Insert);
1469 return true;
1470}
1471
1472/// Try to combine a scalar binop + 2 scalar compares of extracted elements of
1473/// a vector into vector operations followed by extract. Note: The SLP pass
1474/// may miss this pattern because of implementation problems.
1475bool VectorCombine::foldExtractedCmps(Instruction &I) {
1476 auto *BI = dyn_cast<BinaryOperator>(Val: &I);
1477
1478 // We are looking for a scalar binop of booleans.
1479 // binop i1 (cmp Pred I0, C0), (cmp Pred I1, C1)
1480 if (!BI || !I.getType()->isIntegerTy(BitWidth: 1))
1481 return false;
1482
1483 // The compare predicates should match, and each compare should have a
1484 // constant operand.
1485 Value *B0 = I.getOperand(i: 0), *B1 = I.getOperand(i: 1);
1486 Instruction *I0, *I1;
1487 Constant *C0, *C1;
1488 CmpPredicate P0, P1;
1489 if (!match(V: B0, P: m_Cmp(Pred&: P0, L: m_Instruction(I&: I0), R: m_Constant(C&: C0))) ||
1490 !match(V: B1, P: m_Cmp(Pred&: P1, L: m_Instruction(I&: I1), R: m_Constant(C&: C1))))
1491 return false;
1492
1493 auto MatchingPred = CmpPredicate::getMatching(A: P0, B: P1);
1494 if (!MatchingPred)
1495 return false;
1496
1497 // The compare operands must be extracts of the same vector with constant
1498 // extract indexes.
1499 Value *X;
1500 uint64_t Index0, Index1;
1501 if (!match(V: I0, P: m_ExtractElt(Val: m_Value(V&: X), Idx: m_ConstantInt(V&: Index0))) ||
1502 !match(V: I1, P: m_ExtractElt(Val: m_Specific(V: X), Idx: m_ConstantInt(V&: Index1))))
1503 return false;
1504
1505 auto *Ext0 = cast<ExtractElementInst>(Val: I0);
1506 auto *Ext1 = cast<ExtractElementInst>(Val: I1);
1507 ExtractElementInst *ConvertToShuf = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex: CostKind);
1508 if (!ConvertToShuf)
1509 return false;
1510 assert((ConvertToShuf == Ext0 || ConvertToShuf == Ext1) &&
1511 "Unknown ExtractElementInst");
1512
1513 // The original scalar pattern is:
1514 // binop i1 (cmp Pred (ext X, Index0), C0), (cmp Pred (ext X, Index1), C1)
1515 CmpInst::Predicate Pred = *MatchingPred;
1516 unsigned CmpOpcode =
1517 CmpInst::isFPPredicate(P: Pred) ? Instruction::FCmp : Instruction::ICmp;
1518 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
1519 if (!VecTy)
1520 return false;
1521
1522 if (Index0 >= VecTy->getNumElements() || Index1 >= VecTy->getNumElements())
1523 return false;
1524
1525 InstructionCost Ext0Cost =
1526 TTI.getVectorInstrCost(I: *Ext0, Val: VecTy, CostKind, Index: Index0);
1527 InstructionCost Ext1Cost =
1528 TTI.getVectorInstrCost(I: *Ext1, Val: VecTy, CostKind, Index: Index1);
1529 InstructionCost CmpCost = TTI.getCmpSelInstrCost(
1530 Opcode: CmpOpcode, ValTy: I0->getType(), CondTy: CmpInst::makeCmpResultType(opnd_type: I0->getType()), VecPred: Pred,
1531 CostKind);
1532
1533 InstructionCost OldCost =
1534 Ext0Cost + Ext1Cost + CmpCost * 2 +
1535 TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: I.getType(), CostKind);
1536
1537 // The proposed vector pattern is:
1538 // vcmp = cmp Pred X, VecC
1539 // ext (binop vNi1 vcmp, (shuffle vcmp, Index1)), Index0
1540 int CheapIndex = ConvertToShuf == Ext0 ? Index1 : Index0;
1541 int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
1542 auto *CmpTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
1543 InstructionCost NewCost = TTI.getCmpSelInstrCost(
1544 Opcode: CmpOpcode, ValTy: VecTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VecTy), VecPred: Pred, CostKind);
1545 SmallVector<int, 32> ShufMask(VecTy->getNumElements(), PoisonMaskElem);
1546 ShufMask[CheapIndex] = ExpensiveIndex;
1547 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: CmpTy,
1548 SrcTy: CmpTy, Mask: ShufMask, CostKind);
1549 NewCost += TTI.getArithmeticInstrCost(Opcode: I.getOpcode(), Ty: CmpTy, CostKind);
1550 NewCost += TTI.getVectorInstrCost(I: *Ext0, Val: CmpTy, CostKind, Index: CheapIndex);
1551 NewCost += Ext0->hasOneUse() ? 0 : Ext0Cost;
1552 NewCost += Ext1->hasOneUse() ? 0 : Ext1Cost;
1553
1554 // Aggressively form vector ops if the cost is equal because the transform
1555 // may enable further optimization.
1556 // Codegen can reverse this transform (scalarize) if it was not profitable.
1557 if (OldCost < NewCost || !NewCost.isValid())
1558 return false;
1559
1560 // Create a vector constant from the 2 scalar constants.
1561 SmallVector<Constant *, 32> CmpC(VecTy->getNumElements(),
1562 PoisonValue::get(T: VecTy->getElementType()));
1563 CmpC[Index0] = C0;
1564 CmpC[Index1] = C1;
1565 Value *VCmp = Builder.CreateCmp(Pred, LHS: X, RHS: ConstantVector::get(V: CmpC));
1566 Value *Shuf = createShiftShuffle(Vec: VCmp, OldIndex: ExpensiveIndex, NewIndex: CheapIndex, Builder);
1567 Value *LHS = ConvertToShuf == Ext0 ? Shuf : VCmp;
1568 Value *RHS = ConvertToShuf == Ext0 ? VCmp : Shuf;
1569 Value *VecLogic = Builder.CreateBinOp(Opc: BI->getOpcode(), LHS, RHS);
1570 Value *NewExt = Builder.CreateExtractElement(Vec: VecLogic, Idx: CheapIndex);
1571 replaceValue(Old&: I, New&: *NewExt);
1572 ++NumVecCmpBO;
1573 return true;
1574}
1575
1576/// Try to fold scalar selects that select between extracted elements and zero
1577/// into extracting from a vector select. This is rooted at the bitcast.
1578///
1579/// This pattern arises when a vector is bitcast to a smaller element type,
1580/// elements are extracted, and then conditionally selected with zero:
1581///
1582/// %bc = bitcast <4 x i32> %src to <16 x i8>
1583/// %e0 = extractelement <16 x i8> %bc, i32 0
1584/// %s0 = select i1 %cond, i8 %e0, i8 0
1585/// %e1 = extractelement <16 x i8> %bc, i32 1
1586/// %s1 = select i1 %cond, i8 %e1, i8 0
1587/// ...
1588///
1589/// Transforms to:
1590/// %sel = select i1 %cond, <4 x i32> %src, <4 x i32> zeroinitializer
1591/// %bc = bitcast <4 x i32> %sel to <16 x i8>
1592/// %e0 = extractelement <16 x i8> %bc, i32 0
1593/// %e1 = extractelement <16 x i8> %bc, i32 1
1594/// ...
1595///
1596/// This is profitable because vector select on wider types produces fewer
1597/// select/cndmask instructions than scalar selects on each element.
1598bool VectorCombine::foldSelectsFromBitcast(Instruction &I) {
1599 auto *BC = dyn_cast<BitCastInst>(Val: &I);
1600 if (!BC)
1601 return false;
1602
1603 FixedVectorType *SrcVecTy = dyn_cast<FixedVectorType>(Val: BC->getSrcTy());
1604 FixedVectorType *DstVecTy = dyn_cast<FixedVectorType>(Val: BC->getDestTy());
1605 if (!SrcVecTy || !DstVecTy)
1606 return false;
1607
1608 // Source must be 32-bit or 64-bit elements, destination must be smaller
1609 // integer elements. Zero in all these types is all-bits-zero.
1610 Type *SrcEltTy = SrcVecTy->getElementType();
1611 Type *DstEltTy = DstVecTy->getElementType();
1612 unsigned SrcEltBits = SrcEltTy->getPrimitiveSizeInBits();
1613 unsigned DstEltBits = DstEltTy->getPrimitiveSizeInBits();
1614
1615 if (SrcEltBits != 32 && SrcEltBits != 64)
1616 return false;
1617
1618 if (!DstEltTy->isIntegerTy() || DstEltBits >= SrcEltBits)
1619 return false;
1620
1621 // Check profitability using TTI before collecting users.
1622 Type *CondTy = CmpInst::makeCmpResultType(opnd_type: DstEltTy);
1623 Type *VecCondTy = CmpInst::makeCmpResultType(opnd_type: SrcVecTy);
1624
1625 InstructionCost ScalarSelCost =
1626 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: DstEltTy, CondTy,
1627 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1628 InstructionCost VecSelCost =
1629 TTI.getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: SrcVecTy, CondTy: VecCondTy,
1630 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1631
1632 // We need at least this many selects for vectorization to be profitable.
1633 // VecSelCost < ScalarSelCost * NumSelects => NumSelects > VecSelCost /
1634 // ScalarSelCost
1635 if (!ScalarSelCost.isValid() || ScalarSelCost == 0)
1636 return false;
1637
1638 unsigned MinSelects = (VecSelCost.getValue() / ScalarSelCost.getValue()) + 1;
1639
1640 // Quick check: if bitcast doesn't have enough users, bail early.
1641 if (!BC->hasNUsesOrMore(N: MinSelects))
1642 return false;
1643
1644 // Collect all select users that match the pattern, grouped by condition.
1645 // Pattern: select i1 %cond, (extractelement %bc, idx), 0
1646 DenseMap<Value *, SmallVector<SelectInst *, 8>> CondToSelects;
1647
1648 for (User *U : BC->users()) {
1649 auto *Ext = dyn_cast<ExtractElementInst>(Val: U);
1650 if (!Ext)
1651 continue;
1652
1653 for (User *ExtUser : Ext->users()) {
1654 Value *Cond;
1655 // Match: select i1 %cond, %ext, 0
1656 if (match(V: ExtUser, P: m_Select(C: m_Value(V&: Cond), L: m_Specific(V: Ext), R: m_Zero())) &&
1657 Cond->getType()->isIntegerTy(BitWidth: 1))
1658 CondToSelects[Cond].push_back(Elt: cast<SelectInst>(Val: ExtUser));
1659 }
1660 }
1661
1662 if (CondToSelects.empty())
1663 return false;
1664
1665 bool MadeChange = false;
1666 Value *SrcVec = BC->getOperand(i_nocapture: 0);
1667
1668 // Process each group of selects with the same condition.
1669 for (auto [Cond, Selects] : CondToSelects) {
1670 // Only profitable if vector select cost < total scalar select cost.
1671 if (Selects.size() < MinSelects) {
1672 LLVM_DEBUG(dbgs() << "VectorCombine: foldSelectsFromBitcast not "
1673 << "profitable (VecCost=" << VecSelCost
1674 << ", ScalarCost=" << ScalarSelCost
1675 << ", NumSelects=" << Selects.size() << ")\n");
1676 continue;
1677 }
1678
1679 // Create the vector select and bitcast once for this condition.
1680 auto InsertPt = std::next(x: BC->getIterator());
1681
1682 if (auto *CondInst = dyn_cast<Instruction>(Val: Cond))
1683 if (DT.dominates(Def: BC, User: CondInst))
1684 InsertPt = std::next(x: CondInst->getIterator());
1685
1686 Builder.SetInsertPoint(InsertPt);
1687 Value *VecSel =
1688 Builder.CreateSelect(C: Cond, True: SrcVec, False: Constant::getNullValue(Ty: SrcVecTy));
1689 Value *NewBC = Builder.CreateBitCast(V: VecSel, DestTy: DstVecTy);
1690
1691 // Replace each scalar select with an extract from the new bitcast.
1692 for (SelectInst *Sel : Selects) {
1693 auto *Ext = cast<ExtractElementInst>(Val: Sel->getTrueValue());
1694 Value *Idx = Ext->getIndexOperand();
1695
1696 Builder.SetInsertPoint(Sel);
1697 Value *NewExt = Builder.CreateExtractElement(Vec: NewBC, Idx);
1698 replaceValue(Old&: *Sel, New&: *NewExt);
1699 MadeChange = true;
1700 }
1701
1702 LLVM_DEBUG(dbgs() << "VectorCombine: folded " << Selects.size()
1703 << " selects into vector select\n");
1704 }
1705
1706 return MadeChange;
1707}
1708
1709static void analyzeCostOfVecReduction(const IntrinsicInst &II,
1710 TTI::TargetCostKind CostKind,
1711 const TargetTransformInfo &TTI,
1712 InstructionCost &CostBeforeReduction,
1713 InstructionCost &CostAfterReduction) {
1714 Instruction *Op0, *Op1;
1715 auto *RedOp = dyn_cast<Instruction>(Val: II.getOperand(i_nocapture: 0));
1716 auto *VecRedTy = cast<VectorType>(Val: II.getOperand(i_nocapture: 0)->getType());
1717 unsigned ReductionOpc =
1718 getArithmeticReductionInstruction(RdxID: II.getIntrinsicID());
1719 if (RedOp && match(V: RedOp, P: m_ZExtOrSExt(Op: m_Value()))) {
1720 bool IsUnsigned = isa<ZExtInst>(Val: RedOp);
1721 auto *ExtType = cast<VectorType>(Val: RedOp->getOperand(i: 0)->getType());
1722
1723 CostBeforeReduction =
1724 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: ExtType,
1725 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1726 CostAfterReduction =
1727 TTI.getExtendedReductionCost(Opcode: ReductionOpc, IsUnsigned, ResTy: II.getType(),
1728 Ty: ExtType, FMF: FastMathFlags(), CostKind);
1729 return;
1730 }
1731 if (RedOp && II.getIntrinsicID() == Intrinsic::vector_reduce_add &&
1732 match(V: RedOp,
1733 P: m_ZExtOrSExt(Op: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) &&
1734 match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
1735 Op0->getOpcode() == Op1->getOpcode() &&
1736 Op0->getOperand(i: 0)->getType() == Op1->getOperand(i: 0)->getType() &&
1737 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
1738 // Matched reduce.add(ext(mul(ext(A), ext(B)))
1739 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
1740 auto *ExtType = cast<VectorType>(Val: Op0->getOperand(i: 0)->getType());
1741 VectorType *MulType = VectorType::get(ElementType: Op0->getType(), Other: VecRedTy);
1742
1743 InstructionCost ExtCost =
1744 TTI.getCastInstrCost(Opcode: Op0->getOpcode(), Dst: MulType, Src: ExtType,
1745 CCH: TTI::CastContextHint::None, CostKind, I: Op0);
1746 InstructionCost MulCost =
1747 TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: MulType, CostKind);
1748 InstructionCost Ext2Cost =
1749 TTI.getCastInstrCost(Opcode: RedOp->getOpcode(), Dst: VecRedTy, Src: MulType,
1750 CCH: TTI::CastContextHint::None, CostKind, I: RedOp);
1751
1752 CostBeforeReduction = ExtCost * 2 + MulCost + Ext2Cost;
1753 CostAfterReduction = TTI.getMulAccReductionCost(
1754 IsUnsigned, RedOpcode: ReductionOpc, ResTy: II.getType(), Ty: ExtType, CostKind);
1755 return;
1756 }
1757 CostAfterReduction = TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecRedTy,
1758 FMF: std::nullopt, CostKind);
1759}
1760
1761bool VectorCombine::foldBinopOfReductions(Instruction &I) {
1762 Instruction::BinaryOps BinOpOpc = cast<BinaryOperator>(Val: &I)->getOpcode();
1763 Intrinsic::ID ReductionIID = getReductionForBinop(Opc: BinOpOpc);
1764 if (BinOpOpc == Instruction::Sub)
1765 ReductionIID = Intrinsic::vector_reduce_add;
1766 if (ReductionIID == Intrinsic::not_intrinsic)
1767 return false;
1768 // FP reductions have a start-value operand that this fold doesn't handle.
1769 if (ReductionIID == Intrinsic::vector_reduce_fadd ||
1770 ReductionIID == Intrinsic::vector_reduce_fmul)
1771 return false;
1772
1773 auto checkIntrinsicAndGetItsArgument = [](Value *V,
1774 Intrinsic::ID IID) -> Value * {
1775 auto *II = dyn_cast<IntrinsicInst>(Val: V);
1776 if (!II)
1777 return nullptr;
1778 if (II->getIntrinsicID() == IID && II->hasOneUse())
1779 return II->getArgOperand(i: 0);
1780 return nullptr;
1781 };
1782
1783 Value *V0 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 0), ReductionIID);
1784 if (!V0)
1785 return false;
1786 Value *V1 = checkIntrinsicAndGetItsArgument(I.getOperand(i: 1), ReductionIID);
1787 if (!V1)
1788 return false;
1789
1790 auto *VTy = cast<VectorType>(Val: V0->getType());
1791 if (V1->getType() != VTy)
1792 return false;
1793 const auto &II0 = *cast<IntrinsicInst>(Val: I.getOperand(i: 0));
1794 const auto &II1 = *cast<IntrinsicInst>(Val: I.getOperand(i: 1));
1795 unsigned ReductionOpc =
1796 getArithmeticReductionInstruction(RdxID: II0.getIntrinsicID());
1797
1798 InstructionCost OldCost = 0;
1799 InstructionCost NewCost = 0;
1800 InstructionCost CostOfRedOperand0 = 0;
1801 InstructionCost CostOfRed0 = 0;
1802 InstructionCost CostOfRedOperand1 = 0;
1803 InstructionCost CostOfRed1 = 0;
1804 analyzeCostOfVecReduction(II: II0, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand0, CostAfterReduction&: CostOfRed0);
1805 analyzeCostOfVecReduction(II: II1, CostKind, TTI, CostBeforeReduction&: CostOfRedOperand1, CostAfterReduction&: CostOfRed1);
1806 OldCost = CostOfRed0 + CostOfRed1 + TTI.getInstructionCost(U: &I, CostKind);
1807 NewCost =
1808 CostOfRedOperand0 + CostOfRedOperand1 +
1809 TTI.getArithmeticInstrCost(Opcode: BinOpOpc, Ty: VTy, CostKind) +
1810 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VTy, FMF: std::nullopt, CostKind);
1811 if (NewCost >= OldCost || !NewCost.isValid())
1812 return false;
1813
1814 LLVM_DEBUG(dbgs() << "Found two mergeable reductions: " << I
1815 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
1816 << "\n");
1817 Value *VectorBO;
1818 if (BinOpOpc == Instruction::Or)
1819 VectorBO = Builder.CreateOr(LHS: V0, RHS: V1, Name: "",
1820 IsDisjoint: cast<PossiblyDisjointInst>(Val&: I).isDisjoint());
1821 else
1822 VectorBO = Builder.CreateBinOp(Opc: BinOpOpc, LHS: V0, RHS: V1);
1823
1824 Value *Rdx = Builder.CreateIntrinsic(ID: ReductionIID, OverloadTypes: {VTy}, Args: {VectorBO});
1825 replaceValue(Old&: I, New&: *Rdx);
1826 return true;
1827}
1828
1829// Check if memory loc modified between two instrs in the same BB
1830static bool isMemModifiedBetween(BasicBlock::iterator Begin,
1831 BasicBlock::iterator End,
1832 const MemoryLocation &Loc, AAResults &AA) {
1833 unsigned NumScanned = 0;
1834 return std::any_of(first: Begin, last: End, pred: [&](const Instruction &Instr) {
1835 return isModSet(MRI: AA.getModRefInfo(I: &Instr, OptLoc: Loc)) ||
1836 ++NumScanned > MaxInstrsToScan;
1837 });
1838}
1839
1840namespace {
1841/// Helper class to indicate whether a vector index can be safely scalarized and
1842/// if a freeze needs to be inserted.
1843class ScalarizationResult {
1844 enum class StatusTy { Unsafe, Safe, SafeWithFreeze };
1845
1846 StatusTy Status;
1847 Value *ToFreeze;
1848
1849 ScalarizationResult(StatusTy Status, Value *ToFreeze = nullptr)
1850 : Status(Status), ToFreeze(ToFreeze) {}
1851
1852public:
1853 ScalarizationResult(const ScalarizationResult &Other) = default;
1854 ~ScalarizationResult() {
1855 assert(!ToFreeze && "freeze() not called with ToFreeze being set");
1856 }
1857
1858 static ScalarizationResult unsafe() { return {StatusTy::Unsafe}; }
1859 static ScalarizationResult safe() { return {StatusTy::Safe}; }
1860 static ScalarizationResult safeWithFreeze(Value *ToFreeze) {
1861 return {StatusTy::SafeWithFreeze, ToFreeze};
1862 }
1863
1864 /// Returns true if the index can be scalarize without requiring a freeze.
1865 bool isSafe() const { return Status == StatusTy::Safe; }
1866 /// Returns true if the index cannot be scalarized.
1867 bool isUnsafe() const { return Status == StatusTy::Unsafe; }
1868 /// Returns true if the index can be scalarize, but requires inserting a
1869 /// freeze.
1870 bool isSafeWithFreeze() const { return Status == StatusTy::SafeWithFreeze; }
1871
1872 /// Reset the state of Unsafe and clear ToFreze if set.
1873 void discard() {
1874 ToFreeze = nullptr;
1875 Status = StatusTy::Unsafe;
1876 }
1877
1878 /// Freeze the ToFreeze and update the use in \p User to use it.
1879 void freeze(IRBuilderBase &Builder, Instruction &UserI) {
1880 assert(isSafeWithFreeze() &&
1881 "should only be used when freezing is required");
1882 assert(is_contained(ToFreeze->users(), &UserI) &&
1883 "UserI must be a user of ToFreeze");
1884 IRBuilder<>::InsertPointGuard Guard(Builder);
1885 Builder.SetInsertPoint(cast<Instruction>(Val: &UserI));
1886 Value *Frozen =
1887 Builder.CreateFreeze(V: ToFreeze, Name: ToFreeze->getName() + ".frozen");
1888 for (Use &U : make_early_inc_range(Range: (UserI.operands())))
1889 if (U.get() == ToFreeze)
1890 U.set(Frozen);
1891
1892 ToFreeze = nullptr;
1893 }
1894};
1895} // namespace
1896
1897/// Check if it is legal to scalarize a memory access to \p VecTy at index \p
1898/// Idx. \p Idx must access a valid vector element.
1899static ScalarizationResult canScalarizeAccess(VectorType *VecTy, Value *Idx,
1900 const SimplifyQuery &SQ) {
1901 // We do checks for both fixed vector types and scalable vector types.
1902 // This is the number of elements of fixed vector types,
1903 // or the minimum number of elements of scalable vector types.
1904 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1905 unsigned IntWidth = Idx->getType()->getScalarSizeInBits();
1906
1907 if (auto *C = dyn_cast<ConstantInt>(Val: Idx)) {
1908 if (C->getValue().ult(RHS: NumElements))
1909 return ScalarizationResult::safe();
1910 return ScalarizationResult::unsafe();
1911 }
1912
1913 // Always unsafe if the index type can't handle all inbound values.
1914 if (!llvm::isUIntN(N: IntWidth, x: NumElements))
1915 return ScalarizationResult::unsafe();
1916
1917 APInt Zero(IntWidth, 0);
1918 APInt MaxElts(IntWidth, NumElements);
1919 ConstantRange ValidIndices(Zero, MaxElts);
1920 ConstantRange IdxRange(IntWidth, true);
1921
1922 if (isGuaranteedNotToBePoison(V: Idx, AC: SQ.AC, CtxI: SQ.CxtI, DT: SQ.DT)) {
1923 if (ValidIndices.contains(
1924 CR: computeConstantRange(V: Idx, /*ForSigned=*/false, SQ)))
1925 return ScalarizationResult::safe();
1926 return ScalarizationResult::unsafe();
1927 }
1928
1929 // If the index may be poison, check if we can insert a freeze before the
1930 // range of the index is restricted.
1931 Value *IdxBase;
1932 ConstantInt *CI;
1933 if (match(V: Idx, P: m_And(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1934 IdxRange = IdxRange.binaryAnd(Other: CI->getValue());
1935 } else if (match(V: Idx, P: m_URem(L: m_Value(V&: IdxBase), R: m_ConstantInt(CI)))) {
1936 IdxRange = IdxRange.urem(Other: CI->getValue());
1937 }
1938
1939 if (ValidIndices.contains(CR: IdxRange))
1940 return ScalarizationResult::safeWithFreeze(ToFreeze: IdxBase);
1941 return ScalarizationResult::unsafe();
1942}
1943
1944/// The memory operation on a vector of \p ScalarType had alignment of
1945/// \p VectorAlignment. Compute the maximal, but conservatively correct,
1946/// alignment that will be valid for the memory operation on a single scalar
1947/// element of the same type with index \p Idx.
1948static Align computeAlignmentAfterScalarization(Align VectorAlignment,
1949 Type *ScalarType, Value *Idx,
1950 const DataLayout &DL) {
1951 if (auto *C = dyn_cast<ConstantInt>(Val: Idx))
1952 return commonAlignment(A: VectorAlignment,
1953 Offset: C->getZExtValue() * DL.getTypeStoreSize(Ty: ScalarType));
1954 return commonAlignment(A: VectorAlignment, Offset: DL.getTypeStoreSize(Ty: ScalarType));
1955}
1956
1957// Combine patterns like:
1958// %0 = load <4 x i32>, <4 x i32>* %a
1959// %1 = insertelement <4 x i32> %0, i32 %b, i32 1
1960// store <4 x i32> %1, <4 x i32>* %a
1961// to:
1962// %0 = bitcast <4 x i32>* %a to i32*
1963// %1 = getelementptr inbounds i32, i32* %0, i64 0, i64 1
1964// store i32 %b, i32* %1
1965bool VectorCombine::foldSingleElementStore(Instruction &I) {
1966 if (!TTI.allowVectorElementIndexingUsingGEP())
1967 return false;
1968 auto *SI = cast<StoreInst>(Val: &I);
1969 if (!SI->isSimple() || !isa<VectorType>(Val: SI->getValueOperand()->getType()))
1970 return false;
1971
1972 // TODO: Combine more complicated patterns (multiple insert) by referencing
1973 // TargetTransformInfo.
1974 Instruction *Source;
1975 Value *NewElement;
1976 Value *Idx;
1977 if (!match(V: SI->getValueOperand(),
1978 P: m_InsertElt(Val: m_Instruction(I&: Source), Elt: m_Value(V&: NewElement),
1979 Idx: m_Value(V&: Idx))))
1980 return false;
1981
1982 if (auto *Load = dyn_cast<LoadInst>(Val: Source)) {
1983 auto VecTy = cast<VectorType>(Val: SI->getValueOperand()->getType());
1984 Value *SrcAddr = Load->getPointerOperand()->stripPointerCasts();
1985 // Don't optimize for atomic/volatile load or store. Ensure memory is not
1986 // modified between, vector type matches store size, and index is inbounds.
1987 if (!Load->isSimple() || Load->getParent() != SI->getParent() ||
1988 !DL->typeSizeEqualsStoreSize(Ty: Load->getType()->getScalarType()) ||
1989 SrcAddr != SI->getPointerOperand()->stripPointerCasts())
1990 return false;
1991
1992 if (isMemModifiedBetween(Begin: Load->getIterator(), End: SI->getIterator(),
1993 Loc: MemoryLocation::get(SI), AA))
1994 return false;
1995 auto ScalarizableIdx =
1996 canScalarizeAccess(VecTy, Idx, SQ: SQ.getWithInstruction(I: Load));
1997 if (ScalarizableIdx.isUnsafe())
1998 return false;
1999
2000 // Ensure we add the load back to the worklist BEFORE its users so they can
2001 // erased in the correct order.
2002 Worklist.push(I: Load);
2003
2004 if (ScalarizableIdx.isSafeWithFreeze())
2005 ScalarizableIdx.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2006 Value *GEP = Builder.CreateInBoundsGEP(
2007 Ty: SI->getValueOperand()->getType(), Ptr: SI->getPointerOperand(),
2008 IdxList: {ConstantInt::get(Ty: Idx->getType(), V: 0), Idx});
2009 StoreInst *NSI = Builder.CreateStore(Val: NewElement, Ptr: GEP);
2010 NSI->copyMetadata(SrcInst: *SI);
2011 Align ScalarOpAlignment = computeAlignmentAfterScalarization(
2012 VectorAlignment: std::max(a: SI->getAlign(), b: Load->getAlign()), ScalarType: NewElement->getType(), Idx,
2013 DL: *DL);
2014 NSI->setAlignment(ScalarOpAlignment);
2015 replaceValue(Old&: I, New&: *NSI);
2016 eraseInstruction(I);
2017 return true;
2018 }
2019
2020 return false;
2021}
2022
2023/// Try to scalarize vector loads feeding extractelement or bitcast
2024/// instructions.
2025bool VectorCombine::scalarizeLoad(Instruction &I) {
2026 Value *Ptr;
2027 if (!match(V: &I, P: m_Load(Op: m_Value(V&: Ptr))))
2028 return false;
2029
2030 auto *LI = cast<LoadInst>(Val: &I);
2031 auto *VecTy = cast<VectorType>(Val: LI->getType());
2032
2033 // The isSimple() check could be isUnordered(), but for now we cowardly
2034 // refuse to handle even unordered atomics.
2035 if (!LI->isSimple() || !DL->typeSizeEqualsStoreSize(Ty: VecTy->getScalarType()))
2036 return false;
2037
2038 bool AllExtracts = true;
2039 bool AllBitcasts = true;
2040 Instruction *LastCheckedInst = LI;
2041 unsigned NumInstChecked = 0;
2042
2043 // Check what type of users we have (must either all be extracts or
2044 // bitcasts) and ensure no memory modifications between the load and
2045 // its users.
2046 for (User *U : LI->users()) {
2047 auto *UI = dyn_cast<Instruction>(Val: U);
2048 if (!UI || UI->getParent() != LI->getParent())
2049 return false;
2050
2051 // If any user is waiting to be erased, then bail out as this will
2052 // distort the cost calculation and possibly lead to infinite loops.
2053 if (UI->use_empty())
2054 return false;
2055
2056 if (!isa<ExtractElementInst>(Val: UI))
2057 AllExtracts = false;
2058 if (!isa<BitCastInst>(Val: UI))
2059 AllBitcasts = false;
2060
2061 // Check if any instruction between the load and the user may modify memory.
2062 if (LastCheckedInst->comesBefore(Other: UI)) {
2063 for (Instruction &I :
2064 make_range(x: std::next(x: LI->getIterator()), y: UI->getIterator())) {
2065 // Bail out if we reached the check limit or the instruction may write
2066 // to memory.
2067 if (NumInstChecked == MaxInstrsToScan || I.mayWriteToMemory())
2068 return false;
2069 NumInstChecked++;
2070 }
2071 LastCheckedInst = UI;
2072 }
2073 }
2074
2075 if (AllExtracts)
2076 return scalarizeLoadExtract(LI, VecTy, Ptr);
2077 if (AllBitcasts)
2078 return scalarizeLoadBitcast(LI, VecTy, Ptr);
2079 return false;
2080}
2081
2082/// Try to scalarize vector loads feeding extractelement instructions.
2083bool VectorCombine::scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy,
2084 Value *Ptr) {
2085 if (!TTI.allowVectorElementIndexingUsingGEP())
2086 return false;
2087
2088 DenseMap<ExtractElementInst *, ScalarizationResult> NeedFreeze;
2089 llvm::scope_exit FailureGuard([&]() {
2090 // If the transform is aborted, discard the ScalarizationResults.
2091 for (auto &Pair : NeedFreeze)
2092 Pair.second.discard();
2093 });
2094
2095 InstructionCost OriginalCost =
2096 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2097 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2098 InstructionCost ScalarizedCost = 0;
2099
2100 for (User *U : LI->users()) {
2101 auto *UI = cast<ExtractElementInst>(Val: U);
2102
2103 auto ScalarIdx = canScalarizeAccess(VecTy, Idx: UI->getIndexOperand(),
2104 SQ: SQ.getWithInstruction(I: LI));
2105 if (ScalarIdx.isUnsafe())
2106 return false;
2107 if (ScalarIdx.isSafeWithFreeze()) {
2108 NeedFreeze.try_emplace(Key: UI, Args&: ScalarIdx);
2109 ScalarIdx.discard();
2110 }
2111
2112 auto *Index = dyn_cast<ConstantInt>(Val: UI->getIndexOperand());
2113 OriginalCost +=
2114 TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: VecTy, CostKind,
2115 Index: Index ? Index->getZExtValue() : -1);
2116 ScalarizedCost +=
2117 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy->getElementType(),
2118 Alignment: Align(1), AddressSpace: LI->getPointerAddressSpace(), CostKind);
2119 ScalarizedCost += TTI.getAddressComputationCost(PtrTy: LI->getPointerOperandType(),
2120 SE: nullptr, Ptr: nullptr, CostKind);
2121 }
2122
2123 LLVM_DEBUG(dbgs() << "Found all extractions of a vector load: " << *LI
2124 << "\n LoadExtractCost: " << OriginalCost
2125 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2126
2127 if (ScalarizedCost >= OriginalCost)
2128 return false;
2129
2130 // Ensure we add the load back to the worklist BEFORE its users so they can
2131 // erased in the correct order.
2132 Worklist.push(I: LI);
2133
2134 Type *ElemType = VecTy->getElementType();
2135
2136 // Replace extracts with narrow scalar loads.
2137 for (User *U : LI->users()) {
2138 auto *EI = cast<ExtractElementInst>(Val: U);
2139 Value *Idx = EI->getIndexOperand();
2140
2141 // Insert 'freeze' for poison indexes.
2142 auto It = NeedFreeze.find(Val: EI);
2143 if (It != NeedFreeze.end())
2144 It->second.freeze(Builder, UserI&: *cast<Instruction>(Val: Idx));
2145
2146 Builder.SetInsertPoint(EI);
2147 Value *GEP =
2148 Builder.CreateInBoundsGEP(Ty: VecTy, Ptr, IdxList: {Builder.getInt32(C: 0), Idx});
2149 auto *NewLoad = cast<LoadInst>(
2150 Val: Builder.CreateLoad(Ty: ElemType, Ptr: GEP, Name: EI->getName() + ".scalar"));
2151
2152 Align ScalarOpAlignment =
2153 computeAlignmentAfterScalarization(VectorAlignment: LI->getAlign(), ScalarType: ElemType, Idx, DL: *DL);
2154 NewLoad->setAlignment(ScalarOpAlignment);
2155
2156 if (auto *ConstIdx = dyn_cast<ConstantInt>(Val: Idx)) {
2157 size_t Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(Ty: ElemType);
2158 AAMDNodes OldAAMD = LI->getAAMetadata();
2159 NewLoad->setAAMetadata(OldAAMD.adjustForAccess(Offset, AccessTy: ElemType, DL: *DL));
2160 }
2161
2162 replaceValue(Old&: *EI, New&: *NewLoad, Erase: false);
2163 }
2164
2165 FailureGuard.release();
2166 return true;
2167}
2168
2169/// Try to scalarize vector loads feeding bitcast instructions.
2170bool VectorCombine::scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy,
2171 Value *Ptr) {
2172 InstructionCost OriginalCost =
2173 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: VecTy, Alignment: LI->getAlign(),
2174 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2175
2176 Type *TargetScalarType = nullptr;
2177 unsigned VecBitWidth = DL->getTypeSizeInBits(Ty: VecTy);
2178
2179 for (User *U : LI->users()) {
2180 auto *BC = cast<BitCastInst>(Val: U);
2181
2182 Type *DestTy = BC->getDestTy();
2183 if (!DestTy->isIntegerTy() && !DestTy->isFloatingPointTy())
2184 return false;
2185
2186 unsigned DestBitWidth = DL->getTypeSizeInBits(Ty: DestTy);
2187 if (DestBitWidth != VecBitWidth)
2188 return false;
2189
2190 // All bitcasts must target the same scalar type.
2191 if (!TargetScalarType)
2192 TargetScalarType = DestTy;
2193 else if (TargetScalarType != DestTy)
2194 return false;
2195
2196 OriginalCost +=
2197 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: TargetScalarType, Src: VecTy,
2198 CCH: TTI.getCastContextHint(I: BC), CostKind, I: BC);
2199 }
2200
2201 if (!TargetScalarType)
2202 return false;
2203
2204 assert(!LI->user_empty() && "Unexpected load without bitcast users");
2205 InstructionCost ScalarizedCost =
2206 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: TargetScalarType, Alignment: LI->getAlign(),
2207 AddressSpace: LI->getPointerAddressSpace(), CostKind);
2208
2209 LLVM_DEBUG(dbgs() << "Found vector load feeding only bitcasts: " << *LI
2210 << "\n OriginalCost: " << OriginalCost
2211 << " vs ScalarizedCost: " << ScalarizedCost << "\n");
2212
2213 if (ScalarizedCost >= OriginalCost)
2214 return false;
2215
2216 // Ensure we add the load back to the worklist BEFORE its users so they can
2217 // erased in the correct order.
2218 Worklist.push(I: LI);
2219
2220 Builder.SetInsertPoint(LI);
2221 auto *ScalarLoad =
2222 Builder.CreateLoad(Ty: TargetScalarType, Ptr, Name: LI->getName() + ".scalar");
2223 ScalarLoad->setAlignment(LI->getAlign());
2224 ScalarLoad->copyMetadata(SrcInst: *LI);
2225
2226 // Replace all bitcast users with the scalar load.
2227 for (User *U : LI->users()) {
2228 auto *BC = cast<BitCastInst>(Val: U);
2229 replaceValue(Old&: *BC, New&: *ScalarLoad, Erase: false);
2230 }
2231
2232 return true;
2233}
2234
2235bool VectorCombine::scalarizeExtExtract(Instruction &I) {
2236 if (!TTI.allowVectorElementIndexingUsingGEP())
2237 return false;
2238 auto *Ext = dyn_cast<ZExtInst>(Val: &I);
2239 if (!Ext)
2240 return false;
2241
2242 // Try to convert a vector zext feeding only extracts to a set of scalar
2243 // (Src << ExtIdx *Size) & (Size -1)
2244 // if profitable .
2245 auto *SrcTy = dyn_cast<FixedVectorType>(Val: Ext->getOperand(i_nocapture: 0)->getType());
2246 if (!SrcTy)
2247 return false;
2248 auto *DstTy = cast<FixedVectorType>(Val: Ext->getType());
2249
2250 Type *ScalarDstTy = DstTy->getElementType();
2251 if (DL->getTypeSizeInBits(Ty: SrcTy) != DL->getTypeSizeInBits(Ty: ScalarDstTy))
2252 return false;
2253
2254 InstructionCost VectorCost =
2255 TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: DstTy, Src: SrcTy,
2256 CCH: TTI::CastContextHint::None, CostKind, I: Ext);
2257 unsigned ExtCnt = 0;
2258 bool ExtLane0 = false;
2259 for (User *U : Ext->users()) {
2260 uint64_t Idx;
2261 if (!match(V: U, P: m_ExtractElt(Val: m_Value(), Idx: m_ConstantInt(V&: Idx))))
2262 return false;
2263 if (cast<Instruction>(Val: U)->use_empty())
2264 continue;
2265 ExtCnt += 1;
2266 ExtLane0 |= !Idx;
2267 VectorCost += TTI.getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: DstTy,
2268 CostKind, Index: Idx, Op0: U);
2269 }
2270
2271 InstructionCost ScalarCost =
2272 ExtCnt * TTI.getArithmeticInstrCost(
2273 Opcode: Instruction::And, Ty: ScalarDstTy, CostKind,
2274 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2275 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None}) +
2276 (ExtCnt - ExtLane0) *
2277 TTI.getArithmeticInstrCost(
2278 Opcode: Instruction::LShr, Ty: ScalarDstTy, CostKind,
2279 Opd1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2280 Opd2Info: {.Kind: TTI::OK_NonUniformConstantValue, .Properties: TTI::OP_None});
2281 if (ScalarCost > VectorCost)
2282 return false;
2283
2284 Value *ScalarV = Ext->getOperand(i_nocapture: 0);
2285 if (!isGuaranteedNotToBePoison(V: ScalarV, AC: SQ.AC, CtxI: dyn_cast<Instruction>(Val: ScalarV),
2286 DT: SQ.DT)) {
2287 // Check wether all lanes are extracted, all extracts trigger UB
2288 // on poison, and the last extract (and hence all previous ones)
2289 // are guaranteed to execute if Ext executes. If so, we do not
2290 // need to insert a freeze.
2291 SmallDenseSet<ConstantInt *, 8> ExtractedLanes;
2292 bool AllExtractsTriggerUB = true;
2293 ExtractElementInst *LastExtract = nullptr;
2294 BasicBlock *ExtBB = Ext->getParent();
2295 for (User *U : Ext->users()) {
2296 auto *Extract = cast<ExtractElementInst>(Val: U);
2297 if (Extract->getParent() != ExtBB || !programUndefinedIfPoison(Inst: Extract)) {
2298 AllExtractsTriggerUB = false;
2299 break;
2300 }
2301 ExtractedLanes.insert(V: cast<ConstantInt>(Val: Extract->getIndexOperand()));
2302 if (!LastExtract || LastExtract->comesBefore(Other: Extract))
2303 LastExtract = Extract;
2304 }
2305 if (ExtractedLanes.size() != DstTy->getNumElements() ||
2306 !AllExtractsTriggerUB ||
2307 !isGuaranteedToTransferExecutionToSuccessor(Begin: Ext->getIterator(),
2308 End: LastExtract->getIterator()))
2309 ScalarV = Builder.CreateFreeze(V: ScalarV);
2310 }
2311 ScalarV = Builder.CreateBitCast(
2312 V: ScalarV,
2313 DestTy: IntegerType::get(C&: SrcTy->getContext(), NumBits: DL->getTypeSizeInBits(Ty: SrcTy)));
2314 uint64_t SrcEltSizeInBits = DL->getTypeSizeInBits(Ty: SrcTy->getElementType());
2315 uint64_t TotalBits = DL->getTypeSizeInBits(Ty: SrcTy);
2316 APInt EltBitMask = APInt::getLowBitsSet(numBits: TotalBits, loBitsSet: SrcEltSizeInBits);
2317 Type *PackedTy = IntegerType::get(C&: SrcTy->getContext(), NumBits: TotalBits);
2318 Value *Mask = ConstantInt::get(Ty: PackedTy, V: EltBitMask);
2319 for (User *U : Ext->users()) {
2320 auto *Extract = cast<ExtractElementInst>(Val: U);
2321 uint64_t Idx =
2322 cast<ConstantInt>(Val: Extract->getIndexOperand())->getZExtValue();
2323 uint64_t ShiftAmt =
2324 DL->isBigEndian()
2325 ? (TotalBits - SrcEltSizeInBits - Idx * SrcEltSizeInBits)
2326 : (Idx * SrcEltSizeInBits);
2327 Value *LShr = Builder.CreateLShr(LHS: ScalarV, RHS: ShiftAmt);
2328 Value *And = Builder.CreateAnd(LHS: LShr, RHS: Mask);
2329 U->replaceAllUsesWith(V: And);
2330 }
2331 return true;
2332}
2333
2334/// Try to fold "(or (zext (bitcast X)), (shl (zext (bitcast Y)), C))"
2335/// to "(bitcast (concat X, Y))"
2336/// where X/Y are bitcasted from i1 mask vectors.
2337bool VectorCombine::foldConcatOfBoolMasks(Instruction &I) {
2338 Type *Ty = I.getType();
2339 if (!Ty->isIntegerTy())
2340 return false;
2341
2342 // TODO: Add big endian test coverage
2343 if (DL->isBigEndian())
2344 return false;
2345
2346 // Restrict to disjoint cases so the mask vectors aren't overlapping.
2347 Instruction *X, *Y;
2348 if (!match(V: &I, P: m_DisjointOr(L: m_Instruction(I&: X), R: m_Instruction(I&: Y))))
2349 return false;
2350
2351 // Allow both sources to contain shl, to handle more generic pattern:
2352 // "(or (shl (zext (bitcast X)), C1), (shl (zext (bitcast Y)), C2))"
2353 Value *SrcX;
2354 uint64_t ShAmtX = 0;
2355 if (!match(V: X, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX)))))) &&
2356 !match(V: X, P: m_OneUse(
2357 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcX))))),
2358 R: m_ConstantInt(V&: ShAmtX)))))
2359 return false;
2360
2361 Value *SrcY;
2362 uint64_t ShAmtY = 0;
2363 if (!match(V: Y, P: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY)))))) &&
2364 !match(V: Y, P: m_OneUse(
2365 SubPattern: m_Shl(L: m_OneUse(SubPattern: m_ZExt(Op: m_OneUse(SubPattern: m_BitCast(Op: m_Value(V&: SrcY))))),
2366 R: m_ConstantInt(V&: ShAmtY)))))
2367 return false;
2368
2369 // Canonicalize larger shift to the RHS.
2370 if (ShAmtX > ShAmtY) {
2371 std::swap(a&: X, b&: Y);
2372 std::swap(a&: SrcX, b&: SrcY);
2373 std::swap(a&: ShAmtX, b&: ShAmtY);
2374 }
2375
2376 // Ensure both sources are matching vXi1 bool mask types, and that the shift
2377 // difference is the mask width so they can be easily concatenated together.
2378 uint64_t ShAmtDiff = ShAmtY - ShAmtX;
2379 unsigned NumSHL = (ShAmtX > 0) + (ShAmtY > 0);
2380 unsigned BitWidth = Ty->getPrimitiveSizeInBits();
2381 auto *MaskTy = dyn_cast<FixedVectorType>(Val: SrcX->getType());
2382 if (!MaskTy || SrcX->getType() != SrcY->getType() ||
2383 !MaskTy->getElementType()->isIntegerTy(BitWidth: 1) ||
2384 MaskTy->getNumElements() != ShAmtDiff ||
2385 MaskTy->getNumElements() > (BitWidth / 2))
2386 return false;
2387
2388 auto *ConcatTy = FixedVectorType::getDoubleElementsVectorType(VTy: MaskTy);
2389 auto *ConcatIntTy =
2390 Type::getIntNTy(C&: Ty->getContext(), N: ConcatTy->getNumElements());
2391 auto *MaskIntTy = Type::getIntNTy(C&: Ty->getContext(), N: ShAmtDiff);
2392
2393 SmallVector<int, 32> ConcatMask(ConcatTy->getNumElements());
2394 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
2395
2396 // TODO: Is it worth supporting multi use cases?
2397 InstructionCost OldCost = 0;
2398 OldCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Or, Ty, CostKind);
2399 OldCost +=
2400 NumSHL * TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2401 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: MaskIntTy,
2402 CCH: TTI::CastContextHint::None, CostKind);
2403 OldCost += 2 * TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: MaskIntTy, Src: MaskTy,
2404 CCH: TTI::CastContextHint::None, CostKind);
2405
2406 InstructionCost NewCost = 0;
2407 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ConcatTy,
2408 SrcTy: MaskTy, Mask: ConcatMask, CostKind);
2409 NewCost += TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: ConcatIntTy, Src: ConcatTy,
2410 CCH: TTI::CastContextHint::None, CostKind);
2411 if (Ty != ConcatIntTy)
2412 NewCost += TTI.getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: ConcatIntTy,
2413 CCH: TTI::CastContextHint::None, CostKind);
2414 if (ShAmtX > 0)
2415 NewCost += TTI.getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind);
2416
2417 LLVM_DEBUG(dbgs() << "Found a concatenation of bitcasted bool masks: " << I
2418 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2419 << "\n");
2420
2421 if (NewCost > OldCost)
2422 return false;
2423
2424 // Build bool mask concatenation, bitcast back to scalar integer, and perform
2425 // any residual zero-extension or shifting.
2426 Value *Concat = Builder.CreateShuffleVector(V1: SrcX, V2: SrcY, Mask: ConcatMask);
2427 Worklist.pushValue(V: Concat);
2428
2429 Value *Result = Builder.CreateBitCast(V: Concat, DestTy: ConcatIntTy);
2430
2431 if (Ty != ConcatIntTy) {
2432 Worklist.pushValue(V: Result);
2433 Result = Builder.CreateZExt(V: Result, DestTy: Ty);
2434 }
2435
2436 if (ShAmtX > 0) {
2437 Worklist.pushValue(V: Result);
2438 Result = Builder.CreateShl(LHS: Result, RHS: ShAmtX);
2439 }
2440
2441 replaceValue(Old&: I, New&: *Result);
2442 return true;
2443}
2444
2445/// Try to convert "shuffle (binop (shuffle, shuffle)), undef"
2446/// --> "binop (shuffle), (shuffle)".
2447bool VectorCombine::foldPermuteOfBinops(Instruction &I) {
2448 BinaryOperator *BinOp;
2449 ArrayRef<int> OuterMask;
2450 if (!match(V: &I, P: m_Shuffle(v1: m_BinOp(I&: BinOp), v2: m_Undef(), mask: m_Mask(OuterMask))))
2451 return false;
2452
2453 // Don't introduce poison into div/rem.
2454 if (BinOp->isIntDivRem() && llvm::is_contained(Range&: OuterMask, Element: PoisonMaskElem))
2455 return false;
2456
2457 Value *Op00, *Op01, *Op10, *Op11;
2458 ArrayRef<int> Mask0, Mask1;
2459 bool Match0 = match(V: BinOp->getOperand(i_nocapture: 0),
2460 P: m_Shuffle(v1: m_Value(V&: Op00), v2: m_Value(V&: Op01), mask: m_Mask(Mask0)));
2461 bool Match1 = match(V: BinOp->getOperand(i_nocapture: 1),
2462 P: m_Shuffle(v1: m_Value(V&: Op10), v2: m_Value(V&: Op11), mask: m_Mask(Mask1)));
2463 if (!Match0 && !Match1)
2464 return false;
2465
2466 Op00 = Match0 ? Op00 : BinOp->getOperand(i_nocapture: 0);
2467 Op01 = Match0 ? Op01 : BinOp->getOperand(i_nocapture: 0);
2468 Op10 = Match1 ? Op10 : BinOp->getOperand(i_nocapture: 1);
2469 Op11 = Match1 ? Op11 : BinOp->getOperand(i_nocapture: 1);
2470
2471 Instruction::BinaryOps Opcode = BinOp->getOpcode();
2472 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2473 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: BinOp->getType());
2474 auto *Op0Ty = dyn_cast<FixedVectorType>(Val: Op00->getType());
2475 auto *Op1Ty = dyn_cast<FixedVectorType>(Val: Op10->getType());
2476 if (!ShuffleDstTy || !BinOpTy || !Op0Ty || !Op1Ty)
2477 return false;
2478
2479 unsigned NumSrcElts = BinOpTy->getNumElements();
2480
2481 // Don't accept shuffles that reference the second operand in
2482 // div/rem or if its an undef arg.
2483 if ((BinOp->isIntDivRem() || !isa<PoisonValue>(Val: I.getOperand(i: 1))) &&
2484 any_of(Range&: OuterMask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
2485 return false;
2486
2487 // Merge outer / inner (or identity if no match) shuffles.
2488 SmallVector<int> NewMask0, NewMask1;
2489 for (int M : OuterMask) {
2490 if (M < 0 || M >= (int)NumSrcElts) {
2491 NewMask0.push_back(Elt: PoisonMaskElem);
2492 NewMask1.push_back(Elt: PoisonMaskElem);
2493 } else {
2494 NewMask0.push_back(Elt: Match0 ? Mask0[M] : M);
2495 NewMask1.push_back(Elt: Match1 ? Mask1[M] : M);
2496 }
2497 }
2498
2499 unsigned NumOpElts = Op0Ty->getNumElements();
2500 bool IsIdentity0 = ShuffleDstTy == Op0Ty &&
2501 all_of(Range&: NewMask0, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2502 ShuffleVectorInst::isIdentityMask(Mask: NewMask0, NumSrcElts: NumOpElts);
2503 bool IsIdentity1 = ShuffleDstTy == Op1Ty &&
2504 all_of(Range&: NewMask1, P: [NumOpElts](int M) { return M < (int)NumOpElts; }) &&
2505 ShuffleVectorInst::isIdentityMask(Mask: NewMask1, NumSrcElts: NumOpElts);
2506
2507 InstructionCost NewCost = 0;
2508 // Try to merge shuffles across the binop if the new shuffles are not costly.
2509 InstructionCost BinOpCost =
2510 TTI.getArithmeticInstrCost(Opcode, Ty: BinOpTy, CostKind);
2511 InstructionCost OldCost =
2512 BinOpCost + TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
2513 DstTy: ShuffleDstTy, SrcTy: BinOpTy, Mask: OuterMask, CostKind,
2514 Index: 0, SubTp: nullptr, Args: {BinOp}, CxtI: &I);
2515 if (!BinOp->hasOneUse())
2516 NewCost += BinOpCost;
2517
2518 if (Match0) {
2519 InstructionCost Shuf0Cost = TTI.getShuffleCost(
2520 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op0Ty, Mask: Mask0, CostKind,
2521 Index: 0, SubTp: nullptr, Args: {Op00, Op01}, CxtI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 0)));
2522 OldCost += Shuf0Cost;
2523 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 0)->hasOneUse())
2524 NewCost += Shuf0Cost;
2525 }
2526 if (Match1) {
2527 InstructionCost Shuf1Cost = TTI.getShuffleCost(
2528 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: BinOpTy, SrcTy: Op1Ty, Mask: Mask1, CostKind,
2529 Index: 0, SubTp: nullptr, Args: {Op10, Op11}, CxtI: cast<Instruction>(Val: BinOp->getOperand(i_nocapture: 1)));
2530 OldCost += Shuf1Cost;
2531 if (!BinOp->hasOneUse() || !BinOp->getOperand(i_nocapture: 1)->hasOneUse())
2532 NewCost += Shuf1Cost;
2533 }
2534
2535 NewCost += TTI.getArithmeticInstrCost(Opcode, Ty: ShuffleDstTy, CostKind);
2536
2537 if (!IsIdentity0)
2538 NewCost +=
2539 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2540 SrcTy: Op0Ty, Mask: NewMask0, CostKind, Index: 0, SubTp: nullptr, Args: {Op00, Op01});
2541 if (!IsIdentity1)
2542 NewCost +=
2543 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
2544 SrcTy: Op1Ty, Mask: NewMask1, CostKind, Index: 0, SubTp: nullptr, Args: {Op10, Op11});
2545
2546 LLVM_DEBUG(dbgs() << "Found a shuffle feeding a shuffled binop: " << I
2547 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2548 << "\n");
2549
2550 // If costs are equal, still fold as we reduce instruction count.
2551 if (NewCost > OldCost)
2552 return false;
2553
2554 Value *LHS =
2555 IsIdentity0 ? Op00 : Builder.CreateShuffleVector(V1: Op00, V2: Op01, Mask: NewMask0);
2556 Value *RHS =
2557 IsIdentity1 ? Op10 : Builder.CreateShuffleVector(V1: Op10, V2: Op11, Mask: NewMask1);
2558 Value *NewBO = Builder.CreateBinOp(Opc: Opcode, LHS, RHS);
2559
2560 // Intersect flags from the old binops.
2561 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO))
2562 NewInst->copyIRFlags(V: BinOp);
2563
2564 Worklist.pushValue(V: LHS);
2565 Worklist.pushValue(V: RHS);
2566 replaceValue(Old&: I, New&: *NewBO);
2567 return true;
2568}
2569
2570/// Try to convert "shuffle (binop), (binop)" into "binop (shuffle), (shuffle)".
2571/// Try to convert "shuffle (cmpop), (cmpop)" into "cmpop (shuffle), (shuffle)".
2572bool VectorCombine::foldShuffleOfBinops(Instruction &I) {
2573 ArrayRef<int> OldMask;
2574 Instruction *LHS, *RHS;
2575 if (!match(V: &I, P: m_Shuffle(v1: m_Instruction(I&: LHS), v2: m_Instruction(I&: RHS),
2576 mask: m_Mask(OldMask))))
2577 return false;
2578
2579 // TODO: Add support for addlike etc.
2580 if (LHS->getOpcode() != RHS->getOpcode())
2581 return false;
2582
2583 Value *X, *Y, *Z, *W;
2584 bool IsCommutative = false;
2585 CmpPredicate PredLHS = CmpInst::BAD_ICMP_PREDICATE;
2586 CmpPredicate PredRHS = CmpInst::BAD_ICMP_PREDICATE;
2587 if (match(V: LHS, P: m_BinOp(L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2588 match(V: RHS, P: m_BinOp(L: m_Value(V&: Z), R: m_Value(V&: W)))) {
2589 auto *BO = cast<BinaryOperator>(Val: LHS);
2590 // Don't introduce poison into div/rem.
2591 if (llvm::is_contained(Range&: OldMask, Element: PoisonMaskElem) && BO->isIntDivRem())
2592 return false;
2593 IsCommutative = BinaryOperator::isCommutative(Opcode: BO->getOpcode());
2594 } else if (match(V: LHS, P: m_Cmp(Pred&: PredLHS, L: m_Value(V&: X), R: m_Value(V&: Y))) &&
2595 match(V: RHS, P: m_Cmp(Pred&: PredRHS, L: m_Value(V&: Z), R: m_Value(V&: W))) &&
2596 (CmpInst::Predicate)PredLHS == (CmpInst::Predicate)PredRHS) {
2597 IsCommutative = cast<CmpInst>(Val: LHS)->isCommutative();
2598 } else
2599 return false;
2600
2601 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2602 auto *BinResTy = dyn_cast<FixedVectorType>(Val: LHS->getType());
2603 auto *BinOpTy = dyn_cast<FixedVectorType>(Val: X->getType());
2604 if (!ShuffleDstTy || !BinResTy || !BinOpTy || X->getType() != Z->getType())
2605 return false;
2606
2607 bool SameBinOp = LHS == RHS;
2608 unsigned NumSrcElts = BinOpTy->getNumElements();
2609
2610 // If we have something like "add X, Y" and "add Z, X", swap ops to match.
2611 if (IsCommutative && X != Z && Y != W && (X == W || Y == Z))
2612 std::swap(a&: X, b&: Y);
2613
2614 auto ConvertToUnary = [NumSrcElts](int &M) {
2615 if (M >= (int)NumSrcElts)
2616 M -= NumSrcElts;
2617 };
2618
2619 SmallVector<int> NewMask0(OldMask);
2620 TargetTransformInfo::ShuffleKind SK0 = TargetTransformInfo::SK_PermuteTwoSrc;
2621 TTI::OperandValueInfo Op0Info = TTI.commonOperandInfo(X, Y: Z);
2622 if (X == Z) {
2623 llvm::for_each(Range&: NewMask0, F: ConvertToUnary);
2624 SK0 = TargetTransformInfo::SK_PermuteSingleSrc;
2625 Z = PoisonValue::get(T: BinOpTy);
2626 }
2627
2628 SmallVector<int> NewMask1(OldMask);
2629 TargetTransformInfo::ShuffleKind SK1 = TargetTransformInfo::SK_PermuteTwoSrc;
2630 TTI::OperandValueInfo Op1Info = TTI.commonOperandInfo(X: Y, Y: W);
2631 if (Y == W) {
2632 llvm::for_each(Range&: NewMask1, F: ConvertToUnary);
2633 SK1 = TargetTransformInfo::SK_PermuteSingleSrc;
2634 W = PoisonValue::get(T: BinOpTy);
2635 }
2636
2637 // Try to replace a binop with a shuffle if the shuffle is not costly.
2638 // When SameBinOp, only count the binop cost once.
2639 InstructionCost LHSCost = TTI.getInstructionCost(U: LHS, CostKind);
2640 InstructionCost RHSCost = TTI.getInstructionCost(U: RHS, CostKind);
2641
2642 InstructionCost OldCost = LHSCost;
2643 if (!SameBinOp) {
2644 OldCost += RHSCost;
2645 }
2646 OldCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc,
2647 DstTy: ShuffleDstTy, SrcTy: BinResTy, Mask: OldMask, CostKind, Index: 0,
2648 SubTp: nullptr, Args: {LHS, RHS}, CxtI: &I);
2649
2650 // Handle shuffle(binop(shuffle(x),y),binop(z,shuffle(w))) style patterns
2651 // where one use shuffles have gotten split across the binop/cmp. These
2652 // often allow a major reduction in total cost that wouldn't happen as
2653 // individual folds.
2654 auto MergeInner = [&](Value *&Op, int Offset, MutableArrayRef<int> Mask,
2655 TTI::TargetCostKind CostKind) -> bool {
2656 Value *InnerOp;
2657 ArrayRef<int> InnerMask;
2658 if (match(V: Op, P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: InnerOp), v2: m_Undef(),
2659 mask: m_Mask(InnerMask)))) &&
2660 InnerOp->getType() == Op->getType() &&
2661 all_of(Range&: InnerMask,
2662 P: [NumSrcElts](int M) { return M < (int)NumSrcElts; })) {
2663 for (int &M : Mask)
2664 if (Offset <= M && M < (int)(Offset + NumSrcElts)) {
2665 M = InnerMask[M - Offset];
2666 M = 0 <= M ? M + Offset : M;
2667 }
2668 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: Op), CostKind);
2669 Op = InnerOp;
2670 return true;
2671 }
2672 return false;
2673 };
2674 bool ReducedInstCount = false;
2675 ReducedInstCount |= MergeInner(X, 0, NewMask0, CostKind);
2676 ReducedInstCount |= MergeInner(Y, 0, NewMask1, CostKind);
2677 ReducedInstCount |= MergeInner(Z, NumSrcElts, NewMask0, CostKind);
2678 ReducedInstCount |= MergeInner(W, NumSrcElts, NewMask1, CostKind);
2679 bool SingleSrcBinOp = (X == Y) && (Z == W) && (NewMask0 == NewMask1);
2680 // SingleSrcBinOp only reduces instruction count if we also eliminate the
2681 // original binop(s). If binops have multiple uses, they won't be eliminated.
2682 ReducedInstCount |= SingleSrcBinOp && LHS->hasOneUser() && RHS->hasOneUser();
2683
2684 // For concat shuffles of i1 vectors where both binops are one-use, the
2685 // transform keeps the same instruction count but canonicalises to a single
2686 // wider binop, enabling downstream folds (e.g. NOT(XOR(concat(a,b),
2687 // concat(c,d))) -> XNOR(concat(a,b),concat(c,d)) on AVX-512 mask regs).
2688 // Restrict to BinaryOperator (not CmpInst) since narrow comparisons may
2689 // be cheaper than wide ones on some targets (e.g. AVX-512 vpcmpeq).
2690 ReducedInstCount |= cast<ShuffleVectorInst>(Val: &I)->isConcat() &&
2691 I.getType()->getScalarType()->isIntegerTy(BitWidth: 1) &&
2692 isa<BinaryOperator>(Val: LHS) && LHS->hasOneUser() &&
2693 RHS->hasOneUser();
2694
2695 auto *ShuffleCmpTy =
2696 FixedVectorType::get(ElementType: BinOpTy->getElementType(), FVTy: ShuffleDstTy);
2697 InstructionCost NewCost = TTI.getShuffleCost(
2698 Kind: SK0, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, Mask: NewMask0, CostKind, Index: 0, SubTp: nullptr, Args: {X, Z});
2699 if (!SingleSrcBinOp)
2700 NewCost += TTI.getShuffleCost(Kind: SK1, DstTy: ShuffleCmpTy, SrcTy: BinOpTy, Mask: NewMask1,
2701 CostKind, Index: 0, SubTp: nullptr, Args: {Y, W});
2702
2703 if (PredLHS == CmpInst::BAD_ICMP_PREDICATE) {
2704 NewCost += TTI.getArithmeticInstrCost(Opcode: LHS->getOpcode(), Ty: ShuffleDstTy,
2705 CostKind, Opd1Info: Op0Info, Opd2Info: Op1Info);
2706 } else {
2707 NewCost +=
2708 TTI.getCmpSelInstrCost(Opcode: LHS->getOpcode(), ValTy: ShuffleCmpTy, CondTy: ShuffleDstTy,
2709 VecPred: PredLHS, CostKind, Op1Info: Op0Info, Op2Info: Op1Info);
2710 }
2711 // If LHS/RHS have other uses, we need to account for the cost of keeping
2712 // the original instructions. When SameBinOp, only add the cost once.
2713 if (!LHS->hasOneUser())
2714 NewCost += LHSCost;
2715 if (!SameBinOp && !RHS->hasOneUser())
2716 NewCost += RHSCost;
2717
2718 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two binops: " << I
2719 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2720 << "\n");
2721
2722 // If either shuffle will constant fold away, then fold for the same cost as
2723 // we will reduce the instruction count.
2724 ReducedInstCount |= (isa<Constant>(Val: X) && isa<Constant>(Val: Z)) ||
2725 (isa<Constant>(Val: Y) && isa<Constant>(Val: W));
2726 if (ReducedInstCount ? (NewCost > OldCost) : (NewCost >= OldCost))
2727 return false;
2728
2729 Value *Shuf0 = Builder.CreateShuffleVector(V1: X, V2: Z, Mask: NewMask0);
2730 Value *Shuf1 =
2731 SingleSrcBinOp ? Shuf0 : Builder.CreateShuffleVector(V1: Y, V2: W, Mask: NewMask1);
2732 Value *NewBO = PredLHS == CmpInst::BAD_ICMP_PREDICATE
2733 ? Builder.CreateBinOp(
2734 Opc: cast<BinaryOperator>(Val: LHS)->getOpcode(), LHS: Shuf0, RHS: Shuf1)
2735 : Builder.CreateCmp(Pred: PredLHS, LHS: Shuf0, RHS: Shuf1);
2736
2737 // Intersect flags from the old binops.
2738 if (auto *NewInst = dyn_cast<Instruction>(Val: NewBO)) {
2739 NewInst->copyIRFlags(V: LHS);
2740 NewInst->andIRFlags(V: RHS);
2741 }
2742
2743 Worklist.pushValue(V: Shuf0);
2744 Worklist.pushValue(V: Shuf1);
2745 replaceValue(Old&: I, New&: *NewBO);
2746 return true;
2747}
2748
2749/// Try to convert,
2750/// (shuffle(select(c1,t1,f1)), (select(c2,t2,f2)), m) into
2751/// (select (shuffle c1,c2,m), (shuffle t1,t2,m), (shuffle f1,f2,m))
2752bool VectorCombine::foldShuffleOfSelects(Instruction &I) {
2753 ArrayRef<int> Mask;
2754 Value *C1, *T1, *F1, *C2, *T2, *F2;
2755 if (!match(V: &I, P: m_Shuffle(v1: m_Select(C: m_Value(V&: C1), L: m_Value(V&: T1), R: m_Value(V&: F1)),
2756 v2: m_Select(C: m_Value(V&: C2), L: m_Value(V&: T2), R: m_Value(V&: F2)),
2757 mask: m_Mask(Mask))))
2758 return false;
2759
2760 auto *Sel1 = cast<Instruction>(Val: I.getOperand(i: 0));
2761 auto *Sel2 = cast<Instruction>(Val: I.getOperand(i: 1));
2762
2763 auto *C1VecTy = dyn_cast<FixedVectorType>(Val: C1->getType());
2764 auto *C2VecTy = dyn_cast<FixedVectorType>(Val: C2->getType());
2765 if (!C1VecTy || !C2VecTy || C1VecTy != C2VecTy)
2766 return false;
2767
2768 auto *SI0FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 0));
2769 auto *SI1FOp = dyn_cast<FPMathOperator>(Val: I.getOperand(i: 1));
2770 // SelectInsts must have the same FMF.
2771 if (((SI0FOp == nullptr) != (SI1FOp == nullptr)) ||
2772 ((SI0FOp != nullptr) &&
2773 (SI0FOp->getFastMathFlags() != SI1FOp->getFastMathFlags())))
2774 return false;
2775
2776 auto *SrcVecTy = cast<FixedVectorType>(Val: T1->getType());
2777 auto *DstVecTy = cast<FixedVectorType>(Val: I.getType());
2778 auto SK = TargetTransformInfo::SK_PermuteTwoSrc;
2779 auto SelOp = Instruction::Select;
2780
2781 InstructionCost CostSel1 = TTI.getCmpSelInstrCost(
2782 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C1VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2783 InstructionCost CostSel2 = TTI.getCmpSelInstrCost(
2784 Opcode: SelOp, ValTy: SrcVecTy, CondTy: C2VecTy, VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2785
2786 InstructionCost OldCost =
2787 CostSel1 + CostSel2 +
2788 TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, Mask, CostKind, Index: 0, SubTp: nullptr,
2789 Args: {I.getOperand(i: 0), I.getOperand(i: 1)}, CxtI: &I);
2790
2791 InstructionCost NewCost = TTI.getShuffleCost(
2792 Kind: SK, DstTy: FixedVectorType::get(ElementType: C1VecTy->getScalarType(), NumElts: Mask.size()), SrcTy: C1VecTy,
2793 Mask, CostKind, Index: 0, SubTp: nullptr, Args: {C1, C2});
2794 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, Mask, CostKind, Index: 0,
2795 SubTp: nullptr, Args: {T1, T2});
2796 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: SrcVecTy, Mask, CostKind, Index: 0,
2797 SubTp: nullptr, Args: {F1, F2});
2798 auto *C1C2ShuffledVecTy = FixedVectorType::get(
2799 ElementType: Type::getInt1Ty(C&: I.getContext()), NumElts: DstVecTy->getNumElements());
2800 NewCost += TTI.getCmpSelInstrCost(Opcode: SelOp, ValTy: DstVecTy, CondTy: C1C2ShuffledVecTy,
2801 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
2802
2803 if (!Sel1->hasOneUse())
2804 NewCost += CostSel1;
2805 if (!Sel2->hasOneUse())
2806 NewCost += CostSel2;
2807
2808 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two selects: " << I
2809 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2810 << "\n");
2811 if (NewCost > OldCost)
2812 return false;
2813
2814 Value *ShuffleCmp = Builder.CreateShuffleVector(V1: C1, V2: C2, Mask);
2815 Value *ShuffleTrue = Builder.CreateShuffleVector(V1: T1, V2: T2, Mask);
2816 Value *ShuffleFalse = Builder.CreateShuffleVector(V1: F1, V2: F2, Mask);
2817 Value *NewSel;
2818 // We presuppose that the SelectInsts have the same FMF.
2819 if (SI0FOp)
2820 NewSel = Builder.CreateSelectFMF(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse,
2821 FMFSource: SI0FOp->getFastMathFlags());
2822 else
2823 NewSel = Builder.CreateSelect(C: ShuffleCmp, True: ShuffleTrue, False: ShuffleFalse);
2824
2825 Worklist.pushValue(V: ShuffleCmp);
2826 Worklist.pushValue(V: ShuffleTrue);
2827 Worklist.pushValue(V: ShuffleFalse);
2828 replaceValue(Old&: I, New&: *NewSel);
2829 return true;
2830}
2831
2832/// Try to convert "shuffle (castop), (castop)" with a shared castop operand
2833/// into "castop (shuffle)".
2834bool VectorCombine::foldShuffleOfCastops(Instruction &I) {
2835 Value *V0, *V1;
2836 ArrayRef<int> OldMask;
2837 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
2838 return false;
2839
2840 // Check whether this is a binary shuffle.
2841 bool IsBinaryShuffle = !isa<UndefValue>(Val: V1);
2842
2843 auto *C0 = dyn_cast<CastInst>(Val: V0);
2844 auto *C1 = dyn_cast<CastInst>(Val: V1);
2845 if (!C0 || (IsBinaryShuffle && !C1))
2846 return false;
2847
2848 Instruction::CastOps Opcode = C0->getOpcode();
2849
2850 // If this is allowed, foldShuffleOfCastops can get stuck in a loop
2851 // with foldBitcastOfShuffle. Reject in favor of foldBitcastOfShuffle.
2852 if (!IsBinaryShuffle && Opcode == Instruction::BitCast)
2853 return false;
2854
2855 if (IsBinaryShuffle) {
2856 if (C0->getSrcTy() != C1->getSrcTy())
2857 return false;
2858 // Handle shuffle(zext_nneg(x), sext(y)) -> sext(shuffle(x,y)) folds.
2859 if (Opcode != C1->getOpcode()) {
2860 if (match(V: C0, P: m_SExtLike(Op: m_Value())) && match(V: C1, P: m_SExtLike(Op: m_Value())))
2861 Opcode = Instruction::SExt;
2862 else
2863 return false;
2864 }
2865 }
2866
2867 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2868 auto *CastDstTy = dyn_cast<FixedVectorType>(Val: C0->getDestTy());
2869 auto *CastSrcTy = dyn_cast<FixedVectorType>(Val: C0->getSrcTy());
2870 if (!ShuffleDstTy || !CastDstTy || !CastSrcTy)
2871 return false;
2872
2873 unsigned NumSrcElts = CastSrcTy->getNumElements();
2874 unsigned NumDstElts = CastDstTy->getNumElements();
2875 assert((NumDstElts == NumSrcElts || Opcode == Instruction::BitCast) &&
2876 "Only bitcasts expected to alter src/dst element counts");
2877
2878 // Check for bitcasting of unscalable vector types.
2879 // e.g. <32 x i40> -> <40 x i32>
2880 if (NumDstElts != NumSrcElts && (NumSrcElts % NumDstElts) != 0 &&
2881 (NumDstElts % NumSrcElts) != 0)
2882 return false;
2883
2884 SmallVector<int, 16> NewMask;
2885 if (NumSrcElts >= NumDstElts) {
2886 // The bitcast is from wide to narrow/equal elements. The shuffle mask can
2887 // always be expanded to the equivalent form choosing narrower elements.
2888 assert(NumSrcElts % NumDstElts == 0 && "Unexpected shuffle mask");
2889 unsigned ScaleFactor = NumSrcElts / NumDstElts;
2890 narrowShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask);
2891 } else {
2892 // The bitcast is from narrow elements to wide elements. The shuffle mask
2893 // must choose consecutive elements to allow casting first.
2894 assert(NumDstElts % NumSrcElts == 0 && "Unexpected shuffle mask");
2895 unsigned ScaleFactor = NumDstElts / NumSrcElts;
2896 if (!widenShuffleMaskElts(Scale: ScaleFactor, Mask: OldMask, ScaledMask&: NewMask))
2897 return false;
2898 }
2899
2900 auto *NewShuffleDstTy =
2901 FixedVectorType::get(ElementType: CastSrcTy->getScalarType(), NumElts: NewMask.size());
2902
2903 // Try to replace a castop with a shuffle if the shuffle is not costly.
2904 InstructionCost CostC0 =
2905 TTI.getCastInstrCost(Opcode: C0->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
2906 CCH: TTI::CastContextHint::None, CostKind, I: C0);
2907
2908 TargetTransformInfo::ShuffleKind ShuffleKind;
2909 if (IsBinaryShuffle)
2910 ShuffleKind = TargetTransformInfo::SK_PermuteTwoSrc;
2911 else
2912 ShuffleKind = TargetTransformInfo::SK_PermuteSingleSrc;
2913
2914 InstructionCost OldCost = CostC0;
2915 OldCost += TTI.getShuffleCost(Kind: ShuffleKind, DstTy: ShuffleDstTy, SrcTy: CastDstTy, Mask: OldMask,
2916 CostKind, Index: 0, SubTp: nullptr, Args: {}, CxtI: &I);
2917
2918 InstructionCost NewCost = TTI.getShuffleCost(Kind: ShuffleKind, DstTy: NewShuffleDstTy,
2919 SrcTy: CastSrcTy, Mask: NewMask, CostKind);
2920 NewCost += TTI.getCastInstrCost(Opcode, Dst: ShuffleDstTy, Src: NewShuffleDstTy,
2921 CCH: TTI::CastContextHint::None, CostKind);
2922 if (!C0->hasOneUse())
2923 NewCost += CostC0;
2924 if (IsBinaryShuffle) {
2925 InstructionCost CostC1 =
2926 TTI.getCastInstrCost(Opcode: C1->getOpcode(), Dst: CastDstTy, Src: CastSrcTy,
2927 CCH: TTI::CastContextHint::None, CostKind, I: C1);
2928 OldCost += CostC1;
2929 if (!C1->hasOneUse())
2930 NewCost += CostC1;
2931 }
2932
2933 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two casts: " << I
2934 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
2935 << "\n");
2936 if (NewCost > OldCost)
2937 return false;
2938
2939 Value *Shuf;
2940 if (IsBinaryShuffle)
2941 Shuf = Builder.CreateShuffleVector(V1: C0->getOperand(i_nocapture: 0), V2: C1->getOperand(i_nocapture: 0),
2942 Mask: NewMask);
2943 else
2944 Shuf = Builder.CreateShuffleVector(V: C0->getOperand(i_nocapture: 0), Mask: NewMask);
2945
2946 Value *Cast = Builder.CreateCast(Op: Opcode, V: Shuf, DestTy: ShuffleDstTy);
2947
2948 // Intersect flags from the old casts.
2949 if (auto *NewInst = dyn_cast<Instruction>(Val: Cast)) {
2950 NewInst->copyIRFlags(V: C0);
2951 if (IsBinaryShuffle)
2952 NewInst->andIRFlags(V: C1);
2953 }
2954
2955 Worklist.pushValue(V: Shuf);
2956 replaceValue(Old&: I, New&: *Cast);
2957 return true;
2958}
2959
2960/// Try to convert any of:
2961/// "shuffle (shuffle x, y), (shuffle y, x)"
2962/// "shuffle (shuffle x, undef), (shuffle y, undef)"
2963/// "shuffle (shuffle x, undef), y"
2964/// "shuffle x, (shuffle y, undef)"
2965/// into "shuffle x, y".
2966bool VectorCombine::foldShuffleOfShuffles(Instruction &I) {
2967 ArrayRef<int> OuterMask;
2968 Value *OuterV0, *OuterV1;
2969 if (!match(V: &I,
2970 P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1), mask: m_Mask(OuterMask))))
2971 return false;
2972
2973 ArrayRef<int> InnerMask0, InnerMask1;
2974 Value *X0, *X1, *Y0, *Y1;
2975 bool Match0 =
2976 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: X0), v2: m_Value(V&: Y0), mask: m_Mask(InnerMask0)));
2977 bool Match1 =
2978 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: X1), v2: m_Value(V&: Y1), mask: m_Mask(InnerMask1)));
2979 if (!Match0 && !Match1)
2980 return false;
2981
2982 // If the outer shuffle is a permute, then create a fake inner all-poison
2983 // shuffle. This is easier than accounting for length-changing shuffles below.
2984 SmallVector<int, 16> PoisonMask1;
2985 if (!Match1 && isa<PoisonValue>(Val: OuterV1)) {
2986 X1 = X0;
2987 Y1 = Y0;
2988 PoisonMask1.append(NumInputs: InnerMask0.size(), Elt: PoisonMaskElem);
2989 InnerMask1 = PoisonMask1;
2990 Match1 = true; // fake match
2991 }
2992
2993 X0 = Match0 ? X0 : OuterV0;
2994 Y0 = Match0 ? Y0 : OuterV0;
2995 X1 = Match1 ? X1 : OuterV1;
2996 Y1 = Match1 ? Y1 : OuterV1;
2997 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
2998 auto *ShuffleSrcTy = dyn_cast<FixedVectorType>(Val: X0->getType());
2999 auto *ShuffleImmTy = dyn_cast<FixedVectorType>(Val: OuterV0->getType());
3000 if (!ShuffleDstTy || !ShuffleSrcTy || !ShuffleImmTy ||
3001 X0->getType() != X1->getType())
3002 return false;
3003
3004 unsigned NumSrcElts = ShuffleSrcTy->getNumElements();
3005 unsigned NumImmElts = ShuffleImmTy->getNumElements();
3006
3007 // Attempt to merge shuffles, matching upto 2 source operands.
3008 // Replace index to a poison arg with PoisonMaskElem.
3009 // Bail if either inner masks reference an undef arg.
3010 SmallVector<int, 16> NewMask(OuterMask);
3011 Value *NewX = nullptr, *NewY = nullptr;
3012 for (int &M : NewMask) {
3013 Value *Src = nullptr;
3014 if (0 <= M && M < (int)NumImmElts) {
3015 Src = OuterV0;
3016 if (Match0) {
3017 M = InnerMask0[M];
3018 Src = M >= (int)NumSrcElts ? Y0 : X0;
3019 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3020 }
3021 } else if (M >= (int)NumImmElts) {
3022 Src = OuterV1;
3023 M -= NumImmElts;
3024 if (Match1) {
3025 M = InnerMask1[M];
3026 Src = M >= (int)NumSrcElts ? Y1 : X1;
3027 M = M >= (int)NumSrcElts ? (M - NumSrcElts) : M;
3028 }
3029 }
3030 if (Src && M != PoisonMaskElem) {
3031 assert(0 <= M && M < (int)NumSrcElts && "Unexpected shuffle mask index");
3032 if (isa<UndefValue>(Val: Src)) {
3033 // We've referenced an undef element - if its poison, update the shuffle
3034 // mask, else bail.
3035 if (!isa<PoisonValue>(Val: Src))
3036 return false;
3037 M = PoisonMaskElem;
3038 continue;
3039 }
3040 if (!NewX || NewX == Src) {
3041 NewX = Src;
3042 continue;
3043 }
3044 if (!NewY || NewY == Src) {
3045 M += NumSrcElts;
3046 NewY = Src;
3047 continue;
3048 }
3049 return false;
3050 }
3051 }
3052
3053 if (!NewX) {
3054 replaceValue(Old&: I, New&: *PoisonValue::get(T: ShuffleDstTy));
3055 return true;
3056 }
3057
3058 if (!NewY)
3059 NewY = PoisonValue::get(T: ShuffleSrcTy);
3060
3061 // Have we folded to an Identity shuffle?
3062 if (ShuffleVectorInst::isIdentityMask(Mask: NewMask, NumSrcElts)) {
3063 replaceValue(Old&: I, New&: *NewX);
3064 return true;
3065 }
3066
3067 // Try to merge the shuffles if the new shuffle is not costly.
3068 InstructionCost InnerCost0 = 0;
3069 if (Match0)
3070 InnerCost0 = TTI.getInstructionCost(U: cast<User>(Val: OuterV0), CostKind);
3071
3072 InstructionCost InnerCost1 = 0;
3073 if (Match1)
3074 InnerCost1 = TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3075
3076 InstructionCost OuterCost = TTI.getInstructionCost(U: &I, CostKind);
3077
3078 InstructionCost OldCost = InnerCost0 + InnerCost1 + OuterCost;
3079
3080 bool IsUnary = all_of(Range&: NewMask, P: [&](int M) { return M < (int)NumSrcElts; });
3081 TargetTransformInfo::ShuffleKind SK =
3082 IsUnary ? TargetTransformInfo::SK_PermuteSingleSrc
3083 : TargetTransformInfo::SK_PermuteTwoSrc;
3084 InstructionCost NewCost =
3085 TTI.getShuffleCost(Kind: SK, DstTy: ShuffleDstTy, SrcTy: ShuffleSrcTy, Mask: NewMask, CostKind, Index: 0,
3086 SubTp: nullptr, Args: {NewX, NewY});
3087 if (!OuterV0->hasOneUse())
3088 NewCost += InnerCost0;
3089 if (!OuterV1->hasOneUse())
3090 NewCost += InnerCost1;
3091
3092 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two shuffles: " << I
3093 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3094 << "\n");
3095 if (NewCost > OldCost)
3096 return false;
3097
3098 Value *Shuf = Builder.CreateShuffleVector(V1: NewX, V2: NewY, Mask: NewMask);
3099 replaceValue(Old&: I, New&: *Shuf);
3100 return true;
3101}
3102
3103/// Try to convert a chain of length-preserving shuffles that are fed by
3104/// length-changing shuffles from the same source, e.g. a chain of length 3:
3105///
3106/// "shuffle (shuffle (shuffle x, (shuffle y, undef)),
3107/// (shuffle y, undef)),
3108// (shuffle y, undef)"
3109///
3110/// into a single shuffle fed by a length-changing shuffle:
3111///
3112/// "shuffle x, (shuffle y, undef)"
3113///
3114/// Such chains arise e.g. from folding extract/insert sequences.
3115bool VectorCombine::foldShufflesOfLengthChangingShuffles(Instruction &I) {
3116 FixedVectorType *TrunkType = dyn_cast<FixedVectorType>(Val: I.getType());
3117 if (!TrunkType)
3118 return false;
3119
3120 unsigned ChainLength = 0;
3121 SmallVector<int> Mask;
3122 SmallVector<int> YMask;
3123 InstructionCost OldCost = 0;
3124 InstructionCost NewCost = 0;
3125 Value *Trunk = &I;
3126 unsigned NumTrunkElts = TrunkType->getNumElements();
3127 Value *Y = nullptr;
3128
3129 for (;;) {
3130 // Match the current trunk against (commutations of) the pattern
3131 // "shuffle trunk', (shuffle y, undef)"
3132 ArrayRef<int> OuterMask;
3133 Value *OuterV0, *OuterV1;
3134 if (ChainLength != 0 && !Trunk->hasOneUse())
3135 break;
3136 if (!match(V: Trunk, P: m_Shuffle(v1: m_Value(V&: OuterV0), v2: m_Value(V&: OuterV1),
3137 mask: m_Mask(OuterMask))))
3138 break;
3139 if (OuterV0->getType() != TrunkType) {
3140 // This shuffle is not length-preserving, so it cannot be part of the
3141 // chain.
3142 break;
3143 }
3144
3145 ArrayRef<int> InnerMask0, InnerMask1;
3146 Value *A0, *A1, *B0, *B1;
3147 bool Match0 =
3148 match(V: OuterV0, P: m_Shuffle(v1: m_Value(V&: A0), v2: m_Value(V&: B0), mask: m_Mask(InnerMask0)));
3149 bool Match1 =
3150 match(V: OuterV1, P: m_Shuffle(v1: m_Value(V&: A1), v2: m_Value(V&: B1), mask: m_Mask(InnerMask1)));
3151 bool Match0Leaf = Match0 && A0->getType() != I.getType();
3152 bool Match1Leaf = Match1 && A1->getType() != I.getType();
3153 if (Match0Leaf == Match1Leaf) {
3154 // Only handle the case of exactly one leaf in each step. The "two leaves"
3155 // case is handled by foldShuffleOfShuffles.
3156 break;
3157 }
3158
3159 SmallVector<int> CommutedOuterMask;
3160 if (Match0Leaf) {
3161 std::swap(a&: OuterV0, b&: OuterV1);
3162 std::swap(a&: InnerMask0, b&: InnerMask1);
3163 std::swap(a&: A0, b&: A1);
3164 std::swap(a&: B0, b&: B1);
3165 llvm::append_range(C&: CommutedOuterMask, R&: OuterMask);
3166 for (int &M : CommutedOuterMask) {
3167 if (M == PoisonMaskElem)
3168 continue;
3169 if (M < (int)NumTrunkElts)
3170 M += NumTrunkElts;
3171 else
3172 M -= NumTrunkElts;
3173 }
3174 OuterMask = CommutedOuterMask;
3175 }
3176 if (!OuterV1->hasOneUse())
3177 break;
3178
3179 if (!isa<UndefValue>(Val: A1)) {
3180 if (!Y)
3181 Y = A1;
3182 else if (Y != A1)
3183 break;
3184 }
3185 if (!isa<UndefValue>(Val: B1)) {
3186 if (!Y)
3187 Y = B1;
3188 else if (Y != B1)
3189 break;
3190 }
3191
3192 auto *YType = cast<FixedVectorType>(Val: A1->getType());
3193 int NumLeafElts = YType->getNumElements();
3194 SmallVector<int> LocalYMask(InnerMask1);
3195 for (int &M : LocalYMask) {
3196 if (M >= NumLeafElts)
3197 M -= NumLeafElts;
3198 }
3199
3200 InstructionCost LocalOldCost =
3201 TTI.getInstructionCost(U: cast<User>(Val: Trunk), CostKind) +
3202 TTI.getInstructionCost(U: cast<User>(Val: OuterV1), CostKind);
3203
3204 // Handle the initial (start of chain) case.
3205 if (!ChainLength) {
3206 Mask.assign(AR: OuterMask);
3207 YMask.assign(RHS: LocalYMask);
3208 OldCost = NewCost = LocalOldCost;
3209 Trunk = OuterV0;
3210 ChainLength++;
3211 continue;
3212 }
3213
3214 // For the non-root case, first attempt to combine masks.
3215 SmallVector<int> NewYMask(YMask);
3216 bool Valid = true;
3217 for (auto [CombinedM, LeafM] : llvm::zip(t&: NewYMask, u&: LocalYMask)) {
3218 if (LeafM == -1 || CombinedM == LeafM)
3219 continue;
3220 if (CombinedM == -1) {
3221 CombinedM = LeafM;
3222 } else {
3223 Valid = false;
3224 break;
3225 }
3226 }
3227 if (!Valid)
3228 break;
3229
3230 SmallVector<int> NewMask;
3231 NewMask.reserve(N: NumTrunkElts);
3232 for (int M : Mask) {
3233 if (M < 0 || M >= static_cast<int>(NumTrunkElts))
3234 NewMask.push_back(Elt: M);
3235 else
3236 NewMask.push_back(Elt: OuterMask[M]);
3237 }
3238
3239 // Break the chain if adding this new step complicates the shuffles such
3240 // that it would increase the new cost by more than the old cost of this
3241 // step.
3242 InstructionCost LocalNewCost =
3243 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: TrunkType,
3244 SrcTy: YType, Mask: NewYMask, CostKind) +
3245 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: TrunkType,
3246 SrcTy: TrunkType, Mask: NewMask, CostKind);
3247
3248 if (LocalNewCost >= NewCost && LocalOldCost < LocalNewCost - NewCost)
3249 break;
3250
3251 LLVM_DEBUG({
3252 if (ChainLength == 1) {
3253 dbgs() << "Found chain of shuffles fed by length-changing shuffles: "
3254 << I << '\n';
3255 }
3256 dbgs() << " next chain link: " << *Trunk << '\n'
3257 << " old cost: " << (OldCost + LocalOldCost)
3258 << " new cost: " << LocalNewCost << '\n';
3259 });
3260
3261 Mask = NewMask;
3262 YMask = NewYMask;
3263 OldCost += LocalOldCost;
3264 NewCost = LocalNewCost;
3265 Trunk = OuterV0;
3266 ChainLength++;
3267 }
3268 if (ChainLength <= 1)
3269 return false;
3270
3271 // Bail out if all leaves were poison.
3272 if (!Y)
3273 return false;
3274
3275 if (llvm::all_of(Range&: Mask, P: [&](int M) {
3276 return M < 0 || M >= static_cast<int>(NumTrunkElts);
3277 })) {
3278 // Produce a canonical simplified form if all elements are sourced from Y.
3279 for (int &M : Mask) {
3280 if (M >= static_cast<int>(NumTrunkElts))
3281 M = YMask[M - NumTrunkElts];
3282 }
3283 Value *Root =
3284 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask);
3285 replaceValue(Old&: I, New&: *Root);
3286 return true;
3287 }
3288
3289 Value *Leaf =
3290 Builder.CreateShuffleVector(V1: Y, V2: PoisonValue::get(T: Y->getType()), Mask: YMask);
3291 Value *Root = Builder.CreateShuffleVector(V1: Trunk, V2: Leaf, Mask);
3292 replaceValue(Old&: I, New&: *Root);
3293 return true;
3294}
3295
3296/// Try to convert
3297/// "shuffle (intrinsic), (intrinsic)" into "intrinsic (shuffle), (shuffle)".
3298bool VectorCombine::foldShuffleOfIntrinsics(Instruction &I) {
3299 Value *V0, *V1;
3300 ArrayRef<int> OldMask;
3301 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Value(V&: V1), mask: m_Mask(OldMask))))
3302 return false;
3303
3304 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3305 auto *II1 = dyn_cast<IntrinsicInst>(Val: V1);
3306 if (!II0 || !II1)
3307 return false;
3308
3309 Intrinsic::ID IID = II0->getIntrinsicID();
3310 if (IID != II1->getIntrinsicID())
3311 return false;
3312 InstructionCost CostII0 =
3313 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3314 InstructionCost CostII1 =
3315 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II1), CostKind);
3316
3317 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3318 auto *II0Ty = dyn_cast<FixedVectorType>(Val: II0->getType());
3319 if (!ShuffleDstTy || !II0Ty)
3320 return false;
3321
3322 if (!isTriviallyVectorizable(ID: IID))
3323 return false;
3324
3325 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3326 Value *Arg0 = II0->getArgOperand(i: I);
3327 Value *Arg1 = II1->getArgOperand(i: I);
3328 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3329 // Scalar operands must be identical.
3330 if (Arg0 != Arg1)
3331 return false;
3332 } else if (Arg0->getType() != Arg1->getType()) {
3333 // The corresponding vector operands are shuffled together, so they must
3334 // share the same type. For intrinsics overloaded on their operand type
3335 // (e.g. llvm.fptosi.sat), two calls can produce the same result type
3336 // from different operand types; shuffling those would be invalid.
3337 return false;
3338 }
3339 }
3340
3341 InstructionCost OldCost =
3342 CostII0 + CostII1 +
3343 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ShuffleDstTy,
3344 SrcTy: II0Ty, Mask: OldMask, CostKind, Index: 0, SubTp: nullptr, Args: {II0, II1}, CxtI: &I);
3345
3346 SmallVector<Type *> NewArgsTy;
3347 InstructionCost NewCost = 0;
3348 SmallDenseSet<std::pair<Value *, Value *>> SeenOperandPairs;
3349 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3350 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3351 NewArgsTy.push_back(Elt: II0->getArgOperand(i: I)->getType());
3352 } else {
3353 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: I)->getType());
3354 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3355 NumElts: ShuffleDstTy->getNumElements());
3356 NewArgsTy.push_back(Elt: ArgTy);
3357 std::pair<Value *, Value *> OperandPair =
3358 std::make_pair(x: II0->getArgOperand(i: I), y: II1->getArgOperand(i: I));
3359 if (!SeenOperandPairs.insert(V: OperandPair).second) {
3360 // We've already computed the cost for this operand pair.
3361 continue;
3362 }
3363 NewCost += TTI.getShuffleCost(
3364 Kind: TargetTransformInfo::SK_PermuteTwoSrc, DstTy: ArgTy, SrcTy: VecTy, Mask: OldMask,
3365 CostKind, Index: 0, SubTp: nullptr, Args: {II0->getArgOperand(i: I), II1->getArgOperand(i: I)});
3366 }
3367 }
3368 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3369
3370 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3371 if (!II0->hasOneUse())
3372 NewCost += CostII0;
3373 if (II1 != II0 && !II1->hasOneUse())
3374 NewCost += CostII1;
3375
3376 LLVM_DEBUG(dbgs() << "Found a shuffle feeding two intrinsics: " << I
3377 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
3378 << "\n");
3379
3380 if (NewCost > OldCost)
3381 return false;
3382
3383 SmallVector<Value *> NewArgs;
3384 SmallDenseMap<std::pair<Value *, Value *>, Value *> ShuffleCache;
3385 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I)
3386 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3387 NewArgs.push_back(Elt: II0->getArgOperand(i: I));
3388 } else {
3389 std::pair<Value *, Value *> OperandPair =
3390 std::make_pair(x: II0->getArgOperand(i: I), y: II1->getArgOperand(i: I));
3391 auto It = ShuffleCache.find(Val: OperandPair);
3392 if (It != ShuffleCache.end()) {
3393 // Reuse previously created shuffle for this operand pair.
3394 NewArgs.push_back(Elt: It->second);
3395 continue;
3396 }
3397 Value *Shuf = Builder.CreateShuffleVector(V1: II0->getArgOperand(i: I),
3398 V2: II1->getArgOperand(i: I), Mask: OldMask);
3399 ShuffleCache[OperandPair] = Shuf;
3400 NewArgs.push_back(Elt: Shuf);
3401 Worklist.pushValue(V: Shuf);
3402 }
3403 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3404
3405 // Intersect flags from the old intrinsics.
3406 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic)) {
3407 NewInst->copyIRFlags(V: II0);
3408 NewInst->andIRFlags(V: II1);
3409 }
3410
3411 replaceValue(Old&: I, New&: *NewIntrinsic);
3412 return true;
3413}
3414
3415/// Try to convert
3416/// "shuffle (intrinsic), (poison/undef)" into "intrinsic (shuffle)".
3417bool VectorCombine::foldPermuteOfIntrinsic(Instruction &I) {
3418 Value *V0;
3419 ArrayRef<int> Mask;
3420 if (!match(V: &I, P: m_Shuffle(v1: m_Value(V&: V0), v2: m_Undef(), mask: m_Mask(Mask))))
3421 return false;
3422
3423 auto *II0 = dyn_cast<IntrinsicInst>(Val: V0);
3424 if (!II0)
3425 return false;
3426
3427 auto *ShuffleDstTy = dyn_cast<FixedVectorType>(Val: I.getType());
3428 auto *IntrinsicSrcTy = dyn_cast<FixedVectorType>(Val: II0->getType());
3429 if (!ShuffleDstTy || !IntrinsicSrcTy)
3430 return false;
3431
3432 // Validate it's a pure permute, mask should only reference the first vector
3433 unsigned NumSrcElts = IntrinsicSrcTy->getNumElements();
3434 if (any_of(Range&: Mask, P: [NumSrcElts](int M) { return M >= (int)NumSrcElts; }))
3435 return false;
3436
3437 Intrinsic::ID IID = II0->getIntrinsicID();
3438 if (!isTriviallyVectorizable(ID: IID))
3439 return false;
3440
3441 // Cost analysis
3442 InstructionCost IntrinsicCost =
3443 TTI.getIntrinsicInstrCost(ICA: IntrinsicCostAttributes(IID, *II0), CostKind);
3444 InstructionCost OldCost =
3445 IntrinsicCost +
3446 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc, DstTy: ShuffleDstTy,
3447 SrcTy: IntrinsicSrcTy, Mask, CostKind, Index: 0, SubTp: nullptr, Args: {V0}, CxtI: &I);
3448
3449 SmallVector<Type *> NewArgsTy;
3450 InstructionCost NewCost = 0;
3451 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3452 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3453 NewArgsTy.push_back(Elt: II0->getArgOperand(i: I)->getType());
3454 } else {
3455 auto *VecTy = cast<FixedVectorType>(Val: II0->getArgOperand(i: I)->getType());
3456 auto *ArgTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
3457 NumElts: ShuffleDstTy->getNumElements());
3458 NewArgsTy.push_back(Elt: ArgTy);
3459 NewCost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
3460 DstTy: ArgTy, SrcTy: VecTy, Mask, CostKind, Index: 0, SubTp: nullptr,
3461 Args: {II0->getArgOperand(i: I)});
3462 }
3463 }
3464 IntrinsicCostAttributes NewAttr(IID, ShuffleDstTy, NewArgsTy);
3465 NewCost += TTI.getIntrinsicInstrCost(ICA: NewAttr, CostKind);
3466
3467 // If the intrinsic has multiple uses, we need to account for the cost of
3468 // keeping the original intrinsic around.
3469 if (!II0->hasOneUse())
3470 NewCost += IntrinsicCost;
3471
3472 LLVM_DEBUG(dbgs() << "Found a permute of intrinsic: " << I << "\n OldCost: "
3473 << OldCost << " vs NewCost: " << NewCost << "\n");
3474
3475 if (NewCost > OldCost)
3476 return false;
3477
3478 // Transform
3479 SmallVector<Value *> NewArgs;
3480 for (unsigned I = 0, E = II0->arg_size(); I != E; ++I) {
3481 if (isVectorIntrinsicWithScalarOpAtArg(ID: IID, ScalarOpdIdx: I, TTI: &TTI)) {
3482 NewArgs.push_back(Elt: II0->getArgOperand(i: I));
3483 } else {
3484 Value *Shuf = Builder.CreateShuffleVector(V: II0->getArgOperand(i: I), Mask);
3485 NewArgs.push_back(Elt: Shuf);
3486 Worklist.pushValue(V: Shuf);
3487 }
3488 }
3489
3490 Value *NewIntrinsic = Builder.CreateIntrinsic(RetTy: ShuffleDstTy, ID: IID, Args: NewArgs);
3491
3492 if (auto *NewInst = dyn_cast<Instruction>(Val: NewIntrinsic))
3493 NewInst->copyIRFlags(V: II0);
3494
3495 replaceValue(Old&: I, New&: *NewIntrinsic);
3496 return true;
3497}
3498
3499using InstLane = std::pair<Value *, int>;
3500
3501static InstLane lookThroughShuffles(Value *V, int Lane) {
3502 while (auto *SV = dyn_cast<ShuffleVectorInst>(Val: V)) {
3503 unsigned NumElts =
3504 cast<FixedVectorType>(Val: SV->getOperand(i_nocapture: 0)->getType())->getNumElements();
3505 int M = SV->getMaskValue(Elt: Lane);
3506 if (M < 0)
3507 return {nullptr, PoisonMaskElem};
3508 if (static_cast<unsigned>(M) < NumElts) {
3509 V = SV->getOperand(i_nocapture: 0);
3510 Lane = M;
3511 } else {
3512 V = SV->getOperand(i_nocapture: 1);
3513 Lane = M - NumElts;
3514 }
3515 }
3516 return InstLane{V, Lane};
3517}
3518
3519static SmallVector<InstLane>
3520generateInstLaneVectorFromOperand(ArrayRef<InstLane> Item, int Op) {
3521 SmallVector<InstLane> NItem;
3522 for (InstLane IL : Item) {
3523 auto [U, Lane] = IL;
3524 InstLane OpLane =
3525 U ? lookThroughShuffles(V: cast<Instruction>(Val: U)->getOperand(i: Op), Lane)
3526 : InstLane{nullptr, PoisonMaskElem};
3527 NItem.emplace_back(Args&: OpLane);
3528 }
3529 return NItem;
3530}
3531
3532/// Detect concat of multiple values into a vector
3533static bool isFreeConcat(ArrayRef<InstLane> Item, TTI::TargetCostKind CostKind,
3534 const TargetTransformInfo &TTI) {
3535 auto *Ty = cast<FixedVectorType>(Val: Item.front().first->getType());
3536 unsigned NumElts = Ty->getNumElements();
3537 if (Item.size() == NumElts || NumElts == 1 || Item.size() % NumElts != 0)
3538 return false;
3539
3540 // Check that the concat is free, usually meaning that the type will be split
3541 // during legalization.
3542 SmallVector<int, 16> ConcatMask(NumElts * 2);
3543 std::iota(first: ConcatMask.begin(), last: ConcatMask.end(), value: 0);
3544 if (TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc,
3545 DstTy: FixedVectorType::get(ElementType: Ty->getScalarType(), NumElts: NumElts * 2),
3546 SrcTy: Ty, Mask: ConcatMask, CostKind) != 0)
3547 return false;
3548
3549 unsigned NumSlices = Item.size() / NumElts;
3550 // Currently we generate a tree of shuffles for the concats, which limits us
3551 // to a power2.
3552 if (!isPowerOf2_32(Value: NumSlices))
3553 return false;
3554 for (unsigned Slice = 0; Slice < NumSlices; ++Slice) {
3555 Value *SliceV = Item[Slice * NumElts].first;
3556 if (!SliceV || SliceV->getType() != Ty)
3557 return false;
3558 for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
3559 auto [V, Lane] = Item[Slice * NumElts + Elt];
3560 if (Lane != static_cast<int>(Elt) || SliceV != V)
3561 return false;
3562 }
3563 }
3564 return true;
3565}
3566
3567static Value *
3568generateNewInstTree(ArrayRef<InstLane> Item, Use *From,
3569 const DenseSet<std::pair<Value *, Use *>> &IdentityLeafs,
3570 const DenseSet<std::pair<Value *, Use *>> &SplatLeafs,
3571 const DenseSet<std::pair<Value *, Use *>> &ConcatLeafs,
3572 IRBuilderBase &Builder, const TargetTransformInfo *TTI) {
3573 auto [FrontV, FrontLane] = Item.front();
3574
3575 if (IdentityLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3576 return FrontV;
3577 }
3578 if (SplatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3579 SmallVector<int, 16> Mask(Item.size(), FrontLane);
3580 return Builder.CreateShuffleVector(V: FrontV, Mask);
3581 }
3582 if (ConcatLeafs.contains(V: std::make_pair(x&: FrontV, y&: From))) {
3583 unsigned NumElts =
3584 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements();
3585 SmallVector<Value *> Values(Item.size() / NumElts, nullptr);
3586 for (unsigned S = 0; S < Values.size(); ++S)
3587 Values[S] = Item[S * NumElts].first;
3588
3589 while (Values.size() > 1) {
3590 NumElts *= 2;
3591 SmallVector<int, 16> Mask(NumElts, 0);
3592 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
3593 SmallVector<Value *> NewValues(Values.size() / 2, nullptr);
3594 for (unsigned S = 0; S < NewValues.size(); ++S)
3595 NewValues[S] =
3596 Builder.CreateShuffleVector(V1: Values[S * 2], V2: Values[S * 2 + 1], Mask);
3597 Values = NewValues;
3598 }
3599 return Values[0];
3600 }
3601
3602 auto *I = cast<Instruction>(Val: FrontV);
3603
3604 // Handle vector bitcasts that change element count. We cannot use
3605 // generateInstLaneVectorFromOperand for these because the lane indices
3606 // don't map 1:1 through the bitcast.
3607 if (auto *BitCast = dyn_cast<BitCastInst>(Val: I)) {
3608 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3609 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3610 if (BCDstTy && BCSrcTy &&
3611 BCDstTy->getElementCount() != BCSrcTy->getElementCount()) {
3612 unsigned DstElts = BCDstTy->getNumElements();
3613 unsigned SrcElts = BCSrcTy->getNumElements();
3614 SmallVector<InstLane> NewItem;
3615 if (DstElts > SrcElts) {
3616 // Widening: compress operand Item.
3617 unsigned R = DstElts / SrcElts;
3618 if (Item.size() % R != 0)
3619 return nullptr;
3620 for (unsigned Idx = 0, E = Item.size(); Idx < E; Idx += R) {
3621 auto [V, Lane] = Item[Idx];
3622 if (!V) {
3623 NewItem.push_back(Elt: {nullptr, PoisonMaskElem});
3624 continue;
3625 }
3626 NewItem.push_back(
3627 Elt: lookThroughShuffles(V: cast<Operator>(Val: V)->getOperand(i: 0), Lane: Lane / R));
3628 }
3629 } else {
3630 // Narrowing: expand operand Item.
3631 unsigned R = SrcElts / DstElts;
3632 for (auto [V, Lane] : Item) {
3633 if (!V) {
3634 NewItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
3635 continue;
3636 }
3637 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
3638 for (unsigned J = 0; J < R; ++J)
3639 NewItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
3640 }
3641 }
3642 Value *Op = generateNewInstTree(Item: NewItem, From: &BitCast->getOperandUse(i: 0),
3643 IdentityLeafs, SplatLeafs, ConcatLeafs,
3644 Builder, TTI);
3645 return Builder.CreateBitCast(
3646 V: Op, DestTy: FixedVectorType::get(ElementType: BCDstTy->getScalarType(), NumElts: Item.size()));
3647 }
3648 }
3649 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3650 unsigned NumOps = I->getNumOperands() - (II ? 1 : 0);
3651 SmallVector<Value *> Ops(NumOps);
3652 for (unsigned Idx = 0; Idx < NumOps; Idx++) {
3653 if (II &&
3654 isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Idx, TTI)) {
3655 Ops[Idx] = II->getOperand(i_nocapture: Idx);
3656 continue;
3657 }
3658 Ops[Idx] = generateNewInstTree(Item: generateInstLaneVectorFromOperand(Item, Op: Idx),
3659 From: &I->getOperandUse(i: Idx), IdentityLeafs,
3660 SplatLeafs, ConcatLeafs, Builder, TTI);
3661 }
3662
3663 SmallVector<Value *, 8> ValueList;
3664 for (const auto &Lane : Item)
3665 if (Lane.first)
3666 ValueList.push_back(Elt: Lane.first);
3667
3668 Type *DstTy =
3669 FixedVectorType::get(ElementType: I->getType()->getScalarType(), NumElts: Item.size());
3670 if (auto *BI = dyn_cast<BinaryOperator>(Val: I)) {
3671 auto *Value = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)BI->getOpcode(),
3672 LHS: Ops[0], RHS: Ops[1]);
3673 propagateIRFlags(I: Value, VL: ValueList);
3674 return Value;
3675 }
3676 if (auto *CI = dyn_cast<CmpInst>(Val: I)) {
3677 auto *Value = Builder.CreateCmp(Pred: CI->getPredicate(), LHS: Ops[0], RHS: Ops[1]);
3678 propagateIRFlags(I: Value, VL: ValueList);
3679 return Value;
3680 }
3681 if (auto *SI = dyn_cast<SelectInst>(Val: I)) {
3682 auto *Value = Builder.CreateSelect(C: Ops[0], True: Ops[1], False: Ops[2], Name: "", MDFrom: SI);
3683 propagateIRFlags(I: Value, VL: ValueList);
3684 return Value;
3685 }
3686 if (auto *CI = dyn_cast<CastInst>(Val: I)) {
3687 auto *Value = Builder.CreateCast(Op: CI->getOpcode(), V: Ops[0], DestTy: DstTy);
3688 propagateIRFlags(I: Value, VL: ValueList);
3689 return Value;
3690 }
3691 if (II) {
3692 auto *Value = Builder.CreateIntrinsic(RetTy: DstTy, ID: II->getIntrinsicID(), Args: Ops);
3693 propagateIRFlags(I: Value, VL: ValueList);
3694 return Value;
3695 }
3696 assert(isa<UnaryInstruction>(I) && "Unexpected instruction type in Generate");
3697 auto *Value =
3698 Builder.CreateUnOp(Opc: (Instruction::UnaryOps)I->getOpcode(), V: Ops[0]);
3699 propagateIRFlags(I: Value, VL: ValueList);
3700 return Value;
3701}
3702
3703// Starting from a shuffle, look up through operands tracking the shuffled index
3704// of each lane. If we can simplify away the shuffles to identities then
3705// do so.
3706bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
3707 auto *Ty = dyn_cast<FixedVectorType>(Val: I.getType());
3708 if (!Ty || I.use_empty())
3709 return false;
3710
3711 SmallVector<InstLane> Start(Ty->getNumElements());
3712 for (unsigned M = 0, E = Ty->getNumElements(); M < E; ++M)
3713 Start[M] = lookThroughShuffles(V: &I, Lane: M);
3714
3715 SmallVector<std::pair<SmallVector<InstLane>, Use *>> Worklist;
3716 Worklist.push_back(Elt: std::make_pair(x&: Start, y: &*I.use_begin()));
3717 DenseSet<std::pair<Value *, Use *>> IdentityLeafs, SplatLeafs, ConcatLeafs;
3718 unsigned NumVisited = 0;
3719 bool TraversedElCountChangingBitcast = false;
3720
3721 while (!Worklist.empty()) {
3722 if (++NumVisited > MaxInstrsToScan)
3723 return false;
3724
3725 auto ItemFrom = Worklist.pop_back_val();
3726 auto Item = ItemFrom.first;
3727 auto From = ItemFrom.second;
3728 auto [FrontV, FrontLane] = Item.front();
3729
3730 // If we found an undef first lane then bail out to keep things simple.
3731 if (!FrontV)
3732 return false;
3733
3734 // Helper to peek through bitcasts to the same value.
3735 auto IsEquiv = [&](Value *X, Value *Y) {
3736 return X->getType() == Y->getType() &&
3737 peekThroughBitcasts(V: X) == peekThroughBitcasts(V: Y);
3738 };
3739
3740 // Look for an identity value.
3741 if (FrontLane == 0 &&
3742 cast<FixedVectorType>(Val: FrontV->getType())->getNumElements() ==
3743 Item.size() &&
3744 all_of(Range: drop_begin(RangeOrContainer: enumerate(First&: Item)), P: [IsEquiv, Item](const auto &E) {
3745 Value *FrontV = Item.front().first;
3746 return !E.value().first || (IsEquiv(E.value().first, FrontV) &&
3747 E.value().second == (int)E.index());
3748 })) {
3749 IdentityLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3750 continue;
3751 }
3752 // Look for constants, for the moment only supporting constant splats.
3753 if (auto *C = dyn_cast<Constant>(Val: FrontV);
3754 C && C->getSplatValue() &&
3755 all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3756 Value *FrontV = Item.front().first;
3757 Value *V = IL.first;
3758 return !V || (isa<Constant>(Val: V) &&
3759 cast<Constant>(Val: V)->getSplatValue() ==
3760 cast<Constant>(Val: FrontV)->getSplatValue());
3761 })) {
3762 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3763 continue;
3764 }
3765 // Look for a splat value.
3766 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item](InstLane &IL) {
3767 auto [FrontV, FrontLane] = Item.front();
3768 auto [V, Lane] = IL;
3769 return !V || (V == FrontV && Lane == FrontLane);
3770 })) {
3771 SplatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3772 continue;
3773 }
3774
3775 // We need each element to be the same type of value, and check that each
3776 // element has a single use.
3777 auto CheckLaneIsEquivalentToFirst = [Item](InstLane IL) {
3778 Value *FrontV = Item.front().first;
3779 if (!IL.first)
3780 return true;
3781 Value *V = IL.first;
3782 if (auto *I = dyn_cast<Instruction>(Val: V); I && !I->hasOneUser())
3783 return false;
3784 if (V->getValueID() != FrontV->getValueID())
3785 return false;
3786 if (auto *CI = dyn_cast<CmpInst>(Val: V))
3787 if (CI->getPredicate() != cast<CmpInst>(Val: FrontV)->getPredicate())
3788 return false;
3789 if (auto *CI = dyn_cast<CastInst>(Val: V))
3790 if (CI->getSrcTy()->getScalarType() !=
3791 cast<CastInst>(Val: FrontV)->getSrcTy()->getScalarType())
3792 return false;
3793 if (auto *SI = dyn_cast<SelectInst>(Val: V))
3794 if (!isa<VectorType>(Val: SI->getOperand(i_nocapture: 0)->getType()) ||
3795 SI->getOperand(i_nocapture: 0)->getType() !=
3796 cast<SelectInst>(Val: FrontV)->getOperand(i_nocapture: 0)->getType())
3797 return false;
3798 if (isa<CallInst>(Val: V) && !isa<IntrinsicInst>(Val: V))
3799 return false;
3800 auto *II = dyn_cast<IntrinsicInst>(Val: V);
3801 return !II || (isa<IntrinsicInst>(Val: FrontV) &&
3802 II->getIntrinsicID() ==
3803 cast<IntrinsicInst>(Val: FrontV)->getIntrinsicID() &&
3804 !II->hasOperandBundles());
3805 };
3806 if (all_of(Range: drop_begin(RangeOrContainer&: Item), P: CheckLaneIsEquivalentToFirst)) {
3807 // Check the operator is one that we support.
3808 if (isa<BinaryOperator, CmpInst>(Val: FrontV)) {
3809 // We exclude div/rem in case they hit UB from poison lanes.
3810 if (auto *BO = dyn_cast<BinaryOperator>(Val: FrontV);
3811 BO && BO->isIntDivRem())
3812 return false;
3813 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3814 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3815 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
3816 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 1));
3817 continue;
3818 } else if (isa<UnaryOperator, TruncInst, ZExtInst, SExtInst, FPToSIInst,
3819 FPToUIInst, SIToFPInst, UIToFPInst>(Val: FrontV)) {
3820 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3821 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: 0));
3822 continue;
3823 } else if (auto *BitCast = dyn_cast<BitCastInst>(Val: FrontV)) {
3824 auto *BCDstTy = dyn_cast<FixedVectorType>(Val: BitCast->getDestTy());
3825 auto *BCSrcTy = dyn_cast<FixedVectorType>(Val: BitCast->getSrcTy());
3826 if (BCDstTy && BCSrcTy) {
3827 ElementCount DstEC = BCDstTy->getElementCount();
3828 ElementCount SrcEC = BCSrcTy->getElementCount();
3829 if (DstEC == SrcEC) {
3830 // Same element count - simple pass-through.
3831 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3832 Args: &BitCast->getOperandUse(i: 0));
3833 continue;
3834 }
3835 unsigned DstElts = DstEC.getFixedValue();
3836 unsigned SrcElts = SrcEC.getFixedValue();
3837 if (DstElts > SrcElts && DstElts % SrcElts == 0) {
3838 // Widening bitcast (e.g. <2 x i32> -> <4 x i16>). Compress
3839 // consecutive groups of R destination lanes into one source
3840 // lane.
3841 unsigned R = DstElts / SrcElts;
3842 SmallVector<InstLane> NItem;
3843 bool Valid = Item.size() % R == 0;
3844 for (unsigned Idx = 0, E = Item.size(); Valid && Idx < E;
3845 Idx += R) {
3846 auto [V0, L0] = Item[Idx];
3847 if (!V0) {
3848 if (any_of(Range: ArrayRef(Item).slice(N: Idx + 1, M: R - 1),
3849 P: [](InstLane IL) { return IL.first != nullptr; })) {
3850 Valid = false;
3851 break;
3852 }
3853 NItem.push_back(Elt: {nullptr, PoisonMaskElem});
3854 continue;
3855 }
3856 if (L0 % R != 0) {
3857 Valid = false;
3858 break;
3859 }
3860 for (unsigned J = 1; J < R; ++J) {
3861 auto [VJ, LJ] = Item[Idx + J];
3862 if (!VJ || VJ != V0 || LJ != L0 + (int)J) {
3863 Valid = false;
3864 break;
3865 }
3866 }
3867 if (!Valid)
3868 break;
3869 NItem.push_back(Elt: lookThroughShuffles(
3870 V: cast<Operator>(Val: V0)->getOperand(i: 0), Lane: L0 / R));
3871 }
3872 if (Valid) {
3873 TraversedElCountChangingBitcast = true;
3874 Worklist.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
3875 continue;
3876 }
3877 } else if (SrcElts > DstElts && SrcElts % DstElts == 0) {
3878 // Narrowing bitcast (e.g. <4 x i16> -> <2 x i32>). Expand
3879 // each destination lane into R source lanes.
3880 unsigned R = SrcElts / DstElts;
3881 SmallVector<InstLane> NItem;
3882 for (auto [V, Lane] : Item) {
3883 if (!V) {
3884 NItem.append(NumInputs: R, Elt: {nullptr, PoisonMaskElem});
3885 continue;
3886 }
3887 Value *Op = cast<Operator>(Val: V)->getOperand(i: 0);
3888 for (unsigned J = 0; J < R; ++J)
3889 NItem.push_back(Elt: lookThroughShuffles(V: Op, Lane: Lane * R + J));
3890 }
3891 TraversedElCountChangingBitcast = true;
3892 Worklist.emplace_back(Args&: NItem, Args: &BitCast->getOperandUse(i: 0));
3893 continue;
3894 }
3895 }
3896 } else if (auto *Sel = dyn_cast<SelectInst>(Val: FrontV)) {
3897 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 0),
3898 Args: &Sel->getOperandUse(i: 0));
3899 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 1),
3900 Args: &Sel->getOperandUse(i: 1));
3901 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op: 2),
3902 Args: &Sel->getOperandUse(i: 2));
3903 continue;
3904 } else if (auto *II = dyn_cast<IntrinsicInst>(Val: FrontV);
3905 II && isTriviallyVectorizable(ID: II->getIntrinsicID()) &&
3906 !II->hasOperandBundles()) {
3907 for (unsigned Op = 0, E = II->getNumOperands() - 1; Op < E; Op++) {
3908 if (isVectorIntrinsicWithScalarOpAtArg(ID: II->getIntrinsicID(), ScalarOpdIdx: Op,
3909 TTI: &TTI)) {
3910 if (!all_of(Range: drop_begin(RangeOrContainer&: Item), P: [Item, Op](InstLane &IL) {
3911 Value *FrontV = Item.front().first;
3912 Value *V = IL.first;
3913 return !V || (cast<Instruction>(Val: V)->getOperand(i: Op) ==
3914 cast<Instruction>(Val: FrontV)->getOperand(i: Op));
3915 }))
3916 return false;
3917 continue;
3918 }
3919 Worklist.emplace_back(Args: generateInstLaneVectorFromOperand(Item, Op),
3920 Args: &cast<Instruction>(Val: FrontV)->getOperandUse(i: Op));
3921 }
3922 continue;
3923 }
3924 }
3925
3926 if (isFreeConcat(Item, CostKind, TTI)) {
3927 ConcatLeafs.insert(V: std::make_pair(x&: FrontV, y&: From));
3928 continue;
3929 }
3930
3931 return false;
3932 }
3933
3934 if (NumVisited <= 1)
3935 return false;
3936
3937 // If the only non-leaf node traversed was a single bitcast that changes
3938 // element count, the fold would just commute the bitcast and shuffle.
3939 // foldBitcastShuffle does the reverse transform, causing an infinite loop.
3940 if (NumVisited == 2 && TraversedElCountChangingBitcast)
3941 return false;
3942
3943 LLVM_DEBUG(dbgs() << "Found a superfluous identity shuffle: " << I << "\n");
3944
3945 // If we got this far, we know the shuffles are superfluous and can be
3946 // removed. Scan through again and generate the new tree of instructions.
3947 Builder.SetInsertPoint(&I);
3948 Value *V = generateNewInstTree(Item: Start, From: &*I.use_begin(), IdentityLeafs,
3949 SplatLeafs, ConcatLeafs, Builder, TTI: &TTI);
3950 replaceValue(Old&: I, New&: *V);
3951 return true;
3952}
3953
3954/// Given a commutative reduction, the order of the input lanes does not alter
3955/// the results. We can use this to remove certain shuffles feeding the
3956/// reduction, removing the need to shuffle at all.
3957bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
3958 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
3959 if (!II)
3960 return false;
3961 switch (II->getIntrinsicID()) {
3962 case Intrinsic::vector_reduce_add:
3963 case Intrinsic::vector_reduce_mul:
3964 case Intrinsic::vector_reduce_and:
3965 case Intrinsic::vector_reduce_or:
3966 case Intrinsic::vector_reduce_xor:
3967 case Intrinsic::vector_reduce_smin:
3968 case Intrinsic::vector_reduce_smax:
3969 case Intrinsic::vector_reduce_umin:
3970 case Intrinsic::vector_reduce_umax:
3971 break;
3972 default:
3973 return false;
3974 }
3975
3976 // Find all the inputs when looking through operations that do not alter the
3977 // lane order (binops, for example). Currently we look for a single shuffle,
3978 // and can ignore splat values.
3979 std::queue<Value *> Worklist;
3980 SmallPtrSet<Value *, 4> Visited;
3981 ShuffleVectorInst *Shuffle = nullptr;
3982 if (auto *Op = dyn_cast<Instruction>(Val: I.getOperand(i: 0)))
3983 Worklist.push(x: Op);
3984
3985 while (!Worklist.empty()) {
3986 Value *CV = Worklist.front();
3987 Worklist.pop();
3988 if (Visited.contains(Ptr: CV))
3989 continue;
3990
3991 // Splats don't change the order, so can be safely ignored.
3992 if (isSplatValue(V: CV))
3993 continue;
3994
3995 Visited.insert(Ptr: CV);
3996
3997 if (auto *CI = dyn_cast<Instruction>(Val: CV)) {
3998 if (CI->isBinaryOp()) {
3999 for (auto *Op : CI->operand_values())
4000 Worklist.push(x: Op);
4001 continue;
4002 } else if (auto *SV = dyn_cast<ShuffleVectorInst>(Val: CI)) {
4003 if (Shuffle && Shuffle != SV)
4004 return false;
4005 Shuffle = SV;
4006 continue;
4007 }
4008 }
4009
4010 // Anything else is currently an unknown node.
4011 return false;
4012 }
4013
4014 if (!Shuffle)
4015 return false;
4016
4017 // Check all uses of the binary ops and shuffles are also included in the
4018 // lane-invariant operations (Visited should be the list of lanewise
4019 // instructions, including the shuffle that we found).
4020 for (auto *V : Visited)
4021 for (auto *U : V->users())
4022 if (!Visited.contains(Ptr: U) && U != &I)
4023 return false;
4024
4025 FixedVectorType *VecType =
4026 dyn_cast<FixedVectorType>(Val: II->getOperand(i_nocapture: 0)->getType());
4027 if (!VecType)
4028 return false;
4029 FixedVectorType *ShuffleInputType =
4030 dyn_cast<FixedVectorType>(Val: Shuffle->getOperand(i_nocapture: 0)->getType());
4031 if (!ShuffleInputType)
4032 return false;
4033 unsigned NumInputElts = ShuffleInputType->getNumElements();
4034
4035 // Find the mask from sorting the lanes into order. This is most likely to
4036 // become a identity or concat mask. Undef elements are pushed to the end.
4037 SmallVector<int> ConcatMask;
4038 Shuffle->getShuffleMask(Result&: ConcatMask);
4039 sort(C&: ConcatMask, Comp: [](int X, int Y) { return (unsigned)X < (unsigned)Y; });
4040 bool UsesSecondVec =
4041 any_of(Range&: ConcatMask, P: [&](int M) { return M >= (int)NumInputElts; });
4042
4043 InstructionCost OldCost = TTI.getShuffleCost(
4044 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4045 SrcTy: ShuffleInputType, Mask: Shuffle->getShuffleMask(), CostKind);
4046 InstructionCost NewCost = TTI.getShuffleCost(
4047 Kind: UsesSecondVec ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc, DstTy: VecType,
4048 SrcTy: ShuffleInputType, Mask: ConcatMask, CostKind);
4049
4050 LLVM_DEBUG(dbgs() << "Found a reduction feeding from a shuffle: " << *Shuffle
4051 << "\n");
4052 LLVM_DEBUG(dbgs() << " OldCost: " << OldCost << " vs NewCost: " << NewCost
4053 << "\n");
4054 bool MadeChanges = false;
4055 if (NewCost < OldCost) {
4056 Builder.SetInsertPoint(Shuffle);
4057 Value *NewShuffle = Builder.CreateShuffleVector(
4058 V1: Shuffle->getOperand(i_nocapture: 0), V2: Shuffle->getOperand(i_nocapture: 1), Mask: ConcatMask);
4059 LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
4060 replaceValue(Old&: *Shuffle, New&: *NewShuffle);
4061 return true;
4062 }
4063
4064 // See if we can re-use foldSelectShuffle, getting it to reduce the size of
4065 // the shuffle into a nicer order, as it can ignore the order of the shuffles.
4066 MadeChanges |= foldSelectShuffle(I&: *Shuffle, FromReduction: true);
4067 return MadeChanges;
4068}
4069
4070/// Try to fold a chain of shuffles and ops feeding extractelement(..., 0)
4071/// into llvm.vector.reduce.*, by tracking which lanes contribute to the
4072/// extracted lane and reducing the widest vector whose lanes each contribute
4073/// once.
4074///
4075/// For example:
4076///
4077/// %lo = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 0, i32 1>
4078/// %hi = shufflevector <4 x i32> %a, poison, <2 x i32> <i32 2, i32 3>
4079/// %s = add <2 x i32> %lo, %hi
4080/// %sh = shufflevector <2 x i32> %s, poison, <2 x i32> <i32 1, i32 poison>
4081/// %r = add <2 x i32> %s, %sh
4082/// %e = extractelement <2 x i32> %r, i64 0
4083///
4084/// transforms to:
4085///
4086/// %e = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
4087bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
4088 Value *VecOpEE;
4089 if (!match(V: &I, P: m_ExtractElt(Val: m_Value(V&: VecOpEE), Idx: m_Zero())))
4090 return false;
4091
4092 auto *FVT = dyn_cast<FixedVectorType>(Val: VecOpEE->getType());
4093 if (!FVT)
4094 return false;
4095
4096 if (FVT->getNumElements() < 2)
4097 return false;
4098
4099 std::optional<Instruction::BinaryOps> CommonBinOp;
4100 std::optional<Intrinsic::ID> CommonCallOp;
4101
4102 if (auto *BO = dyn_cast<BinaryOperator>(Val: VecOpEE)) {
4103 if (!getReductionForBinop(Opc: BO->getOpcode()))
4104 return false;
4105 CommonBinOp = BO->getOpcode();
4106 } else if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: VecOpEE)) {
4107 CommonCallOp = MMI->getIntrinsicID();
4108 } else {
4109 return false;
4110 }
4111
4112 // For floating-point reductions, track FMF intersection across all binops.
4113 FastMathFlags CommonFMF;
4114 bool IsFloatReduction = false;
4115
4116 // A chain node is one we walk through, either a matching-opcode binop/min-max
4117 // or a single-source shuffle. Anything else is a leaf source.
4118 auto IsChainNode = [&](Value *V) {
4119 if (auto *BO = dyn_cast<BinaryOperator>(Val: V))
4120 return CommonBinOp && BO->getOpcode() == *CommonBinOp;
4121 if (auto *MMI = dyn_cast<MinMaxIntrinsic>(Val: V))
4122 return CommonCallOp && MMI->getIntrinsicID() == *CommonCallOp;
4123 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V))
4124 return isa<PoisonValue>(Val: SVI->getOperand(i_nocapture: 1));
4125 return false;
4126 };
4127
4128 // Collect the chain, building Nodes in postorder. Bail if the chain is empty
4129 // or exceeds MaxChainNodes.
4130 constexpr unsigned MaxChainNodes = 32;
4131 SmallSetVector<Value *, 16> Nodes;
4132 SmallSetVector<Value *, 4> Sources;
4133 unsigned NumVisited = 0;
4134 auto AddSource = [&](Value *V) {
4135 if (!isa<FixedVectorType>(Val: V->getType()))
4136 return false;
4137 Sources.insert(X: V);
4138 return true;
4139 };
4140 auto Walk = [&](Value *V, auto &&Walk) -> bool {
4141 if (Nodes.contains(key: V) || Sources.contains(key: V))
4142 return true;
4143 if (++NumVisited > MaxChainNodes)
4144 return false;
4145 if (!IsChainNode(V))
4146 return AddSource(V);
4147 // Chain shuffles always have poison as op1, so only op0 matters.
4148 auto *U = cast<Instruction>(Val: V);
4149 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4150 for (unsigned I = 0; I != NumOps; ++I)
4151 if (!Walk(U->getOperand(i: I), Walk))
4152 return false;
4153 if (isa<ShuffleVectorInst>(Val: U) || Nodes.contains(key: U->getOperand(i: 0)) ||
4154 Nodes.contains(key: U->getOperand(i: 1))) {
4155 Nodes.insert(X: V);
4156 return true;
4157 }
4158 // Both operands are leaves so treat this binop as a source rather than
4159 // walking into it.
4160 return AddSource(V);
4161 };
4162 if (!Walk(VecOpEE, Walk) || Nodes.empty())
4163 return false;
4164
4165 bool IsIdempotent =
4166 CommonCallOp || (CommonBinOp && Instruction::isIdempotent(Opcode: *CommonBinOp));
4167
4168 // For FP reductions, require reassoc on every binop and collect FMF.
4169 for (Value *V : Nodes) {
4170 auto *BinOp = dyn_cast<BinaryOperator>(Val: V);
4171 if (!BinOp || !BinOp->getType()->isFPOrFPVectorTy())
4172 continue;
4173 if (!BinOp->hasAllowReassoc())
4174 return false;
4175 if (!IsFloatReduction) {
4176 CommonFMF = BinOp->getFastMathFlags();
4177 IsFloatReduction = true;
4178 } else {
4179 CommonFMF &= BinOp->getFastMathFlags();
4180 }
4181 }
4182
4183 // Top-down demanded elements. For each chain value, track which lanes feed
4184 // the extracted lane 0 and which feed it more than once. Reverse postorder
4185 // visits every use before its value. A binop forwards its demand to both
4186 // operands and a shuffle follows its mask back to the source lane.
4187 struct Demand {
4188 APInt Lanes;
4189 APInt Duplicates;
4190 };
4191 DenseMap<Value *, Demand> Demands;
4192 auto DemandOf = [&](Value *V) -> Demand & {
4193 unsigned N = cast<FixedVectorType>(Val: V->getType())->getNumElements();
4194 Demand &D = Demands[V];
4195 if (D.Lanes.getBitWidth() != N)
4196 D.Lanes = D.Duplicates = APInt::getZero(numBits: N);
4197 return D;
4198 };
4199 DemandOf(VecOpEE).Lanes.setBit(0);
4200 for (Value *V : reverse(C&: Nodes)) {
4201 Demand DV = Demands.lookup(Val: V);
4202 if (DV.Lanes.isZero())
4203 continue;
4204 if (auto *SVI = dyn_cast<ShuffleVectorInst>(Val: V)) {
4205 ArrayRef<int> Mask = SVI->getShuffleMask();
4206 Demand &DS = DemandOf(SVI->getOperand(i_nocapture: 0));
4207 for (unsigned I = 0, E = Mask.size(); I != E; ++I) {
4208 // Skip lanes that are undemanded or map to poison.
4209 if (!DV.Lanes[I] || Mask[I] < 0 ||
4210 (unsigned)Mask[I] >= DS.Lanes.getBitWidth())
4211 continue;
4212 if (DS.Lanes[Mask[I]] || DV.Duplicates[I])
4213 DS.Duplicates.setBit(Mask[I]);
4214 DS.Lanes.setBit(Mask[I]);
4215 }
4216 } else {
4217 auto *U = cast<User>(Val: V);
4218 for (Value *Op : {U->getOperand(i: 0), U->getOperand(i: 1)}) {
4219 Demand &DOp = DemandOf(Op);
4220 // Lanes demanded through more than one path accumulate in Duplicates.
4221 DOp.Duplicates |= DV.Duplicates | (DOp.Lanes & DV.Lanes);
4222 DOp.Lanes |= DV.Lanes;
4223 }
4224 }
4225 }
4226
4227 // Reducing V replaces the entire chain, so every contribution to the result
4228 // must flow through V. Reject if anything above V reads outside the chain.
4229 auto CoversChain = [&](Value *V) {
4230 SmallVector<Value *, 8> Worklist(1, VecOpEE);
4231 SmallPtrSet<Value *, 8> Seen;
4232 Seen.insert(Ptr: VecOpEE);
4233 while (!Worklist.empty()) {
4234 auto *U = cast<Instruction>(Val: Worklist.pop_back_val());
4235 unsigned NumOps = isa<ShuffleVectorInst>(Val: U) ? 1 : 2;
4236 for (unsigned I = 0; I != NumOps; ++I) {
4237 Value *Op = U->getOperand(i: I);
4238 if (Op == V || !Seen.insert(Ptr: Op).second)
4239 continue;
4240 if (!Nodes.contains(key: Op))
4241 return false;
4242 Worklist.push_back(Elt: Op);
4243 }
4244 }
4245 return true;
4246 };
4247
4248 // Reduce a single cleanly demanded source if there is one, otherwise the
4249 // deepest intermediate that covers the chain.
4250 struct ReductionCut {
4251 Value *Src;
4252 APInt Elts;
4253 };
4254 std::optional<ReductionCut> Cut;
4255 for (Value *S : Sources) {
4256 auto It = Demands.find(Val: S);
4257 if (It == Demands.end() || It->second.Lanes.isZero())
4258 continue;
4259 if (Cut || (!IsIdempotent && !It->second.Duplicates.isZero())) {
4260 Cut.reset();
4261 break;
4262 }
4263 Cut = ReductionCut{.Src: S, .Elts: It->second.Lanes};
4264 }
4265 if (!Cut) {
4266 for (Value *V : Nodes) {
4267 if (!isa<BinaryOperator>(Val: V) && !isa<MinMaxIntrinsic>(Val: V))
4268 continue;
4269 auto It = Demands.find(Val: V);
4270 if (It == Demands.end() || !It->second.Lanes.isAllOnes())
4271 continue;
4272 if (!IsIdempotent && !It->second.Duplicates.isZero())
4273 continue;
4274 if (!CoversChain(V))
4275 continue;
4276 Cut = ReductionCut{.Src: V, .Elts: It->second.Lanes};
4277 break;
4278 }
4279 }
4280 // Reducing one lane is just an extract and can refold forever.
4281 if (!Cut || Cut->Elts.popcount() < 2)
4282 return false;
4283
4284 Intrinsic::ID ReducedOp =
4285 (CommonCallOp ? getMinMaxReductionIntrinsicID(IID: *CommonCallOp)
4286 : getReductionForBinop(Opc: *CommonBinOp));
4287 if (!ReducedOp)
4288 return false;
4289
4290 InstructionCost OrigCost = 0;
4291 for (Value *V : Nodes)
4292 OrigCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4293
4294 auto *SrcVT = cast<FixedVectorType>(Val: Cut->Src->getType());
4295 bool IsPartialReduction = !Cut->Elts.isAllOnes();
4296 FixedVectorType *ReduceVecTy =
4297 IsPartialReduction
4298 ? FixedVectorType::get(ElementType: FVT->getElementType(), NumElts: Cut->Elts.popcount())
4299 : SrcVT;
4300
4301 SmallVector<int> ExtractMask;
4302 InstructionCost NewCost = 0;
4303 if (IsPartialReduction) {
4304 for (unsigned I = 0, E = Cut->Elts.getBitWidth(); I != E; ++I)
4305 if (Cut->Elts[I])
4306 ExtractMask.push_back(Elt: I);
4307 unsigned SubIdx = 0, SubLen;
4308 auto SK = Cut->Elts.isShiftedMask(MaskIdx&: SubIdx, MaskLen&: SubLen)
4309 ? TargetTransformInfo::SK_ExtractSubvector
4310 : TargetTransformInfo::SK_PermuteSingleSrc;
4311 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: ReduceVecTy, SrcTy: SrcVT, Mask: ExtractMask, CostKind,
4312 Index: SubIdx, SubTp: ReduceVecTy);
4313 }
4314
4315 IntrinsicCostAttributes ICA(
4316 ReducedOp, ReduceVecTy->getElementType(),
4317 IsFloatReduction
4318 ? SmallVector<Type *, 2>{ReduceVecTy->getElementType(), ReduceVecTy}
4319 : SmallVector<Type *, 2>{ReduceVecTy},
4320 IsFloatReduction ? CommonFMF : FastMathFlags());
4321 NewCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
4322
4323 LLVM_DEBUG(dbgs() << "Found reduction shuffle chain: " << I << "\n OldCost : "
4324 << OrigCost << " vs NewCost: " << NewCost << "\n");
4325
4326 if (!OrigCost.isValid() || !NewCost.isValid())
4327 return false;
4328
4329 if (VecOpEE->hasOneUse() ? (NewCost > OrigCost) : (NewCost >= OrigCost))
4330 return false;
4331
4332 Value *ReduceInput = Cut->Src;
4333 if (IsPartialReduction)
4334 ReduceInput = Builder.CreateShuffleVector(V: Cut->Src, Mask: ExtractMask);
4335
4336 Value *ReducedResult;
4337 if (IsFloatReduction) {
4338 Value *Identity = ConstantExpr::getBinOpIdentity(
4339 Opcode: *CommonBinOp, Ty: ReduceVecTy->getElementType(), /*AllowRHSConstant=*/false,
4340 NSZ: CommonFMF.noSignedZeros());
4341 ReducedResult = Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy},
4342 Args: {Identity, ReduceInput}, FMFSource: CommonFMF);
4343 } else {
4344 ReducedResult =
4345 Builder.CreateIntrinsic(ID: ReducedOp, OverloadTypes: {ReduceVecTy}, Args: {ReduceInput});
4346 }
4347 replaceValue(Old&: I, New&: *ReducedResult);
4348
4349 return true;
4350}
4351
4352/// Determine if its more efficient to fold:
4353/// reduce(trunc(x)) -> trunc(reduce(x)).
4354/// reduce(sext(x)) -> sext(reduce(x)).
4355/// reduce(zext(x)) -> zext(reduce(x)).
4356bool VectorCombine::foldCastFromReductions(Instruction &I) {
4357 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
4358 if (!II)
4359 return false;
4360
4361 bool TruncOnly = false;
4362 Intrinsic::ID IID = II->getIntrinsicID();
4363 switch (IID) {
4364 case Intrinsic::vector_reduce_add:
4365 case Intrinsic::vector_reduce_mul:
4366 TruncOnly = true;
4367 break;
4368 case Intrinsic::vector_reduce_and:
4369 case Intrinsic::vector_reduce_or:
4370 case Intrinsic::vector_reduce_xor:
4371 break;
4372 default:
4373 return false;
4374 }
4375
4376 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
4377 Value *ReductionSrc = I.getOperand(i: 0);
4378
4379 Value *Src;
4380 if (!match(V: ReductionSrc, P: m_OneUse(SubPattern: m_Trunc(Op: m_Value(V&: Src)))) &&
4381 (TruncOnly || !match(V: ReductionSrc, P: m_OneUse(SubPattern: m_ZExtOrSExt(Op: m_Value(V&: Src))))))
4382 return false;
4383
4384 auto CastOpc =
4385 (Instruction::CastOps)cast<Instruction>(Val: ReductionSrc)->getOpcode();
4386
4387 auto *SrcTy = cast<VectorType>(Val: Src->getType());
4388 auto *ReductionSrcTy = cast<VectorType>(Val: ReductionSrc->getType());
4389 Type *ResultTy = I.getType();
4390
4391 InstructionCost OldCost = TTI.getArithmeticReductionCost(
4392 Opcode: ReductionOpc, Ty: ReductionSrcTy, FMF: std::nullopt, CostKind);
4393 OldCost += TTI.getCastInstrCost(Opcode: CastOpc, Dst: ReductionSrcTy, Src: SrcTy,
4394 CCH: TTI::CastContextHint::None, CostKind,
4395 I: cast<CastInst>(Val: ReductionSrc));
4396 InstructionCost NewCost =
4397 TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: SrcTy, FMF: std::nullopt,
4398 CostKind) +
4399 TTI.getCastInstrCost(Opcode: CastOpc, Dst: ResultTy, Src: ReductionSrcTy->getScalarType(),
4400 CCH: TTI::CastContextHint::None, CostKind);
4401
4402 if (OldCost <= NewCost || !NewCost.isValid())
4403 return false;
4404
4405 Value *NewReduction = Builder.CreateIntrinsic(RetTy: SrcTy->getScalarType(),
4406 ID: II->getIntrinsicID(), Args: {Src});
4407 Value *NewCast = Builder.CreateCast(Op: CastOpc, V: NewReduction, DestTy: ResultTy);
4408 replaceValue(Old&: I, New&: *NewCast);
4409 return true;
4410}
4411
4412/// Fold:
4413/// icmp pred (reduce.{add,or,and,umax,umin}(signbit_extract(x))), C
4414/// into:
4415/// icmp sgt/slt (reduce.{or,umax,and,umin}(x)), -1/0
4416///
4417/// Sign-bit reductions produce values with known semantics:
4418/// - reduce.{or,umax}: 0 if no element is negative, 1 if any is
4419/// - reduce.{and,umin}: 1 if all elements are negative, 0 if any isn't
4420/// - reduce.add: count of negative elements (0 to NumElts)
4421///
4422/// Both lshr and ashr are supported:
4423/// - lshr produces 0 or 1, so reduce.add range is [0, N]
4424/// - ashr produces 0 or -1, so reduce.add range is [-N, 0]
4425///
4426/// The fold generalizes to multiple source vectors combined with the same
4427/// operation as the reduction. For example:
4428/// reduce.or(or(shr A, shr B)) conceptually extends the vector
4429/// For reduce.add, this changes the count to M*N where M is the number of
4430/// source vectors.
4431///
4432/// We transform to a direct sign check on the original vector using
4433/// reduce.{or,umax} or reduce.{and,umin}.
4434///
4435/// In spirit, it's similar to foldSignBitCheck in InstCombine.
4436bool VectorCombine::foldSignBitReductionCmp(Instruction &I) {
4437 CmpPredicate Pred;
4438 IntrinsicInst *ReduceOp;
4439 const APInt *CmpVal;
4440 if (!match(V: &I,
4441 P: m_ICmp(Pred, L: m_OneUse(SubPattern: m_AnyIntrinsic(I&: ReduceOp)), R: m_APInt(Res&: CmpVal))))
4442 return false;
4443
4444 Intrinsic::ID OrigIID = ReduceOp->getIntrinsicID();
4445 switch (OrigIID) {
4446 case Intrinsic::vector_reduce_or:
4447 case Intrinsic::vector_reduce_umax:
4448 case Intrinsic::vector_reduce_and:
4449 case Intrinsic::vector_reduce_umin:
4450 case Intrinsic::vector_reduce_add:
4451 break;
4452 default:
4453 return false;
4454 }
4455
4456 Value *ReductionSrc = ReduceOp->getArgOperand(i: 0);
4457 auto *VecTy = dyn_cast<FixedVectorType>(Val: ReductionSrc->getType());
4458 if (!VecTy)
4459 return false;
4460
4461 unsigned BitWidth = VecTy->getScalarSizeInBits();
4462 if (BitWidth == 1)
4463 return false;
4464
4465 unsigned NumElts = VecTy->getNumElements();
4466
4467 // Determine the expected tree opcode for multi-vector patterns.
4468 // The tree opcode must match the reduction's underlying operation.
4469 //
4470 // TODO: for pairs of equivalent operators, we should match both,
4471 // not only the most common.
4472 Instruction::BinaryOps TreeOpcode;
4473 switch (OrigIID) {
4474 case Intrinsic::vector_reduce_or:
4475 case Intrinsic::vector_reduce_umax:
4476 TreeOpcode = Instruction::Or;
4477 break;
4478 case Intrinsic::vector_reduce_and:
4479 case Intrinsic::vector_reduce_umin:
4480 TreeOpcode = Instruction::And;
4481 break;
4482 case Intrinsic::vector_reduce_add:
4483 TreeOpcode = Instruction::Add;
4484 break;
4485 default:
4486 llvm_unreachable("Unexpected intrinsic");
4487 }
4488
4489 // Collect sign-bit extraction leaves from an associative tree of TreeOpcode.
4490 // The tree conceptually extends the vector being reduced.
4491 SmallVector<Value *, 8> Worklist;
4492 SmallVector<Value *, 8> Sources; // Original vectors (X in shr X, BW-1)
4493 Worklist.push_back(Elt: ReductionSrc);
4494 std::optional<bool> IsAShr;
4495 constexpr unsigned MaxSources = 8;
4496
4497 // Calculate old cost: all shifts + tree ops + reduction
4498 InstructionCost OldCost = TTI.getInstructionCost(U: ReduceOp, CostKind);
4499
4500 while (!Worklist.empty() && Worklist.size() <= MaxSources &&
4501 Sources.size() <= MaxSources) {
4502 Value *V = Worklist.pop_back_val();
4503
4504 // Try to match sign-bit extraction: shr X, (bitwidth-1)
4505 Value *X;
4506 if (match(V, P: m_OneUse(SubPattern: m_Shr(L: m_Value(V&: X), R: m_SpecificInt(V: BitWidth - 1))))) {
4507 auto *Shr = cast<Instruction>(Val: V);
4508
4509 // All shifts must be the same type (all lshr or all ashr)
4510 bool ThisIsAShr = Shr->getOpcode() == Instruction::AShr;
4511 if (!IsAShr)
4512 IsAShr = ThisIsAShr;
4513 else if (*IsAShr != ThisIsAShr)
4514 return false;
4515
4516 Sources.push_back(Elt: X);
4517
4518 // As part of the fold, we remove all of the shifts, so we need to keep
4519 // track of their costs.
4520 OldCost += TTI.getInstructionCost(U: Shr, CostKind);
4521
4522 continue;
4523 }
4524
4525 // Try to extend through a tree node of the expected opcode
4526 Value *A, *B;
4527 if (!match(V, P: m_OneUse(SubPattern: m_BinOp(Opcode: TreeOpcode, L: m_Value(V&: A), R: m_Value(V&: B)))))
4528 return false;
4529
4530 // We are potentially replacing these operations as well, so we add them
4531 // to the costs.
4532 OldCost += TTI.getInstructionCost(U: cast<Instruction>(Val: V), CostKind);
4533
4534 Worklist.push_back(Elt: A);
4535 Worklist.push_back(Elt: B);
4536 }
4537
4538 // Must have at least one source and not exceed limit
4539 if (Sources.empty() || Sources.size() > MaxSources ||
4540 Worklist.size() > MaxSources || !IsAShr)
4541 return false;
4542
4543 unsigned NumSources = Sources.size();
4544
4545 // For reduce.add, the total count must fit as a signed integer.
4546 // Range is [0, M*N] for lshr or [-M*N, 0] for ashr.
4547 if (OrigIID == Intrinsic::vector_reduce_add &&
4548 !isIntN(N: BitWidth, x: NumSources * NumElts))
4549 return false;
4550
4551 // Compute the boundary value when all elements are negative:
4552 // - Per-element contribution: 1 for lshr, -1 for ashr
4553 // - For add: M*N (total elements across all sources); for others: just 1
4554 unsigned Count =
4555 (OrigIID == Intrinsic::vector_reduce_add) ? NumSources * NumElts : 1;
4556 APInt NegativeVal(CmpVal->getBitWidth(), Count);
4557 if (*IsAShr)
4558 NegativeVal.negate();
4559
4560 // Range is [min(0, AllNegVal), max(0, AllNegVal)]
4561 APInt Zero = APInt::getZero(numBits: CmpVal->getBitWidth());
4562 APInt RangeLow = APIntOps::smin(A: Zero, B: NegativeVal);
4563 APInt RangeHigh = APIntOps::smax(A: Zero, B: NegativeVal);
4564
4565 // Determine comparison semantics:
4566 // - IsEq: true for equality test, false for inequality
4567 // - TestsNegative: true if testing against AllNegVal, false for zero
4568 //
4569 // In addition to EQ/NE against 0 or AllNegVal, we support inequalities
4570 // that fold to boundary tests given the narrow value range:
4571 // < RangeHigh -> != RangeHigh
4572 // > RangeHigh-1 -> == RangeHigh
4573 // > RangeLow -> != RangeLow
4574 // < RangeLow+1 -> == RangeLow
4575 //
4576 // For inequalities, we work with signed predicates only. Unsigned predicates
4577 // are canonicalized to signed when the range is non-negative (where they are
4578 // equivalent). When the range includes negative values, unsigned predicates
4579 // would have different semantics due to wrap-around, so we reject them.
4580 if (!ICmpInst::isEquality(P: Pred) && !ICmpInst::isSigned(Pred)) {
4581 if (RangeLow.isNegative())
4582 return false;
4583 Pred = ICmpInst::getSignedPredicate(Pred);
4584 }
4585
4586 bool IsEq;
4587 bool TestsNegative;
4588 if (ICmpInst::isEquality(P: Pred)) {
4589 if (CmpVal->isZero()) {
4590 TestsNegative = false;
4591 } else if (*CmpVal == NegativeVal) {
4592 TestsNegative = true;
4593 } else {
4594 return false;
4595 }
4596 IsEq = Pred == ICmpInst::ICMP_EQ;
4597 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeHigh) {
4598 IsEq = false;
4599 TestsNegative = (RangeHigh == NegativeVal);
4600 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeHigh - 1) {
4601 IsEq = true;
4602 TestsNegative = (RangeHigh == NegativeVal);
4603 } else if (Pred == ICmpInst::ICMP_SGT && *CmpVal == RangeLow) {
4604 IsEq = false;
4605 TestsNegative = (RangeLow == NegativeVal);
4606 } else if (Pred == ICmpInst::ICMP_SLT && *CmpVal == RangeLow + 1) {
4607 IsEq = true;
4608 TestsNegative = (RangeLow == NegativeVal);
4609 } else {
4610 return false;
4611 }
4612
4613 // For this fold we support four types of checks:
4614 //
4615 // 1. All lanes are negative - AllNeg
4616 // 2. All lanes are non-negative - AllNonNeg
4617 // 3. At least one negative lane - AnyNeg
4618 // 4. At least one non-negative lane - AnyNonNeg
4619 //
4620 // For each case, we can generate the following code:
4621 //
4622 // 1. AllNeg - reduce.and/umin(X) < 0
4623 // 2. AllNonNeg - reduce.or/umax(X) > -1
4624 // 3. AnyNeg - reduce.or/umax(X) < 0
4625 // 4. AnyNonNeg - reduce.and/umin(X) > -1
4626 //
4627 // The table below shows the aggregation of all supported cases
4628 // using these four cases.
4629 //
4630 // Reduction | == 0 | != 0 | == MAX | != MAX
4631 // ------------+-----------+-----------+-----------+-----------
4632 // or/umax | AllNonNeg | AnyNeg | AnyNeg | AllNonNeg
4633 // and/umin | AnyNonNeg | AllNeg | AllNeg | AnyNonNeg
4634 // add | AllNonNeg | AnyNeg | AllNeg | AnyNonNeg
4635 //
4636 // NOTE: MAX = 1 for or/and/umax/umin, and the vector size N for add
4637 //
4638 // For easier codegen and check inversion, we use the following encoding:
4639 //
4640 // 1. Bit-3 === requires or/umax (1) or and/umin (0) check
4641 // 2. Bit-2 === checks < 0 (1) or > -1 (0)
4642 // 3. Bit-1 === universal (1) or existential (0) check
4643 //
4644 // AnyNeg = 0b110: uses or/umax, checks negative, any-check
4645 // AllNonNeg = 0b101: uses or/umax, checks non-neg, all-check
4646 // AnyNonNeg = 0b000: uses and/umin, checks non-neg, any-check
4647 // AllNeg = 0b011: uses and/umin, checks negative, all-check
4648 //
4649 // XOR with 0b011 inverts the check (swaps all/any and neg/non-neg).
4650 //
4651 enum CheckKind : unsigned {
4652 AnyNonNeg = 0b000,
4653 AllNeg = 0b011,
4654 AllNonNeg = 0b101,
4655 AnyNeg = 0b110,
4656 };
4657 // Return true if we fold this check into or/umax and false for and/umin
4658 auto RequiresOr = [](CheckKind C) -> bool { return C & 0b100; };
4659 // Return true if we should check if result is negative and false otherwise
4660 auto IsNegativeCheck = [](CheckKind C) -> bool { return C & 0b010; };
4661 // Logically invert the check
4662 auto Invert = [](CheckKind C) { return CheckKind(C ^ 0b011); };
4663
4664 CheckKind Base;
4665 switch (OrigIID) {
4666 case Intrinsic::vector_reduce_or:
4667 case Intrinsic::vector_reduce_umax:
4668 Base = TestsNegative ? AnyNeg : AllNonNeg;
4669 break;
4670 case Intrinsic::vector_reduce_and:
4671 case Intrinsic::vector_reduce_umin:
4672 Base = TestsNegative ? AllNeg : AnyNonNeg;
4673 break;
4674 case Intrinsic::vector_reduce_add:
4675 Base = TestsNegative ? AllNeg : AllNonNeg;
4676 break;
4677 default:
4678 llvm_unreachable("Unexpected intrinsic");
4679 }
4680
4681 CheckKind Check = IsEq ? Base : Invert(Base);
4682
4683 auto PickCheaper = [&](Intrinsic::ID Arith, Intrinsic::ID MinMax) {
4684 InstructionCost ArithCost =
4685 TTI.getArithmeticReductionCost(Opcode: getArithmeticReductionInstruction(RdxID: Arith),
4686 Ty: VecTy, FMF: std::nullopt, CostKind);
4687 InstructionCost MinMaxCost =
4688 TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: MinMax), Ty: VecTy,
4689 FMF: FastMathFlags(), CostKind);
4690 return ArithCost <= MinMaxCost ? std::make_pair(x&: Arith, y&: ArithCost)
4691 : std::make_pair(x&: MinMax, y&: MinMaxCost);
4692 };
4693
4694 // Choose output reduction based on encoding's MSB
4695 auto [NewIID, NewCost] = RequiresOr(Check)
4696 ? PickCheaper(Intrinsic::vector_reduce_or,
4697 Intrinsic::vector_reduce_umax)
4698 : PickCheaper(Intrinsic::vector_reduce_and,
4699 Intrinsic::vector_reduce_umin);
4700
4701 // Add cost of combining multiple sources with or/and
4702 if (NumSources > 1) {
4703 unsigned CombineOpc =
4704 RequiresOr(Check) ? Instruction::Or : Instruction::And;
4705 NewCost += TTI.getArithmeticInstrCost(Opcode: CombineOpc, Ty: VecTy, CostKind) *
4706 (NumSources - 1);
4707 }
4708
4709 LLVM_DEBUG(dbgs() << "Found sign-bit reduction cmp: " << I << "\n OldCost: "
4710 << OldCost << " vs NewCost: " << NewCost << "\n");
4711
4712 if (NewCost > OldCost)
4713 return false;
4714
4715 // Generate the combined input and reduction
4716 Builder.SetInsertPoint(&I);
4717 Type *ScalarTy = VecTy->getScalarType();
4718
4719 Value *Input;
4720 if (NumSources == 1) {
4721 Input = Sources[0];
4722 } else {
4723 // Combine sources with or/and based on check type
4724 Input = RequiresOr(Check) ? Builder.CreateOr(Ops: Sources)
4725 : Builder.CreateAnd(Ops: Sources);
4726 }
4727
4728 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: NewIID, Args: {Input});
4729 Value *NewCmp = IsNegativeCheck(Check) ? Builder.CreateIsNeg(Arg: NewReduce)
4730 : Builder.CreateIsNotNeg(Arg: NewReduce);
4731 replaceValue(Old&: I, New&: *NewCmp);
4732 return true;
4733}
4734
4735/// Fold a zero test of reduce.or or reduce.umax into a boolean reduction.
4736///
4737/// Vectorization may produce IR that compares the result of a scalar reduction
4738/// with zero. Depending on the target, lowering a reduction and a scalar
4739/// comparison separately can cost more than reducing lane-wise comparison
4740/// results. This fold creates the latter form only when it is not costlier.
4741///
4742/// Before:
4743/// %r = call iT @llvm.vector.reduce.or.vNiT(<N x iT> %x)
4744/// %cmp = icmp ne iT %r, 0
4745///
4746/// After:
4747/// %lane.cmp = icmp ne <N x iT> %x, zeroinitializer
4748/// %cmp = call i1 @llvm.vector.reduce.or.vNi1(<N x i1> %lane.cmp)
4749///
4750/// `reduce.or` and `reduce.umax` are non-zero when at least one lane is
4751/// non-zero. Therefore, `icmp ne` uses the existential `reduce.or` test.
4752/// Conversely, `icmp eq` must check that every lane is zero, so it uses the
4753/// universal `reduce.and` test.
4754///
4755/// Before:
4756/// %r = call iT @llvm.vector.reduce.umax.vNiT(<N x iT> %x)
4757/// %cmp = icmp eq iT %r, 0
4758///
4759/// After:
4760/// %lane.cmp = icmp eq <N x iT> %x, zeroinitializer
4761/// %cmp = call i1 @llvm.vector.reduce.and.vNi1(<N x i1> %lane.cmp)
4762bool VectorCombine::foldReductionZeroTest(Instruction &I) {
4763 CmpPredicate Pred;
4764 Value *Op;
4765
4766 if (!match(V: &I, P: m_c_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
4767 !ICmpInst::isEquality(P: Pred))
4768 return false;
4769
4770 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
4771 if (!II || !II->hasOneUse())
4772 return false;
4773
4774 auto ReduceID = II->getIntrinsicID();
4775 if (ReduceID != Intrinsic::vector_reduce_or &&
4776 ReduceID != Intrinsic::vector_reduce_umax)
4777 return false;
4778
4779 Value *Vec = II->getArgOperand(i: 0);
4780 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
4781 if (!VecTy || !VecTy->getElementType()->isIntegerTy())
4782 return false;
4783
4784 // Map the scalar zero test to an any-lane or all-lane boolean reduction.
4785 Intrinsic::ID NewIID = (Pred == ICmpInst::ICMP_NE)
4786 ? Intrinsic::vector_reduce_or
4787 : Intrinsic::vector_reduce_and;
4788
4789 // This is not an unconditional canonicalization: compare the cost of the
4790 // original scalar reduction and compare with the vector compare and i1
4791 // reduction replacement for both reduce.or and reduce.umax.
4792 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
4793 TTI.getInstructionCost(U: &I, CostKind);
4794
4795 auto *CmpTy = cast<VectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecTy));
4796 InstructionCost NewCost =
4797 TTI.getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: VecTy, CondTy: CmpTy, VecPred: Pred, CostKind);
4798 NewCost += TTI.getArithmeticReductionCost(
4799 Opcode: getArithmeticReductionInstruction(RdxID: NewIID), Ty: CmpTy, FMF: std::nullopt, CostKind);
4800
4801 LLVM_DEBUG(dbgs() << "Found a reduction zero test: " << I << "\n OldCost: "
4802 << OldCost << " vs NewCost: " << NewCost << "\n");
4803
4804 if (!OldCost.isValid() || !NewCost.isValid() || NewCost > OldCost)
4805 return false;
4806
4807 Builder.SetInsertPoint(&I);
4808 Value *NewCmp = Builder.CreateICmp(P: Pred, LHS: Vec, RHS: Constant::getNullValue(Ty: VecTy));
4809 Value *NewReduce = Builder.CreateIntrinsic(ID: NewIID, OverloadTypes: {CmpTy}, Args: {NewCmp});
4810 replaceValue(Old&: I, New&: *NewReduce);
4811 return true;
4812}
4813
4814/// vector.reduce.OP f(X_i) == 0 -> vector.reduce.OP X_i == 0
4815///
4816/// We can prove it for cases when:
4817///
4818/// 1. OP X_i == 0 <=> \forall i \in [1, N] X_i == 0
4819/// 1'. OP X_i == 0 <=> \exists j \in [1, N] X_j == 0
4820/// 2. f(x) == 0 <=> x == 0
4821///
4822/// From 1 and 2 (or 1' and 2), we can infer that
4823///
4824/// OP f(X_i) == 0 <=> OP X_i == 0.
4825///
4826/// (1)
4827/// OP f(X_i) == 0 <=> \forall i \in [1, N] f(X_i) == 0
4828/// (2)
4829/// <=> \forall i \in [1, N] X_i == 0
4830/// (1)
4831/// <=> OP(X_i) == 0
4832///
4833/// For some of the OP's and f's, we need to have domain constraints on X
4834/// to ensure properties 1 (or 1') and 2.
4835bool VectorCombine::foldICmpEqZeroVectorReduce(Instruction &I) {
4836 CmpPredicate Pred;
4837 Value *Op;
4838 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: Op), R: m_Zero())) ||
4839 !ICmpInst::isEquality(P: Pred))
4840 return false;
4841
4842 auto *II = dyn_cast<IntrinsicInst>(Val: Op);
4843 if (!II)
4844 return false;
4845
4846 switch (II->getIntrinsicID()) {
4847 case Intrinsic::vector_reduce_add:
4848 case Intrinsic::vector_reduce_or:
4849 case Intrinsic::vector_reduce_umin:
4850 case Intrinsic::vector_reduce_umax:
4851 case Intrinsic::vector_reduce_smin:
4852 case Intrinsic::vector_reduce_smax:
4853 break;
4854 default:
4855 return false;
4856 }
4857
4858 Value *InnerOp = II->getArgOperand(i: 0);
4859
4860 // TODO: fixed vector type might be too restrictive
4861 if (!II->hasOneUse() || !isa<FixedVectorType>(Val: InnerOp->getType()))
4862 return false;
4863
4864 Value *X = nullptr;
4865
4866 // Check for zero-preserving operations where f(x) = 0 <=> x = 0
4867 //
4868 // 1. f(x) = shl nuw x, y for arbitrary y
4869 // 2. f(x) = mul nuw x, c for defined c != 0
4870 // 3. f(x) = zext x
4871 // 4. f(x) = sext x
4872 // 5. f(x) = neg x
4873 //
4874 if (!(match(V: InnerOp, P: m_NUWShl(L: m_Value(V&: X), R: m_Value())) || // Case 1
4875 match(V: InnerOp, P: m_NUWMul(L: m_Value(V&: X), R: m_NonZeroInt())) || // Case 2
4876 match(V: InnerOp, P: m_ZExt(Op: m_Value(V&: X))) || // Case 3
4877 match(V: InnerOp, P: m_SExt(Op: m_Value(V&: X))) || // Case 4
4878 match(V: InnerOp, P: m_Neg(V: m_Value(V&: X))) // Case 5
4879 ))
4880 return false;
4881
4882 SimplifyQuery S = SQ.getWithInstruction(I: &I);
4883 auto *XTy = cast<FixedVectorType>(Val: X->getType());
4884
4885 // Check for domain constraints for all supported reductions.
4886 //
4887 // a. OR X_i - has property 1 for every X
4888 // b. UMAX X_i - has property 1 for every X
4889 // c. UMIN X_i - has property 1' for every X
4890 // d. SMAX X_i - has property 1 for X >= 0
4891 // e. SMIN X_i - has property 1' for X >= 0
4892 // f. ADD X_i - has property 1 for X >= 0 && ADD X_i doesn't sign wrap
4893 //
4894 // In order for the proof to work, we need 1 (or 1') to be true for both
4895 // OP f(X_i) and OP X_i and that's why below we check constraints twice.
4896 //
4897 // NOTE: ADD X_i holds property 1 for a mirror case as well, i.e. when
4898 // X <= 0 && ADD X_i doesn't sign wrap. However, due to the nature
4899 // of known bits, we can't reasonably hold knowledge of "either 0
4900 // or negative".
4901 switch (II->getIntrinsicID()) {
4902 case Intrinsic::vector_reduce_add: {
4903 // We need to check that both X_i and f(X_i) have enough leading
4904 // zeros to not overflow.
4905 KnownBits KnownX = computeKnownBits(V: X, Q: S);
4906 KnownBits KnownFX = computeKnownBits(V: InnerOp, Q: S);
4907 unsigned NumElems = XTy->getNumElements();
4908 // Adding N elements loses at most ceil(log2(N)) leading bits.
4909 unsigned LostBits = Log2_32_Ceil(Value: NumElems);
4910 unsigned LeadingZerosX = KnownX.countMinLeadingZeros();
4911 unsigned LeadingZerosFX = KnownFX.countMinLeadingZeros();
4912 // Need at least one leading zero left after summation to ensure no overflow
4913 if (LeadingZerosX <= LostBits || LeadingZerosFX <= LostBits)
4914 return false;
4915
4916 // We are not checking whether X or f(X) are positive explicitly because
4917 // we implicitly checked for it when we checked if both cases have enough
4918 // leading zeros to not wrap addition.
4919 break;
4920 }
4921 case Intrinsic::vector_reduce_smin:
4922 case Intrinsic::vector_reduce_smax:
4923 // Check whether X >= 0 and f(X) >= 0
4924 if (!isKnownNonNegative(V: InnerOp, SQ: S) || !isKnownNonNegative(V: X, SQ: S))
4925 return false;
4926
4927 break;
4928 default:
4929 break;
4930 };
4931
4932 LLVM_DEBUG(dbgs() << "Found a reduction to 0 comparison with removable op: "
4933 << *II << "\n");
4934
4935 // For zext/sext, check if the transform is profitable using cost model.
4936 // For other operations (shl, mul, neg), we're removing an instruction
4937 // while keeping the same reduction type, so it's always profitable.
4938 if (isa<ZExtInst>(Val: InnerOp) || isa<SExtInst>(Val: InnerOp)) {
4939 auto *FXTy = cast<FixedVectorType>(Val: InnerOp->getType());
4940 Intrinsic::ID IID = II->getIntrinsicID();
4941
4942 InstructionCost ExtCost = TTI.getCastInstrCost(
4943 Opcode: cast<CastInst>(Val: InnerOp)->getOpcode(), Dst: FXTy, Src: XTy,
4944 CCH: TTI::CastContextHint::None, CostKind, I: cast<CastInst>(Val: InnerOp));
4945
4946 InstructionCost OldReduceCost, NewReduceCost;
4947 switch (IID) {
4948 case Intrinsic::vector_reduce_add:
4949 case Intrinsic::vector_reduce_or:
4950 OldReduceCost = TTI.getArithmeticReductionCost(
4951 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: FXTy, FMF: std::nullopt, CostKind);
4952 NewReduceCost = TTI.getArithmeticReductionCost(
4953 Opcode: getArithmeticReductionInstruction(RdxID: IID), Ty: XTy, FMF: std::nullopt, CostKind);
4954 break;
4955 case Intrinsic::vector_reduce_umin:
4956 case Intrinsic::vector_reduce_umax:
4957 case Intrinsic::vector_reduce_smin:
4958 case Intrinsic::vector_reduce_smax:
4959 OldReduceCost = TTI.getMinMaxReductionCost(
4960 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: FXTy, FMF: FastMathFlags(), CostKind);
4961 NewReduceCost = TTI.getMinMaxReductionCost(
4962 IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: XTy, FMF: FastMathFlags(), CostKind);
4963 break;
4964 default:
4965 llvm_unreachable("Unexpected reduction");
4966 }
4967
4968 InstructionCost OldCost = OldReduceCost + ExtCost;
4969 InstructionCost NewCost =
4970 NewReduceCost + (InnerOp->hasOneUse() ? 0 : ExtCost);
4971
4972 LLVM_DEBUG(dbgs() << "Found a removable extension before reduction: "
4973 << *InnerOp << "\n OldCost: " << OldCost
4974 << " vs NewCost: " << NewCost << "\n");
4975
4976 // We consider transformation to still be potentially beneficial even
4977 // when the costs are the same because we might remove a use from f(X)
4978 // and unlock other optimizations. Equal costs would just mean that we
4979 // didn't make it worse in the worst case.
4980 if (NewCost > OldCost)
4981 return false;
4982 }
4983
4984 // Since we support zext and sext as f, we might change the scalar type
4985 // of the intrinsic.
4986 Type *Ty = XTy->getScalarType();
4987 Value *NewReduce = Builder.CreateIntrinsic(RetTy: Ty, ID: II->getIntrinsicID(), Args: {X});
4988 Value *NewCmp =
4989 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty));
4990 replaceValue(Old&: I, New&: *NewCmp);
4991 return true;
4992}
4993
4994/// Fold comparisons of reduce.or/reduce.and with reduce.umax/reduce.umin
4995/// based on cost, preserving the comparison semantics.
4996///
4997/// We use two fundamental properties for each pair:
4998///
4999/// 1. or(X) == 0 <=> umax(X) == 0
5000/// 2. or(X) == 1 <=> umax(X) == 1
5001/// 3. sign(or(X)) == sign(umax(X))
5002///
5003/// 1. and(X) == -1 <=> umin(X) == -1
5004/// 2. and(X) == -2 <=> umin(X) == -2
5005/// 3. sign(and(X)) == sign(umin(X))
5006///
5007/// From these we can infer the following transformations:
5008/// a. or(X) ==/!= 0 <-> umax(X) ==/!= 0
5009/// b. or(X) s< 0 <-> umax(X) s< 0
5010/// c. or(X) s> -1 <-> umax(X) s> -1
5011/// d. or(X) s< 1 <-> umax(X) s< 1
5012/// e. or(X) ==/!= 1 <-> umax(X) ==/!= 1
5013/// f. or(X) s< 2 <-> umax(X) s< 2
5014/// g. and(X) ==/!= -1 <-> umin(X) ==/!= -1
5015/// h. and(X) s< 0 <-> umin(X) s< 0
5016/// i. and(X) s> -1 <-> umin(X) s> -1
5017/// j. and(X) s> -2 <-> umin(X) s> -2
5018/// k. and(X) ==/!= -2 <-> umin(X) ==/!= -2
5019/// l. and(X) s> -3 <-> umin(X) s> -3
5020///
5021bool VectorCombine::foldEquivalentReductionCmp(Instruction &I) {
5022 CmpPredicate Pred;
5023 Value *ReduceOp;
5024 const APInt *CmpVal;
5025 if (!match(V: &I, P: m_ICmp(Pred, L: m_Value(V&: ReduceOp), R: m_APInt(Res&: CmpVal))))
5026 return false;
5027
5028 auto *II = dyn_cast<IntrinsicInst>(Val: ReduceOp);
5029 if (!II || !II->hasOneUse())
5030 return false;
5031
5032 const auto IsValidOrUmaxCmp = [&]() {
5033 // or === umax for i1
5034 if (CmpVal->getBitWidth() == 1)
5035 return true;
5036
5037 // Cases a and e
5038 bool IsEquality =
5039 (CmpVal->isZero() || CmpVal->isOne()) && ICmpInst::isEquality(P: Pred);
5040 // Case c
5041 bool IsPositive = CmpVal->isAllOnes() && Pred == ICmpInst::ICMP_SGT;
5042 // Cases b, d, and f
5043 bool IsNegative = (CmpVal->isZero() || CmpVal->isOne() || *CmpVal == 2) &&
5044 Pred == ICmpInst::ICMP_SLT;
5045 return IsEquality || IsPositive || IsNegative;
5046 };
5047
5048 const auto IsValidAndUminCmp = [&]() {
5049 // and === umin for i1
5050 if (CmpVal->getBitWidth() == 1)
5051 return true;
5052
5053 const auto LeadingOnes = CmpVal->countl_one();
5054
5055 // Cases g and k
5056 bool IsEquality =
5057 (CmpVal->isAllOnes() || LeadingOnes + 1 == CmpVal->getBitWidth()) &&
5058 ICmpInst::isEquality(P: Pred);
5059 // Case h
5060 bool IsNegative = CmpVal->isZero() && Pred == ICmpInst::ICMP_SLT;
5061 // Cases i, j, and l
5062 bool IsPositive =
5063 // if the number has at least N - 2 leading ones
5064 // and the two LSBs are:
5065 // - 1 x 1 -> -1
5066 // - 1 x 0 -> -2
5067 // - 0 x 1 -> -3
5068 LeadingOnes + 2 >= CmpVal->getBitWidth() &&
5069 ((*CmpVal)[0] || (*CmpVal)[1]) && Pred == ICmpInst::ICMP_SGT;
5070 return IsEquality || IsNegative || IsPositive;
5071 };
5072
5073 Intrinsic::ID OriginalIID = II->getIntrinsicID();
5074 Intrinsic::ID AlternativeIID;
5075
5076 // Check if this is a valid comparison pattern and determine the alternate
5077 // reduction intrinsic.
5078 switch (OriginalIID) {
5079 case Intrinsic::vector_reduce_or:
5080 if (!IsValidOrUmaxCmp())
5081 return false;
5082 AlternativeIID = Intrinsic::vector_reduce_umax;
5083 break;
5084 case Intrinsic::vector_reduce_umax:
5085 if (!IsValidOrUmaxCmp())
5086 return false;
5087 AlternativeIID = Intrinsic::vector_reduce_or;
5088 break;
5089 case Intrinsic::vector_reduce_and:
5090 if (!IsValidAndUminCmp())
5091 return false;
5092 AlternativeIID = Intrinsic::vector_reduce_umin;
5093 break;
5094 case Intrinsic::vector_reduce_umin:
5095 if (!IsValidAndUminCmp())
5096 return false;
5097 AlternativeIID = Intrinsic::vector_reduce_and;
5098 break;
5099 default:
5100 return false;
5101 }
5102
5103 Value *X = II->getArgOperand(i: 0);
5104 auto *VecTy = dyn_cast<FixedVectorType>(Val: X->getType());
5105 if (!VecTy)
5106 return false;
5107
5108 const auto GetReductionCost = [&](Intrinsic::ID IID) -> InstructionCost {
5109 unsigned ReductionOpc = getArithmeticReductionInstruction(RdxID: IID);
5110 if (ReductionOpc != Instruction::ICmp)
5111 return TTI.getArithmeticReductionCost(Opcode: ReductionOpc, Ty: VecTy, FMF: std::nullopt,
5112 CostKind);
5113 return TTI.getMinMaxReductionCost(IID: getMinMaxReductionIntrinsicOp(RdxID: IID), Ty: VecTy,
5114 FMF: FastMathFlags(), CostKind);
5115 };
5116
5117 InstructionCost OrigCost = GetReductionCost(OriginalIID);
5118 InstructionCost AltCost = GetReductionCost(AlternativeIID);
5119
5120 LLVM_DEBUG(dbgs() << "Found equivalent reduction cmp: " << I
5121 << "\n OrigCost: " << OrigCost
5122 << " vs AltCost: " << AltCost << "\n");
5123
5124 if (AltCost >= OrigCost)
5125 return false;
5126
5127 Builder.SetInsertPoint(&I);
5128 Type *ScalarTy = VecTy->getScalarType();
5129 Value *NewReduce = Builder.CreateIntrinsic(RetTy: ScalarTy, ID: AlternativeIID, Args: {X});
5130 Value *NewCmp =
5131 Builder.CreateICmp(P: Pred, LHS: NewReduce, RHS: ConstantInt::get(Ty: ScalarTy, V: *CmpVal));
5132
5133 replaceValue(Old&: I, New&: *NewCmp);
5134 return true;
5135}
5136
5137/// Used by foldReduceAddCmpZero to check if we can prove that a value is
5138/// non-positive.
5139/// KnownBits cannot see sext <? x i1> as non-positive: each top bit equals a
5140/// single unknown input bit, which a per-bit lattice cannot track. The fold's
5141/// target shape is popcount-style sums of <N x i1> valid/invalid masks (e.g.
5142/// ray-intersection hits) tested for any-hit.
5143/// Previous attempts to approximate the known bits of such expressions were
5144/// using a fully recursive value tracking approach to infer a constant range
5145/// but ultimately turned to be too expensive in compile time.
5146static bool isKnownNonPositive(const Value *V, const SimplifyQuery &SQ,
5147 unsigned Depth = 0) {
5148 constexpr unsigned MaxLocalDepth = 2;
5149 if (Depth > MaxLocalDepth)
5150 return false;
5151
5152 auto NumSignBits = [&](const Value *X) {
5153 return ComputeNumSignBits(Op: X, DL: SQ.DL, AC: SQ.AC, CxtI: SQ.CxtI, DT: SQ.DT);
5154 };
5155 if (NumSignBits(V) == V->getType()->getScalarSizeInBits())
5156 return true;
5157
5158 Value *A, *B;
5159 if (match(V, P: m_Add(L: m_Value(V&: A), R: m_Value(V&: B))))
5160 return NumSignBits(A) >= 2 && NumSignBits(B) >= 2 &&
5161 isKnownNonPositive(V: A, SQ, Depth: Depth + 1) &&
5162 isKnownNonPositive(V: B, SQ, Depth: Depth + 1);
5163
5164 return computeKnownBits(V, Q: SQ).isNonPositive();
5165}
5166
5167/// Fold (icmp pred (reduce.add X), 0) to (icmp pred' (reduce.or X), 0) when X
5168/// has lanes known to all be non-negative or all non-positive, so that
5169/// sum == 0 iff every lane is 0. Falls back to reduce.umax if reduce.or is
5170/// more expensive on the target.
5171bool VectorCombine::foldReduceAddCmpZero(Instruction &I) {
5172 CmpPredicate Pred;
5173 Value *Vec;
5174 if (!match(V: &I, P: m_ICmp(Pred,
5175 L: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vector_reduce_add>(
5176 Ops: m_Value(V&: Vec))),
5177 R: m_Zero())))
5178 return false;
5179
5180 auto *VecTy = dyn_cast<FixedVectorType>(Val: Vec->getType());
5181 if (!VecTy || VecTy->getNumElements() < 2)
5182 return false;
5183
5184 SimplifyQuery Q = SQ.getWithInstruction(I: &I);
5185 bool IsNonNegative = isKnownNonNegative(V: Vec, SQ: Q);
5186 bool IsNonPositive = !IsNonNegative && isKnownNonPositive(V: Vec, SQ: Q);
5187 if (!IsNonNegative && !IsNonPositive)
5188 return false;
5189
5190 // Summing NumElts lanes can consume up to log2(NumElts) sign bits. Require
5191 // strictly more headroom than that so the sum cannot wrap to zero.
5192 unsigned NumElts = VecTy->getNumElements();
5193 unsigned NumSignBits = ComputeNumSignBits(Op: Vec, DL: *DL, AC: SQ.AC, CxtI: &I, DT: &DT);
5194 if (Log2_32(Value: NumElts) >= NumSignBits)
5195 return false;
5196
5197 ICmpInst::Predicate NewPred;
5198 switch (Pred) {
5199 case ICmpInst::ICMP_EQ:
5200 case ICmpInst::ICMP_ULE:
5201 case ICmpInst::ICMP_SLE:
5202 case ICmpInst::ICMP_SGE:
5203 NewPred = ICmpInst::ICMP_EQ;
5204 break;
5205 case ICmpInst::ICMP_NE:
5206 case ICmpInst::ICMP_UGT:
5207 case ICmpInst::ICMP_SGT:
5208 case ICmpInst::ICMP_SLT:
5209 NewPred = ICmpInst::ICMP_NE;
5210 break;
5211 default:
5212 return false;
5213 }
5214
5215 // SGT and SLE on a non-positive tree, and SLT and SGE on a non-negative
5216 // tree, are tautologies (always true or always false). Leave those to
5217 // InstCombine rather than mapping them here. Remaining signed inequalities
5218 // also need one extra sign bit so the sum cannot flip sign.
5219 if (!IsNonNegative &&
5220 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE))
5221 return false;
5222 if (!IsNonPositive &&
5223 (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE))
5224 return false;
5225 if ((Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE ||
5226 Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE) &&
5227 Log2_32(Value: NumElts) >= NumSignBits - 1)
5228 return false;
5229
5230 InstructionCost OrigCost = TTI.getArithmeticReductionCost(
5231 Opcode: Instruction::Add, Ty: VecTy, FMF: std::nullopt, CostKind);
5232 InstructionCost OrCost = TTI.getArithmeticReductionCost(
5233 Opcode: Instruction::Or, Ty: VecTy, FMF: std::nullopt, CostKind);
5234 InstructionCost UmaxCost = TTI.getMinMaxReductionCost(
5235 IID: Intrinsic::umax, Ty: VecTy, FMF: FastMathFlags(), CostKind);
5236 if (!OrCost.isValid() && !UmaxCost.isValid())
5237 return false;
5238 bool UseOr = OrCost.isValid() && (!UmaxCost.isValid() || OrCost <= UmaxCost);
5239 InstructionCost AltCost = UseOr ? OrCost : UmaxCost;
5240 if (AltCost > OrigCost)
5241 return false;
5242
5243 Builder.SetInsertPoint(&I);
5244 Value *NewReduce = UseOr ? Builder.CreateOrReduce(Src: Vec)
5245 : Builder.CreateIntrinsic(
5246 ID: Intrinsic::vector_reduce_umax, OverloadTypes: {VecTy}, Args: {Vec});
5247 Worklist.pushValue(V: NewReduce);
5248 Value *NewCmp = Builder.CreateICmp(
5249 P: NewPred, LHS: NewReduce, RHS: ConstantInt::getNullValue(Ty: VecTy->getScalarType()));
5250 replaceValue(Old&: I, New&: *NewCmp);
5251 return true;
5252}
5253
5254/// Returns true if this ShuffleVectorInst eventually feeds into a
5255/// vector reduction intrinsic (e.g., vector_reduce_add) by only following
5256/// chains of shuffles and binary operators (in any combination/order).
5257/// The search does not go deeper than the given Depth.
5258static bool feedsIntoVectorReduction(ShuffleVectorInst *SVI) {
5259 constexpr unsigned MaxVisited = 32;
5260 SmallPtrSet<Instruction *, 8> Visited;
5261 SmallVector<Instruction *, 4> WorkList;
5262 bool FoundReduction = false;
5263
5264 WorkList.push_back(Elt: SVI);
5265 while (!WorkList.empty()) {
5266 Instruction *I = WorkList.pop_back_val();
5267 for (User *U : I->users()) {
5268 auto *UI = cast<Instruction>(Val: U);
5269 if (!UI || !Visited.insert(Ptr: UI).second)
5270 continue;
5271 if (Visited.size() > MaxVisited)
5272 return false;
5273 if (auto *II = dyn_cast<IntrinsicInst>(Val: UI)) {
5274 // More than one reduction reached
5275 if (FoundReduction)
5276 return false;
5277 switch (II->getIntrinsicID()) {
5278 case Intrinsic::vector_reduce_add:
5279 case Intrinsic::vector_reduce_mul:
5280 case Intrinsic::vector_reduce_and:
5281 case Intrinsic::vector_reduce_or:
5282 case Intrinsic::vector_reduce_xor:
5283 case Intrinsic::vector_reduce_smin:
5284 case Intrinsic::vector_reduce_smax:
5285 case Intrinsic::vector_reduce_umin:
5286 case Intrinsic::vector_reduce_umax:
5287 FoundReduction = true;
5288 continue;
5289 default:
5290 return false;
5291 }
5292 }
5293
5294 if (!isa<BinaryOperator>(Val: UI) && !isa<ShuffleVectorInst>(Val: UI))
5295 return false;
5296
5297 WorkList.emplace_back(Args&: UI);
5298 }
5299 }
5300 return FoundReduction;
5301}
5302
5303/// This method looks for groups of shuffles acting on binops, of the form:
5304/// %x = shuffle ...
5305/// %y = shuffle ...
5306/// %a = binop %x, %y
5307/// %b = binop %x, %y
5308/// shuffle %a, %b, selectmask
5309/// We may, especially if the shuffle is wider than legal, be able to convert
5310/// the shuffle to a form where only parts of a and b need to be computed. On
5311/// architectures with no obvious "select" shuffle, this can reduce the total
5312/// number of operations if the target reports them as cheaper.
5313bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
5314 auto *SVI = cast<ShuffleVectorInst>(Val: &I);
5315 auto *VT = cast<FixedVectorType>(Val: I.getType());
5316 auto *Op0 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 0));
5317 auto *Op1 = dyn_cast<Instruction>(Val: SVI->getOperand(i_nocapture: 1));
5318 if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() ||
5319 VT != Op0->getType())
5320 return false;
5321
5322 auto *SVI0A = dyn_cast<Instruction>(Val: Op0->getOperand(i: 0));
5323 auto *SVI0B = dyn_cast<Instruction>(Val: Op0->getOperand(i: 1));
5324 auto *SVI1A = dyn_cast<Instruction>(Val: Op1->getOperand(i: 0));
5325 auto *SVI1B = dyn_cast<Instruction>(Val: Op1->getOperand(i: 1));
5326 SmallPtrSet<Instruction *, 4> InputShuffles({SVI0A, SVI0B, SVI1A, SVI1B});
5327 auto checkSVNonOpUses = [&](Instruction *I) {
5328 if (!I || I->getOperand(i: 0)->getType() != VT)
5329 return true;
5330 return any_of(Range: I->users(), P: [&](User *U) {
5331 return U != Op0 && U != Op1 &&
5332 !(isa<ShuffleVectorInst>(Val: U) &&
5333 (InputShuffles.contains(Ptr: cast<Instruction>(Val: U)) ||
5334 isInstructionTriviallyDead(I: cast<Instruction>(Val: U))));
5335 });
5336 };
5337 if (checkSVNonOpUses(SVI0A) || checkSVNonOpUses(SVI0B) ||
5338 checkSVNonOpUses(SVI1A) || checkSVNonOpUses(SVI1B))
5339 return false;
5340
5341 // Collect all the uses that are shuffles that we can transform together. We
5342 // may not have a single shuffle, but a group that can all be transformed
5343 // together profitably.
5344 SmallVector<ShuffleVectorInst *> Shuffles;
5345 auto collectShuffles = [&](Instruction *I) {
5346 for (auto *U : I->users()) {
5347 auto *SV = dyn_cast<ShuffleVectorInst>(Val: U);
5348 if (!SV || SV->getType() != VT)
5349 return false;
5350 if ((SV->getOperand(i_nocapture: 0) != Op0 && SV->getOperand(i_nocapture: 0) != Op1) ||
5351 (SV->getOperand(i_nocapture: 1) != Op0 && SV->getOperand(i_nocapture: 1) != Op1))
5352 return false;
5353 if (!llvm::is_contained(Range&: Shuffles, Element: SV))
5354 Shuffles.push_back(Elt: SV);
5355 }
5356 return true;
5357 };
5358 if (!collectShuffles(Op0) || !collectShuffles(Op1))
5359 return false;
5360 // From a reduction, we need to be processing a single shuffle, otherwise the
5361 // other uses will not be lane-invariant.
5362 if (FromReduction && Shuffles.size() > 1)
5363 return false;
5364
5365 // Add any shuffle uses for the shuffles we have found, to include them in our
5366 // cost calculations.
5367 if (!FromReduction) {
5368 for (size_t Idx = 0, E = Shuffles.size(); Idx != E; ++Idx) {
5369 for (auto *U : Shuffles[Idx]->users()) {
5370 ShuffleVectorInst *SSV = dyn_cast<ShuffleVectorInst>(Val: U);
5371 if (SSV && isa<UndefValue>(Val: SSV->getOperand(i_nocapture: 1)) && SSV->getType() == VT)
5372 Shuffles.push_back(Elt: SSV);
5373 }
5374 }
5375 }
5376
5377 // For each of the output shuffles, we try to sort all the first vector
5378 // elements to the beginning, followed by the second array elements at the
5379 // end. If the binops are legalized to smaller vectors, this may reduce total
5380 // number of binops. We compute the ReconstructMask mask needed to convert
5381 // back to the original lane order.
5382 SmallVector<std::pair<int, int>> V1, V2;
5383 SmallVector<SmallVector<int>> OrigReconstructMasks;
5384 int MaxV1Elt = 0, MaxV2Elt = 0;
5385 unsigned NumElts = VT->getNumElements();
5386 for (ShuffleVectorInst *SVN : Shuffles) {
5387 SmallVector<int> Mask;
5388 SVN->getShuffleMask(Result&: Mask);
5389
5390 // Check the operands are the same as the original, or reversed (in which
5391 // case we need to commute the mask).
5392 Value *SVOp0 = SVN->getOperand(i_nocapture: 0);
5393 Value *SVOp1 = SVN->getOperand(i_nocapture: 1);
5394 if (isa<UndefValue>(Val: SVOp1)) {
5395 auto *SSV = cast<ShuffleVectorInst>(Val: SVOp0);
5396 SVOp0 = SSV->getOperand(i_nocapture: 0);
5397 SVOp1 = SSV->getOperand(i_nocapture: 1);
5398 for (int &Elem : Mask) {
5399 if (Elem >= static_cast<int>(SSV->getShuffleMask().size()))
5400 return false;
5401 Elem = Elem < 0 ? Elem : SSV->getMaskValue(Elt: Elem);
5402 }
5403 }
5404 if (SVOp0 == Op1 && SVOp1 == Op0) {
5405 std::swap(a&: SVOp0, b&: SVOp1);
5406 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumElts);
5407 }
5408 if (SVOp0 != Op0 || SVOp1 != Op1)
5409 return false;
5410
5411 // Calculate the reconstruction mask for this shuffle, as the mask needed to
5412 // take the packed values from Op0/Op1 and reconstructing to the original
5413 // order.
5414 SmallVector<int> ReconstructMask;
5415 for (unsigned I = 0; I < Mask.size(); I++) {
5416 if (Mask[I] < 0) {
5417 ReconstructMask.push_back(Elt: -1);
5418 } else if (Mask[I] < static_cast<int>(NumElts)) {
5419 MaxV1Elt = std::max(a: MaxV1Elt, b: Mask[I]);
5420 auto It = find_if(Range&: V1, P: [&](const std::pair<int, int> &A) {
5421 return Mask[I] == A.first;
5422 });
5423 if (It != V1.end())
5424 ReconstructMask.push_back(Elt: It - V1.begin());
5425 else {
5426 ReconstructMask.push_back(Elt: V1.size());
5427 V1.emplace_back(Args&: Mask[I], Args: V1.size());
5428 }
5429 } else {
5430 MaxV2Elt = std::max<int>(a: MaxV2Elt, b: Mask[I] - NumElts);
5431 auto It = find_if(Range&: V2, P: [&](const std::pair<int, int> &A) {
5432 return Mask[I] - static_cast<int>(NumElts) == A.first;
5433 });
5434 if (It != V2.end())
5435 ReconstructMask.push_back(Elt: NumElts + It - V2.begin());
5436 else {
5437 ReconstructMask.push_back(Elt: NumElts + V2.size());
5438 V2.emplace_back(Args: Mask[I] - NumElts, Args: NumElts + V2.size());
5439 }
5440 }
5441 }
5442
5443 // For reductions, we know that the lane ordering out doesn't alter the
5444 // result. In-order can help simplify the shuffle away.
5445 if (FromReduction)
5446 sort(C&: ReconstructMask);
5447 OrigReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5448 }
5449
5450 // If the Maximum element used from V1 and V2 are not larger than the new
5451 // vectors, the vectors are already packes and performing the optimization
5452 // again will likely not help any further. This also prevents us from getting
5453 // stuck in a cycle in case the costs do not also rule it out.
5454 if (V1.empty() || V2.empty() ||
5455 (MaxV1Elt == static_cast<int>(V1.size()) - 1 &&
5456 MaxV2Elt == static_cast<int>(V2.size()) - 1))
5457 return false;
5458
5459 // GetBaseMaskValue takes one of the inputs, which may either be a shuffle, a
5460 // shuffle of another shuffle, or not a shuffle (that is treated like a
5461 // identity shuffle).
5462 auto GetBaseMaskValue = [&](Instruction *I, int M) {
5463 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5464 if (!SV)
5465 return M;
5466 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5467 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5468 if (InputShuffles.contains(Ptr: SSV))
5469 return SSV->getMaskValue(Elt: SV->getMaskValue(Elt: M));
5470 return SV->getMaskValue(Elt: M);
5471 };
5472
5473 // Attempt to sort the inputs my ascending mask values to make simpler input
5474 // shuffles and push complex shuffles down to the uses. We sort on the first
5475 // of the two input shuffle orders, to try and get at least one input into a
5476 // nice order.
5477 auto SortBase = [&](Instruction *A, std::pair<int, int> X,
5478 std::pair<int, int> Y) {
5479 int MXA = GetBaseMaskValue(A, X.first);
5480 int MYA = GetBaseMaskValue(A, Y.first);
5481 return MXA < MYA;
5482 };
5483 stable_sort(Range&: V1, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5484 return SortBase(SVI0A, A, B);
5485 });
5486 stable_sort(Range&: V2, C: [&](std::pair<int, int> A, std::pair<int, int> B) {
5487 return SortBase(SVI1A, A, B);
5488 });
5489 // Calculate our ReconstructMasks from the OrigReconstructMasks and the
5490 // modified order of the input shuffles.
5491 SmallVector<SmallVector<int>> ReconstructMasks;
5492 for (const auto &Mask : OrigReconstructMasks) {
5493 SmallVector<int> ReconstructMask;
5494 for (int M : Mask) {
5495 auto FindIndex = [](const SmallVector<std::pair<int, int>> &V, int M) {
5496 auto It = find_if(Range: V, P: [M](auto A) { return A.second == M; });
5497 assert(It != V.end() && "Expected all entries in Mask");
5498 return std::distance(first: V.begin(), last: It);
5499 };
5500 if (M < 0)
5501 ReconstructMask.push_back(Elt: -1);
5502 else if (M < static_cast<int>(NumElts)) {
5503 ReconstructMask.push_back(Elt: FindIndex(V1, M));
5504 } else {
5505 ReconstructMask.push_back(Elt: NumElts + FindIndex(V2, M));
5506 }
5507 }
5508 ReconstructMasks.push_back(Elt: std::move(ReconstructMask));
5509 }
5510
5511 // Calculate the masks needed for the new input shuffles, which get padded
5512 // with undef
5513 SmallVector<int> V1A, V1B, V2A, V2B;
5514 for (unsigned I = 0; I < V1.size(); I++) {
5515 V1A.push_back(Elt: GetBaseMaskValue(SVI0A, V1[I].first));
5516 V1B.push_back(Elt: GetBaseMaskValue(SVI0B, V1[I].first));
5517 }
5518 for (unsigned I = 0; I < V2.size(); I++) {
5519 V2A.push_back(Elt: GetBaseMaskValue(SVI1A, V2[I].first));
5520 V2B.push_back(Elt: GetBaseMaskValue(SVI1B, V2[I].first));
5521 }
5522 while (V1A.size() < NumElts) {
5523 V1A.push_back(Elt: PoisonMaskElem);
5524 V1B.push_back(Elt: PoisonMaskElem);
5525 }
5526 while (V2A.size() < NumElts) {
5527 V2A.push_back(Elt: PoisonMaskElem);
5528 V2B.push_back(Elt: PoisonMaskElem);
5529 }
5530
5531 auto AddShuffleCost = [&](InstructionCost C, Instruction *I) {
5532 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5533 if (!SV)
5534 return C;
5535 return C + TTI.getShuffleCost(Kind: isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1))
5536 ? TTI::SK_PermuteSingleSrc
5537 : TTI::SK_PermuteTwoSrc,
5538 DstTy: VT, SrcTy: VT, Mask: SV->getShuffleMask(), CostKind);
5539 };
5540 auto AddShuffleMaskCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5541 return C +
5542 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, Mask, CostKind);
5543 };
5544
5545 unsigned ElementSize = VT->getElementType()->getPrimitiveSizeInBits();
5546 unsigned MaxVectorSize =
5547 TTI.getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
5548 unsigned MaxElementsInVector = MaxVectorSize / ElementSize;
5549 if (MaxElementsInVector == 0)
5550 return false;
5551 // When there are multiple shufflevector operations on the same input,
5552 // especially when the vector length is larger than the register size,
5553 // identical shuffle patterns may occur across different groups of elements.
5554 // To avoid overestimating the cost by counting these repeated shuffles more
5555 // than once, we only account for unique shuffle patterns. This adjustment
5556 // prevents inflated costs in the cost model for wide vectors split into
5557 // several register-sized groups.
5558 std::set<SmallVector<int, 4>> UniqueShuffles;
5559 auto AddShuffleMaskAdjustedCost = [&](InstructionCost C, ArrayRef<int> Mask) {
5560 // Compute the cost for performing the shuffle over the full vector.
5561 auto ShuffleCost =
5562 TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: VT, SrcTy: VT, Mask, CostKind);
5563 unsigned NumFullVectors = Mask.size() / MaxElementsInVector;
5564 if (NumFullVectors < 2)
5565 return C + ShuffleCost;
5566 SmallVector<int, 4> SubShuffle(MaxElementsInVector);
5567 unsigned NumUniqueGroups = 0;
5568 unsigned NumGroups = Mask.size() / MaxElementsInVector;
5569 // For each group of MaxElementsInVector contiguous elements,
5570 // collect their shuffle pattern and insert into the set of unique patterns.
5571 for (unsigned I = 0; I < NumFullVectors; ++I) {
5572 for (unsigned J = 0; J < MaxElementsInVector; ++J)
5573 SubShuffle[J] = Mask[MaxElementsInVector * I + J];
5574 if (UniqueShuffles.insert(x: SubShuffle).second)
5575 NumUniqueGroups += 1;
5576 }
5577 return C + ShuffleCost * NumUniqueGroups / NumGroups;
5578 };
5579 auto AddShuffleAdjustedCost = [&](InstructionCost C, Instruction *I) {
5580 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5581 if (!SV)
5582 return C;
5583 SmallVector<int, 16> Mask;
5584 SV->getShuffleMask(Result&: Mask);
5585 return AddShuffleMaskAdjustedCost(C, Mask);
5586 };
5587 // Check that input consists of ShuffleVectors applied to the same input
5588 auto AllShufflesHaveSameOperands =
5589 [](SmallPtrSetImpl<Instruction *> &InputShuffles) {
5590 if (InputShuffles.size() < 2)
5591 return false;
5592 ShuffleVectorInst *FirstSV =
5593 dyn_cast<ShuffleVectorInst>(Val: *InputShuffles.begin());
5594 if (!FirstSV)
5595 return false;
5596
5597 Value *In0 = FirstSV->getOperand(i_nocapture: 0), *In1 = FirstSV->getOperand(i_nocapture: 1);
5598 return std::all_of(
5599 first: std::next(x: InputShuffles.begin()), last: InputShuffles.end(),
5600 pred: [&](Instruction *I) {
5601 ShuffleVectorInst *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5602 return SV && SV->getOperand(i_nocapture: 0) == In0 && SV->getOperand(i_nocapture: 1) == In1;
5603 });
5604 };
5605
5606 // Get the costs of the shuffles + binops before and after with the new
5607 // shuffle masks.
5608 InstructionCost CostBefore =
5609 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: VT, CostKind) +
5610 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: VT, CostKind);
5611 CostBefore += std::accumulate(first: Shuffles.begin(), last: Shuffles.end(),
5612 init: InstructionCost(0), binary_op: AddShuffleCost);
5613 if (AllShufflesHaveSameOperands(InputShuffles)) {
5614 UniqueShuffles.clear();
5615 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5616 init: InstructionCost(0), binary_op: AddShuffleAdjustedCost);
5617 } else {
5618 CostBefore += std::accumulate(first: InputShuffles.begin(), last: InputShuffles.end(),
5619 init: InstructionCost(0), binary_op: AddShuffleCost);
5620 }
5621
5622 // The new binops will be unused for lanes past the used shuffle lengths.
5623 // These types attempt to get the correct cost for that from the target.
5624 FixedVectorType *Op0SmallVT =
5625 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V1.size());
5626 FixedVectorType *Op1SmallVT =
5627 FixedVectorType::get(ElementType: VT->getScalarType(), NumElts: V2.size());
5628 InstructionCost CostAfter =
5629 TTI.getArithmeticInstrCost(Opcode: Op0->getOpcode(), Ty: Op0SmallVT, CostKind) +
5630 TTI.getArithmeticInstrCost(Opcode: Op1->getOpcode(), Ty: Op1SmallVT, CostKind);
5631 UniqueShuffles.clear();
5632 CostAfter += std::accumulate(first: ReconstructMasks.begin(), last: ReconstructMasks.end(),
5633 init: InstructionCost(0), binary_op: AddShuffleMaskAdjustedCost);
5634 std::set<SmallVector<int>> OutputShuffleMasks({V1A, V1B, V2A, V2B});
5635 CostAfter +=
5636 std::accumulate(first: OutputShuffleMasks.begin(), last: OutputShuffleMasks.end(),
5637 init: InstructionCost(0), binary_op: AddShuffleMaskCost);
5638
5639 LLVM_DEBUG(dbgs() << "Found a binop select shuffle pattern: " << I << "\n");
5640 LLVM_DEBUG(dbgs() << " CostBefore: " << CostBefore
5641 << " vs CostAfter: " << CostAfter << "\n");
5642 if (CostBefore < CostAfter ||
5643 (CostBefore == CostAfter && !feedsIntoVectorReduction(SVI)))
5644 return false;
5645
5646 // The cost model has passed, create the new instructions.
5647 auto GetShuffleOperand = [&](Instruction *I, unsigned Op) -> Value * {
5648 auto *SV = dyn_cast<ShuffleVectorInst>(Val: I);
5649 if (!SV)
5650 return I;
5651 if (isa<UndefValue>(Val: SV->getOperand(i_nocapture: 1)))
5652 if (auto *SSV = dyn_cast<ShuffleVectorInst>(Val: SV->getOperand(i_nocapture: 0)))
5653 if (InputShuffles.contains(Ptr: SSV))
5654 return SSV->getOperand(i_nocapture: Op);
5655 return SV->getOperand(i_nocapture: Op);
5656 };
5657 Builder.SetInsertPoint(*SVI0A->getInsertionPointAfterDef());
5658 Value *NSV0A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0A, 0),
5659 V2: GetShuffleOperand(SVI0A, 1), Mask: V1A);
5660 Builder.SetInsertPoint(*SVI0B->getInsertionPointAfterDef());
5661 Value *NSV0B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI0B, 0),
5662 V2: GetShuffleOperand(SVI0B, 1), Mask: V1B);
5663 Builder.SetInsertPoint(*SVI1A->getInsertionPointAfterDef());
5664 Value *NSV1A = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1A, 0),
5665 V2: GetShuffleOperand(SVI1A, 1), Mask: V2A);
5666 Builder.SetInsertPoint(*SVI1B->getInsertionPointAfterDef());
5667 Value *NSV1B = Builder.CreateShuffleVector(V1: GetShuffleOperand(SVI1B, 0),
5668 V2: GetShuffleOperand(SVI1B, 1), Mask: V2B);
5669 Builder.SetInsertPoint(Op0);
5670 Value *NOp0 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op0->getOpcode(),
5671 LHS: NSV0A, RHS: NSV0B);
5672 if (auto *I = dyn_cast<Instruction>(Val: NOp0))
5673 I->copyIRFlags(V: Op0, IncludeWrapFlags: true);
5674 Builder.SetInsertPoint(Op1);
5675 Value *NOp1 = Builder.CreateBinOp(Opc: (Instruction::BinaryOps)Op1->getOpcode(),
5676 LHS: NSV1A, RHS: NSV1B);
5677 if (auto *I = dyn_cast<Instruction>(Val: NOp1))
5678 I->copyIRFlags(V: Op1, IncludeWrapFlags: true);
5679
5680 for (int S = 0, E = ReconstructMasks.size(); S != E; S++) {
5681 Builder.SetInsertPoint(Shuffles[S]);
5682 Value *NSV = Builder.CreateShuffleVector(V1: NOp0, V2: NOp1, Mask: ReconstructMasks[S]);
5683 replaceValue(Old&: *Shuffles[S], New&: *NSV, Erase: false);
5684 }
5685
5686 Worklist.pushValue(V: NSV0A);
5687 Worklist.pushValue(V: NSV0B);
5688 Worklist.pushValue(V: NSV1A);
5689 Worklist.pushValue(V: NSV1B);
5690 return true;
5691}
5692
5693/// Check if instruction depends on ZExt and this ZExt can be moved after the
5694/// instruction. Move ZExt if it is profitable. For example:
5695/// logic(zext(x),y) -> zext(logic(x,trunc(y)))
5696/// lshr((zext(x),y) -> zext(lshr(x,trunc(y)))
5697/// Cost model calculations takes into account if zext(x) has other users and
5698/// whether it can be propagated through them too.
5699bool VectorCombine::shrinkType(Instruction &I) {
5700 Value *ZExted, *OtherOperand;
5701 if (!match(V: &I, P: m_c_BitwiseLogic(L: m_ZExt(Op: m_Value(V&: ZExted)),
5702 R: m_Value(V&: OtherOperand))) &&
5703 !match(V: &I, P: m_LShr(L: m_ZExt(Op: m_Value(V&: ZExted)), R: m_Value(V&: OtherOperand))))
5704 return false;
5705
5706 Value *ZExtOperand = I.getOperand(i: I.getOperand(i: 0) == OtherOperand ? 1 : 0);
5707
5708 auto *BigTy = cast<FixedVectorType>(Val: I.getType());
5709 auto *SmallTy = cast<FixedVectorType>(Val: ZExted->getType());
5710 unsigned BW = SmallTy->getElementType()->getPrimitiveSizeInBits();
5711
5712 if (I.getOpcode() == Instruction::LShr) {
5713 // Check that the shift amount is less than the number of bits in the
5714 // smaller type. Otherwise, the smaller lshr will return a poison value.
5715 KnownBits ShAmtKB = computeKnownBits(V: I.getOperand(i: 1), DL: *DL);
5716 if (ShAmtKB.getMaxValue().uge(RHS: BW))
5717 return false;
5718 } else {
5719 // Check that the expression overall uses at most the same number of bits as
5720 // ZExted
5721 KnownBits KB = computeKnownBits(V: &I, DL: *DL);
5722 if (KB.countMaxActiveBits() > BW)
5723 return false;
5724 }
5725
5726 // Calculate costs of leaving current IR as it is and moving ZExt operation
5727 // later, along with adding truncates if needed
5728 InstructionCost ZExtCost = TTI.getCastInstrCost(
5729 Opcode: Instruction::ZExt, Dst: BigTy, Src: SmallTy,
5730 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5731 InstructionCost CurrentCost = ZExtCost;
5732 InstructionCost ShrinkCost = 0;
5733
5734 // Calculate total cost and check that we can propagate through all ZExt users
5735 for (User *U : ZExtOperand->users()) {
5736 auto *UI = cast<Instruction>(Val: U);
5737 if (UI == &I) {
5738 CurrentCost +=
5739 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5740 ShrinkCost +=
5741 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5742 ShrinkCost += ZExtCost;
5743 continue;
5744 }
5745
5746 if (!Instruction::isBinaryOp(Opcode: UI->getOpcode()))
5747 return false;
5748
5749 // Check if we can propagate ZExt through its other users
5750 KnownBits KB = computeKnownBits(V: UI, DL: *DL);
5751 if (KB.countMaxActiveBits() > BW)
5752 return false;
5753
5754 CurrentCost += TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: BigTy, CostKind);
5755 ShrinkCost +=
5756 TTI.getArithmeticInstrCost(Opcode: UI->getOpcode(), Ty: SmallTy, CostKind);
5757 ShrinkCost += ZExtCost;
5758 }
5759
5760 // If the other instruction operand is not a constant, we'll need to
5761 // generate a truncate instruction. So we have to adjust cost
5762 if (!isa<Constant>(Val: OtherOperand))
5763 ShrinkCost += TTI.getCastInstrCost(
5764 Opcode: Instruction::Trunc, Dst: SmallTy, Src: BigTy,
5765 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5766
5767 // If the cost of shrinking types and leaving the IR is the same, we'll lean
5768 // towards modifying the IR because shrinking opens opportunities for other
5769 // shrinking optimisations.
5770 if (ShrinkCost > CurrentCost)
5771 return false;
5772
5773 Builder.SetInsertPoint(&I);
5774 Value *Op0 = ZExted;
5775 Value *Op1 = Builder.CreateTrunc(V: OtherOperand, DestTy: SmallTy);
5776 // Keep the order of operands the same
5777 if (I.getOperand(i: 0) == OtherOperand)
5778 std::swap(a&: Op0, b&: Op1);
5779 Value *NewBinOp =
5780 Builder.CreateBinOp(Opc: (Instruction::BinaryOps)I.getOpcode(), LHS: Op0, RHS: Op1);
5781 cast<Instruction>(Val: NewBinOp)->copyIRFlags(V: &I);
5782 cast<Instruction>(Val: NewBinOp)->copyMetadata(SrcInst: I);
5783 Value *NewZExtr = Builder.CreateZExt(V: NewBinOp, DestTy: BigTy);
5784 replaceValue(Old&: I, New&: *NewZExtr);
5785 return true;
5786}
5787
5788/// insert (DstVec, (extract SrcVec, ExtIdx), InsIdx) -->
5789/// shuffle (DstVec, SrcVec, Mask)
5790bool VectorCombine::foldInsExtVectorToShuffle(Instruction &I) {
5791 Value *DstVec, *SrcVec;
5792 uint64_t ExtIdx, InsIdx;
5793 if (!match(V: &I,
5794 P: m_InsertElt(Val: m_Value(V&: DstVec),
5795 Elt: m_ExtractElt(Val: m_Value(V&: SrcVec), Idx: m_ConstantInt(V&: ExtIdx)),
5796 Idx: m_ConstantInt(V&: InsIdx))))
5797 return false;
5798
5799 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: I.getType());
5800 auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcVec->getType());
5801 // We can try combining vectors with different element sizes.
5802 if (!DstVecTy || !SrcVecTy ||
5803 SrcVecTy->getElementType() != DstVecTy->getElementType())
5804 return false;
5805
5806 unsigned NumDstElts = DstVecTy->getNumElements();
5807 unsigned NumSrcElts = SrcVecTy->getNumElements();
5808 if (InsIdx >= NumDstElts || ExtIdx >= NumSrcElts || NumDstElts == 1)
5809 return false;
5810
5811 // Insertion into poison is a cheaper single operand shuffle.
5812 TargetTransformInfo::ShuffleKind SK;
5813 SmallVector<int> Mask(NumDstElts, PoisonMaskElem);
5814
5815 bool NeedExpOrNarrow = NumSrcElts != NumDstElts;
5816 bool NeedDstSrcSwap = isa<PoisonValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec);
5817 if (NeedDstSrcSwap) {
5818 SK = TargetTransformInfo::SK_PermuteSingleSrc;
5819 Mask[InsIdx] = ExtIdx % NumDstElts;
5820 std::swap(a&: DstVec, b&: SrcVec);
5821 } else {
5822 SK = TargetTransformInfo::SK_PermuteTwoSrc;
5823 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
5824 Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
5825 }
5826
5827 // Cost
5828 auto *Ins = cast<InsertElementInst>(Val: &I);
5829 auto *Ext = cast<ExtractElementInst>(Val: I.getOperand(i: 1));
5830 InstructionCost InsCost =
5831 TTI.getVectorInstrCost(I: *Ins, Val: DstVecTy, CostKind, Index: InsIdx);
5832 InstructionCost ExtCost =
5833 TTI.getVectorInstrCost(I: *Ext, Val: DstVecTy, CostKind, Index: ExtIdx);
5834 InstructionCost OldCost = ExtCost + InsCost;
5835
5836 InstructionCost NewCost = 0;
5837 SmallVector<int> ExtToVecMask;
5838 if (!NeedExpOrNarrow) {
5839 // Ignore 'free' identity insertion shuffle.
5840 // TODO: getShuffleCost should return TCC_Free for Identity shuffles.
5841 if (!ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts))
5842 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, Mask, CostKind, Index: 0,
5843 SubTp: nullptr, Args: {DstVec, SrcVec});
5844 } else {
5845 // When creating a length-changing-vector, always try to keep the relevant
5846 // element in an equivalent position, so that bulk shuffles are more likely
5847 // to be useful.
5848 ExtToVecMask.assign(NumElts: NumDstElts, Elt: PoisonMaskElem);
5849 ExtToVecMask[ExtIdx % NumDstElts] = ExtIdx;
5850 // Add cost for expanding or narrowing
5851 NewCost = TTI.getShuffleCost(Kind: TargetTransformInfo::SK_PermuteSingleSrc,
5852 DstTy: DstVecTy, SrcTy: SrcVecTy, Mask: ExtToVecMask, CostKind);
5853 NewCost += TTI.getShuffleCost(Kind: SK, DstTy: DstVecTy, SrcTy: DstVecTy, Mask, CostKind);
5854 }
5855
5856 if (!Ext->hasOneUse())
5857 NewCost += ExtCost;
5858
5859 LLVM_DEBUG(dbgs() << "Found a insert/extract shuffle-like pair: " << I
5860 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
5861 << "\n");
5862
5863 if (OldCost < NewCost)
5864 return false;
5865
5866 if (NeedExpOrNarrow) {
5867 if (!NeedDstSrcSwap)
5868 SrcVec = Builder.CreateShuffleVector(V: SrcVec, Mask: ExtToVecMask);
5869 else
5870 DstVec = Builder.CreateShuffleVector(V: DstVec, Mask: ExtToVecMask);
5871 }
5872
5873 // Canonicalize undef param to RHS to help further folds.
5874 if (isa<UndefValue>(Val: DstVec) && !isa<UndefValue>(Val: SrcVec)) {
5875 ShuffleVectorInst::commuteShuffleMask(Mask, InVecNumElts: NumDstElts);
5876 std::swap(a&: DstVec, b&: SrcVec);
5877 }
5878
5879 Value *Shuf = Builder.CreateShuffleVector(V1: DstVec, V2: SrcVec, Mask);
5880 replaceValue(Old&: I, New&: *Shuf);
5881
5882 return true;
5883}
5884
5885/// If we're interleaving 2 constant splats, for instance `<vscale x 8 x i32>
5886/// <splat of 666>` and `<vscale x 8 x i32> <splat of 777>`, we can create a
5887/// larger splat `<vscale x 8 x i64> <splat of ((777 << 32) | 666)>` first
5888/// before casting it back into `<vscale x 16 x i32>`.
5889bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
5890 const APInt *SplatVal0, *SplatVal1;
5891 if (!match(V: &I, P: m_Intrinsic<Intrinsic::vector_interleave2>(
5892 Ops: m_APInt(Res&: SplatVal0), Ops: m_APInt(Res&: SplatVal1))))
5893 return false;
5894
5895 LLVM_DEBUG(dbgs() << "VC: Folding interleave2 with two splats: " << I
5896 << "\n");
5897
5898 auto *VTy =
5899 cast<VectorType>(Val: cast<IntrinsicInst>(Val&: I).getArgOperand(i: 0)->getType());
5900 auto *ExtVTy = VectorType::getExtendedElementVectorType(VTy);
5901 unsigned Width = VTy->getElementType()->getIntegerBitWidth();
5902
5903 // Just in case the cost of interleave2 intrinsic and bitcast are both
5904 // invalid, in which case we want to bail out, we use <= rather
5905 // than < here. Even they both have valid and equal costs, it's probably
5906 // not a good idea to emit a high-cost constant splat.
5907 if (TTI.getInstructionCost(U: &I, CostKind) <=
5908 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: I.getType(), Src: ExtVTy,
5909 CCH: TTI::CastContextHint::None, CostKind)) {
5910 LLVM_DEBUG(dbgs() << "VC: The cost to cast from " << *ExtVTy << " to "
5911 << *I.getType() << " is too high.\n");
5912 return false;
5913 }
5914
5915 APInt NewSplatVal = SplatVal1->zext(width: Width * 2);
5916 NewSplatVal <<= Width;
5917 NewSplatVal |= SplatVal0->zext(width: Width * 2);
5918 auto *NewSplat = ConstantVector::getSplat(
5919 EC: ExtVTy->getElementCount(), Elt: ConstantInt::get(Context&: F.getContext(), V: NewSplatVal));
5920
5921 IRBuilder<> Builder(&I);
5922 replaceValue(Old&: I, New&: *Builder.CreateBitCast(V: NewSplat, DestTy: I.getType()));
5923 return true;
5924}
5925
5926/// Given this sequence:
5927/// ```
5928/// %d = llvm.vector.deinterleave2 <vscale x 16 x i32> %v
5929/// %f0 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 0
5930/// %f1 = extractvalue { <vscale x 8 x i32>, <vscale x 8 x i32> } %d, 1
5931///
5932/// %low0 = and <vscale x 8 x i32> %f0, splat (i32 65535)
5933/// %low1 = shl <vscale x 8 x i32> %f1, splat (i32 16)
5934/// %merge0 = or disjoint <vscale x 8 x i32> %low0, %low1
5935///
5936/// %high0 = and <vscale x 8 x i32> %f1, splat (i32 -65536)
5937/// %high1 = lshr <vscale x 8 x i32> %f0, splat (i32 16)
5938/// %merge1 = or disjoint <vscale x 8 x i32> %high0, %high1
5939/// ```
5940/// It is actually just de-interleaving a 16-bit vector with double the
5941/// vector length. More generally speaking, it's de-interleaving on a vector
5942/// with half the element width as the original vector.
5943///
5944/// Therefore, we can turn it into:
5945/// ```
5946/// %narrow.v = bitcast <vscale x 16 x i32> %v to <vscale x 32 x i16>
5947/// %d = llvm.vector.deinterleave2 <vscale x 32 x i16> %narrow.v
5948/// %f0 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 0
5949/// %f1 = extractvalue { <vscale x 16 x i16>, <vscale x 16 x i16> } %d, 1
5950///
5951/// %merge0 = bitcast <vscale x 16 x i16> %f0 to <vscale x 8 x i32>
5952/// %merge1 = bitcast <vscale x 16 x i16> %f1 to <vscale x 8 x i32>
5953/// ```
5954bool VectorCombine::foldDeinterleaveIntrinsics(Instruction &I) {
5955 // This pattern involves bitcast that is not compatible with big endian.
5956 if (DL->isBigEndian())
5957 return false;
5958
5959 using namespace PatternMatch;
5960 Value *DeinterleavedVal;
5961 if (!match(V: &I, P: m_Deinterleave2(Op: m_Value(V&: DeinterleavedVal))))
5962 return false;
5963
5964 VectorType *VecTy = cast<VectorType>(Val: DeinterleavedVal->getType());
5965 IntegerType *ElementTy = dyn_cast<IntegerType>(Val: VecTy->getElementType());
5966 if (!ElementTy)
5967 return false;
5968 unsigned ElementWidth = ElementTy->getBitWidth();
5969 if (ElementWidth < 2 || !isPowerOf2_32(Value: ElementWidth))
5970 return false;
5971 unsigned HalfElementWidth = ElementWidth / 2;
5972
5973 if (!I.hasNUses(N: 2))
5974 return false;
5975 std::array<ExtractValueInst *, 2> OrigFields{};
5976 for (User *Usr : I.users()) {
5977 auto *E = dyn_cast<ExtractValueInst>(Val: Usr);
5978 // The deinterleave result can only be used by extractions.
5979 if (!E || E->getNumIndices() != 1)
5980 return false;
5981 unsigned Idx = *E->idx_begin();
5982 // A single field cannot be extracted more than once.
5983 if (Idx >= 2 || OrigFields[Idx] || !E->hasNUses(N: 2))
5984 return false;
5985 OrigFields[Idx] = E;
5986 }
5987
5988 // Find the merge instruction (i.e. OR) first.
5989 SmallVector<Instruction *, 2> MergeInsts;
5990 for (auto *FieldUsr : OrigFields[0]->users()) {
5991 if (!FieldUsr->hasOneUse() || !isa<Instruction>(Val: FieldUsr->user_back()))
5992 return false;
5993 MergeInsts.push_back(Elt: cast<Instruction>(Val: FieldUsr->user_back()));
5994 }
5995 assert(MergeInsts.size() == 2);
5996
5997 // Pattern match bottom-up from the merge instructions.
5998 auto MatchMerge = [&](void) -> bool {
5999 APInt LoMask = APInt::getLowBitsSet(numBits: ElementWidth, loBitsSet: HalfElementWidth);
6000 APInt HiMask = APInt::getHighBitsSet(numBits: ElementWidth, hiBitsSet: HalfElementWidth);
6001 return match(V: MergeInsts[0],
6002 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[0]), R: m_SpecificInt(V: LoMask)),
6003 R: m_Shl(L: m_Specific(V: OrigFields[1]),
6004 R: m_SpecificInt(V: HalfElementWidth)))) &&
6005 match(V: MergeInsts[1],
6006 P: m_c_Or(L: m_And(L: m_Specific(V: OrigFields[1]), R: m_SpecificInt(V: HiMask)),
6007 R: m_LShr(L: m_Specific(V: OrigFields[0]),
6008 R: m_SpecificInt(V: HalfElementWidth))));
6009 };
6010 if (!MatchMerge()) {
6011 std::swap(a&: MergeInsts[0], b&: MergeInsts[1]);
6012 if (!MatchMerge())
6013 return false;
6014 }
6015
6016 // Profitability check.
6017 InstructionCost OldCost =
6018 TTI.getInstructionCost(U: MergeInsts[0], CostKind) +
6019 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 0)),
6020 CostKind) +
6021 TTI.getInstructionCost(U: cast<Instruction>(Val: MergeInsts[0]->getOperand(i: 1)),
6022 CostKind);
6023 // There are two fields (assuming SHL has the same cost as LSHR).
6024 OldCost *= 2;
6025
6026 auto *NewFieldTy = VecTy->getWithNewBitWidth(NewBitWidth: HalfElementWidth);
6027 auto *NewVecTy =
6028 VectorType::getDoubleElementsVectorType(VTy: cast<VectorType>(Val: NewFieldTy));
6029 InstructionCost NewCost =
6030 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: VecTy, Src: NewVecTy,
6031 CCH: TTI::CastContextHint::None, CostKind) +
6032 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: NewFieldTy,
6033 Src: MergeInsts[0]->getType(), CCH: TTI::CastContextHint::None,
6034 CostKind) *
6035 2;
6036 if (OldCost <= NewCost || !NewCost.isValid()) {
6037 LLVM_DEBUG(
6038 dbgs() << "VC: New deinterleave2 sequence cost (" << NewCost << ")"
6039 << " is higher than that of the old one (" << OldCost << ")\n");
6040 return false;
6041 }
6042
6043 // Do the replacement.
6044 IRBuilder<> Builder(&I);
6045 Value *NewVecCast = Builder.CreateBitCast(V: DeinterleavedVal, DestTy: NewVecTy);
6046 Value *NewDeinterleave = Builder.CreateIntrinsic(
6047 ID: Intrinsic::vector_deinterleave2, OverloadTypes: {NewVecTy}, Args: {NewVecCast});
6048 for (auto [Idx, MergeInst] : enumerate(First&: MergeInsts)) {
6049 Value *NewField = Builder.CreateExtractValue(Agg: NewDeinterleave, Idxs: Idx);
6050 NewField = Builder.CreateBitCast(V: NewField, DestTy: MergeInst->getType());
6051 replaceValue(Old&: *MergeInst, New&: *NewField);
6052 }
6053
6054 return true;
6055}
6056
6057bool VectorCombine::foldBitcastOfVPLoad(Instruction &I) {
6058 const DataLayout &DL = I.getDataLayout();
6059 auto *Cast = dyn_cast<CastInst>(Val: &I);
6060 if (!Cast || !Cast->isNoopCast(DL) || !isa<VectorType>(Val: Cast->getDestTy()))
6061 return false;
6062
6063 // Fold away bit casts of the loaded value by loading the desired type,
6064 // if the mask is all-ones.
6065 Value *EVL;
6066 auto *II = dyn_cast<VPIntrinsic>(Val: I.getOperand(i: 0));
6067 if (!II || !match(V: II, P: m_OneUse(SubPattern: m_Intrinsic<Intrinsic::vp_load>(
6068 Ops: m_Value(), Ops: m_AllOnes(), Ops: m_Value(V&: EVL)))))
6069 return false;
6070
6071 VectorType *OrigVecTy = cast<VectorType>(Val: II->getType());
6072 Align OrigAlign =
6073 DL.getValueOrABITypeAlignment(Alignment: II->getPointerAlignment(), Ty: OrigVecTy);
6074 ElementCount OrigVecCnt = OrigVecTy->getElementCount();
6075 VectorType *NewVecTy = cast<VectorType>(Val: Cast->getDestTy());
6076 ElementCount NewVecCnt = NewVecTy->getElementCount();
6077
6078 // Right now we only support cases where the NewVec is longer, because for
6079 // cases where it's shorter, we have to be sure that EVL can be exactly
6080 // divided, otherwise it might yield incorrect results or even page faults
6081 // (if we round-up during the division).
6082 if (!(OrigVecCnt.isScalable() == NewVecCnt.isScalable() &&
6083 NewVecCnt.hasKnownScalarFactor(RHS: OrigVecCnt)))
6084 return false;
6085
6086 InstructionCost OldCost =
6087 TTI.getMemIntrinsicInstrCost(MICA: {Intrinsic::vp_load, OrigVecTy,
6088 II->getMemoryPointerParam(), false,
6089 OrigAlign},
6090 CostKind) +
6091 TTI.getCastInstrCost(Opcode: Instruction::BitCast, Dst: Cast->getType(), Src: OrigVecTy,
6092 CCH: TTI::CastContextHint::None, CostKind);
6093 InstructionCost NewCost = TTI.getMemIntrinsicInstrCost(
6094 MICA: {Intrinsic::vp_load, NewVecTy, II->getMemoryPointerParam(), false,
6095 OrigAlign},
6096 CostKind);
6097 LLVM_DEBUG(dbgs() << "foldBitcastOfVPLoad: OldCost=" << OldCost
6098 << " NewCost=" << NewCost << "\n");
6099 if (NewCost > OldCost || !NewCost.isValid())
6100 return false;
6101
6102 unsigned Factor = NewVecCnt.getKnownScalarFactor(RHS: OrigVecCnt);
6103 Value *NewEVL = Builder.CreateNUWMul(LHS: EVL, RHS: Builder.getInt32(C: Factor));
6104 Value *NewMask = Builder.CreateVectorSplat(EC: NewVecCnt, V: Builder.getTrue());
6105 CallInst *NewVP = Builder.CreateIntrinsicWithoutFolding(
6106 RetTy: NewVecTy, ID: Intrinsic::vp_load,
6107 Args: {II->getMemoryPointerParam(), NewMask, NewEVL});
6108 // Preserve the original alignment.
6109 NewVP->addParamAttrs(
6110 ArgNo: 0, B: AttrBuilder(II->getContext()).addAlignmentAttr(Align: OrigAlign));
6111 replaceValue(Old&: *Cast, New&: *NewVP);
6112 return true;
6113}
6114/// Fold the following cases into a single byte-level bit-reverse operation
6115/// and accepts bswap and bitreverse intrinsics:
6116/// bswap(bitreverse(x)) <--> bitcast(bitreverse(bitcast(x)))
6117/// bitreverse(bswap(x)) <--> bitcast(bitreverse(bitcast(x)))
6118/// The direction of the fold is cost-model driven.
6119bool VectorCombine::foldBitOrderReverseAndSwap(Instruction &I) {
6120 Value *X;
6121
6122 if (match(V: &I, P: m_BitCast(Op: m_BitReverse(Op0: m_BitCast(Op: m_Value(V&: X)))))) {
6123 Type *Ty = X->getType();
6124 Type *VecTy = I.getOperand(i: 0)->getType();
6125 if (Ty->isIntegerTy() && Ty == I.getType() && isa<FixedVectorType>(Val: VecTy) &&
6126 cast<FixedVectorType>(Val: VecTy)->getElementType()->isIntegerTy(BitWidth: 8) &&
6127 Ty->getIntegerBitWidth() % 16 == 0) {
6128 auto *InnerCall = dyn_cast<Instruction>(Val: I.getOperand(i: 0));
6129 if (!InnerCall)
6130 return false;
6131 auto *InnerBitCast = dyn_cast<BitCastInst>(Val: InnerCall->getOperand(i: 0));
6132 if (!InnerBitCast)
6133 return false;
6134 InstructionCost OldCost = TTI.getInstructionCost(U: InnerBitCast, CostKind) +
6135 TTI.getInstructionCost(U: InnerCall, CostKind) +
6136 TTI.getInstructionCost(U: &I, CostKind);
6137 IntrinsicCostAttributes ICABSwap(Intrinsic::bswap, Ty, {Ty});
6138 IntrinsicCostAttributes ICABRev(Intrinsic::bitreverse, Ty, {Ty});
6139 InstructionCost NewCost = TTI.getIntrinsicInstrCost(ICA: ICABSwap, CostKind) +
6140 TTI.getIntrinsicInstrCost(ICA: ICABRev, CostKind);
6141 if (!InnerCall->hasOneUse())
6142 NewCost += TTI.getInstructionCost(U: InnerCall, CostKind) +
6143 TTI.getInstructionCost(U: InnerBitCast, CostKind);
6144 else if (!InnerBitCast->hasOneUse())
6145 NewCost += TTI.getInstructionCost(U: InnerBitCast, CostKind);
6146 LLVM_DEBUG(dbgs() << "Found bitreverse vector roundtrip: " << I
6147 << "\n OldCost: " << OldCost
6148 << " vs NewCost: " << NewCost << "\n");
6149 if (NewCost.isValid() && NewCost < OldCost) {
6150 Builder.SetInsertPoint(&I);
6151 Value *BSwap = Builder.CreateUnaryIntrinsic(ID: Intrinsic::bswap, Op: X);
6152 Worklist.pushValue(V: BSwap);
6153 Value *BRev =
6154 Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: BSwap);
6155 replaceValue(Old&: I, New&: *BRev);
6156 return true;
6157 }
6158 }
6159 }
6160
6161 if (!match(V: &I, P: m_BitReverse(Op0: m_BSwap(Op0: m_Value(V&: X)))) &&
6162 !match(V: &I, P: m_BSwap(Op0: m_BitReverse(Op0: m_Value(V&: X)))))
6163 return false;
6164 Type *Ty = I.getType();
6165 Type *I8Ty = Builder.getInt8Ty();
6166 TypeSize ElementSize = DL->getTypeStoreSize(Ty);
6167 ElementCount NewVecCnt = ElementCount::get(MinVal: ElementSize.getKnownMinValue(),
6168 Scalable: ElementSize.isScalable());
6169 Type *NewVecTy = VectorType::get(ElementType: I8Ty, EC: NewVecCnt);
6170 auto *II = cast<IntrinsicInst>(Val: &I);
6171 auto *InnerII = cast<IntrinsicInst>(Val: II->getArgOperand(i: 0));
6172 // OldCost = cost of bitreverse/bswap + cost of bswap/bitreverse
6173 InstructionCost OldCost = TTI.getInstructionCost(U: II, CostKind) +
6174 TTI.getInstructionCost(U: InnerII, CostKind);
6175 // NewCost = cost of bitcast to byte vector +
6176 // cost of bitreverse/bswap on byte vector +
6177 // cost of bitcast back to original type
6178 InstructionCost CastToVecCost = TTI.getCastInstrCost(
6179 Opcode: Instruction::BitCast, Dst: NewVecTy, Src: Ty, CCH: TTI::CastContextHint::None, CostKind);
6180 InstructionCost CastToOrigCost = TTI.getCastInstrCost(
6181 Opcode: Instruction::BitCast, Dst: Ty, Src: NewVecTy, CCH: TTI::CastContextHint::None, CostKind);
6182 IntrinsicCostAttributes ICANew(Intrinsic::bitreverse, NewVecTy, {NewVecTy});
6183 InstructionCost NewIntrinsicCost =
6184 TTI.getIntrinsicInstrCost(ICA: ICANew, CostKind);
6185 InstructionCost NewCost = CastToVecCost + NewIntrinsicCost + CastToOrigCost;
6186 if (!InnerII->hasOneUse())
6187 NewCost += TTI.getInstructionCost(U: InnerII, CostKind);
6188 LLVM_DEBUG(dbgs() << "Found bitorder reverse and swap: " << I
6189 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6190 << "\n");
6191 if (!NewCost.isValid() || NewCost >= OldCost)
6192 return false;
6193 // Perform transform: bitcast(arg, <N x i8>), bitreverse, bitcast back
6194 Builder.SetInsertPoint(II);
6195 Value *CastToVec = Builder.CreateBitCast(V: X, DestTy: NewVecTy);
6196 Value *NewCall =
6197 Builder.CreateUnaryIntrinsic(ID: Intrinsic::bitreverse, Op: CastToVec);
6198 Value *CastToOrig = Builder.CreateBitCast(V: NewCall, DestTy: Ty);
6199 replaceValue(Old&: I, New&: *CastToOrig);
6200 return true;
6201}
6202// Attempt to shrink loads that are only used by shufflevector instructions.
6203bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
6204 auto *OldLoad = dyn_cast<LoadInst>(Val: &I);
6205 if (!OldLoad || !OldLoad->isSimple())
6206 return false;
6207
6208 auto *OldLoadTy = dyn_cast<FixedVectorType>(Val: OldLoad->getType());
6209 if (!OldLoadTy)
6210 return false;
6211
6212 unsigned const OldNumElements = OldLoadTy->getNumElements();
6213
6214 // Search all uses of load. If all uses are shufflevector instructions, and
6215 // the second operands are all poison values, find the minimum and maximum
6216 // indices of the vector elements referenced by all shuffle masks.
6217 // Otherwise return `std::nullopt`.
6218 using IndexRange = std::pair<int, int>;
6219 auto GetIndexRangeInShuffles = [&]() -> std::optional<IndexRange> {
6220 IndexRange OutputRange = IndexRange(OldNumElements, -1);
6221 for (llvm::Use &Use : I.uses()) {
6222 // Ensure all uses match the required pattern.
6223 User *Shuffle = Use.getUser();
6224 ArrayRef<int> Mask;
6225
6226 if (!match(V: Shuffle,
6227 P: m_Shuffle(v1: m_Specific(V: OldLoad), v2: m_Undef(), mask: m_Mask(Mask))))
6228 return std::nullopt;
6229
6230 // Ignore shufflevector instructions that have no uses.
6231 if (Shuffle->use_empty())
6232 continue;
6233
6234 // Find the min and max indices used by the shufflevector instruction.
6235 for (int Index : Mask) {
6236 if (Index >= 0 && Index < static_cast<int>(OldNumElements)) {
6237 OutputRange.first = std::min(a: Index, b: OutputRange.first);
6238 OutputRange.second = std::max(a: Index, b: OutputRange.second);
6239 }
6240 }
6241 }
6242
6243 if (OutputRange.second < OutputRange.first)
6244 return std::nullopt;
6245
6246 return OutputRange;
6247 };
6248
6249 // Get the range of vector elements used by shufflevector instructions.
6250 if (std::optional<IndexRange> Indices = GetIndexRangeInShuffles()) {
6251 unsigned const NewNumElements = Indices->second + 1u;
6252
6253 // If the range of vector elements is smaller than the full load, attempt
6254 // to create a smaller load.
6255 if (NewNumElements < OldNumElements) {
6256 IRBuilder Builder(&I);
6257 Builder.SetCurrentDebugLocation(I.getDebugLoc());
6258
6259 // Calculate costs of old and new ops.
6260 Type *ElemTy = OldLoadTy->getElementType();
6261 FixedVectorType *NewLoadTy = FixedVectorType::get(ElementType: ElemTy, NumElts: NewNumElements);
6262 Value *PtrOp = OldLoad->getPointerOperand();
6263
6264 InstructionCost OldCost = TTI.getMemoryOpCost(
6265 Opcode: Instruction::Load, Src: OldLoad->getType(), Alignment: OldLoad->getAlign(),
6266 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6267 InstructionCost NewCost =
6268 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: NewLoadTy, Alignment: OldLoad->getAlign(),
6269 AddressSpace: OldLoad->getPointerAddressSpace(), CostKind);
6270
6271 using UseEntry = std::pair<ShuffleVectorInst *, std::vector<int>>;
6272 SmallVector<UseEntry, 4u> NewUses;
6273 unsigned const MaxIndex = NewNumElements * 2u;
6274
6275 for (llvm::Use &Use : I.uses()) {
6276 auto *Shuffle = cast<ShuffleVectorInst>(Val: Use.getUser());
6277
6278 // Ignore shufflevector instructions that have no uses.
6279 if (Shuffle->use_empty())
6280 continue;
6281
6282 ArrayRef<int> OldMask = Shuffle->getShuffleMask();
6283
6284 // Create entry for new use.
6285 NewUses.push_back(Elt: {Shuffle, OldMask});
6286
6287 // Validate mask indices.
6288 for (int Index : OldMask) {
6289 if (Index >= static_cast<int>(MaxIndex))
6290 return false;
6291 }
6292
6293 // Update costs.
6294 OldCost +=
6295 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6296 SrcTy: OldLoadTy, Mask: OldMask, CostKind);
6297 NewCost +=
6298 TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: Shuffle->getType(),
6299 SrcTy: NewLoadTy, Mask: OldMask, CostKind);
6300 }
6301
6302 LLVM_DEBUG(
6303 dbgs() << "Found a load used only by shufflevector instructions: "
6304 << I << "\n OldCost: " << OldCost
6305 << " vs NewCost: " << NewCost << "\n");
6306
6307 if (OldCost < NewCost || !NewCost.isValid())
6308 return false;
6309
6310 // Create new load of smaller vector.
6311 auto *NewLoad = cast<LoadInst>(
6312 Val: Builder.CreateAlignedLoad(Ty: NewLoadTy, Ptr: PtrOp, Align: OldLoad->getAlign()));
6313 NewLoad->copyMetadata(SrcInst: I);
6314
6315 // Replace all uses.
6316 for (UseEntry &Use : NewUses) {
6317 ShuffleVectorInst *Shuffle = Use.first;
6318 std::vector<int> &NewMask = Use.second;
6319
6320 Builder.SetInsertPoint(Shuffle);
6321 Builder.SetCurrentDebugLocation(Shuffle->getDebugLoc());
6322 Value *NewShuffle = Builder.CreateShuffleVector(
6323 V1: NewLoad, V2: PoisonValue::get(T: NewLoadTy), Mask: NewMask);
6324
6325 replaceValue(Old&: *Shuffle, New&: *NewShuffle, Erase: false);
6326 }
6327
6328 return true;
6329 }
6330 }
6331 return false;
6332}
6333
6334// Attempt to narrow a phi of shufflevector instructions where the two incoming
6335// values have the same operands but different masks. If the two shuffle masks
6336// are offsets of one another we can use one branch to rotate the incoming
6337// vector and perform one larger shuffle after the phi.
6338bool VectorCombine::shrinkPhiOfShuffles(Instruction &I) {
6339 auto *Phi = dyn_cast<PHINode>(Val: &I);
6340 if (!Phi || Phi->getNumIncomingValues() != 2u)
6341 return false;
6342
6343 Value *Op = nullptr;
6344 ArrayRef<int> Mask0;
6345 ArrayRef<int> Mask1;
6346
6347 if (!match(V: Phi->getOperand(i_nocapture: 0u),
6348 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Value(V&: Op), v2: m_Poison(), mask: m_Mask(Mask0)))) ||
6349 !match(V: Phi->getOperand(i_nocapture: 1u),
6350 P: m_OneUse(SubPattern: m_Shuffle(v1: m_Specific(V: Op), v2: m_Poison(), mask: m_Mask(Mask1)))))
6351 return false;
6352
6353 auto *Shuf = cast<ShuffleVectorInst>(Val: Phi->getOperand(i_nocapture: 0u));
6354
6355 // Ensure result vectors are wider than the argument vector.
6356 auto *InputVT = cast<FixedVectorType>(Val: Op->getType());
6357 auto *ResultVT = cast<FixedVectorType>(Val: Shuf->getType());
6358 auto const InputNumElements = InputVT->getNumElements();
6359
6360 if (InputNumElements >= ResultVT->getNumElements())
6361 return false;
6362
6363 // Take the difference of the two shuffle masks at each index. Ignore poison
6364 // values at the same index in both masks.
6365 SmallVector<int, 16> NewMask;
6366 NewMask.reserve(N: Mask0.size());
6367
6368 for (auto [M0, M1] : zip(t&: Mask0, u&: Mask1)) {
6369 if (M0 >= 0 && M1 >= 0)
6370 NewMask.push_back(Elt: M0 - M1);
6371 else if (M0 == -1 && M1 == -1)
6372 continue;
6373 else
6374 return false;
6375 }
6376
6377 // Ensure all elements of the new mask are equal. If the difference between
6378 // the incoming mask elements is the same, the two must be constant offsets
6379 // of one another.
6380 if (NewMask.empty() || !all_equal(Range&: NewMask))
6381 return false;
6382
6383 // Create new mask using difference of the two incoming masks.
6384 int MaskOffset = NewMask[0u];
6385 unsigned Index = (InputNumElements + MaskOffset) % InputNumElements;
6386 NewMask.clear();
6387
6388 for (unsigned I = 0u; I < InputNumElements; ++I) {
6389 NewMask.push_back(Elt: Index);
6390 Index = (Index + 1u) % InputNumElements;
6391 }
6392
6393 // Calculate costs for worst cases and compare.
6394 auto const Kind = TTI::SK_PermuteSingleSrc;
6395 auto OldCost =
6396 std::max(a: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, Mask: Mask0, CostKind),
6397 b: TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, Mask: Mask1, CostKind));
6398 auto NewCost = TTI.getShuffleCost(Kind, DstTy: InputVT, SrcTy: InputVT, Mask: NewMask, CostKind) +
6399 TTI.getShuffleCost(Kind, DstTy: ResultVT, SrcTy: InputVT, Mask: Mask1, CostKind);
6400
6401 LLVM_DEBUG(dbgs() << "Found a phi of mergeable shuffles: " << I
6402 << "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
6403 << "\n");
6404
6405 if (NewCost > OldCost)
6406 return false;
6407
6408 // Create new shuffles and narrowed phi.
6409 auto Builder = IRBuilder(Shuf);
6410 Builder.SetCurrentDebugLocation(Shuf->getDebugLoc());
6411 auto *PoisonVal = PoisonValue::get(T: InputVT);
6412 auto *NewShuf0 = Builder.CreateShuffleVector(V1: Op, V2: PoisonVal, Mask: NewMask);
6413 Worklist.push(I: cast<Instruction>(Val: NewShuf0));
6414
6415 Builder.SetInsertPoint(Phi);
6416 Builder.SetCurrentDebugLocation(Phi->getDebugLoc());
6417 auto *NewPhi = Builder.CreatePHI(Ty: NewShuf0->getType(), NumReservedValues: 2u);
6418 NewPhi->addIncoming(V: NewShuf0, BB: Phi->getIncomingBlock(i: 0u));
6419 NewPhi->addIncoming(V: Op, BB: Phi->getIncomingBlock(i: 1u));
6420
6421 Builder.SetInsertPoint(*NewPhi->getInsertionPointAfterDef());
6422 PoisonVal = PoisonValue::get(T: NewPhi->getType());
6423 auto *NewShuf1 = Builder.CreateShuffleVector(V1: NewPhi, V2: PoisonVal, Mask: Mask1);
6424
6425 replaceValue(Old&: *Phi, New&: *NewShuf1);
6426 return true;
6427}
6428
6429/// This is the entry point for all transforms. Pass manager differences are
6430/// handled in the callers of this function.
6431bool VectorCombine::run() {
6432 if (DisableVectorCombine)
6433 return false;
6434
6435 // Don't attempt vectorization if the target does not support vectors.
6436 if (!TTI.getNumberOfRegisters(ClassID: TTI.getRegisterClassForType(/*Vector*/ true)))
6437 return false;
6438
6439 LLVM_DEBUG(dbgs() << "\n\nVECTORCOMBINE on " << F.getName() << "\n");
6440
6441 auto FoldInst = [this](Instruction &I) {
6442 Builder.SetInsertPoint(&I);
6443 bool IsVectorType = isa<VectorType>(Val: I.getType());
6444 bool IsFixedVectorType = isa<FixedVectorType>(Val: I.getType());
6445 auto Opcode = I.getOpcode();
6446
6447 LLVM_DEBUG(dbgs() << "VC: Visiting: " << I << '\n');
6448
6449 // These folds should be beneficial regardless of when this pass is run
6450 // in the optimization pipeline.
6451 // The type checking is for run-time efficiency. We can avoid wasting time
6452 // dispatching to folding functions if there's no chance of matching.
6453 if (IsFixedVectorType) {
6454 switch (Opcode) {
6455 case Instruction::InsertElement:
6456 if (vectorizeLoadInsert(I))
6457 return true;
6458 break;
6459 case Instruction::ShuffleVector:
6460 if (widenSubvectorLoad(I))
6461 return true;
6462 break;
6463 default:
6464 break;
6465 }
6466 }
6467
6468 // This transform works with scalable and fixed vectors
6469 // TODO: Identify and allow other scalable transforms
6470 if (IsVectorType) {
6471 if (scalarizeOpOrCmp(I))
6472 return true;
6473 if (scalarizeLoad(I))
6474 return true;
6475 if (scalarizeExtExtract(I))
6476 return true;
6477 if (scalarizeVPIntrinsic(I))
6478 return true;
6479 if (foldInterleaveIntrinsics(I))
6480 return true;
6481 if (foldBitcastOfVPLoad(I))
6482 return true;
6483 }
6484
6485 if (foldDeinterleaveIntrinsics(I))
6486 return true;
6487
6488 if (Opcode == Instruction::Store)
6489 if (foldSingleElementStore(I))
6490 return true;
6491
6492 // If this is an early pipeline invocation of this pass, we are done.
6493 if (TryEarlyFoldsOnly)
6494 return false;
6495
6496 if (Opcode == Instruction::Call)
6497 if (foldBitOrderReverseAndSwap(I))
6498 return true;
6499 if (Opcode == Instruction::BitCast)
6500 if (foldBitOrderReverseAndSwap(I))
6501 return true;
6502
6503 // Otherwise, try folds that improve codegen but may interfere with
6504 // early IR canonicalizations.
6505 // The type checking is for run-time efficiency. We can avoid wasting time
6506 // dispatching to folding functions if there's no chance of matching.
6507 if (IsFixedVectorType) {
6508 switch (Opcode) {
6509 case Instruction::InsertElement:
6510 if (foldInsExtFNeg(I))
6511 return true;
6512 if (foldInsExtBinop(I))
6513 return true;
6514 if (foldInsExtVectorToShuffle(I))
6515 return true;
6516 break;
6517 case Instruction::ShuffleVector:
6518 if (foldPermuteOfBinops(I))
6519 return true;
6520 if (foldShuffleOfBinops(I))
6521 return true;
6522 if (foldShuffleOfSelects(I))
6523 return true;
6524 if (foldShuffleOfCastops(I))
6525 return true;
6526 if (foldShuffleOfShuffles(I))
6527 return true;
6528 if (foldPermuteOfIntrinsic(I))
6529 return true;
6530 if (foldShufflesOfLengthChangingShuffles(I))
6531 return true;
6532 if (foldShuffleOfIntrinsics(I))
6533 return true;
6534 if (foldSelectShuffle(I))
6535 return true;
6536 if (foldShuffleToIdentity(I))
6537 return true;
6538 break;
6539 case Instruction::Load:
6540 if (shrinkLoadForShuffles(I))
6541 return true;
6542 break;
6543 case Instruction::BitCast:
6544 if (foldBitcastShuffle(I))
6545 return true;
6546 if (foldSelectsFromBitcast(I))
6547 return true;
6548 break;
6549 case Instruction::And:
6550 case Instruction::Or:
6551 case Instruction::Xor:
6552 if (foldBitOpOfCastops(I))
6553 return true;
6554 if (foldBitOpOfCastConstant(I))
6555 return true;
6556 break;
6557 case Instruction::PHI:
6558 if (shrinkPhiOfShuffles(I))
6559 return true;
6560 break;
6561 default:
6562 if (shrinkType(I))
6563 return true;
6564 break;
6565 }
6566 } else {
6567 switch (Opcode) {
6568 case Instruction::Call:
6569 if (foldShuffleFromReductions(I))
6570 return true;
6571 if (foldCastFromReductions(I))
6572 return true;
6573 break;
6574 case Instruction::ExtractElement:
6575 if (foldShuffleChainsToReduce(I))
6576 return true;
6577 break;
6578 case Instruction::ICmp:
6579 if (foldSignBitReductionCmp(I))
6580 return true;
6581 if (foldICmpEqZeroVectorReduce(I))
6582 return true;
6583 if (foldReductionZeroTest(I))
6584 return true;
6585 if (foldEquivalentReductionCmp(I))
6586 return true;
6587 if (foldReduceAddCmpZero(I))
6588 return true;
6589 [[fallthrough]];
6590 case Instruction::FCmp:
6591 if (foldExtractExtract(I))
6592 return true;
6593 break;
6594 case Instruction::Or:
6595 if (foldConcatOfBoolMasks(I))
6596 return true;
6597 [[fallthrough]];
6598 default:
6599 if (Instruction::isBinaryOp(Opcode)) {
6600 if (foldExtractExtract(I))
6601 return true;
6602 if (foldExtractedCmps(I))
6603 return true;
6604 if (foldBinopOfReductions(I))
6605 return true;
6606 }
6607 break;
6608 }
6609 }
6610 return false;
6611 };
6612
6613 bool MadeChange = false;
6614 for (BasicBlock &BB : F) {
6615 // Ignore unreachable basic blocks.
6616 if (!DT.isReachableFromEntry(A: &BB))
6617 continue;
6618 // Use early increment range so that we can erase instructions in loop.
6619 // make_early_inc_range is not applicable here, as the next iterator may
6620 // be invalidated by RecursivelyDeleteTriviallyDeadInstructions.
6621 // We manually maintain the next instruction and update it when it is about
6622 // to be deleted.
6623 Instruction *I = &BB.front();
6624 while (I) {
6625 NextInst = I->getNextNode();
6626 if (!I->isDebugOrPseudoInst())
6627 MadeChange |= FoldInst(*I);
6628 I = NextInst;
6629 }
6630 }
6631
6632 NextInst = nullptr;
6633
6634 while (!Worklist.isEmpty()) {
6635 Instruction *I = Worklist.removeOne();
6636 if (!I)
6637 continue;
6638
6639 if (isInstructionTriviallyDead(I)) {
6640 eraseInstruction(I&: *I);
6641 continue;
6642 }
6643
6644 MadeChange |= FoldInst(*I);
6645 }
6646
6647 return MadeChange;
6648}
6649
6650PreservedAnalyses VectorCombinePass::run(Function &F,
6651 FunctionAnalysisManager &FAM) {
6652 auto &AC = FAM.getResult<AssumptionAnalysis>(IR&: F);
6653 TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(IR&: F);
6654 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(IR&: F);
6655 AAResults &AA = FAM.getResult<AAManager>(IR&: F);
6656 const DataLayout *DL = &F.getDataLayout();
6657 TTI::TargetCostKind CostKind =
6658 F.hasOptSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput;
6659 VectorCombine Combiner(F, TTI, DT, AA, AC, DL, CostKind, TryEarlyFoldsOnly);
6660 if (!Combiner.run())
6661 return PreservedAnalyses::all();
6662 PreservedAnalyses PA;
6663 PA.preserveSet<CFGAnalyses>();
6664 return PA;
6665}
6666